diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9918875d9..addafb9f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,9 +7,16 @@ on: permissions: read-all jobs: + eval: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: cachix/install-nix-action@v30 + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json tests: - needs: [check_secrets] strategy: fail-fast: false matrix: @@ -23,87 +30,24 @@ jobs: - uses: cachix/install-nix-action@v30 with: # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: "sandbox = true" - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' + extra_nix_config: | + sandbox = true + max-jobs = 1 + - uses: DeterminateSystems/magic-nix-cache-action@main + # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: + # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces + - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + if: matrix.os == 'ubuntu-latest' + - run: scripts/build-checks + - run: scripts/prepare-installer-for-github-actions + - name: Upload installer tarball + uses: actions/upload-artifact@v4 with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - if: matrix.os == 'ubuntu-latest' - run: | - free -h - swapon --show - swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') - echo "Found swap: $swap" - sudo swapoff $swap - # resize it (fallocate) - sudo fallocate -l 10G $swap - sudo mkswap $swap - sudo swapon $swap - free -h - ( - while sleep 60; do - free -h - done - ) & - - run: nix --experimental-features 'nix-command flakes' flake check -L - - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json - - # Steps to test CI automation in your own fork. - # Cachix: - # 1. Sign-up for https://www.cachix.org/ - # 2. Create a cache for $githubuser-nix-install-tests - # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN - # Dockerhub: - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check Cachix and Docker secrets present for installer tests - runs-on: ubuntu-latest - outputs: - cachix: ${{ steps.secret.outputs.cachix }} - docker: ${{ steps.secret.outputs.docker }} - steps: - - name: Check for secrets - id: secret - env: - _CACHIX_SECRETS: ${{ secrets.CACHIX_SIGNING_KEY }}${{ secrets.CACHIX_AUTH_TOKEN }} - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "::set-output name=cachix::${{ env._CACHIX_SECRETS != '' }}" - echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - - installer: - needs: [tests, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' - runs-on: ubuntu-latest - outputs: - installerURL: ${{ steps.prepare-installer.outputs.installerURL }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: cachix/cachix-action@v15 - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - cachixArgs: '-v' - - id: prepare-installer - run: scripts/prepare-installer-for-github-actions + name: installer-${{matrix.os}} + path: out/* installer_test: - needs: [installer, check_secrets] - if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' + needs: [tests] strategy: fail-fast: false matrix: @@ -111,11 +55,18 @@ jobs: runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - name: Download installer tarball + uses: actions/download-artifact@v4 + with: + name: installer-${{matrix.os}} + path: out + - name: Serving installer + id: serving_installer + run: ./scripts/serve-installer-for-github-actions - uses: cachix/install-nix-action@v30 with: - install_url: '${{needs.installer.outputs.installerURL}}' - install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" + install_url: 'http://localhost:8126/install' + install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-latest' - run: brew install fish @@ -127,32 +78,50 @@ jobs: - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + # Steps to test CI automation in your own fork. + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork + check_secrets: + permissions: + contents: none + name: Check Docker secrets present for installer tests + runs-on: ubuntu-latest + outputs: + docker: ${{ steps.secret.outputs.docker }} + steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" + docker_push_image: - needs: [check_secrets, tests, vm_tests] + needs: [tests, vm_tests, check_secrets] permissions: contents: read packages: write if: >- + needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && - github.ref_name == 'master' && - needs.check_secrets.outputs.cachix == 'true' && - needs.check_secrets.outputs.docker == 'true' + github.ref_name == 'master' runs-on: ubuntu-latest steps: + - name: Check for secrets + id: secret + env: + _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} + run: | + echo "::set-output name=docker::${{ env._DOCKER_SECRETS != '' }}" - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV + - uses: DeterminateSystems/magic-nix-cache-action@main - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v15 - if: needs.check_secrets.outputs.cachix == 'true' - with: - name: '${{ env.CACHIX_NAME }}' - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION diff --git a/.gitignore b/.gitignore index de1183977..337a7c154 100644 --- a/.gitignore +++ b/.gitignore @@ -1,110 +1,12 @@ -Makefile.config -perl/Makefile.config - -# / -/aclocal.m4 -/autom4te.cache -/precompiled-headers.h.gch -/config.* -/configure -/stamp-h1 -/svn-revision -/libtool -/config/config.* # Default meson build dir /build -# /doc/manual/ -/doc/manual/*.1 -/doc/manual/*.5 -/doc/manual/*.8 -/doc/manual/generated/* -/doc/manual/nix.json -/doc/manual/conf-file.json -/doc/manual/language.json -/doc/manual/xp-features.json -/doc/manual/source/SUMMARY.md -/doc/manual/source/SUMMARY-rl-next.md -/doc/manual/source/store/types/* -!/doc/manual/source/store/types/index.md.in -/doc/manual/source/command-ref/new-cli -/doc/manual/source/command-ref/conf-file.md -/doc/manual/source/command-ref/experimental-features-shortlist.md -/doc/manual/source/contributing/experimental-feature-descriptions.md -/doc/manual/source/language/builtins.md -/doc/manual/source/language/builtin-constants.md -/doc/manual/source/release-notes/rl-next.md - -# /scripts/ -/scripts/nix-profile.sh -/scripts/nix-profile-daemon.sh -/scripts/nix-profile.fish -/scripts/nix-profile-daemon.fish - -# /src/libexpr/ -/src/libexpr/lexer-tab.cc -/src/libexpr/lexer-tab.hh -/src/libexpr/parser-tab.cc -/src/libexpr/parser-tab.hh -/src/libexpr/parser-tab.output -/src/libexpr/nix.tbl -/src/libexpr/tests -/src/libexpr-tests/libnixexpr-tests - -# /src/libfetchers -/src/libfetchers-tests/libnixfetchers-tests - -# /src/libflake -/src/libflake-tests/libnixflake-tests - -# /src/libstore/ -*.gen.* -/src/libstore/tests -/src/libstore-tests/libnixstore-tests - -# /src/libutil/ -/src/libutil/tests -/src/libutil-tests/libnixutil-tests - -/src/nix/nix - -/src/nix/generated-doc - -# /src/nix-env/ -/src/nix-env/nix-env - -# /src/nix-instantiate/ -/src/nix-instantiate/nix-instantiate - -# /src/nix-store/ -/src/nix-store/nix-store - -/src/nix-prefetch-url/nix-prefetch-url - -/src/nix-collect-garbage/nix-collect-garbage - -# /src/nix-channel/ -/src/nix-channel/nix-channel - -# /src/nix-build/ -/src/nix-build/nix-build - -/src/nix-copy-closure/nix-copy-closure - -/src/error-demo/error-demo - -/src/build-remote/build-remote - # /tests/functional/ -/tests/functional/test-tmp /tests/functional/common/subst-vars.sh -/tests/functional/result* /tests/functional/restricted-innocent -/tests/functional/shell -/tests/functional/shell.drv -/tests/functional/repl-result-out /tests/functional/debugger-test-out /tests/functional/test-libstoreconsumer/test-libstoreconsumer +/tests/functional/nix-shell # /tests/functional/lang/ /tests/functional/lang/*.out @@ -112,27 +14,9 @@ perl/Makefile.config /tests/functional/lang/*.err /tests/functional/lang/*.ast -/perl/lib/Nix/Config.pm -/perl/lib/Nix/Store.cc - -/misc/systemd/nix-daemon.service -/misc/systemd/nix-daemon.socket -/misc/systemd/nix-daemon.conf -/misc/upstart/nix-daemon.conf - outputs/ -*.a -*.o -*.o.tmp -*.so -*.dylib -*.dll -*.exe -*.dep *~ -*.pc -*.plist # GNU Global GPATH @@ -147,8 +31,6 @@ GTAGS compile_commands.json *.compile_commands.json -nix-rust/target - result result-* @@ -163,3 +45,5 @@ result-* # Mac OS .DS_Store + +flake-regressions diff --git a/.mergify.yml b/.mergify.yml index c297d3d5e..70fccae49 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,13 +2,11 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=installer - - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-latest) - check-success=tests (macos-latest) - check-success=tests (ubuntu-latest) + - check-success=installer_test (macos-latest) + - check-success=installer_test (ubuntu-latest) - check-success=vm_tests - merge_method: rebase batch_size: 5 pull_request_rules: @@ -90,3 +88,13 @@ pull_request_rules: - "2.24-maintenance" labels: - merge-queue + + - name: backport patches to 2.25 + conditions: + - label=backport 2.25-maintenance + actions: + backport: + branches: + - "2.25-maintenance" + labels: + - merge-queue diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 3630e2dc8..c4cc9b893 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -199,6 +199,7 @@ nix3_manpages = [ 'nix3-build', 'nix3-bundle', 'nix3-config', + 'nix3-config-check', 'nix3-config-show', 'nix3-copy', 'nix3-daemon', @@ -206,8 +207,8 @@ nix3_manpages = [ 'nix3-derivation', 'nix3-derivation-show', 'nix3-develop', - #'nix3-doctor', 'nix3-edit', + 'nix3-env-shell', 'nix3-eval', 'nix3-flake-archive', 'nix3-flake-check', @@ -224,6 +225,7 @@ nix3_manpages = [ 'nix3-fmt', 'nix3-hash-file', 'nix3-hash', + 'nix3-hash-convert', 'nix3-hash-path', 'nix3-hash-to-base16', 'nix3-hash-to-base32', @@ -238,6 +240,7 @@ nix3_manpages = [ 'nix3-nar-cat', 'nix3-nar-dump-path', 'nix3-nar-ls', + 'nix3-nar-pack', 'nix3-nar', 'nix3-path-info', 'nix3-print-dev-env', @@ -260,7 +263,7 @@ nix3_manpages = [ 'nix3-repl', 'nix3-run', 'nix3-search', - #'nix3-shell', + 'nix3-store-add', 'nix3-store-add-file', 'nix3-store-add-path', 'nix3-store-cat', @@ -270,6 +273,7 @@ nix3_manpages = [ 'nix3-store-diff-closures', 'nix3-store-dump-path', 'nix3-store-gc', + 'nix3-store-info', 'nix3-store-ls', 'nix3-store-make-content-addressed', 'nix3-store', diff --git a/doc/manual/rl-next/ignore-local-registries.md b/doc/manual/rl-next/ignore-local-registries.md new file mode 100644 index 000000000..8d5e333dd --- /dev/null +++ b/doc/manual/rl-next/ignore-local-registries.md @@ -0,0 +1,22 @@ +--- +synopsis: "Flake lock file generation now ignores local registries" +prs: [12019] +--- + +When resolving indirect flake references like `nixpkgs` in `flake.nix` files, Nix will no longer use the system and user flake registries. It will only use the global flake registry and overrides given on the command line via `--override-flake`. + +This avoids accidents where users have local registry overrides that map `nixpkgs` to a `path:` flake in the local file system, which then end up in committed lock files pushed to other users. + +In the future, we may remove the use of the registry during lock file generation altogether. It's better to explicitly specify the URL of a flake input. For example, instead of +```nix +{ + outputs = { self, nixpkgs }: { ... }; +} +``` +write +```nix +{ + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; + outputs = { self, nixpkgs }: { ... }; +} +``` diff --git a/doc/manual/rl-next/nix-sshopts-parsing.md b/doc/manual/rl-next/nix-sshopts-parsing.md new file mode 100644 index 000000000..65fe6f562 --- /dev/null +++ b/doc/manual/rl-next/nix-sshopts-parsing.md @@ -0,0 +1,21 @@ +--- +synopsis: "Improved `NIX_SSHOPTS` parsing for better SSH option handling" +issues: [5181] +prs: [12020] +--- + +The parsing of the `NIX_SSHOPTS` environment variable has been improved to handle spaces and quotes correctly. +Previously, incorrectly split SSH options could cause failures in CLIs like `nix-copy-closure`, +especially when using complex ssh invocations such as `-o ProxyCommand="ssh -W %h:%p ..."`. + +This change introduces a `shellSplitString` function to ensure +that `NIX_SSHOPTS` is parsed in a manner consistent with shell +behavior, addressing common parsing errors. + +For example, the following now works as expected: + +```bash +export NIX_SSHOPTS='-o ProxyCommand="ssh -W %h:%p ..."' +``` + +This update improves the reliability of SSH-related operations using `NIX_SSHOPTS` across Nix CLIs. diff --git a/docker.nix b/docker.nix index 19479e57c..e2e9da728 100644 --- a/docker.nix +++ b/docker.nix @@ -258,14 +258,14 @@ let mkdir -p $out/nix/var/nix/profiles/per-user/${uname} ln -s ${profile} $out/nix/var/nix/profiles/default-1-link - ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default ln -s /nix/var/nix/profiles/default $out${userHome}/.nix-profile ln -s ${channel} $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link - ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels + ln -s /nix/var/nix/profiles/per-user/${uname}/channels-1-link $out/nix/var/nix/profiles/per-user/${uname}/channels mkdir -p $out${userHome}/.nix-defexpr - ln -s $out/nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels + ln -s /nix/var/nix/profiles/per-user/${uname}/channels $out${userHome}/.nix-defexpr/channels echo "${channelURL} ${channelName}" > $out${userHome}/.nix-channels mkdir -p $out/bin $out/usr/bin diff --git a/flake.nix b/flake.nix index 06025e3b7..44a764dd7 100644 --- a/flake.nix +++ b/flake.nix @@ -124,18 +124,36 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { - inherit (final) lib; - inherit officialRelease; - src = self; - }); + nixComponents = + lib.makeScopeWithSplicing' + { + inherit (final) splicePackages; + inherit (final.nixDependencies) newScope; + } + { + otherSplices = final.generateSplicesForMkScope "nixComponents"; + f = import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; + }; + }; # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. - nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { - inherit inputs stdenv; - pkgs = final; - }); + nixDependencies = + lib.makeScopeWithSplicing' + { + inherit (final) splicePackages; + inherit (final) newScope; # layered directly on pkgs, unlike nixComponents above + } + { + otherSplices = final.generateSplicesForMkScope "nixDependencies"; + f = import ./packaging/dependencies.nix { + inherit inputs stdenv; + pkgs = final; + }; + }; nix = final.nixComponents.nix-cli; @@ -168,7 +186,7 @@ }; checks = forAllSystems (system: { - binaryTarball = self.hydraJobs.binaryTarball.${system}; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = @@ -183,11 +201,7 @@ # Some perl dependencies are broken on i686-linux. # Since the support is only best-effort there, disable the perl # bindings - - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable this. - #perlBindings = self.hydraJobs.perlBindings.${system}; + perlBindings = self.hydraJobs.perlBindings.${system}; } # Add "passthru" tests // flatMapAttrs ({ @@ -219,6 +233,8 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix; + installerScriptForGHA = self.hydraJobs.installerScriptForGHA.${system}; + binaryTarball = self.hydraJobs.binaryTarball.${system}; # TODO probably should be `nix-cli` nix = self.packages.${system}.nix-everything; nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index fdb031302..1a134b91a 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -12,6 +12,8 @@ hooks = { clang-format = { enable = true; + # https://github.com/cachix/git-hooks.nix/pull/532 + package = pkgs.llvmPackages_latest.clang-tools; excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' @@ -496,7 +498,6 @@ ''^scripts/create-darwin-volume\.sh$'' ''^scripts/install-darwin-multi-user\.sh$'' ''^scripts/install-multi-user\.sh$'' - ''^scripts/install-nix-from-closure\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' diff --git a/meson.build b/meson.build index 8985b631e..49adf9832 100644 --- a/meson.build +++ b/meson.build @@ -34,6 +34,7 @@ endif subproject('libutil-c') subproject('libstore-c') subproject('libexpr-c') +subproject('libflake-c') subproject('libmain-c') # Language Bindings diff --git a/misc/launchd/meson.build b/misc/launchd/meson.build new file mode 100644 index 000000000..5168131d1 --- /dev/null +++ b/misc/launchd/meson.build @@ -0,0 +1,13 @@ +configure_file( + input : 'org.nixos.nix-daemon.plist.in', + output : 'org.nixos.nix-daemon.plist', + install : true, + install_dir : get_option('prefix') / 'Library/LaunchDaemons', + install_mode : 'rw-r--r--', + configuration : { + # TODO: unhardcode paths with something like: + # 'storedir' : store_dir, + # 'localstatedir' : localstatedir, + # 'bindir' : bindir, + }, +) diff --git a/misc/meson.build b/misc/meson.build index a6d1f944b..82f2b0c65 100644 --- a/misc/meson.build +++ b/misc/meson.build @@ -2,4 +2,10 @@ subdir('bash') subdir('fish') subdir('zsh') -subdir('systemd') +if host_machine.system() == 'linux' + subdir('systemd') +endif + +if host_machine.system() == 'darwin' + subdir('launchd') +endif diff --git a/build-utils-meson/common/meson.build b/nix-meson-build-support/common/meson.build similarity index 100% rename from build-utils-meson/common/meson.build rename to nix-meson-build-support/common/meson.build diff --git a/build-utils-meson/deps-lists/meson.build b/nix-meson-build-support/deps-lists/meson.build similarity index 100% rename from build-utils-meson/deps-lists/meson.build rename to nix-meson-build-support/deps-lists/meson.build diff --git a/build-utils-meson/export-all-symbols/meson.build b/nix-meson-build-support/export-all-symbols/meson.build similarity index 100% rename from build-utils-meson/export-all-symbols/meson.build rename to nix-meson-build-support/export-all-symbols/meson.build diff --git a/build-utils-meson/export/meson.build b/nix-meson-build-support/export/meson.build similarity index 100% rename from build-utils-meson/export/meson.build rename to nix-meson-build-support/export/meson.build diff --git a/build-utils-meson/generate-header/meson.build b/nix-meson-build-support/generate-header/meson.build similarity index 100% rename from build-utils-meson/generate-header/meson.build rename to nix-meson-build-support/generate-header/meson.build diff --git a/build-utils-meson/libatomic/meson.build b/nix-meson-build-support/libatomic/meson.build similarity index 100% rename from build-utils-meson/libatomic/meson.build rename to nix-meson-build-support/libatomic/meson.build diff --git a/build-utils-meson/subprojects/meson.build b/nix-meson-build-support/subprojects/meson.build similarity index 100% rename from build-utils-meson/subprojects/meson.build rename to nix-meson-build-support/subprojects/meson.build diff --git a/build-utils-meson/windows-version/meson.build b/nix-meson-build-support/windows-version/meson.build similarity index 100% rename from build-utils-meson/windows-version/meson.build rename to nix-meson-build-support/windows-version/meson.build diff --git a/packaging/components.nix b/packaging/components.nix index c29e04ae9..e1f661be8 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -44,6 +44,7 @@ in nix-expr-tests = callPackage ../src/libexpr-tests/package.nix { }; nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-c = callPackage ../src/libflake-c/package.nix { }; nix-flake-tests = callPackage ../src/libflake-tests/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; diff --git a/packaging/everything.nix b/packaging/everything.nix index b09b9d2a9..0b04d2c6d 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -19,6 +19,7 @@ nix-expr-tests, nix-flake, + nix-flake-c, nix-flake-tests, nix-main, @@ -53,6 +54,7 @@ let nix-expr-c nix-fetchers nix-flake + nix-flake-c nix-main nix-main-c nix-store @@ -86,6 +88,7 @@ let "nix-expr-c" "nix-fetchers" "nix-flake" + "nix-flake-c" "nix-main" "nix-main-c" "nix-store" @@ -169,6 +172,7 @@ in nix-expr nix-expr-c nix-flake + nix-flake-c nix-main nix-main-c ; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 81406a249..5b1e47559 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -18,12 +18,8 @@ let testNixVersions = pkgs: daemon: pkgs.nixComponents.nix-functional-tests.override { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast pkgs.nix.version "2.4pre20211005") - "-${pkgs.nix.version}-against-${daemon.version}"; + pname = "nix-daemon-compat-tests"; + version = "${pkgs.nix.version}-with-daemon-${daemon.version}"; test-daemon = daemon; }; @@ -127,15 +123,10 @@ in self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; + + installerScriptForGHA = forAllSystems (system: nixpkgsFor.${system}.native.callPackage ../scripts/installer.nix { + tarballs = [ self.hydraJobs.binaryTarball.${system} ]; + }); # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 104189b0c..9de90b7fb 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -23,7 +23,7 @@ in runCommand "nix-binary-tarball-${version}" env '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ + substitute ${./install-nix-from-tarball.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} @@ -65,7 +65,7 @@ runCommand "nix-binary-tarball-${version}" env '' fn=$out/$dir.tar.xz mkdir -p $out/nix-support echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ + tar cfJ $fn \ --owner=0 --group=0 --mode=u+rw,uga+r \ --mtime='1970-01-01' \ --absolute-names \ diff --git a/scripts/build-checks b/scripts/build-checks new file mode 100755 index 000000000..e0ee70631 --- /dev/null +++ b/scripts/build-checks @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +system=$(nix eval --raw --impure --expr builtins.currentSystem) +nix eval --json ".#checks.$system" --apply builtins.attrNames | \ + jq -r '.[]' | \ + xargs -P0 -I '{}' sh -c "nix build -L .#checks.$system.{} || { echo 'FAILED: \033[0;31mnix build -L .#checks.$system.{}\\033[0m'; kill 0; }" diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index 103e1e391..7a61764d4 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -463,7 +463,7 @@ EOF EDITOR="$SCRATCH/ex_cleanroom_wrapper" _sudo "to add nix to fstab" "$@" < "$SCRATCH/.nix-channels" _sudo "to set up the default system channel (part 1)" \ - install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" + install -m 0644 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" fi } @@ -964,7 +969,7 @@ $NIX_EXTRA_CONF build-users-group = $NIX_BUILD_GROUP_NAME EOF _sudo "to place the default nix daemon configuration (part 2)" \ - install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf + install -m 0644 "$SCRATCH/nix.conf" /etc/nix/nix.conf } diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-tarball.sh similarity index 95% rename from scripts/install-nix-from-closure.sh rename to scripts/install-nix-from-tarball.sh index 794622530..8d127a9c5 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-tarball.sh @@ -9,6 +9,8 @@ self="$(dirname "$0")" nix="@nix@" cacert="@cacert@" +# allow to override identity change command +readonly NIX_BECOME="${NIX_BECOME:-sudo}" if ! [ -e "$self/.reginfo" ]; then echo "$0: incomplete installer (.reginfo is missing)" >&2 @@ -48,15 +50,14 @@ case "$(uname -s)" in INSTALL_MODE=no-daemon;; esac -# space-separated string -ACTIONS= +ACTION= # handle the command line flags while [ $# -gt 0 ]; do case $1 in --daemon) INSTALL_MODE=daemon - ACTIONS="${ACTIONS}install " + ACTION=install ;; --no-daemon) if [ "$(uname -s)" = "Darwin" ]; then @@ -64,19 +65,14 @@ while [ $# -gt 0 ]; do exit 1 fi INSTALL_MODE=no-daemon - # intentional tail space - ACTIONS="${ACTIONS}install " + ACTION=install ;; - # --uninstall) - # # intentional tail space - # ACTIONS="${ACTIONS}uninstall " - # ;; --yes) export NIX_INSTALLER_YES=1;; --no-channel-add) export NIX_INSTALLER_NO_CHANNEL_ADD=1;; --daemon-user-count) - export NIX_USER_COUNT=$2 + export NIX_USER_COUNT="$2" shift;; --no-modify-profile) NIX_INSTALLER_NO_MODIFY_PROFILE=1;; @@ -128,7 +124,7 @@ done if [ "$INSTALL_MODE" = "daemon" ]; then printf '\e[1;31mSwitching to the Multi-user Installer\e[0m\n' - exec "$self/install-multi-user" $ACTIONS # let ACTIONS split + exec "$self/install-multi-user" $ACTION exit 0 fi @@ -140,8 +136,8 @@ echo "performing a single-user installation of Nix..." >&2 if ! [ -e "$dest" ]; then cmd="mkdir -m 0755 $dest && chown $USER $dest" - echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 - if ! sudo sh -c "$cmd"; then + echo "directory $dest does not exist; creating it by running '$cmd' using $NIX_BECOME" >&2 + if ! $NIX_BECOME sh -c "$cmd"; then echo "$0: please manually run '$cmd' as root to create $dest" >&2 exit 1 fi diff --git a/scripts/prepare-installer-for-github-actions b/scripts/prepare-installer-for-github-actions index 4b994a753..0fbecf25c 100755 --- a/scripts/prepare-installer-for-github-actions +++ b/scripts/prepare-installer-for-github-actions @@ -1,10 +1,11 @@ #!/usr/bin/env bash -set -e +set -euo pipefail -script=$(nix-build -A outputs.hydraJobs.installerScriptForGHA --no-out-link) -installerHash=$(echo "$script" | cut -b12-43 -) +nix build -L ".#installerScriptForGHA" ".#binaryTarball" -installerURL=https://$CACHIX_NAME.cachix.org/serve/$installerHash/install - -echo "::set-output name=installerURL::$installerURL" +mkdir -p out +cp ./result/install "out/install" +name="$(basename "$(realpath ./result-1)")" +# everything before the first dash +cp -r ./result-1 "out/${name%%-*}" diff --git a/scripts/serve-installer-for-github-actions b/scripts/serve-installer-for-github-actions new file mode 100755 index 000000000..2efd2aa32 --- /dev/null +++ b/scripts/serve-installer-for-github-actions @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -euo pipefail +if [[ ! -d out ]]; then + echo "run prepare-installer-for-github-actions first" + exit 1 +fi +cd out +PORT=${PORT:-8126} +nohup python -m http.server "$PORT" >/dev/null 2>&1 & +pid=$! + +while ! curl -s "http://localhost:$PORT"; do + sleep 1 + if ! kill -0 $pid; then + echo "Failed to start http server" + exit 1 + fi +done + +echo 'To install nix, run the following command:' +echo "sh <(curl http://localhost:$PORT/install) --tarball-url-prefix http://localhost:$PORT" diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 8e235dae5..3af2f5b81 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -40,6 +40,7 @@ GENERATE_LATEX = NO INPUT = \ @src@/src/libutil-c \ @src@/src/libexpr-c \ + @src@/src/libflake-c \ @src@/src/libstore-c \ @src@/src/external-api-docs/README.md diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index 0c592955a..57c5138cf 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -30,6 +30,7 @@ mkMesonDerivation (finalAttrs: { # Source is not compiled, but still must be available for Doxygen # to gather comments. (cpp ../libexpr-c) + (cpp ../libflake-c) (cpp ../libstore-c) (cpp ../libutil-c) ]; diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libcmd/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f2b27af7c..250cd1413 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -858,7 +858,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() applyDefaultInstallables(rawInstallables); std::vector res; res.reserve(rawInstallables.size()); - for (auto i : rawInstallables) + for (const auto & i : rawInstallables) res.push_back(parseFlakeRefWithFragment( fetchSettings, expandTilde(i), diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 1f27c1614..222817c81 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -14,7 +14,7 @@ project('nix-cmd', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -28,7 +28,7 @@ deps_public_maybe_subproject = [ dependency('nix-flake'), dependency('nix-main'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -70,7 +70,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'built-path.cc', @@ -125,4 +125,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libcmd/nix-meson-build-support b/src/libcmd/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libcmd/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index 244179ee4..53e54d2f8 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -39,8 +39,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 5bcca29e0..1556dae51 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -14,7 +14,7 @@ project('nix-expr-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_expr.cc', @@ -72,8 +72,8 @@ headers = [config_h] + files( # TODO move this header to libexpr, maybe don't use it in tests? headers += files('nix_api_expr_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixexprc', @@ -89,4 +89,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-c/nix-meson-build-support b/src/libexpr-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 6144a7986..a024248cd 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -6,6 +6,7 @@ #include "eval-gc.hh" #include "globals.hh" #include "eval-settings.hh" +#include "ref.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -18,6 +19,29 @@ # include #endif +/** + * @brief Allocate and initialize using self-reference + * + * This allows a brace initializer to reference the object being constructed. + * + * @warning Use with care, as the pointer points to an object that is not fully constructed yet. + * + * @tparam T Type to allocate + * @tparam F A function type for `init`, taking a T* and returning the initializer for T + * @param init Function that takes a T* and returns the initializer for T + * @return Pointer to allocated and initialized object + */ +template +static T * unsafe_new_with_self(F && init) +{ + // Allocate + void * p = ::operator new( + sizeof(T), + static_cast(alignof(T))); + // Initialize with placement new + return new (p) T(init(static_cast(p))); +} + nix_err nix_libexpr_init(nix_c_context * context) { if (context) @@ -93,7 +117,42 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val NIXC_CATCH_ERRS } -EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return unsafe_new_with_self([&](auto * self) { + return nix_eval_state_builder{ + .store = nix::ref(store->ptr), + .settings = nix::EvalSettings{/* &bool */ self->readOnlyMode}, + .fetchSettings = nix::fetchers::Settings{}, + .readOnlyMode = true, + }; + }); + } + NIXC_CATCH_ERRS_NULL +} + +void nix_eval_state_builder_free(nix_eval_state_builder * builder) +{ + delete builder; +} + +nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) +{ + if (context) + context->last_err_code = NIX_OK; + try { + // TODO: load in one go? + builder->settings.readOnlyMode = nix::settings.readOnlyMode; + loadConfFile(builder->settings); + loadConfFile(builder->fetchSettings); + } + NIXC_CATCH_ERRS +} + +nix_err nix_eval_state_builder_set_lookup_path(nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath_c) { if (context) context->last_err_code = NIX_OK; @@ -102,28 +161,47 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c if (lookupPath_c != nullptr) for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); + builder->lookupPath = nix::LookupPath::parse(lookupPath); + } + NIXC_CATCH_ERRS +} - void * p = ::operator new( - sizeof(EvalState), - static_cast(alignof(EvalState))); - auto * p2 = static_cast(p); - new (p) EvalState { - .fetchSettings = nix::fetchers::Settings{}, - .settings = nix::EvalSettings{ - nix::settings.readOnlyMode, - }, - .state = nix::EvalState( - nix::LookupPath::parse(lookupPath), - store->ptr, - p2->fetchSettings, - p2->settings), - }; - loadConfFile(p2->settings); - return p2; +EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return unsafe_new_with_self([&](auto * self) { + return EvalState{ + .fetchSettings = std::move(builder->fetchSettings), + .settings = std::move(builder->settings), + .state = nix::EvalState( + builder->lookupPath, + builder->store, + self->fetchSettings, + self->settings), + }; + }); } NIXC_CATCH_ERRS_NULL } +EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c, Store * store) +{ + auto builder = nix_eval_state_builder_new(context, store); + if (builder == nullptr) + return nullptr; + + if (nix_eval_state_builder_load(context, builder) != NIX_OK) + return nullptr; + + if (nix_eval_state_builder_set_lookup_path(context, builder, lookupPath_c) + != NIX_OK) + return nullptr; + + return nix_eval_state_build(context, builder); +} + void nix_state_free(EvalState * state) { delete state; diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index e680f5ff1..f8d181452 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -30,6 +30,11 @@ extern "C" { // cffi start // Type definitions +/** + * @brief Builder for EvalState + */ +typedef struct nix_eval_state_builder nix_eval_state_builder; + /** * @brief Represents a state of the Nix language evaluator. * @@ -174,12 +179,70 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); /** - * @brief Create a new Nix language evaluator state. + * @brief Create a new nix_eval_state_builder + * + * The settings are initialized to their default value. + * Values can be sourced elsewhere with nix_eval_state_builder_load. + * + * @param[out] context Optional, stores error information + * @param[in] store The Nix store to use. + * @return A new nix_eval_state_builder or NULL on failure. + */ +nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Store * store); + +/** + * @brief Read settings from the ambient environment + * + * Settings are sourced from environment variables and configuration files, + * as documented in the Nix manual. + * + * @param[out] context Optional, stores error information + * @param[out] builder The builder to modify. + * @return NIX_OK if successful, an error code otherwise. + */ +nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder); + +/** + * @brief Set the lookup path for `<...>` expressions + * + * @param[in] context Optional, stores error information + * @param[in] builder The builder to modify. + * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. + */ +nix_err nix_eval_state_builder_set_lookup_path( + nix_c_context * context, nix_eval_state_builder * builder, const char ** lookupPath); + +/** + * @brief Create a new Nix language evaluator state + * + * Remember to nix_eval_state_builder_free after building the state. + * + * @param[out] context Optional, stores error information + * @param[in] builder The builder to use and free + * @return A new Nix state or NULL on failure. + * @see nix_eval_state_builder_new, nix_eval_state_builder_free + */ +EvalState * nix_eval_state_build(nix_c_context * context, nix_eval_state_builder * builder); + +/** + * @brief Free a nix_eval_state_builder + * + * Does not fail. + * + * @param[in] builder The builder to free. + */ +void nix_eval_state_builder_free(nix_eval_state_builder * builder); + +/** + * @brief Create a new Nix language evaluator state + * + * For more control, use nix_eval_state_builder * * @param[out] context Optional, stores error information * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. * @param[in] store The Nix store to use. * @return A new Nix state or NULL on failure. + * @see nix_state_builder_new */ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath, Store * store); diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 12f24b6eb..f59664011 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -6,6 +6,17 @@ #include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" +#include "search-path.hh" + +struct nix_eval_state_builder +{ + nix::ref store; + nix::EvalSettings settings; + nix::fetchers::Settings fetchSettings; + nix::LookupPath lookupPath; + // TODO: make an EvalSettings setting own this instead? + bool readOnlyMode; +}; struct EvalState { diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 8a0813ebe..711b0adbc 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -213,7 +213,7 @@ nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_ /** @brief Get path as string * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect - * @return string + * @return string, if the type is NIX_TYPE_PATH * @return NULL in case of error. */ const char * nix_get_path_string(nix_c_context * context, const nix_value * value); diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index df49a8bdc..727b3a811 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-test-support/build-utils-meson b/src/libexpr-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index bdfd435a8..64d4fe218 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-expr-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,8 +24,9 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-store-test-support'), dependency('nix-expr'), + dependency('nix-expr-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -39,7 +40,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/value/context.cc', @@ -53,8 +54,8 @@ headers = files( 'tests/value/context.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-expr-test-support', @@ -72,4 +73,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr-test-support/nix-meson-build-support b/src/libexpr-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index bcf6118e0..4842f5f17 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -4,6 +4,7 @@ , nix-store-test-support , nix-expr +, nix-expr-c , rapidcheck @@ -22,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-store-test-support nix-expr + nix-expr-c rapidcheck ]; diff --git a/src/libexpr-tests/build-utils-meson b/src/libexpr-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-tests/error_traces.cc b/src/libexpr-tests/error_traces.cc index be379a909..2aa13cf62 100644 --- a/src/libexpr-tests/error_traces.cc +++ b/src/libexpr-tests/error_traces.cc @@ -691,15 +691,15 @@ namespace nix { ASSERT_TRACE2("elemAt \"foo\" (-1)", TypeError, HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.elemAt'")); ASSERT_TRACE1("elemAt [] (-1)", Error, - HintFmt("list index %d is out of bounds", -1)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", -1, 0)); ASSERT_TRACE1("elemAt [\"foo\"] 3", Error, - HintFmt("list index %d is out of bounds", 3)); + HintFmt("'builtins.elemAt' called with index %d on a list of size %d", 3, 1)); } @@ -708,11 +708,11 @@ namespace nix { ASSERT_TRACE2("head 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("while evaluating the first argument passed to 'builtins.head'")); ASSERT_TRACE1("head []", Error, - HintFmt("list index %d is out of bounds", 0)); + HintFmt("'builtins.head' called on an empty list")); } @@ -721,11 +721,11 @@ namespace nix { ASSERT_TRACE2("tail 1", TypeError, HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), - HintFmt("while evaluating the first argument passed to builtins.tail")); + HintFmt("while evaluating the first argument passed to 'builtins.tail'")); ASSERT_TRACE1("tail []", Error, - HintFmt("'tail' called on an empty list")); + HintFmt("'builtins.tail' called on an empty list")); } diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index b50c18c9c..f37e85e57 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -14,7 +14,7 @@ project('nix-expr-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'derived-path.cc', diff --git a/src/libexpr-tests/nix-meson-build-support b/src/libexpr-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index b37ac44b3..5ed78d2fc 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -7,12 +7,49 @@ #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" +#include "file-system.hh" #include #include namespace nixC { +TEST_F(nix_api_store_test, nix_eval_state_lookup_path) +{ + auto tmpDir = nix::createTempDir(); + auto delTmpDir = std::make_unique(tmpDir, true); + auto nixpkgs = tmpDir + "/pkgs"; + auto nixos = tmpDir + "/cfg"; + std::filesystem::create_directories(nixpkgs); + std::filesystem::create_directories(nixos); + + std::string nixpkgsEntry = "nixpkgs=" + nixpkgs; + std::string nixosEntry = "nixos-config=" + nixos; + const char * lookupPath[] = {nixpkgsEntry.c_str(), nixosEntry.c_str(), nullptr}; + + auto builder = nix_eval_state_builder_new(ctx, store); + assert_ctx_ok(); + + ASSERT_EQ(NIX_OK, nix_eval_state_builder_set_lookup_path(ctx, builder, lookupPath)); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + + nix_eval_state_builder_free(builder); + + Value * value = nix_alloc_value(ctx, state); + nix_expr_eval_from_string(ctx, state, "builtins.seq ", ".", value); + assert_ctx_ok(); + + ASSERT_EQ(nix_get_type(ctx, value), NIX_TYPE_PATH); + assert_ctx_ok(); + + auto pathStr = nix_get_path_string(ctx, value); + assert_ctx_ok(); + ASSERT_EQ(0, strcmp(pathStr, nixpkgs.c_str())); +} + TEST_F(nix_api_expr_test, nix_expr_eval_from_string) { nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value); diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 959d6b84e..70e497b7e 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -27,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr-tests/trivial.cc b/src/libexpr-tests/trivial.cc index e455a571b..d77b4d53b 100644 --- a/src/libexpr-tests/trivial.cc +++ b/src/libexpr-tests/trivial.cc @@ -177,6 +177,57 @@ namespace nix { ) ); +// The following macros ultimately define 48 tests (16 variations on three +// templates). Each template tests an expression that can be written in 2^4 +// different ways, by making four choices about whether to write a particular +// attribute path segment as `x.y = ...;` (collapsed) or `x = { y = ...; };` +// (expanded). +// +// The nestedAttrsetMergeXXXX tests check that the expression +// `{ a.b.c = 1; a.b.d = 2; }` has the same value regardless of how it is +// expanded. (That exact expression is exercised in test +// nestedAttrsetMerge0000, because it is fully collapsed. The test +// nestedAttrsetMerge1001 would instead examine +// `{ a = { b.c = 1; }; a.b = { d = 2; }; }`.) +// +// The nestedAttrsetMergeDupXXXX tests check that the expression +// `{ a.b.c = 1; a.b.c = 2; }` throws a duplicate attribute error, again +// regardless of how it is expanded. +// +// The nestedAttrsetMergeLetXXXX tests check that the expression +// `let a.b.c = 1; a.b.d = 2; in a` has the same value regardless of how it is +// expanded. +#define X_EXPAND_IF0(k, v) k "." v +#define X_EXPAND_IF1(k, v) k " = { " v " };" +#define X4(w, x, y, z) \ + TEST_F(TrivialExpressionTest, nestedAttrsetMerge##w##x##y##z) { \ + auto v = eval("{ a.b = { c = 1; d = 2; }; } == { " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " }"); \ + ASSERT_THAT(v, IsTrue()); \ + }; \ + TEST_F(TrivialExpressionTest, nestedAttrsetMergeDup##w##x##y##z) { \ + ASSERT_THROW(eval("{ " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "c = 2;")) " }"), Error); \ + }; \ + TEST_F(TrivialExpressionTest, nestedAttrsetMergeLet##w##x##y##z) { \ + auto v = eval("{ b = { c = 1; d = 2; }; } == (let " \ + X_EXPAND_IF##w("a", X_EXPAND_IF##x("b", "c = 1;")) " " \ + X_EXPAND_IF##y("a", X_EXPAND_IF##z("b", "d = 2;")) " in a)"); \ + ASSERT_THAT(v, IsTrue()); \ + }; +#define X3(...) X4(__VA_ARGS__, 0) X4(__VA_ARGS__, 1) +#define X2(...) X3(__VA_ARGS__, 0) X3(__VA_ARGS__, 1) +#define X1(...) X2(__VA_ARGS__, 0) X2(__VA_ARGS__, 1) + X1(0) X1(1) +#undef X_EXPAND_IF0 +#undef X_EXPAND_IF1 +#undef X1 +#undef X2 +#undef X3 +#undef X4 + TEST_F(TrivialExpressionTest, functor) { auto v = eval("{ __functor = self: arg: self.v + arg; v = 10; } 5"); ASSERT_THAT(v, IsIntEq(15)); diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 2f67260c5..822ec7620 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -129,7 +129,6 @@ std::pair findPackageFilename(EvalState & state, Value & v try { auto colon = fn.rfind(':'); if (colon == std::string::npos) fail(); - std::string filename(fn, 0, colon); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libexpr/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 05f58957e..fe5f05ab8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -347,6 +347,16 @@ void EvalState::allowPath(const StorePath & storePath) rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath))); } +void EvalState::allowClosure(const StorePath & storePath) +{ + if (!rootFS.dynamic_pointer_cast()) return; + + StorePathSet closure; + store->computeFSClosure(storePath, closure); + for (auto & p : closure) + allowPath(p); +} + void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) { allowPath(storePath); @@ -3099,10 +3109,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat allowPath(path.path.abs()); if (store->isInStore(path.path.abs())) { try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure); - for (auto & p : closure) - allowPath(p); + allowClosure(store->toStorePath(path.path.abs()).first); } catch (InvalidPath &) { } } } @@ -3178,5 +3185,14 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } +void forceNoNullByte(std::string_view s) +{ + if (s.find('\0') != s.npos) { + using namespace std::string_view_literals; + auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); + throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + } +} + } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 3ac3c8a8a..84b7d823c 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -400,6 +400,11 @@ public: */ void allowPath(const StorePath & storePath); + /** + * Allow access to the closure of a store path. + */ + void allowClosure(const StorePath & storePath); + /** * Allow access to a store path and return it as a string. */ diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 9ac56541a..17cab7ad5 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -50,6 +50,7 @@ class JSONSax : nlohmann::json_sax { public: void key(string_t & name, EvalState & state) { + forceNoNullByte(name); attrs.insert_or_assign(state.symbols.create(name), &value(state)); } }; @@ -122,6 +123,7 @@ public: bool string(string_t & val) override { + forceNoNullByte(val); rs->value(state).mkString(val); rs->add(); return true; diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index caba6e18f..d40f7b874 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -1,5 +1,13 @@ #pragma once +#include + +// inluding the generated headers twice leads to errors +#ifndef BISON_HEADER +# include "lexer-tab.hh" +# include "parser-tab.hh" +#endif + namespace nix::lexer::internal { void initLoc(YYLTYPE * loc); diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 28318579e..b3c559ba7 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -14,7 +14,7 @@ project('nix-expr', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_public_maybe_subproject = [ dependency('nix-store'), dependency('nix-fetchers'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') boost = dependency( 'boost', @@ -77,7 +77,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') parser_tab = custom_target( input : 'parser.y', @@ -121,7 +121,7 @@ lexer_tab = custom_target( install_dir : get_option('includedir') / 'nix', ) -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -205,4 +205,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libexpr/nix-meson-build-support b/src/libexpr/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libexpr/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 2950ff1fd..a7ad580d2 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -168,7 +168,7 @@ struct ExprVar : Expr the set stored in the environment that is `level` levels up from the current one.*/ Level level; - Displacement displ; + Displacement displ = 0; ExprVar(Symbol name) : name(name) { }; ExprVar(const PosIdx & pos, Symbol name) : pos(pos), name(name) { }; @@ -242,7 +242,7 @@ struct ExprAttrs : Expr Kind kind; Expr * e; PosIdx pos; - Displacement displ; // displacement + Displacement displ = 0; // displacement AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain) : kind(kind), e(e), pos(pos) { }; AttrDef() { }; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index d97e7f3a8..5171d70fd 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -40,8 +40,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 8ad0d9ad7..21a880e8e 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -88,6 +88,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); + void addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -120,64 +121,29 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + ExprAttrs * nested; if (i->symbol) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) { - ExprAttrs * attrs2 = dynamic_cast(j->second.e); - if (!attrs2) dupAttr(attrPath, pos, j->second.pos); - attrs = attrs2; - } else + nested = dynamic_cast(j->second.e); + if (!nested) { + attrPath.erase(i + 1, attrPath.end()); dupAttr(attrPath, pos, j->second.pos); + } } else { - ExprAttrs * nested = new ExprAttrs; + nested = new ExprAttrs; attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); - attrs = nested; } } else { - ExprAttrs *nested = new ExprAttrs; + nested = new ExprAttrs; attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); - attrs = nested; } + attrs = nested; } // Expr insertion. // ========================== if (i->symbol) { - ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); - if (j != attrs->attrs.end()) { - // This attr path is already defined. However, if both - // e and the expr pointed by the attr path are two attribute sets, - // we want to merge them. - // Otherwise, throw an error. - auto ae = dynamic_cast(e); - auto jAttrs = dynamic_cast(j->second.e); - if (jAttrs && ae) { - if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) - jAttrs->inheritFromExprs = std::make_unique>(); - for (auto & ad : ae->attrs) { - auto j2 = jAttrs->attrs.find(ad.first); - if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. - dupAttr(ad.first, j2->second.pos, ad.second.pos); - jAttrs->attrs.emplace(ad.first, ad.second); - if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { - auto & sel = dynamic_cast(*ad.second.e); - auto & from = dynamic_cast(*sel.e); - from.displ += jAttrs->inheritFromExprs->size(); - } - } - jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); - if (ae->inheritFromExprs) { - jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), - ae->inheritFromExprs->begin(), ae->inheritFromExprs->end()); - } - } else { - dupAttr(attrPath, pos, j->second.pos); - } - } else { - // This attr path is not defined. Let's create it. - attrs->attrs.emplace(i->symbol, ExprAttrs::AttrDef(e, pos)); - e->setName(i->symbol); - } + addAttr(attrs, attrPath, i->symbol, ExprAttrs::AttrDef(e, pos)); } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } @@ -189,6 +155,60 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const } } +/** + * Precondition: attrPath is used for error messages and should already contain + * symbol as its last element. + */ +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath & attrPath, const Symbol & symbol, ExprAttrs::AttrDef && def) +{ + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(def.e); + auto jAttrs = dynamic_cast(j->second.e); + + // N.B. In a world in which we are less bound by our past mistakes, we + // would also test that jAttrs and ae are not recursive. The effect of + // not doing so is that any `rec` marker on ae is discarded, and any + // `rec` marker on jAttrs will apply to the attributes in ae. + // See https://github.com/NixOS/nix/issues/9020. + if (jAttrs && ae) { + if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) + jAttrs->inheritFromExprs = std::make_unique>(); + for (auto & ad : ae->attrs) { + if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { + auto & sel = dynamic_cast(*ad.second.e); + auto & from = dynamic_cast(*sel.e); + from.displ += jAttrs->inheritFromExprs->size(); + } + attrPath.emplace_back(AttrName(ad.first)); + addAttr(jAttrs, attrPath, ad.first, std::move(ad.second)); + attrPath.pop_back(); + } + ae->attrs.clear(); + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), + std::make_move_iterator(ae->dynamicAttrs.begin()), + std::make_move_iterator(ae->dynamicAttrs.end())); + ae->dynamicAttrs.clear(); + if (ae->inheritFromExprs) { + jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), + std::make_move_iterator(ae->inheritFromExprs->begin()), + std::make_move_iterator(ae->inheritFromExprs->end())); + ae->inheritFromExprs = nullptr; + } + } else { + dupAttr(attrPath, def.pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs.emplace(symbol, def); + def.e->setName(symbol); + } +} + inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) { std::sort(formals->formals.begin(), formals->formals.end(), diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 53bfce6c5..c1f1cf4c1 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -66,14 +66,12 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - auto ctxS = store->printStorePath(o.path); ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, [&](const NixStringContextElem::DrvDeep & d) { /* Treat same as Opaque */ - auto ctxS = store->printStorePath(d.drvPath); ensureValid(d.drvPath); if (maybePathsOut) maybePathsOut->emplace(d.drvPath); @@ -121,11 +119,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); if (isIFD) { - for (auto & outputPath : outputsToCopyAndAllow) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(outputPath); - } + /* Allow access to the output closures of this derivation. */ + for (auto & outputPath : outputsToCopyAndAllow) + allowClosure(outputPath); } return res; @@ -1102,7 +1098,7 @@ static RegisterPrimOp primop_warn({ .name = "__warn", .args = {"e1", "e2"}, .doc = R"( - Evaluate *e1*, which must be a string and print iton standard error as a warning. + Evaluate *e1*, which must be a string, and print it on standard error as a warning. Then return *e2*. This function is useful for non-critical situations where attention is advisable. @@ -3261,23 +3257,19 @@ static RegisterPrimOp primop_isList({ .fun = prim_isList, }); -static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Value & v) -{ - state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); - if (n < 0 || (unsigned int) n >= list.listSize()) - state.error( - "list index %1% is out of bounds", - n - ).atPos(pos).debugThrow(); - state.forceValue(*list.listElems()[n], pos); - v = *list.listElems()[n]; -} - /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; - elemAt(state, pos, *args[0], elem, v); + NixInt::Inner n = state.forceInt(*args[1], pos, "while evaluating the second argument passed to 'builtins.elemAt'").value; + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.elemAt'"); + if (n < 0 || (unsigned int) n >= args[0]->listSize()) + state.error( + "'builtins.elemAt' called with index %d on a list of size %d", + n, + args[0]->listSize() + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[n], pos); + v = *args[0]->listElems()[n]; } static RegisterPrimOp primop_elemAt({ @@ -3293,7 +3285,13 @@ static RegisterPrimOp primop_elemAt({ /* Return the first element of a list. */ static void prim_head(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], 0, v); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.head'"); + if (args[0]->listSize() == 0) + state.error( + "'builtins.head' called on an empty list" + ).atPos(pos).debugThrow(); + state.forceValue(*args[0]->listElems()[0], pos); + v = *args[0]->listElems()[0]; } static RegisterPrimOp primop_head({ @@ -3312,9 +3310,9 @@ static RegisterPrimOp primop_head({ don't want to use it! */ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); + state.forceList(*args[0], pos, "while evaluating the first argument passed to 'builtins.tail'"); if (args[0]->listSize() == 0) - state.error("'tail' called on an empty list").atPos(pos).debugThrow(); + state.error("'builtins.tail' called on an empty list").atPos(pos).debugThrow(); auto list = state.buildList(args[0]->listSize() - 1); for (const auto & [n, v] : enumerate(list)) @@ -4385,7 +4383,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) for (auto i = begin; i != end; ++i) { assert(idx <= 2 * len + 1 - 3); - auto match = *i; + const auto & match = *i; // Add a string for non-matched characters. list[idx++] = mkString(state, match.prefix()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 02683b173..ede7d97ba 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -132,6 +132,8 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V }, [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { /* Reuse original item because we want this to be idempotent. */ + /* FIXME: Suspicious move out of const. This is actually a copy, so the comment + above does not make much sense. */ return std::move(c); }, }, context.begin()->raw) }), diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index fc5bb3145..04b8d0595 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -40,7 +40,7 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor }); } - auto toPath = *toPathMaybe; + const auto & toPath = *toPathMaybe; // check and return diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 264046711..404425054 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -28,8 +28,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V auto attrs = state.buildBindings(size); - for(auto & elem : table) + for(auto & elem : table) { + forceNoNullByte(elem.first); visit(attrs.alloc(elem.first), elem.second); + } v.mkAttrs(attrs); } @@ -54,7 +56,11 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V v.mkFloat(toml::get(t)); break;; case toml::value_t::string: - v.mkString(toml::get(t)); + { + auto s = toml::get(t); + forceNoNullByte(s); + v.mkString(s); + } break;; case toml::value_t::local_datetime: case toml::value_t::offset_datetime: @@ -66,7 +72,9 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V attrs.alloc("_type").mkString("timestamp"); std::ostringstream s; s << t; - attrs.alloc("value").mkString(toView(s)); + auto str = toView(s); + forceNoNullByte(str); + attrs.alloc("value").mkString(str); v.mkAttrs(attrs); } else { throw std::runtime_error("Dates and times are not supported"); diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh index 080ba26b8..9ad54e532 100644 --- a/src/libexpr/print-options.hh +++ b/src/libexpr/print-options.hh @@ -5,6 +5,7 @@ */ #include +#include namespace nix { diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index d98161488..88fcae986 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,4 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); +void forceNoNullByte(std::string_view s); + } diff --git a/src/libfetchers-tests/build-utils-meson b/src/libfetchers-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libfetchers-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index fdab6ba6c..3e82c6111 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-store-test-support'), @@ -22,10 +22,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'public-key.cc', diff --git a/src/libfetchers-tests/nix-meson-build-support b/src/libfetchers-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index 7b2ba8f2c..e18d644ed 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libfetchers/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7006fc6be..d12993168 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -113,7 +113,15 @@ Input Input::fromAttrs(const Settings & settings, Attrs && attrs) std::optional Input::getFingerprint(ref store) const { - return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; + if (!scheme) return std::nullopt; + + if (cachedFingerprint) return *cachedFingerprint; + + auto fingerprint = scheme->getFingerprint(store, *this); + + cachedFingerprint = fingerprint; + + return fingerprint; } ParsedURL Input::toURL() const @@ -313,7 +321,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto accessor = makeStorePathAccessor(store, storePath); - accessor->fingerprint = scheme->getFingerprint(store, *this); + accessor->fingerprint = getFingerprint(store); return {accessor, *this}; } catch (Error & e) { @@ -324,7 +332,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, result] = scheme->getAccessor(store, *this); assert(!accessor->fingerprint); - accessor->fingerprint = scheme->getFingerprint(store, result); + accessor->fingerprint = result.getFingerprint(store); return {accessor, std::move(result)}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 4738fb09d..53d64adc9 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -41,6 +41,11 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; + /** + * Cached result of getFingerprint(). + */ + mutable std::optional> cachedFingerprint; + public: /** * Create an `Input` from a URL. @@ -105,6 +110,11 @@ public: bool operator ==(const Input & other) const noexcept; + bool operator <(const Input & other) const + { + return attrs < other.attrs; + } + bool contains(const Input & other) const; /** diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 74e68fe12..b54416b10 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -5,6 +5,7 @@ #include "signals.hh" #include "users.hh" #include "fs-sink.hh" +#include "sync.hh" #include #include @@ -437,7 +438,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && !(statusFlags & GIT_STATUS_WT_DELETED)) + { info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.dirtyFiles.insert(CanonPath(path)); + } else + info.deletedFiles.insert(CanonPath(path)); if (statusFlags != GIT_STATUS_CURRENT) info.isDirty = true; return 0; @@ -1262,4 +1268,17 @@ ref getTarballCache() return GitRepo::openRepo(repoDir, true, true); } +GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) +{ + static Sync> _cache; + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + auto workdirInfo = GitRepo::openRepo(path)->getWorkdirInfo(); + _cache.lock()->emplace(path, workdirInfo); + return workdirInfo; +} + } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index f45b5a504..ff115143f 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -59,12 +59,20 @@ struct GitRepo modified or added, but excluding deleted files. */ std::set files; + /* All modified or added files. */ + std::set dirtyFiles; + + /* The deleted files. */ + std::set deletedFiles; + /* The submodules listed in .gitmodules of this workdir. */ std::vector submodules; }; virtual WorkdirInfo getWorkdirInfo() = 0; + static WorkdirInfo getCachedWorkdirInfo(const std::filesystem::path & path); + /* Get the ref that HEAD points to. */ virtual std::optional getWorkdirRef() = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 99d91919e..c73f53765 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "finally.hh" #include "fetch-settings.hh" #include "json-utils.hh" +#include "archive.hh" #include #include @@ -430,7 +431,7 @@ struct GitInputScheme : InputScheme // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); + repoInfo.workdirInfo = GitRepo::getCachedWorkdirInfo(repoInfo.url); return repoInfo; } @@ -514,8 +515,6 @@ struct GitInputScheme : InputScheme auto origRev = input.getRev(); - std::string name = input.getName(); - auto originalRef = input.getRef(); auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); input.attrs.insert_or_assign("ref", ref); @@ -795,10 +794,33 @@ struct GitInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { + auto makeFingerprint = [&](const Hash & rev) + { + return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); + }; + if (auto rev = input.getRev()) - return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : ""); - else + return makeFingerprint(*rev); + else { + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + /* Calculate a fingerprint that takes into account the + deleted and modified/added files. */ + HashSink hashSink{HashAlgorithm::SHA512}; + for (auto & file : repoInfo.workdirInfo.dirtyFiles) { + writeString("modified:", hashSink); + writeString(file.abs(), hashSink); + dumpPath(repoInfo.url + "/" + file.abs(), hashSink); + } + for (auto & file : repoInfo.workdirInfo.deletedFiles) { + writeString("deleted:", hashSink); + writeString(file.abs(), hashSink); + } + return makeFingerprint(*repoInfo.workdirInfo.headRev) + + ";d=" + hashSink.finish().first.to_string(HashFormat::Base16, false); + } return std::nullopt; + } } bool isLocked(const Input & input) const override diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 07a1178cc..b4408e943 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -14,7 +14,7 @@ project('nix-fetchers', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -41,7 +41,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'attrs.cc', @@ -90,4 +90,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libfetchers/nix-meson-build-support b/src/libfetchers/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libfetchers/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 70973bdb2..7dad00025 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 7f7a09053..171afcea7 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -94,12 +94,9 @@ void Registry::add( void Registry::remove(const Input & input) { - // FIXME: use C++20 std::erase. - for (auto i = entries.begin(); i != entries.end(); ) - if (i->from == input) - i = entries.erase(i); - else - ++i; + entries.erase( + std::remove_if(entries.begin(), entries.end(), [&](const Entry & entry) { return entry.from == input; }), + entries.end()); } static Path getSystemRegistryPath() @@ -181,7 +178,8 @@ Registries getRegistries(const Settings & settings, ref store) std::pair lookupInRegistries( ref store, - const Input & _input) + const Input & _input, + const RegistryFilter & filter) { Attrs extraAttrs; int n = 0; @@ -193,6 +191,7 @@ std::pair lookupInRegistries( if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); for (auto & registry : getRegistries(*input.settings, store)) { + if (filter && !filter(registry->type)) continue; // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 0d68ac395..8f47e1590 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -65,8 +65,15 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs); +using RegistryFilter = std::function; + +/** + * Rewrite a flakeref using the registries. If `filter` is set, only + * use the registries for which the filter function returns true. + */ std::pair lookupInRegistries( ref store, - const Input & input); + const Input & input, + const RegistryFilter & filter = {}); } diff --git a/src/libflake-c/.version b/src/libflake-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libflake-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build new file mode 100644 index 000000000..b7669fe97 --- /dev/null +++ b/src/libflake-c/meson.build @@ -0,0 +1,93 @@ +project('nix-flake-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('nix-meson-build-support/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), + dependency('nix-flake'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), + dependency('nix-expr-c'), +] +subdir('nix-meson-build-support/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-flake.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + # not generated (yet?) + # '-include', 'config-flake.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + '-include', 'config-flake.h', + language : 'cpp', +) + +subdir('nix-meson-build-support/common') + +sources = files( + 'nix_api_flake.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_flake.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_flake.h') + +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') + +this_library = library( + 'nixflakec', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('nix-meson-build-support/export') diff --git a/src/libflake-c/nix-meson-build-support b/src/libflake-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc new file mode 100644 index 000000000..17cf6572d --- /dev/null +++ b/src/libflake-c/nix_api_flake.cc @@ -0,0 +1,32 @@ +#include "nix_api_flake.h" +#include "nix_api_flake_internal.hh" +#include "nix_api_util_internal.h" + +#include "flake/flake.hh" + +nix_flake_settings * nix_flake_settings_new(nix_c_context * context) +{ + try { + auto settings = nix::make_ref(); + return new nix_flake_settings{settings}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_flake_settings_free(nix_flake_settings * settings) +{ + delete settings; +} + +nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings) +{ + static std::shared_ptr registeredSettings; + try { + if (registeredSettings) + throw nix::Error("nix_flake_init_global already initialized"); + + registeredSettings = settings->settings; + nix::flake::initLib(*registeredSettings); + } + NIXC_CATCH_ERRS +} diff --git a/src/libflake-c/nix_api_flake.h b/src/libflake-c/nix_api_flake.h new file mode 100644 index 000000000..80051298d --- /dev/null +++ b/src/libflake-c/nix_api_flake.h @@ -0,0 +1,46 @@ +#ifndef NIX_API_FLAKE_H +#define NIX_API_FLAKE_H +/** @defgroup libflake libflake + * @brief Bindings to the Nix Flakes library + * + * @{ + */ +/** @file + * @brief Main entry for the libflake C bindings + */ + +#include "nix_api_store.h" +#include "nix_api_util.h" +#include "nix_api_expr.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +typedef struct nix_flake_settings nix_flake_settings; + +// Function prototypes +/** + * Create a nix_flake_settings initialized with default values. + * @param[out] context Optional, stores error information + * @return A new nix_flake_settings or NULL on failure. + * @see nix_flake_settings_free + */ +nix_flake_settings * nix_flake_settings_new(nix_c_context * context); + +/** + * @brief Release the resources associated with a nix_flake_settings. + */ +void nix_flake_settings_free(nix_flake_settings * settings); + +/** + * @brief Register Flakes support process-wide. + */ +nix_err nix_flake_init_global(nix_c_context * context, nix_flake_settings * settings); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/src/libflake-c/nix_api_flake_internal.hh b/src/libflake-c/nix_api_flake_internal.hh new file mode 100644 index 000000000..4c154a342 --- /dev/null +++ b/src/libflake-c/nix_api_flake_internal.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "ref.hh" +#include "flake/settings.hh" + +struct nix_flake_settings +{ + nix::ref settings; +}; diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix new file mode 100644 index 000000000..7425d6140 --- /dev/null +++ b/src/libflake-c/package.nix @@ -0,0 +1,60 @@ +{ lib +, stdenv +, mkMesonLibrary + +, nix-store-c +, nix-expr-c +, nix-flake + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonLibrary (finalAttrs: { + pname = "nix-flake-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../nix-meson-build-support + ./nix-meson-build-support + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + propagatedBuildInputs = [ + nix-expr-c + nix-store-c + nix-flake + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libflake-tests/build-utils-meson b/src/libflake-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libflake-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index c0a9b8847..5c3c58e53 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -14,18 +14,19 @@ project('nix-flake-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-expr-test-support'), dependency('nix-flake'), + dependency('nix-flake-c'), ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,10 +43,11 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flakeref.cc', + 'nix_api_flake.cc', 'url-name.cc', ) @@ -68,6 +70,7 @@ test( this_exe, env : { '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + 'NIX_CONFIG': 'extra-experimental-features = flakes', }, protocol : 'gtest', ) diff --git a/src/libflake-tests/nix-meson-build-support b/src/libflake-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake-tests/nix_api_flake.cc b/src/libflake-tests/nix_api_flake.cc new file mode 100644 index 000000000..21109d181 --- /dev/null +++ b/src/libflake-tests/nix_api_flake.cc @@ -0,0 +1,51 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "nix_api_flake.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include +#include + +namespace nixC { + +TEST_F(nix_api_store_test, nix_api_init_global_getFlake_exists) +{ + nix_libstore_init(ctx); + assert_ctx_ok(); + nix_libexpr_init(ctx); + assert_ctx_ok(); + + auto settings = nix_flake_settings_new(ctx); + assert_ctx_ok(); + ASSERT_NE(nullptr, settings); + + nix_flake_init_global(ctx, settings); + assert_ctx_ok(); + + nix_eval_state_builder * builder = nix_eval_state_builder_new(ctx, store); + ASSERT_NE(nullptr, builder); + assert_ctx_ok(); + + auto state = nix_eval_state_build(ctx, builder); + assert_ctx_ok(); + ASSERT_NE(nullptr, state); + + nix_eval_state_builder_free(builder); + + auto value = nix_alloc_value(ctx, state); + assert_ctx_ok(); + ASSERT_NE(nullptr, value); + + nix_err err = nix_expr_eval_from_string(ctx, state, "builtins.getFlake", ".", value); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, err); + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(ctx, value)); +} + +} // namespace nixC diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 67e716979..f1abbb32d 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -4,6 +4,7 @@ , mkMesonExecutable , nix-flake +, nix-flake-c , nix-expr-test-support , rapidcheck @@ -26,8 +27,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build @@ -38,6 +39,7 @@ mkMesonExecutable (finalAttrs: { buildInputs = [ nix-flake + nix-flake-c nix-expr-test-support rapidcheck gtest @@ -67,6 +69,7 @@ mkMesonExecutable (finalAttrs: { mkdir -p "$HOME" '' + '' export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + export NIX_CONFIG="extra-experimental-features = flakes" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libflake/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 9d5760ed1..e9fa28f06 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -21,23 +21,23 @@ using namespace flake; namespace flake { -typedef std::pair FetchedFlake; -typedef std::vector> FlakeCache; +struct FetchedFlake +{ + FlakeRef lockedRef; + StorePath storePath; +}; + +typedef std::map FlakeCache; static std::optional lookupInFlakeCache( const FlakeCache & flakeCache, const FlakeRef & flakeRef) { - // FIXME: inefficient. - for (auto & i : flakeCache) { - if (flakeRef == i.first) { - debug("mapping '%s' to previously seen input '%s' -> '%s", - flakeRef, i.first, i.second.second); - return i.second; - } - } - - return std::nullopt; + auto i = flakeCache.find(flakeRef); + if (i == flakeCache.end()) return std::nullopt; + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i->first, i->second.lockedRef); + return i->second; } static std::tuple fetchOrSubstituteTree( @@ -51,32 +51,39 @@ static std::tuple fetchOrSubstituteTree( if (!fetched) { if (originalRef.input.isDirect()) { - fetched.emplace(originalRef.fetchTree(state.store)); + auto [storePath, lockedRef] = originalRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); } else { if (allowLookup) { - resolvedRef = originalRef.resolve(state.store); - auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef); - if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store)); - flakeCache.push_back({resolvedRef, *fetchedResolved}); - fetched.emplace(*fetchedResolved); + resolvedRef = originalRef.resolve( + state.store, + [](fetchers::Registry::RegistryType type) { + /* Only use the global registry and CLI flags + to resolve indirect flakerefs. */ + return type == fetchers::Registry::Flag || type == fetchers::Registry::Global; + }); + fetched = lookupInFlakeCache(flakeCache, originalRef); + if (!fetched) { + auto [storePath, lockedRef] = resolvedRef.fetchTree(state.store); + fetched.emplace(FetchedFlake{.lockedRef = lockedRef, .storePath = storePath}); + } + flakeCache.insert_or_assign(resolvedRef, *fetched); } else { throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef); } } - flakeCache.push_back({originalRef, *fetched}); + flakeCache.insert_or_assign(originalRef, *fetched); } - auto [storePath, lockedRef] = *fetched; - debug("got tree '%s' from '%s'", - state.store->printStorePath(storePath), lockedRef); + state.store->printStorePath(fetched->storePath), fetched->lockedRef); - state.allowPath(storePath); + state.allowPath(fetched->storePath); - assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store)); + assert(!originalRef.input.getNarHash() || fetched->storePath == originalRef.input.computeStorePath(*state.store)); - return {std::move(storePath), resolvedRef, lockedRef}; + return {fetched->storePath, resolvedRef, fetched->lockedRef}; } static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) @@ -752,6 +759,8 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { + if (lockFlags.failOnUnlocked) + throw Error("cannot write lock file of flake '%s' because it has an unlocked input ('%s').\n", topRef, *unlockedInput); if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { diff --git a/src/libflake/flake/flake.hh b/src/libflake/flake/flake.hh index cc2bea76e..0dfd9440d 100644 --- a/src/libflake/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -156,6 +156,11 @@ struct LockFlags */ bool writeLockFile = true; + /** + * Throw an exception when the flake has an unlocked input. + */ + bool failOnUnlocked = false; + /** * Whether to use the registries to lookup indirect flake * references like 'nixpkgs'. diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 0ef01fbab..41e991c20 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -3,7 +3,6 @@ #include "url.hh" #include "url-parts.hh" #include "fetchers.hh" -#include "registry.hh" namespace nix { @@ -36,7 +35,9 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -FlakeRef FlakeRef::resolve(ref store) const +FlakeRef FlakeRef::resolve( + ref store, + const fetchers::RegistryFilter & filter) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); @@ -265,8 +266,6 @@ std::pair parseFlakeRefWithFragment( { using namespace fetchers; - std::smatch match; - if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { diff --git a/src/libflake/flake/flakeref.hh b/src/libflake/flake/flakeref.hh index 8c4d5dfe1..32094d381 100644 --- a/src/libflake/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -6,6 +6,7 @@ #include "types.hh" #include "fetchers.hh" #include "outputs-spec.hh" +#include "registry.hh" namespace nix { @@ -48,6 +49,11 @@ struct FlakeRef bool operator ==(const FlakeRef & other) const = default; + bool operator <(const FlakeRef & other) const + { + return std::tie(input, subdir) < std::tie(other.input, other.subdir); + } + FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) { } @@ -57,7 +63,9 @@ struct FlakeRef fetchers::Attrs toAttrs() const; - FlakeRef resolve(ref store) const; + FlakeRef resolve( + ref store, + const fetchers::RegistryFilter & filter = {}) const; static FlakeRef fromAttrs( const fetchers::Settings & fetchSettings, diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 2c1a70a18..f9e217729 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -14,7 +14,7 @@ project('nix-flake', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-fetchers'), dependency('nix-expr'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json @@ -39,7 +39,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'flake/config.cc', @@ -72,4 +72,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libflake/nix-meson-build-support b/src/libflake/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libflake/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libflake/package.nix b/src/libflake/package.nix index fff481720..92445739f 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -23,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libmain-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 3cb1e4baa..5a5684b8d 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -14,7 +14,7 @@ project('nix-main-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -27,7 +27,7 @@ deps_public_maybe_subproject = [ dependency('nix-util-c'), dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -53,7 +53,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_main.cc', @@ -65,8 +65,8 @@ headers = [config_h] + files( 'nix_api_main.h', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixmainc', @@ -82,4 +82,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain-c/nix-meson-build-support b/src/libmain-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libmain-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 5522037f3..d65792921 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libmain/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 6c6298e2b..87fc8b8d2 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -14,7 +14,7 @@ project('nix-main', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -24,7 +24,7 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') pubsetbuf_test = ''' #include @@ -58,7 +58,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-args.cc', @@ -96,4 +96,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libmain/nix-meson-build-support b/src/libmain/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libmain/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7e7b80472..7d9d99b61 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -22,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 44b5fe11d..1ac331ad0 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -14,7 +14,7 @@ project('nix-store-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -25,7 +25,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -49,7 +49,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_store.cc', @@ -64,8 +64,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` headers += files('nix_api_store_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstorec', @@ -81,4 +81,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-c/nix-meson-build-support b/src/libstore-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index fb7391276..bc306e0d0 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -67,6 +67,17 @@ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string NIXC_CATCH_ERRS } +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return call_nix_get_string_callback(store->ptr->storeDir, callback, user_data); + } + NIXC_CATCH_ERRS +} + nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) { @@ -89,6 +100,18 @@ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * NIXC_CATCH_ERRS_RES(false); } +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->toRealPath(path->path); + return call_nix_get_string_callback(res, callback, user_data); + } + NIXC_CATCH_ERRS +} + StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 282ccc285..e55bc3f59 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -48,12 +48,27 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context); * Store instances may share state and resources behind the scenes. * * @param[out] context Optional, stores error information - * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference + * + * @param[in] uri @parblock + * URI of the Nix store, copied. + * + * If `NULL`, the store from the settings will be used. + * Note that `"auto"` holds a strange middle ground, reading part of the general environment, but not all of it. It + * ignores `NIX_REMOTE` and the `store` option. For this reason, `NULL` is most likely the better choice. + * + * For supported store URLs, see [*Store URL format* in the Nix Reference * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format). - * @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint", - * "https://s3.local"}}. See [*Store Types* in the Nix Reference - * Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * + * @param[in] params @parblock + * optional, null-terminated array of key-value pairs, e.g. {{"endpoint", + * "https://s3.local"}}. + * + * See [*Store Types* in the Nix Reference Manual](https://nixos.org/manual/nix/stable/store/types). + * @endparblock + * * @return a Store pointer, NULL in case of errors + * * @see nix_store_free */ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params); @@ -78,7 +93,18 @@ void nix_store_free(Store * store); */ nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); -// returns: owned StorePath* +/** + * @brief get the storeDir of a Nix store, typically `"/nix/store"` + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the URI. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err +nix_store_get_storedir(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + /** * @brief Parse a Nix store path into a StorePath * @@ -123,6 +149,26 @@ void nix_store_path_free(StorePath * p); * @return true or false, error info in context */ bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); + +/** + * @brief Get the physical location of a store path + * + * A store may reside at a different location than its `storeDir` suggests. + * This situation is called a relocated store. + * Relocated stores are used during NixOS installation, as well as in restricted computing environments that don't offer + * a writable `/nix/store`. + * + * Not all types of stores support this operation. + * + * @param[in] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] path the path to get the real path from + * @param[in] callback called with the real path + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +nix_err nix_store_real_path( + nix_c_context * context, Store * store, StorePath * path, nix_get_string_callback callback, void * user_data); + // nix_err nix_store_ensure(Store*, const char*); // nix_err nix_store_build_paths(Store*); /** diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index 896a1a39f..351d4510a 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -20,8 +20,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore-test-support/build-utils-meson b/src/libstore-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index f8308e7bb..2a07e56ac 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -14,7 +14,7 @@ project('nix-store-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] @@ -22,8 +22,9 @@ deps_public_maybe_subproject = [ dependency('nix-util'), dependency('nix-util-test-support'), dependency('nix-store'), + dependency('nix-store-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -36,7 +37,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/derived-path.cc', @@ -55,8 +56,8 @@ headers = files( 'tests/protocol.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-store-test-support', @@ -74,4 +75,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore-test-support/nix-meson-build-support b/src/libstore-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 48f8b5e6b..62f783c4f 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -4,6 +4,7 @@ , nix-util-test-support , nix-store +, nix-store-c , rapidcheck @@ -22,8 +23,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build @@ -35,6 +36,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-util-test-support nix-store + nix-store-c rapidcheck ]; diff --git a/src/libstore-test-support/tests/libstore.hh b/src/libstore-test-support/tests/libstore.hh index 84be52c23..699ba957e 100644 --- a/src/libstore-test-support/tests/libstore.hh +++ b/src/libstore-test-support/tests/libstore.hh @@ -19,12 +19,12 @@ public: protected: LibStoreTest() : store(openStore({ - .variant = - StoreReference::Specified{ - .scheme = "dummy", - }, - .params = {}, - })) + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) { } diff --git a/src/libstore-tests/build-utils-meson b/src/libstore-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fc9152f2f..b706fa12c 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -14,19 +14,21 @@ project('nix-store-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') + +nix_store = dependency('nix-store') deps_private_maybe_subproject = [ - dependency('nix-store'), + nix_store, dependency('nix-store-c'), dependency('nix-store-test-support'), ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite @@ -50,7 +52,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'common-protocol.cc', @@ -90,6 +92,7 @@ this_exe = executable( include_directories : include_dirs, # TODO: -lrapidcheck, see ../libutil-support/build.meson link_args: linker_export_flags + ['-lrapidcheck'], + cpp_args : [ '-DNIX_STORE_DIR="' + nix_store.get_variable('storedir') + '"' ], # get main from gtest install : true, ) diff --git a/src/libstore-tests/nix-meson-build-support b/src/libstore-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index 7c6ec0780..a8b7b8e5f 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -24,6 +24,39 @@ TEST_F(nix_api_store_test, nix_store_get_uri) ASSERT_STREQ("local", str.c_str()); } +TEST_F(nix_api_util_context, nix_store_get_storedir_default) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // skipping test in sandbox because nix_store_open tries to create /nix/var/nix/profiles + GTEST_SKIP(); + } + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, nullptr, nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(NIX_STORE_DIR, str.c_str()); + + nix_store_free(store); +} + +TEST_F(nix_api_store_test, nix_store_get_storedir) +{ + std::string str; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(str)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // These tests run with a unique storeDir, but not a relocated store + ASSERT_STREQ(nixStoreDir.c_str(), str.c_str()); +} + TEST_F(nix_api_store_test, InvalidPathFails) { nix_store_parse_path(ctx, store, "invalid-path"); @@ -86,4 +119,84 @@ TEST_F(nix_api_store_test, nix_store_is_valid_path_not_in_store) ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, path)); } +TEST_F(nix_api_store_test, nix_store_real_path) +{ + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((nixStoreDir + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); } + +TEST_F(nix_api_util_context, nix_store_real_path_relocated) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // Can't open default store from within sandbox + GTEST_SKIP(); + } + auto tmp = nix::createTempDir(); + std::string storeRoot = tmp + "/store"; + std::string stateDir = tmp + "/state"; + std::string logDir = tmp + "/log"; + const char * rootkv[] = {"root", storeRoot.c_str()}; + const char * statekv[] = {"state", stateDir.c_str()}; + const char * logkv[] = {"log", logDir.c_str()}; + // const char * rokv[] = {"read-only", "true"}; + const char ** kvs[] = {rootkv, statekv, logkv, NULL}; + + nix_libstore_init(ctx); + assert_ctx_ok(); + + Store * store = nix_store_open(ctx, "local", kvs); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string nixStoreDir; + auto ret = nix_store_get_storedir(ctx, store, OBSERVE_STRING(nixStoreDir)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(NIX_STORE_DIR, nixStoreDir.c_str()); + + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + + // Assumption: we're not testing with a relocated store + ASSERT_STREQ((storeRoot + NIX_STORE_DIR + PATH_SUFFIX).c_str(), rp.c_str()); + + nix_store_path_free(path); +} + +TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) +{ + if (nix::getEnv("HOME").value_or("") == "/homeless-shelter") { + // TODO: override NIX_CACHE_HOME? + // skipping test in sandbox because narinfo cache can't be written + GTEST_SKIP(); + } + + Store * store = nix_store_open(ctx, "https://cache.nixos.org", nullptr); + assert_ctx_ok(); + ASSERT_NE(store, nullptr); + + std::string path_raw = std::string(NIX_STORE_DIR) + PATH_SUFFIX; + StorePath * path = nix_store_parse_path(ctx, store, path_raw.c_str()); + assert_ctx_ok(); + ASSERT_NE(path, nullptr); + + std::string rp; + auto ret = nix_store_real_path(ctx, store, path, OBSERVE_STRING(rp)); + assert_ctx_ok(); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(path_raw.c_str(), rp.c_str()); +} + +} // namespace nixC diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 3704d8c5c..4937b5329 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -28,8 +28,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b3..896779f85 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -39,15 +39,13 @@ BinaryCacheStore::BinaryCacheStore(const Params & params) void BinaryCacheStore::init() { - std::string cacheInfoFile = "nix-cache-info"; - - auto cacheInfo = getFile(cacheInfoFile); + auto cacheInfo = getNixCacheInfo(); if (!cacheInfo) { upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); } else { for (auto & line : tokenizeString(*cacheInfo, "\n")) { - size_t colon= line.find(':'); - if (colon ==std::string::npos) continue; + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; auto name = line.substr(0, colon); auto value = trim(line.substr(colon + 1, std::string::npos)); if (name == "StoreDir") { @@ -63,6 +61,11 @@ void BinaryCacheStore::init() } } +std::optional BinaryCacheStore::getNixCacheInfo() +{ + return getFile(cacheInfoFile); +} + void BinaryCacheStore::upsertFile(const std::string & path, std::string && data, const std::string & mimeType) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc9252..6bd7fd14a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -64,6 +64,8 @@ protected: // The prefix under which realisation infos will be stored const std::string realisationsPrefix = "realisations"; + const std::string cacheInfoFile = "nix-cache-info"; + BinaryCacheStore(const Params & params); public: @@ -84,6 +86,12 @@ public: */ virtual void getFile(const std::string & path, Sink & sink); + /** + * Get the contents of /nix-cache-info. Return std::nullopt if it + * doesn't exist. + */ + virtual std::optional getNixCacheInfo(); + /** * Fetch the specified file and call the specified callback with * the result. A subclass may implement this asynchronously. diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libstore/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 794be1568..2ff0ef92f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1161,7 +1161,7 @@ HookReply DerivationGoal::tryBuildHook() throw; } }(); - if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, "the build hook", true)) ; else if (s.substr(0, 2) == "# ") { reply = s.substr(2); @@ -1229,7 +1229,7 @@ HookReply DerivationGoal::tryBuildHook() hook->toHook.writeSide.close(); /* Create the log file and pipe. */ - Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = openLogFile(); std::set fds; fds.insert(hook->fromHook.readSide.get()); @@ -1346,9 +1346,9 @@ void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { - auto json = parseJSONMessage(currentHookLine); + auto json = parseJSONMessage(currentHookLine, "the derivation builder"); if (json) { - auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); + auto s = handleJSONLogMessage(*json, worker.act, hook->activities, "the derivation builder", true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. if (s && !isWrittenToLog && logSink) { @@ -1390,7 +1390,7 @@ void DerivationGoal::handleEOF(Descriptor fd) void DerivationGoal::flushLine() { - if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, "the derivation builder", false)) ; else { diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index dedcad2b1..f069c0d94 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -32,7 +32,7 @@ Goal::Co DrvOutputSubstitutionGoal::init() bool substituterFailed = false; - for (auto sub : subs) { + for (const auto & sub : subs) { trace("trying next substituter"); /* The callback of the curl download below can outlive `this` (if diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 315500719..983c86601 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -57,7 +57,7 @@ Goal::Co PathSubstitutionGoal::init() bool substituterFailed = false; - for (auto sub : subs) { + for (const auto & sub : subs) { trace("trying next substituter"); cleanup(); diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index cb36c0c1b..1c62cdfad 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -13,14 +13,9 @@ void Store::exportPaths(const StorePathSet & paths, Sink & sink) auto sorted = topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - std::string doneLabel("paths exported"); - //logger->incExpected(doneLabel, sorted.size()); - for (auto & path : sorted) { - //Activity act(*logger, lvlInfo, "exporting path '%s'", path); sink << 1; exportPath(path, sink); - //logger->incProgress(doneLabel); } sink << 0; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index fc7ac2dea..f32616f94 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -194,6 +194,19 @@ protected: } } + std::optional getNixCacheInfo() override + { + try { + auto result = getFileTransfer()->download(makeRequest(cacheInfoFile)); + return result.data; + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound) + return std::nullopt; + maybeDisable(); + throw; + } + } + /** * This isn't actually necessary read only. We support "upsert" now, so we * have a notion of authentication via HTTP POST/PUT. diff --git a/src/libstore/keys.cc b/src/libstore/keys.cc index 70478e7ad..668725fc7 100644 --- a/src/libstore/keys.cc +++ b/src/libstore/keys.cc @@ -10,12 +10,12 @@ PublicKeys getDefaultPublicKeys() // FIXME: filter duplicates - for (auto s : settings.trustedPublicKeys.get()) { + for (const auto & s : settings.trustedPublicKeys.get()) { PublicKey key(s); publicKeys.emplace(key.name, key); } - for (auto secretKeyFile : settings.secretKeyFiles.get()) { + for (const auto & secretKeyFile : settings.secretKeyFiles.get()) { try { SecretKey secretKey(readFile(secretKeyFile)); publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index b86beba2c..56ff6bef3 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -156,7 +156,7 @@ void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) { auto res = LocalStore::queryValidDerivers(path); - for (auto p : lowerStore->queryValidDerivers(path)) + for (const auto & p : lowerStore->queryValidDerivers(path)) res.insert(p); return res; } diff --git a/src/libstore/meson.build b/src/libstore/meson.build index f836b8d4f..12a0e6376 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -15,7 +15,7 @@ project('nix-store', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -29,7 +29,7 @@ deps_private_maybe_subproject = [ deps_public_maybe_subproject = [ dependency('nix-util'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') run_command('ln', '-s', meson.project_build_root() / '__nothing_link_target', @@ -81,7 +81,7 @@ if host_machine.system() == 'windows' deps_other += [wsock32] endif -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') boost = dependency( 'boost', @@ -131,7 +131,7 @@ if aws_s3.found() endif deps_other += aws_s3 -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/generate-header') generated_headers = [] foreach header : [ @@ -179,7 +179,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'binary-cache-store.cc', @@ -416,8 +416,8 @@ foreach name, value : cpp_str_defines ] endforeach -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstore', @@ -446,4 +446,4 @@ if host_machine.system() != 'darwin' } endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 8b2557060..27fcc2864 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -118,7 +118,7 @@ std::string NarInfo::to_string(const Store & store) const if (deriver) res += "Deriver: " + std::string(deriver->to_string()) + "\n"; - for (auto sig : sigs) + for (const auto & sig : sigs) res += "Sig: " + sig + "\n"; if (ca) diff --git a/src/libstore/nix-meson-build-support b/src/libstore/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libstore/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libstore/package.nix b/src/libstore/package.nix index f04e3b95f..47a203f83 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -32,8 +32,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libstore/path-regex.hh b/src/libstore/path-regex.hh index 56c2cfc1d..e34a305c5 100644 --- a/src/libstore/path-regex.hh +++ b/src/libstore/path-regex.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index bcbf0b55e..cfa713b00 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -48,7 +48,11 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), - s + ": " + outcome.GetError().GetMessage()); + fmt( + "%s: %s (request id: %s)", + s, + outcome.GetError().GetMessage(), + outcome.GetError().GetRequestId())); return outcome.GetResultWithOwnership(); } @@ -121,9 +125,10 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy checkInterrupt(); auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) - printError("AWS error '%s' (%s), will retry in %d ms", + printError("AWS error '%s' (%s; request id: %s), will retry in %d ms", error.GetExceptionName(), error.GetMessage(), + error.GetRequestId(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); return retry; } @@ -454,7 +459,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual debug("got %d keys, next marker '%s'", contents.size(), res.GetNextMarker()); - for (auto object : contents) { + for (const auto & object : contents) { auto & key = object.GetKey(); if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; paths.insert(parseStorePath(storeDir + "/" + key.substr(0, key.size() - 8) + "-" + MissingName)); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index dec733fd5..116a480ba 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -41,8 +41,17 @@ void SSHMaster::addCommonSSHOpts(Strings & args) { auto state(state_.lock()); - for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS").value_or(""))) - args.push_back(i); + std::string sshOpts = getEnv("NIX_SSHOPTS").value_or(""); + + try { + std::list opts = shellSplitString(sshOpts); + for (auto & i : opts) + args.push_back(i); + } catch (Error & e) { + e.addTrace({}, "while splitting NIX_SSHOPTS '%s'", sshOpts); + throw; + } + if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); if (!sshPublicHostKey.empty()) { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 10577fa2a..78cc3b917 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1332,7 +1332,7 @@ ref openStore(StoreReference && storeURI) return std::make_shared(params); }, [&](const StoreReference::Specified & g) { - for (auto implem : *Implementations::registered) + for (const auto & implem : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) return implem.create(g.scheme, g.authority, params); @@ -1363,7 +1363,7 @@ std::list> getDefaultSubstituters() } }; - for (auto uri : settings.substituters.get()) + for (const auto & uri : settings.substituters.get()) addStore(uri); stores.sort([](ref & a, ref & b) { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index dcfaadeef..06a2f85be 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -883,7 +883,7 @@ void LocalDerivationGoal::startBuilder() printMsg(lvlVomit, "setting builder env variable '%1%'='%2%'", i.first, i.second); /* Create the log file. */ - Path logFile = openLogFile(); + [[maybe_unused]] Path logFile = openLogFile(); /* Create a pseudoterminal to get the output of the builder. */ builderOut = posix_openpt(O_RDWR | O_NOCTTY); diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 00761a8c3..197f5a1c4 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -2,10 +2,12 @@ #include "pathlocks.hh" #include "signals.hh" #include "util.hh" -#include -#include -#include -#include "windows-error.hh" + +#ifdef WIN32 +# include +# include +# include +# include "windows-error.hh" namespace nix { @@ -154,3 +156,4 @@ FdLock::FdLock(Descriptor desc, LockType lockType, bool wait, std::string_view w } } +#endif diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-c/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index d44453676..44cec1afc 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -14,7 +14,7 @@ project('nix-util-c', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -23,7 +23,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) @@ -45,7 +45,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'nix_api_util.cc', @@ -60,8 +60,8 @@ headers = [config_h] + files( # TODO don't install this once tests don't use it. headers += files('nix_api_util_internal.h') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutilc', @@ -77,4 +77,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-c/nix-meson-build-support b/src/libutil-c/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-c/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 43f9fa9dc..5f42641d4 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -47,7 +47,7 @@ extern "C" { */ // Error codes /** - * @brief Type for error codes in the NIX system + * @brief Type for error codes in the Nix system * * This type can have one of several predefined constants: * - NIX_OK: No error occurred (0) diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 35533f981..4caf96804 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -19,8 +19,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-test-support/build-utils-meson b/src/libutil-test-support/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-test-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index fa1df7320..03ae63f1a 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -14,14 +14,15 @@ project('nix-util-test-support', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ dependency('nix-util'), + dependency('nix-util-c'), ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') rapidcheck = dependency('rapidcheck') deps_public += rapidcheck @@ -33,7 +34,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'tests/hash.cc', @@ -50,8 +51,8 @@ headers = files( 'tests/string_callback.hh', ) -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-util-test-support', @@ -69,4 +70,4 @@ install_headers(headers, subdir : 'nix', preserve_path : true) libraries_private = [] -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil-test-support/nix-meson-build-support b/src/libutil-test-support/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-test-support/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index 2525e1602..19b5d6b77 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -3,6 +3,7 @@ , mkMesonLibrary , nix-util +, nix-util-c , rapidcheck @@ -21,8 +22,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build @@ -33,6 +34,7 @@ mkMesonLibrary (finalAttrs: { propagatedBuildInputs = [ nix-util + nix-util-c rapidcheck ]; diff --git a/src/libutil-test-support/tests/gtest-with-params.hh b/src/libutil-test-support/tests/gtest-with-params.hh index d72aec4fd..a6e23ad89 100644 --- a/src/libutil-test-support/tests/gtest-with-params.hh +++ b/src/libutil-test-support/tests/gtest-with-params.hh @@ -40,7 +40,7 @@ void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) } else { std::ostringstream ss; printResultMessage(result, ss); - FAIL() << ss.str() << std::endl; + throw std::runtime_error(ss.str()); } } } diff --git a/src/libutil-test-support/tests/nix_api_util.hh b/src/libutil-test-support/tests/nix_api_util.hh index efd200116..006dc497c 100644 --- a/src/libutil-test-support/tests/nix_api_util.hh +++ b/src/libutil-test-support/tests/nix_api_util.hh @@ -26,14 +26,13 @@ protected: inline void assert_ctx_ok() { - if (nix_err_code(ctx) == NIX_OK) { return; } unsigned int n; const char * p = nix_err_msg(nullptr, ctx, &n); std::string msg(p, n); - FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg; + throw std::runtime_error(std::string("nix_err_code(ctx) != NIX_OK, message: ") + msg); } inline void assert_ctx_err() @@ -41,7 +40,7 @@ protected: if (nix_err_code(ctx) != NIX_OK) { return; } - FAIL() << "Got NIX_OK, but expected an error!"; + throw std::runtime_error("Got NIX_OK, but expected an error!"); } }; diff --git a/src/libutil-tests/build-utils-meson b/src/libutil-tests/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil-tests/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-tests/hilite.cc b/src/libutil-tests/hilite.cc index 1ff5980d5..5ef581888 100644 --- a/src/libutil-tests/hilite.cc +++ b/src/libutil-tests/hilite.cc @@ -52,8 +52,7 @@ namespace nix { std::regex("pt"), }; std::vector matches; - for(auto regex : regexes) - { + for (const auto & regex : regexes) { for(auto it = std::sregex_iterator(str.begin(), str.end(), regex); it != std::sregex_iterator(); ++it) { matches.push_back(*it); } diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index f59350774..83ac79e92 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -14,7 +14,7 @@ project('nix-util-tests', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') deps_private_maybe_subproject = [ dependency('nix-util'), @@ -23,10 +23,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') rapidcheck = dependency('rapidcheck') deps_private += rapidcheck @@ -42,7 +42,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'args.cc', diff --git a/src/libutil-tests/nix-meson-build-support b/src/libutil-tests/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil-tests/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index b36f71042..7b77bd87f 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -136,7 +136,6 @@ TEST_F(nix_api_util_context, nix_err_name) // no error EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); - std::string err_msg_ref; try { throw nix::Error("testing error"); } catch (...) { diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index b099037ee..f06704e26 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -26,8 +26,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil-tests/strings.cc b/src/libutil-tests/strings.cc index 8ceb16767..206890bcf 100644 --- a/src/libutil-tests/strings.cc +++ b/src/libutil-tests/strings.cc @@ -2,11 +2,10 @@ #include #include "strings.hh" +#include "error.hh" namespace nix { -using Strings = std::vector; - /* ---------------------------------------------------------------------------- * concatStringsSep * --------------------------------------------------------------------------*/ @@ -345,4 +344,108 @@ RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); } +/* ---------------------------------------------------------------------------- + * shellSplitString + * --------------------------------------------------------------------------*/ + +TEST(shellSplitString, empty) +{ + std::list expected = {}; + + ASSERT_EQ(shellSplitString(""), expected); +} + +TEST(shellSplitString, oneWord) +{ + std::list expected = {"foo"}; + + ASSERT_EQ(shellSplitString("foo"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpaces) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("'foo bar'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithSpacesAndDoubleQuoteInSingleQuote) +{ + std::list expected = {"foo bar\""}; + + ASSERT_EQ(shellSplitString("'foo bar\"'"), expected); +} + +TEST(shellSplitString, oneWordQuotedWithDoubleQuotes) +{ + std::list expected = {"foo bar"}; + + ASSERT_EQ(shellSplitString("\"foo bar\""), expected); +} + +TEST(shellSplitString, twoWords) +{ + std::list expected = {"foo", "bar"}; + + ASSERT_EQ(shellSplitString("foo bar"), expected); +} + +TEST(shellSplitString, twoWordsWithSpacesAndQuotesQuoted) +{ + std::list expected = {"foo bar'", "baz\""}; + + ASSERT_EQ(shellSplitString("\"foo bar'\" 'baz\"'"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedSingleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo '' bar baz ''"), expected); +} + +TEST(shellSplitString, emptyArgumentsAreAllowedDoubleQuotes) +{ + std::list expected = {"foo", "", "bar", "baz", ""}; + + ASSERT_EQ(shellSplitString("foo \"\" bar baz \"\""), expected); +} + +TEST(shellSplitString, singleQuoteDoesNotUseEscapes) +{ + std::list expected = {"foo\\\"bar"}; + + ASSERT_EQ(shellSplitString("'foo\\\"bar'"), expected); +} + +TEST(shellSplitString, doubleQuoteDoesUseEscapes) +{ + std::list expected = {"foo\"bar"}; + + ASSERT_EQ(shellSplitString("\"foo\\\"bar\""), expected); +} + +TEST(shellSplitString, backslashEscapesSpaces) +{ + std::list expected = {"foo bar", "baz", "qux quux"}; + + ASSERT_EQ(shellSplitString("foo\\ bar baz qux\\ quux"), expected); +} + +TEST(shellSplitString, backslashEscapesQuotes) +{ + std::list expected = {"foo\"bar", "baz", "qux'quux"}; + + ASSERT_EQ(shellSplitString("foo\\\"bar baz qux\\'quux"), expected); +} + +TEST(shellSplitString, testUnbalancedQuotes) +{ + ASSERT_THROW(shellSplitString("foo'"), Error); + ASSERT_THROW(shellSplitString("foo\""), Error); + ASSERT_THROW(shellSplitString("foo'bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar"), Error); + ASSERT_THROW(shellSplitString("foo\"bar\\\""), Error); +} + } // namespace nix diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 385b6cd34..05ecf724e 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -348,7 +348,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) /* Now that all the other args are processed, run the deferred completions. */ - for (auto d : deferredCompletions) + for (const auto & d : deferredCompletions) d.completer(*completions, d.n, d.prefix); } diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson deleted file mode 120000 index 5fff21bab..000000000 --- a/src/libutil/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/callback.hh b/src/libutil/callback.hh index 26c386d80..c2cada2f6 100644 --- a/src/libutil/callback.hh +++ b/src/libutil/callback.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index c3aa61ddb..94c2cb2e4 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -13,6 +13,7 @@ */ #include "config.hh" +#include "args.hh" namespace nix { diff --git a/src/libutil/config.hh b/src/libutil/config.hh index c0c59ac68..e98e09bf7 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -115,6 +115,8 @@ public: * Re-applies all previously attempted changes to unknown settings */ void reapplyUnknownSettings(); + + virtual ~AbstractConfig() = default; }; /** diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index ebd522a41..8d665c7df 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -73,7 +73,7 @@ ExecutablePath::findName(const OsString & exe, std::function createTempFile(const Path & prefix) void createSymlink(const Path & target, const Path & link) { - fs::create_symlink(target, link); + try { + fs::create_symlink(target, link); + } catch (fs::filesystem_error & e) { + throw SysError("creating symlink '%1%' -> '%2%'", link, target); + } } void replaceSymlink(const fs::path & target, const fs::path & link) @@ -615,10 +619,16 @@ void replaceSymlink(const fs::path & target, const fs::path & link) fs::create_symlink(target, tmp); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; - throw; + throw SysError("creating symlink '%1%' -> '%2%'", tmp, target); + } + + try { + fs::rename(tmp, link); + } catch (fs::filesystem_error & e) { + if (e.code() == std::errc::file_exists) continue; + throw SysError("renaming '%1%' to '%2%'", tmp, link); } - fs::rename(tmp, link); break; } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4c08cdf58..3c49181a0 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -250,8 +250,6 @@ void setWriteTime(const std::filesystem::path & path, const struct stat & st); /** * Create a symlink. * - * In the process of being deprecated for - * `std::filesystem::create_symlink`. */ void createSymlink(const Path & target, const Path & link); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 748176d33..b69dec685 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -134,7 +134,8 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const Hash Hash::dummy(HashAlgorithm::SHA256); -Hash Hash::parseSRI(std::string_view original) { +Hash Hash::parseSRI(std::string_view original) +{ auto rest = original; // Parse the has type before the separater, if there was one. diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 80c107ef5..a5add5565 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -280,61 +280,72 @@ static Logger::Fields getFields(nlohmann::json & json) return fields; } -std::optional parseJSONMessage(const std::string & msg) +std::optional parseJSONMessage(const std::string & msg, std::string_view source) { if (!hasPrefix(msg, "@nix ")) return std::nullopt; try { return nlohmann::json::parse(std::string(msg, 5)); } catch (std::exception & e) { - printError("bad JSON log message from builder: %s", e.what()); + printError("bad JSON log message from %s: %s", + Uncolored(source), + e.what()); } return std::nullopt; } bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, - bool trusted) + std::string_view source, bool trusted) { - std::string action = json["action"]; + try { + std::string action = json["action"]; - if (action == "start") { - auto type = (ActivityType) json["type"]; - if (trusted || type == actFileTransfer) - activities.emplace(std::piecewise_construct, - std::forward_as_tuple(json["id"]), - std::forward_as_tuple(*logger, (Verbosity) json["level"], type, - json["text"], getFields(json["fields"]), act.id)); + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actFileTransfer) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + return true; + } catch (const nlohmann::json::exception &e) { + warn( + "Unable to handle a JSON message from %s: %s", + Uncolored(source), + e.what() + ); + return false; } - - else if (action == "stop") - activities.erase((ActivityId) json["id"]); - - else if (action == "result") { - auto i = activities.find((ActivityId) json["id"]); - if (i != activities.end()) - i->second.result((ResultType) json["type"], getFields(json["fields"])); - } - - else if (action == "setPhase") { - std::string phase = json["phase"]; - act.result(resSetPhase, phase); - } - - else if (action == "msg") { - std::string msg = json["msg"]; - logger->log((Verbosity) json["level"], msg); - } - - return true; } bool handleJSONLogMessage(const std::string & msg, - const Activity & act, std::map & activities, bool trusted) + const Activity & act, std::map & activities, std::string_view source, bool trusted) { - auto json = parseJSONMessage(msg); + auto json = parseJSONMessage(msg, source); if (!json) return false; - return handleJSONLogMessage(*json, act, activities, trusted); + return handleJSONLogMessage(*json, act, activities, source, trusted); } Activity::~Activity() diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 250f92099..11e4033a5 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -185,14 +185,25 @@ Logger * makeSimpleLogger(bool printBuildLogs = true); Logger * makeJSONLogger(Logger & prevLogger); -std::optional parseJSONMessage(const std::string & msg); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ +std::optional parseJSONMessage(const std::string & msg, std::string_view source); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(nlohmann::json & json, const Activity & act, std::map & activities, + std::string_view source, bool trusted); +/** + * @param source A noun phrase describing the source of the message, e.g. "the builder". + */ bool handleJSONLogMessage(const std::string & msg, const Activity & act, std::map & activities, + std::string_view source, bool trusted); /** diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 11b4ea592..bbe7872cf 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -14,7 +14,7 @@ project('nix-util', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') configdata = configuration_data() @@ -22,7 +22,7 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') # Check for each of these functions, and create a define like `#define # HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 @@ -53,7 +53,7 @@ endforeach configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) -subdir('build-utils-meson/libatomic') +subdir('nix-meson-build-support/libatomic') if host_machine.system() == 'windows' socket = cxx.find_library('ws2_32') @@ -120,7 +120,7 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') +subdir('nix-meson-build-support/common') sources = files( 'archive.cc', @@ -256,8 +256,8 @@ else subdir('unix') endif -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutil', @@ -278,4 +278,4 @@ if host_machine.system() == 'windows' libraries_private += ['-lws2_32'] endif -subdir('build-utils-meson/export') +subdir('nix-meson-build-support/export') diff --git a/src/libutil/nix-meson-build-support b/src/libutil/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/libutil/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 17a156740..69ebbf726 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -25,8 +25,8 @@ mkMesonLibrary (finalAttrs: { workDir = ./.; fileset = fileset.unions [ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/libutil/regex-combinators.hh b/src/libutil/regex-combinators.hh index 87d6aa678..75ccd4e6c 100644 --- a/src/libutil/regex-combinators.hh +++ b/src/libutil/regex-combinators.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include namespace nix::regex { @@ -10,22 +12,23 @@ namespace nix::regex { static inline std::string either(std::string_view a, std::string_view b) { - return std::string { a } + "|" + b; + std::stringstream ss; + ss << a << "|" << b; + return ss.str(); } static inline std::string group(std::string_view a) { - return std::string { "(" } + a + ")"; -} - -static inline std::string many(std::string_view a) -{ - return std::string { "(?:" } + a + ")*"; + std::stringstream ss; + ss << "(" << a << ")"; + return ss.str(); } static inline std::string list(std::string_view a) { - return std::string { a } + many(group("," + a)); + std::stringstream ss; + ss << a << "(," << a << ")*"; + return ss.str(); } } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 168d2ed32..381e7ae38 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -90,7 +90,6 @@ void Source::operator () (std::string_view data) void Source::drainInto(Sink & sink) { - std::string s; std::array buf; while (true) { size_t n; @@ -427,7 +426,7 @@ Error readError(Source & source) auto type = readString(source); assert(type == "Error"); auto level = (Verbosity) readInt(source); - auto name = readString(source); // removed + [[maybe_unused]] auto name = readString(source); // removed auto msg = readString(source); ErrorInfo info { .level = level, diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index e797951c7..d3e304f74 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -84,9 +84,10 @@ CanonPath SourceAccessor::resolveSymlinks( todo.pop_front(); if (c == "" || c == ".") ; - else if (c == "..") - res.pop(); - else { + else if (c == "..") { + if (!res.isRoot()) + res.pop(); + } else { res.push(c); if (mode == SymlinkResolution::Full || !todo.empty()) { if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) { diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index c221a43c6..402b7ae98 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -4,6 +4,7 @@ #include "strings-inline.hh" #include "os-string.hh" +#include "error.hh" namespace nix { @@ -48,4 +49,107 @@ template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const s template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + * + * Read https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html for the + * POSIX shell specification, which is technically what we are implementing here. + */ +std::list shellSplitString(std::string_view s) +{ + std::list result; + std::string current; + bool startedCurrent = false; + bool escaping = false; + + auto pushCurrent = [&]() { + if (startedCurrent) { + result.push_back(current); + current.clear(); + startedCurrent = false; + } + }; + + auto pushChar = [&](char c) { + current.push_back(c); + startedCurrent = true; + }; + + auto pop = [&]() { + auto c = s[0]; + s.remove_prefix(1); + return c; + }; + + auto inDoubleQuotes = [&]() { + startedCurrent = true; + // in double quotes, escaping with backslash is only effective for $, `, ", and backslash + while (!s.empty()) { + auto c = pop(); + if (escaping) { + switch (c) { + case '$': + case '`': + case '"': + case '\\': + pushChar(c); + break; + default: + pushChar('\\'); + pushChar(c); + break; + } + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == '"') { + return; + } else { + pushChar(c); + } + } + if (s.empty()) { + throw Error("unterminated double quote"); + } + }; + + auto inSingleQuotes = [&]() { + startedCurrent = true; + while (!s.empty()) { + auto c = pop(); + if (c == '\'') { + return; + } + pushChar(c); + } + if (s.empty()) { + throw Error("unterminated single quote"); + } + }; + + while (!s.empty()) { + auto c = pop(); + if (escaping) { + pushChar(c); + escaping = false; + } else if (c == '\\') { + escaping = true; + } else if (c == ' ' || c == '\t') { + pushCurrent(); + } else if (c == '"') { + inDoubleQuotes(); + } else if (c == '\'') { + inSingleQuotes(); + } else { + pushChar(c); + } + } + + pushCurrent(); + + return result; +} } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 533126be1..c4fd3daa1 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -71,4 +71,11 @@ extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); +/** + * Shell split string: split a string into shell arguments, respecting quotes and backslashes. + * + * Used for NIX_SSHOPTS handling, which previously used `tokenizeString` and was broken by + * Arguments that need to be passed to ssh with spaces in them. + */ +std::list shellSplitString(std::string_view s); } diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index db7a6fcd1..4c127ddb0 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -26,7 +26,7 @@ bool isTTY() std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) { - std::string t, e; + std::string t; size_t w = 0; auto i = s.begin(); diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 325e3ea73..9f5c75827 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -43,9 +43,11 @@ template struct Explicit { T t; - bool operator ==(const Explicit & other) const + bool operator ==(const Explicit & other) const = default; + + bool operator <(const Explicit & other) const { - return t == other.t; + return t < other.t; } }; diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh index 2193922be..037416e7d 100644 --- a/src/libutil/unix/signals-impl.hh +++ b/src/libutil/unix/signals-impl.hh @@ -14,6 +14,7 @@ #include "error.hh" #include "logging.hh" #include "ansicolor.hh" +#include "signals.hh" #include #include diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9ed49dcbe..63b9734ee 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -77,7 +77,7 @@ std::map decodeQuery(const std::string & query) { std::map result; - for (auto s : tokenizeString(query, "&")) { + for (const auto & s : tokenizeString(query, "&")) { auto e = s.find('='); if (e == std::string::npos) { warn("dubious URI query '%s' is missing equal sign '%s', ignoring", s, "="); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0fb6ff837..4d5683e2b 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -338,7 +338,9 @@ std::string showBytes(uint64_t bytes); */ inline std::string operator + (const std::string & s1, std::string_view s2) { - auto s = s1; + std::string s; + s.reserve(s1.size() + s2.size()); + s.append(s1); s.append(s2); return s; } @@ -351,10 +353,11 @@ inline std::string operator + (std::string && s, std::string_view s2) inline std::string operator + (std::string_view s1, const char * s2) { + auto s2Size = strlen(s2); std::string s; - s.reserve(s1.size() + strlen(s2)); + s.reserve(s1.size() + s2Size); s.append(s1); - s.append(s2); + s.append(s2, s2Size); return s; } diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 5ce8a1395..308a432e4 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,7 @@ #include "environment-variables.hh" -#include "processenv.h" +#ifdef WIN32 +# include "processenv.h" namespace nix { @@ -43,3 +44,4 @@ int setEnvOs(const OsString & name, const OsString & value) } } +#endif diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 7b8a712e8..71f53ccb8 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,6 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" +#ifdef WIN32 #include #include #include @@ -152,3 +153,4 @@ Path windows::handleToPath(HANDLE handle) { #endif } +#endif diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index b15355efe..53271cef3 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,5 +1,6 @@ #include "file-system.hh" +#ifdef WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) @@ -15,3 +16,4 @@ Descriptor openDirectory(const std::filesystem::path & path) } } +#endif diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index 91a321f7c..af7e987e9 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,9 +1,10 @@ -#include -#include "windows-error.hh" +#ifdef WIN32 +# include +# include "windows-error.hh" -#include "logging.hh" -#include "util.hh" -#include "muxable-pipe.hh" +# include "logging.hh" +# include "util.hh" +# include "muxable-pipe.hh" namespace nix { @@ -68,3 +69,4 @@ void MuxablePipePollState::iterate( } } +#endif diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 7507f9030..26ad9cba0 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,6 +7,8 @@ #include "file-path-impl.hh" #include "util.hh" +#ifdef WIN32 + namespace nix { std::string os_string_to_string(PathViewNG::string_view path) @@ -22,3 +24,5 @@ std::filesystem::path::string_type string_to_os_string(std::string_view s) } } + +#endif diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 7f34c5632..e69f1ed45 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,6 +23,8 @@ #include #include +#ifdef WIN32 + #define WIN32_LEAN_AND_MEAN #include @@ -386,3 +388,5 @@ int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * } } + +#endif diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index db6c42df3..2780e45f4 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" +#ifdef WIN32 #define WIN32_LEAN_AND_MEAN #include @@ -50,3 +51,4 @@ bool isRootUser() { } } +#endif diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 4fa57ca36..13b563510 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,6 +1,8 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" +#ifdef WIN32 + namespace nix::windows { void AsyncPipe::createAsyncPipe(HANDLE iocp) @@ -47,3 +49,5 @@ void AsyncPipe::close() } } + +#endif diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 8f554e403..277336ed7 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,6 +2,7 @@ ///@file #include "file-descriptor.hh" +#ifdef WIN32 namespace nix::windows { @@ -25,3 +26,4 @@ public: }; } +#endif diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index aead4af23..4cf4274da 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,5 +1,6 @@ #include "windows-error.hh" +#ifdef WIN32 #include #define WIN32_LEAN_AND_MEAN #include @@ -29,3 +30,4 @@ std::string WinError::renderError(DWORD lastError) } } +#endif diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 624b4c4cb..4e48ee859 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#ifdef WIN32 #include #include "error.hh" @@ -49,3 +50,4 @@ private: }; } +#endif diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index ba2baccee..e9eb52708 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -481,12 +481,13 @@ static void printMissing(EvalState & state, PackageInfos & elems) { std::vector targets; for (auto & i : elems) - if (auto drvPath = i.queryDrvPath()) - targets.emplace_back(DerivedPath::Built{ + if (auto drvPath = i.queryDrvPath()) { + auto path = DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, - }); - else + }; + targets.emplace_back(std::move(path)); + } else targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index c823c930e..b731b25af 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -222,7 +222,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) { auto method = FileIngestionMethod::Flat; - for (auto i : opFlags) + for (const auto & i : opFlags) if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson deleted file mode 120000 index 91937f183..000000000 --- a/src/nix/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 9a95bc695..1736add9a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -611,7 +611,7 @@ struct CmdDevelop : Common, MixEnvironment else if (!command.empty()) { std::vector args; args.reserve(command.size()); - for (auto s : command) + for (const auto & s : command) args.push_back(shellEscape(s)); script += fmt("exec %s\n", concatStringsSep(" ", args)); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ce2faacb0..55aa8971e 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -94,7 +94,7 @@ public: .label="inputs", .optional=true, .handler={[&](std::vector inputsToUpdate){ - for (auto inputToUpdate : inputsToUpdate) { + for (const auto & inputToUpdate : inputsToUpdate) { InputPath inputPath; try { inputPath = flake::parseInputPath(inputToUpdate); @@ -162,6 +162,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; lockFlake(); @@ -643,10 +644,11 @@ struct CmdFlakeCheck : FlakeCommand fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { - drvPaths.push_back(DerivedPath::Built { + auto path = DerivedPath::Built { .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, - }); + }; + drvPaths.push_back(std::move(path)); } } } @@ -936,7 +938,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - fs::create_symlink(target, to2); + createSymlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); diff --git a/src/nix/flake.md b/src/nix/flake.md index 92dc63e10..a55bba4fc 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -165,7 +165,8 @@ can occur in *locked* flake references and are available to Nix code: Currently the `type` attribute can be one of the following: -* `indirect`: *The default*. Indirection through the flake registry. +* `indirect`: *The default*. These are symbolic references to flakes + that are looked up in [the flake registries](./nix3-registry.md). These have the form ``` @@ -256,6 +257,9 @@ Currently the `type` attribute can be one of the following: * `./sub/dir` (when used on the command line and `dir/flake.nix` is in a git repository) * `git+https://example.org/my/repo` * `git+https://example.org/my/repo?dir=flake1` + * `git+https://example.org/my/repo?shallow=1` A shallow clone of the repository. + For large repositories, the shallow clone option can significantly speed up fresh clones compared + to non-shallow clones, while still providing faster updates than other fetch methods such as `tarball:` or `github:`. * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` * `git+file:///home/my-user/some-repo/some-repo` @@ -661,7 +665,7 @@ following fields: * `inputs`: The dependencies of this node, as a mapping from input names (e.g. `nixpkgs`) to node labels (e.g. `n2`). -* `original`: The original input specification from `flake.lock`, as a +* `original`: The original input specification from `flake.nix`, as a set of `builtins.fetchTree` arguments. * `locked`: The locked input specification, as a set of diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 62266fda1..416cd19b3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -79,7 +79,7 @@ struct CmdHashBase : Command void run() override { - for (auto path : paths) { + for (const auto & path : paths) { auto makeSink = [&]() -> std::unique_ptr { if (modulus) return std::make_unique(hashAlgo, *modulus); @@ -163,8 +163,11 @@ struct CmdToBase : Command HashFormat hashFormat; std::optional hashAlgo; std::vector args; + bool legacyCli; - CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) + CmdToBase(HashFormat hashFormat, bool legacyCli = false) + : hashFormat(hashFormat) + , legacyCli(legacyCli) { addFlag(flag::hashAlgoOpt("type", &hashAlgo)); expectArgs("strings", &args); @@ -181,8 +184,9 @@ struct CmdToBase : Command void run() override { - warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); - for (auto s : args) + if (!legacyCli) + warn("The old format conversion subcommands of `nix hash` were deprecated in favor of `nix hash convert`."); + for (const auto & s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -222,11 +226,18 @@ struct CmdHashConvert : Command Category category() override { return catUtility; } void run() override { - for (const auto& s: hashStrings) { - Hash h = Hash::parseAny(s, algo); - if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + for (const auto & s : hashStrings) { + Hash h = + from == HashFormat::SRI + ? Hash::parseSRI(s) + : Hash::parseAny(s, algo); + if (from + && from != HashFormat::SRI + && h.to_string(*from, false) != + (from == HashFormat::Base16 ? toLower(s) : s)) + { auto from_as_string = printHashFormat(*from); - throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + throw BadHash("input hash '%s' does not have the expected format for '--from %s'", s, from_as_string); } logger->cout(h.to_string(to, to == HashFormat::SRI)); } @@ -321,7 +332,7 @@ static int compatNixHash(int argc, char * * argv) } else { - CmdToBase cmd(hashFormat); + CmdToBase cmd(hashFormat, true); cmd.args = ss; if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo; cmd.run(); diff --git a/src/nix/meson.build b/src/nix/meson.build index 5c70c8216..1d4840b12 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -15,7 +15,7 @@ project('nix', 'cpp', cxx = meson.get_compiler('cpp') -subdir('build-utils-meson/deps-lists') +subdir('nix-meson-build-support/deps-lists') nix_store = dependency('nix-store') @@ -30,10 +30,10 @@ deps_private_maybe_subproject = [ ] deps_public_maybe_subproject = [ ] -subdir('build-utils-meson/subprojects') +subdir('nix-meson-build-support/subprojects') -subdir('build-utils-meson/export-all-symbols') -subdir('build-utils-meson/windows-version') +subdir('nix-meson-build-support/export-all-symbols') +subdir('nix-meson-build-support/windows-version') configdata = configuration_data() @@ -63,8 +63,8 @@ add_project_arguments( language : 'cpp', ) -subdir('build-utils-meson/common') -subdir('build-utils-meson/generate-header') +subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/generate-header') nix_sources = [config_h] + files( 'add-to-store.cc', diff --git a/src/nix/nix-meson-build-support b/src/nix/nix-meson-build-support new file mode 120000 index 000000000..0b140f56b --- /dev/null +++ b/src/nix/nix-meson-build-support @@ -0,0 +1 @@ +../../nix-meson-build-support \ No newline at end of file diff --git a/src/nix/package.nix b/src/nix/package.nix index c7b24efce..9bc139c3b 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -22,8 +22,8 @@ mkMesonExecutable (finalAttrs: { workDir = ./.; fileset = fileset.unions ([ - ../../build-utils-meson - ./build-utils-meson + ../../nix-meson-build-support + ./nix-meson-build-support ../../.version ./.version ./meson.build diff --git a/src/nix/registry.md b/src/nix/registry.md index bd3575d1b..d6f8af5e9 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -34,6 +34,8 @@ highest precedence: * Overrides specified on the command line using the option `--override-flake`. +Note that the system and user registries are not used to resolve flake references in `flake.nix`. They are only used to resolve flake references on the command line. + # Registry format A registry is a JSON file with the following format: diff --git a/src/nix/run.cc b/src/nix/run.cc index c9857e13e..a9f9ef60f 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -180,9 +180,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (auto entry : fs::directory_iterator{"/"}) { + for (const auto & entry : fs::directory_iterator{"/"}) { checkInterrupt(); - auto src = entry.path(); + const auto & src = entry.path(); fs::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; auto st = entry.symlink_status(); diff --git a/src/nix/search.cc b/src/nix/search.cc index c8d0b9e96..30b96c500 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -161,7 +161,6 @@ struct CmdSearch : InstallableValueCommand, MixJSON {"description", description}, }; } else { - auto name2 = hiliteMatches(name.name, nameMatches, ANSI_GREEN, "\e[0;2m"); if (results > 1) logger->cout(""); logger->cout( "* %s%s", diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh index 3161553ec..91e260f0b 100644 --- a/src/nix/self-exe.hh +++ b/src/nix/self-exe.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include namespace nix { diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 2afe4b267..134d4f34a 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -41,7 +41,6 @@ struct CmdCopySigs : StorePathsCommand ThreadPool pool; - std::string doneLabel = "done"; std::atomic added{0}; //logger->setExpected(doneLabel, storePaths.size()); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index f54cc59d0..77fb798a7 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -107,7 +107,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); - auto & where = *whereOpt; + const auto & where = whereOpt->parent_path(); printInfo("found Nix in '%s'", where); @@ -126,7 +126,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) - throw Error("directory '%s' does not appear to be part of a Nix profile", where); + throw Error("directory %s does not appear to be part of a Nix profile", where); if (!store->isValidPath(store->parseStorePath(userEnv))) throw Error("directory '%s' is not in the Nix store", userEnv); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 124a05bed..52585fe08 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -129,7 +129,7 @@ struct CmdVerify : StorePathsCommand size_t validSigs = 0; auto doSigs = [&](StringSet sigs) { - for (auto sig : sigs) { + for (const auto & sig : sigs) { if (!sigsSeen.insert(sig).second) continue; if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(*store, publicKeys, sig)) validSigs++; diff --git a/src/perl/meson.build b/src/perl/meson.build index dcb6a68a4..52d85fd60 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -65,7 +65,13 @@ yath = find_program('yath', required : false) # Required Libraries #------------------------------------------------- -bzip2_dep = dependency('bzip2') +bzip2_dep = dependency('bzip2', required: false) +if not bzip2_dep.found() + bzip2_dep = cpp.find_library('bz2') + if not bzip2_dep.found() + error('No "bzip2" pkg-config or "bz2" library found') + endif +endif curl_dep = dependency('libcurl') libsodium_dep = dependency('libsodium') diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 4eccb73e0..3401de1b0 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -8,6 +8,7 @@ TODO_NixOS restartDaemon requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh index c498d46c3..9f91a91a9 100644 --- a/tests/functional/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -5,6 +5,7 @@ # shellcheck disable=SC2154 requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "$busybox" =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index 1a5334577..3231341cb 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -3,6 +3,7 @@ : "${file?must be defined by caller (remote building test case using this)}" requireSandboxSupport +requiresUnprivilegedUserNamespaces [[ "${busybox-}" =~ busybox ]] || skipTest "no busybox" # Avoid store dir being inside sandbox build-dir @@ -27,6 +28,7 @@ builders=( chmod -R +w "$TEST_ROOT/machine"* || true rm -rf "$TEST_ROOT/machine"* || true + # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 46e91f0aa..ccde3e90b 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -40,6 +40,7 @@ EOF cp simple.nix shell.nix simple.builder.sh "${config_nix}" "$flakeDir/" TODO_NixOS + requiresUnprivilegedUserNamespaces outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir") diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 7195149cb..bf3dd2ca8 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -345,4 +345,15 @@ count() { trap onError ERR +requiresUnprivilegedUserNamespaces() { + if [[ -f /proc/sys/kernel/apparmor_restrict_unprivileged_userns ]] && [[ $(< /proc/sys/kernel/apparmor_restrict_unprivileged_userns) -eq 1 ]]; then + skipTest "Unprivileged user namespaces are disabled. Run 'sudo sysctl -w /proc/sys/kernel/apparmor_restrict_unprivileged_userns=0' to allow, and run these tests." + fi +} + +execUnshare () { + requiresUnprivilegedUserNamespaces + exec unshare --mount --map-root-user "$SHELL" "$@" +} + fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index be1a7ae9a..4ff29227f 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,7 +1,7 @@ { hashInvalidator ? "" }: with import ./config.nix; -let { +let input0 = mkDerivation { name = "dependencies-input-0"; @@ -33,16 +33,15 @@ let { outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; }; - body = mkDerivation { - name = "dependencies-top"; - builder = ./dependencies.builder0.sh + "/FOOBAR/../."; - input1 = input1 + "/."; - input2 = "${input2}/."; - input1_drv = input1; - input2_drv = input2; - input0_drv = input0; - fod_input_drv = fod_input; - meta.description = "Random test package"; - }; - +in +mkDerivation { + name = "dependencies-top"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + input2_drv = input2; + input0_drv = input0; + fod_input_drv = fod_input; + meta.description = "Random test package"; } diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 271f17dc6..6707b345c 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -13,7 +13,7 @@ badExitCode=0 store="$TEST_ROOT/store" for nixFile in derivation/*.nix; do - drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + drvPath=$(env -u NIX_STORE nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") testName=$(basename "$nixFile" .nix) got="${store}${drvPath}" expected="derivation/$testName.drv" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index de37ae1b7..6c466a0c7 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -77,6 +77,7 @@ hash1=$(echo "$json" | jq -r .revision) echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index a17a0c2af..ebad332d0 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -30,3 +30,6 @@ git -C "$flake2Dir" add flake.nix echo 456 > "$flake1Dir"/x.nix [[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] + +expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" | + grepQuiet "cannot write lock file.*because it has an unlocked input" diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index 3a099950f..c40cb469c 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -93,15 +93,19 @@ try3() { # Asserting input format fails. # - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] + expectStderr 1 nix hash convert --hash-algo "$1" --from sri "$2" | grepQuiet "is not SRI" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$2" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from base16 "$3" | grepQuiet "input hash" + expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" + # Base-16 hashes can be in uppercase. + nix hash convert --hash-algo "$1" --from base16 "$(echo $2 | tr [a-z] [A-Z])" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" + +# Test SRI hashes that lack trailing '=' characters. These are incorrect but we need to support them for backward compatibility. +[[ $(nix hash convert --from sri "sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0") = sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= ]] +[[ $(nix hash convert --from sri "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ") = sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ== ]] diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix index cc53451cf..770dd86cf 100644 --- a/tests/functional/import-from-derivation.nix +++ b/tests/functional/import-from-derivation.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import ; rec { bar = mkDerivation { @@ -30,4 +30,23 @@ rec { echo -n BLA$(cat $src) > $out ''; }; + + step1 = mkDerivation { + name = "step1"; + buildCommand = '' + mkdir -p $out + echo 'foo' > $out/bla + ''; + }; + + addPathExpr = mkDerivation { + name = "add-path"; + inherit step1; + buildCommand = '' + mkdir -p $out + echo "builtins.path { path = \"$step1\"; sha256 = \"7ptL+pnrZXnSa5hwwB+2SXTLkcSb5264WGGokN8OXto=\"; }" > $out/default.nix + ''; + }; + + importAddPathExpr = import addPathExpr; } diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh index 83ef92a6f..a00761235 100755 --- a/tests/functional/import-from-derivation.sh +++ b/tests/functional/import-from-derivation.sh @@ -6,6 +6,8 @@ TODO_NixOS clearStoreIfPossible +export NIX_PATH=config="${config_nix}" + if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then echo "read-only evaluation of an imported derivation unexpectedly failed" exit 1 @@ -15,6 +17,9 @@ outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) [ "$(cat "$outPath")" = FOO579 ] +# Check that we can have access to the entire closure of a derivation output. +nix build --no-link --restrict-eval -I src=. -f ./import-from-derivation.nix importAddPathExpr -v + # FIXME: the next tests are broken on CA. if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then exit 0 diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp new file mode 100644 index 000000000..d1cdc7b76 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.err.exp @@ -0,0 +1,5 @@ +error: undefined variable 'd' + at /pwd/lang/eval-fail-attrset-merge-drops-later-rec.nix:1:26: + 1| { a.b = 1; a = rec { c = d + 2; d = 3; }; }.c + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix new file mode 100644 index 000000000..fdb314b91 --- /dev/null +++ b/tests/functional/lang/eval-fail-attrset-merge-drops-later-rec.nix @@ -0,0 +1 @@ +{ a.b = 1; a = rec { c = d + 2; d = 3; }; }.c diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp new file mode 100644 index 000000000..a16192c59 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-keyWithNullByte.nix:1:1: + 1| builtins.fromJSON ''{"a\u0000b": 1}'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix new file mode 100644 index 000000000..ffaa6a97d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"a\u0000b": 1}'' diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp new file mode 100644 index 000000000..c5c08e6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-valueWithNullByte.nix:1:1: + 1| builtins.fromJSON ''"a\u0000b"'' + | ^ + 2| + + error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix new file mode 100644 index 000000000..c71ab990d --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromJSON ''"a\u0000b"'' diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp new file mode 100644 index 000000000..dc2180f0b --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-keyWithNullByte.nix:1:1: + 1| builtins.fromTOML ''"a\u0000b" = 1'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix new file mode 100644 index 000000000..b622dc4dc --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-keyWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''"a\u0000b" = 1'' diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp new file mode 100644 index 000000000..0235692a8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-valueWithNullByte.nix:1:1: + 1| builtins.fromTOML ''k = "a\u0000b"'' + | ^ + 2| + + error: while parsing TOML: error: input string 'a␀b' cannot be represented as Nix string because it contains null bytes diff --git a/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix new file mode 100644 index 000000000..183cab6b3 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-valueWithNullByte.nix @@ -0,0 +1 @@ +builtins.fromTOML ''k = "a\u0000b"'' diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp new file mode 100644 index 000000000..1e8b31496 --- /dev/null +++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.exp @@ -0,0 +1 @@ +6 diff --git a/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix new file mode 100644 index 000000000..8df6a2ad8 --- /dev/null +++ b/tests/functional/lang/eval-okay-regrettable-rec-attrset-merge.nix @@ -0,0 +1,3 @@ +# This is for backwards compatibility, not because we like it. +# See https://github.com/NixOS/nix/issues/9020. +{ a = rec { b = c + 1; d = 2; }; a.c = d + 3; }.a.b diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp index a4472156b..49a07323f 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -1,6 +1,6 @@ -error: attribute 'z' already defined at «stdin»:3:16 - at «stdin»:2:3: - 1| { +error: attribute 'x.z' already defined at «stdin»:2:3 + at «stdin»:3:16: 2| x.z = 3; - | ^ 3| x = { y = 3; z = 3; }; + | ^ + 4| } diff --git a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp index ead1f0dbd..36fab2fe6 100644 --- a/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp +++ b/tests/functional/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -1,6 +1,6 @@ -error: attribute 'y' already defined at «stdin»:3:9 - at «stdin»:2:3: - 1| { +error: attribute 'x.y.y' already defined at «stdin»:2:3 + at «stdin»:3:9: 2| x.y.y = 3; - | ^ 3| x = { y.y= 3; z = 3; }; + | ^ + 4| } diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 81ef36237..abb635f11 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -9,6 +9,7 @@ TODO_NixOS clearStore requireSandboxSupport +requiresUnprivilegedUserNamespaces # Note: we need to bind-mount $SHELL into the chroot. Currently we # only support the case where $SHELL is in the Nix store, because diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index b7930e32e..f0c6a151c 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -19,7 +19,7 @@ TODO_NixOS for i in "${storesBad[@]}"; do echo $i - unshare --mount --map-root-user bash <&$NIX_LOG_FD + touch $out + ''; +} diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 0d46f9ce2..933595cd5 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -17,12 +17,10 @@ fs = import('fs') nix = find_program('nix') bash = find_program('bash', native : true) busybox = find_program('busybox', native : true, required : false) -if host_machine.system() == 'windows' - # Because of the state of symlinks on Windows, coreutils.exe doesn't usually exist, but things like ls.exe will - coreutils = find_program('ls', native : true) -else - coreutils = find_program('coreutils', native : true) -endif +# Look up `coreutils` package by searching for `ls` binary. +# Previously we looked up `coreutils` on `linux`, but that is not +# guaranteed to exist either. +coreutils = find_program('ls', native : true) dot = find_program('dot', native : true, required : false) nix_bin_dir = fs.parent(nix.full_path()) diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index 7462d2968..4d4cf125e 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -7,6 +7,7 @@ source common.sh TODO_NixOS requireSandboxSupport +requiresUnprivilegedUserNamespaces start="$TEST_ROOT/start" mkdir -p "$start" diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index e9c40a5d9..7c04e82f5 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -18,6 +18,7 @@ goodStoreUrl () { # whether this test is being run in a derivation as part of the nix build or # being manually run by a developer outside a derivation runNixBuild () { + local storeFun=$1 local altitude=$2 nix-build \ diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 706e0f5db..59d1f1be0 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -309,7 +309,7 @@ runRepl () { for test in $(cd "$testDir/repl"; echo *.in); do test="$(basename "$test" .in)" in="$testDir/repl/$test.in" - actual="$testDir/repl/$test.actual" + actual="$TEST_ROOT/$test.actual" expected="$testDir/repl/$test.expected" (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { echo "FAIL: $test (exit code $?)" >&2 diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index cfc8e4102..51032ff1b 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -52,6 +52,7 @@ if isDaemonNewer "2.20.0pre20231220"; then fi requireSandboxSupport +requiresUnprivilegedUserNamespaces chmod -R u+w "$TEST_ROOT/store0" || true rm -rf "$TEST_ROOT/store0" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index 50259a3e1..400333f7d 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -9,7 +9,7 @@ needLocalStore "The test uses --store always so we would just be bypassing the d TODO_NixOS -unshare --mount --map-root-user -- bash -e -x <