diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index addafb9f8..6169c0924 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: read-all jobs: eval: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: @@ -20,8 +20,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} + include: + - scenario: on ubuntu + runs-on: ubuntu-24.04 + os: linux + - scenario: on macos + runs-on: macos-14 + os: darwin + name: tests ${{ matrix.scenario }} + runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -37,7 +44,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'linux' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,8 +58,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} + include: + - scenario: on ubuntu + runs-on: ubuntu-24.04 + os: linux + - scenario: on macos + runs-on: macos-14 + os: darwin + name: installer test ${{ matrix.scenario }} + runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 - name: Download installer tarball @@ -68,9 +82,9 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'linux' - run: brew install fish - if: matrix.os == 'macos-latest' + if: matrix.os == 'darwin' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" @@ -86,7 +100,7 @@ jobs: permissions: contents: none name: Check Docker secrets present for installer tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: docker: ${{ steps.secret.outputs.docker }} steps: @@ -106,7 +120,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && github.ref_name == 'master' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Check for secrets id: secret @@ -158,7 +172,7 @@ jobs: docker push $IMAGE_ID:master vm_tests: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main @@ -173,7 +187,7 @@ jobs: flake_regressions: needs: vm_tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout nix uses: actions/checkout@v4 diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 34aa4e6bd..23a5d9e51 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -15,7 +15,7 @@ permissions: jobs: labels: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 if: github.repository_owner == 'NixOS' steps: - uses: actions/labeler@v5 diff --git a/.mergify.yml b/.mergify.yml index 70fccae49..5d2bf8520 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,10 +2,10 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=tests (macos-latest) - - check-success=tests (ubuntu-latest) - - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-latest) + - check-success=tests on macos + - check-success=tests on ubuntu + - check-success=installer test on macos + - check-success=installer test on ubuntu - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 2e6fcede3..f8133f2e1 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -3,7 +3,7 @@ , meson , ninja -, lowdown +, lowdown-unsandboxed , mdbook , mdbook-linkcheck , jq @@ -42,7 +42,7 @@ mkMesonDerivation (finalAttrs: { passthru.externalNativeBuildInputs = [ meson ninja - (lib.getBin lowdown) + (lib.getBin lowdown-unsandboxed) mdbook mdbook-linkcheck jq diff --git a/doc/manual/rl-next/nix-instantiate-raw.md b/doc/manual/rl-next/nix-instantiate-raw.md new file mode 100644 index 000000000..fb4a72b88 --- /dev/null +++ b/doc/manual/rl-next/nix-instantiate-raw.md @@ -0,0 +1,8 @@ +--- +synopsis: "`nix-instantiate --eval` now supports `--raw`" +prs: [12119] +--- + +The `nix-instantiate --eval` command now supports a `--raw` flag, when used +the evaluation result must be a string, which is printed verbatim without +quotation marks or escaping. diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index bd05f2816..763179b8e 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,6 +62,15 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. + - [`--max-freed`](#opt-max-freed) *bytes* + + + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. + {{#include ./opt-common.md}} {{#include ./env-common.md}} diff --git a/doc/manual/source/command-ref/nix-copy-closure.md b/doc/manual/source/command-ref/nix-copy-closure.md index 8cfd6ebad..b7e31d93b 100644 --- a/doc/manual/source/command-ref/nix-copy-closure.md +++ b/doc/manual/source/command-ref/nix-copy-closure.md @@ -84,7 +84,7 @@ When using public key authentication, you can avoid typing the passphrase with ` > Copy GNU Hello from a remote machine using a known store path, and run it: > > ```shell-session -> $ storePath="$(nix-instantiate --eval '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath | tr -d '"')" +> $ storePath="$(nix-instantiate --eval --raw '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath)" > $ nix-copy-closure --from alice@itchy.example.org "$storePath" > $ "$storePath"/bin/hello > Hello, world! diff --git a/doc/manual/source/command-ref/nix-instantiate.md b/doc/manual/source/command-ref/nix-instantiate.md index 6f6fcdc1f..487ef8f10 100644 --- a/doc/manual/source/command-ref/nix-instantiate.md +++ b/doc/manual/source/command-ref/nix-instantiate.md @@ -5,7 +5,7 @@ # Synopsis `nix-instantiate` - [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ] + [`--parse` | `--eval` [`--strict`] [`--raw` | `--json` | `--xml`] ] [`--read-write-mode`] [`--arg` *name* *value*] [{`--attr`| `-A`} *attrPath*] @@ -102,6 +102,11 @@ standard input. > This option can cause non-termination, because lazy data > structures can be infinitely large. +- `--raw` + + When used with `--eval`, the evaluation result must be a string, + which is printed verbatim, without quoting, escaping or trailing newline. + - `--json` When used with `--eval`, print the resulting value as an JSON diff --git a/doc/manual/source/command-ref/nix-store/add-fixed.md b/doc/manual/source/command-ref/nix-store/add-fixed.md index bebf15026..2ea90a135 100644 --- a/doc/manual/source/command-ref/nix-store/add-fixed.md +++ b/doc/manual/source/command-ref/nix-store/add-fixed.md @@ -21,6 +21,9 @@ This operation has the following options: Use recursive instead of flat hashing mode, used when adding directories to the store. + *paths* that refer to symlinks are not dereferenced, but added to the store + as symlinks with the same target. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/source/command-ref/nix-store/add.md b/doc/manual/source/command-ref/nix-store/add.md index 87d504cd3..ab4740723 100644 --- a/doc/manual/source/command-ref/nix-store/add.md +++ b/doc/manual/source/command-ref/nix-store/add.md @@ -11,6 +11,9 @@ The operation `--add` adds the specified paths to the Nix store. It prints the resulting paths in the Nix store on standard output. +*paths* that refer to symlinks are not dereferenced, but added to the store +as symlinks with the same target. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/source/development/documentation.md b/doc/manual/source/development/documentation.md index 2e188f232..30cc8adc4 100644 --- a/doc/manual/source/development/documentation.md +++ b/doc/manual/source/development/documentation.md @@ -19,10 +19,11 @@ nix-build -E '(import ./.).packages.${builtins.currentSystem}.nix.doc' or ```console -nix build .#nix^doc +nix build .#nix-manual ``` -and open `./result-doc/share/doc/nix/manual/index.html`. +and open `./result/share/doc/nix/manual/index.html`. + To build the manual incrementally, [enter the development shell](./building.md) and run: diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 30aa7d0d5..d582ce4b4 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup diff --git a/flake.lock b/flake.lock index b5d0b881c..ce484a67a 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", "type": "github" }, "original": { @@ -48,11 +48,11 @@ ] }, "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "lastModified": 1734279981, + "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", "type": "github" }, "original": { @@ -61,35 +61,18 @@ "type": "github" } }, - "libgit2": { - "flake": false, - "locked": { - "lastModified": 1715853528, - "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", - "owner": "libgit2", - "repo": "libgit2", - "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", - "type": "github" - }, - "original": { - "owner": "libgit2", - "ref": "v1.8.1", - "repo": "libgit2", - "type": "github" - } - }, "nixpkgs": { "locked": { - "lastModified": 1723688146, - "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", + "lastModified": 1734359947, + "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", + "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.05", + "ref": "release-24.11", "repo": "nixpkgs", "type": "github" } @@ -131,7 +114,6 @@ "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", - "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" diff --git a/flake.nix b/flake.nix index 44a764dd7..83bdc5ecc 100644 --- a/flake.nix +++ b/flake.nix @@ -1,11 +1,11 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.11"; + inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; @@ -18,7 +18,7 @@ inputs.git-hooks-nix.inputs.flake-compat.follows = ""; inputs.git-hooks-nix.inputs.gitignore.follows = ""; - outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, ... }: let @@ -36,7 +36,8 @@ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" "riscv64-unknown-linux-gnu" - "x86_64-unknown-netbsd" + # Disabled because of https://github.com/NixOS/nixpkgs/issues/344423 + # "x86_64-unknown-netbsd" "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -163,7 +164,6 @@ if prev.stdenv.hostPlatform.system == "i686-linux" then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) else prev.pre-commit; - }; in { diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 deleted file mode 100644 index cdc4ddb40..000000000 --- a/m4/gcc_bug_80431.m4 +++ /dev/null @@ -1,66 +0,0 @@ -# Ensure that this bug is not present in the C++ toolchain we are using. -# -# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 -# -# The test program is from that issue, with only a slight modification -# to set an exit status instead of printing strings. -AC_DEFUN([ENSURE_NO_GCC_BUG_80431], -[ - AC_MSG_CHECKING([that GCC bug 80431 is fixed]) - AC_LANG_PUSH(C++) - AC_RUN_IFELSE( - [AC_LANG_PROGRAM( - [[ - #include - - static bool a = true; - static bool b = true; - - struct Options { }; - - struct Option - { - Option(Options * options) - { - a = false; - } - - ~Option() - { - b = false; - } - }; - - struct MyOptions : Options { }; - - struct MyOptions2 : virtual MyOptions - { - Option foo{this}; - }; - ]], - [[ - { - MyOptions2 opts; - } - return (a << 1) | b; - ]])], - [status_80431=0], - [status_80431=$?], - [status_80431='']) - AC_LANG_POP(C++) - AS_CASE([$status_80431], - [''],[ - AC_MSG_RESULT(cannot check because cross compiling) - AC_MSG_NOTICE(assume we are bug free) - ], - [0],[ - AC_MSG_RESULT(yes) - ], - [2],[ - AC_MSG_RESULT(no) - AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) - ], - [ - AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) - ]) -]) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 1a134b91a..6b311c62e 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -356,6 +356,7 @@ ''^src/libutil/util\.cc$'' ''^src/libutil/util\.hh$'' ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/widecharwidth/widechar_width\.h$'' # vendored source ''^src/libutil/windows/file-descriptor\.cc$'' ''^src/libutil/windows/file-path\.cc$'' ''^src/libutil/windows/processes\.cc$'' diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index f0322183e..67b6658f5 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -16,7 +16,3 @@ add_project_arguments( '-Wno-deprecated-declarations', language : 'cpp', ) - -if get_option('buildtype') not in ['debug'] - add_project_arguments('-O3', language : 'cpp') -endif diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index a8005ce16..4bc7495e7 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -66,6 +66,21 @@ let mesonLayer = finalAttrs: prevAttrs: { + # NOTE: + # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26, + # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default. + # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype. + mesonBuildType = "release"; + # NOTE: + # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the + # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. + # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. + preConfigure = prevAttrs.preConfigure or "" + '' + case "$mesonBuildType" in + release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; + *) appendToVar mesonFlags "-Db_lto=false" ;; + esac + ''; nativeBuildInputs = [ pkgs.buildPackages.meson pkgs.buildPackages.ninja @@ -114,14 +129,6 @@ scope: { requiredSystemFeatures = [ ]; }; - libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = pkgs.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - boehmgc = pkgs.boehmgc.override { enableLargeConfig = true; }; @@ -140,8 +147,6 @@ scope: { }); libgit2 = pkgs.libgit2.overrideAttrs (attrs: { - src = inputs.libgit2; - version = inputs.libgit2.lastModifiedDate; cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; nativeBuildInputs = attrs.nativeBuildInputs or [] @@ -169,36 +174,6 @@ scope: { ]; }); - busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - - # TODO change in Nixpkgs, Windows works fine. First commit of - # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. - toml11 = pkgs.toml11.overrideAttrs (old: { - meta.platforms = lib.platforms.all; - }); - inherit resolvePath filesetToSource; mkMesonDerivation = diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 6a0d05d9f..4566e6ba6 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -16,13 +16,25 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown) { int windowWidth = getWindowSize().second; - struct lowdown_opts opts - { - .type = LOWDOWN_TERM, - .maxdepth = 20, +#if HAVE_LOWDOWN_1_4 + struct lowdown_opts_term opts_term { .cols = (size_t) std::max(windowWidth - 5, 60), .hmargin = 0, .vmargin = 0, + }; +#endif + struct lowdown_opts opts + { + .type = LOWDOWN_TERM, +#if HAVE_LOWDOWN_1_4 + .term = opts_term, +#endif + .maxdepth = 20, +#if !HAVE_LOWDOWN_1_4 + .cols = (size_t) std::max(windowWidth - 5, 60), + .hmargin = 0, + .vmargin = 0, +#endif .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, .oflags = LOWDOWN_TERM_NOLINK, }; diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 222817c81..4145f408a 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -4,8 +4,6 @@ project('nix-cmd', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', @@ -36,6 +34,8 @@ deps_public += nlohmann_json lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) deps_private += lowdown configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) +# The API changed slightly around terminal initialization. +configdata.set('HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int()) readline_flavor = get_option('readline-flavor') if readline_flavor == 'editline' diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 1556dae51..9487132cf 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -4,8 +4,6 @@ project('nix-expr-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index 64d4fe218..56e814cd1 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-expr-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index f37e85e57..667a0d7b7 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -4,8 +4,6 @@ project('nix-expr-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fe5f05ab8..21dd5a294 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3185,12 +3185,16 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } -void forceNoNullByte(std::string_view s) +void forceNoNullByte(std::string_view s, std::function pos) { if (s.find('\0') != s.npos) { using namespace std::string_view_literals; auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); - throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + Error error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + if (pos) { + error.atPos(pos()); + } + throw error; } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a7e44cb72..067f86e01 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -41,16 +41,18 @@ namespace nix { // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. -static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) +// getting the position is expensive and thus it is implemented lazily. +static StringToken unescapeStr(char * const s, size_t length, std::function && pos) { - char * result = s; + bool noNullByte = true; char * t = s; - char c; // the input string is terminated with *two* NULs, so we can safely take // *one* character after the one being checked against. - while ((c = *s++)) { + for (size_t i = 0; i < length; t++) { + char c = s[i++]; + noNullByte &= c != '\0'; if (c == '\\') { - c = *s++; + c = s[i++]; if (c == 'n') *t = '\n'; else if (c == 'r') *t = '\r'; else if (c == 't') *t = '\t'; @@ -59,12 +61,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) else if (c == '\r') { /* Normalise CR and CR/LF into LF. */ *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ + if (s[i] == '\n') i++; /* cr/lf */ } else *t = c; - t++; } - return {result, size_t(t - result)}; + if (!noNullByte) { + forceNoNullByte({s, size_t(t - s)}, std::move(pos)); + } + return {s, size_t(t - s)}; } static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) @@ -175,7 +179,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->str = unescapeStr(state->symbols, yytext, yyleng); + yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; }); return STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -191,6 +195,7 @@ or { return OR_KW; } \'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } ([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->str = {yytext, (size_t) yyleng, true}; + forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; }); return IND_STR; } \'\'\$ | @@ -203,7 +208,7 @@ or { return OR_KW; } return IND_STR; } \'\'\\{ANY} { - yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; }); return IND_STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index b3c559ba7..b33aebc86 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -4,8 +4,6 @@ project('nix-expr', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c1f1cf4c1..a0e2753b5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2045,7 +2045,7 @@ static RegisterPrimOp primop_readFileType({ .args = {"p"}, .doc = R"( Determine the directory entry type of a filesystem node, being - one of "directory", "regular", "symlink", or "unknown". + one of `"directory"`, `"regular"`, `"symlink"`, or `"unknown"`. )", .fun = prim_readFileType, }); @@ -4059,7 +4059,7 @@ static RegisterPrimOp primop_toString({ }); /* `substring start len str' returns the substring of `str' starting - at character position `min(start, stringLength str)' inclusive and + at byte position `min(start, stringLength str)' inclusive and ending at `min(start + len, stringLength str)'. `start' must be non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -4098,7 +4098,7 @@ static RegisterPrimOp primop_substring({ .name = "__substring", .args = {"start", "len", "s"}, .doc = R"( - Return the substring of *s* from character position *start* + Return the substring of *s* from byte position *start* (zero-based) up to but not including *start + len*. If *start* is greater than the length of the string, an empty string is returned. If *start + len* lies beyond the end of the string or *len* is `-1`, diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 8044fe347..5aa4fe4fd 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -108,7 +108,11 @@ json printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { - str << printValueAsJSON(state, strict, v, pos, context, copyToStore); + try { + str << printValueAsJSON(state, strict, v, pos, context, copyToStore); + } catch (nlohmann::json::exception & e) { + throw JSONSerializationError("JSON serialization error: %s", e.what()); + } } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 47ac90313..867c4e3a8 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -16,4 +16,7 @@ nlohmann::json printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); + +MakeError(JSONSerializationError, Error); + } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 88fcae986..8925693e3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,6 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); -void forceNoNullByte(std::string_view s); +void forceNoNullByte(std::string_view s, std::function = nullptr); } diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 3e82c6111..739435501 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -4,8 +4,6 @@ project('nix-fetchers-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b105c252a..87bf0dd71 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -66,7 +66,7 @@ Input Input::fromURL( } } - throw Error("input '%s' is unsupported", url.url); + throw Error("input '%s' is unsupported", url); } Input Input::fromAttrs(const Settings & settings, Attrs && attrs) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 79f2faa98..96c634eba 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -434,7 +434,7 @@ struct GitInputScheme : InputScheme auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - repoInfo.url = repoInfo.isLocal ? url.path : url.base; + repoInfo.url = repoInfo.isLocal ? url.path : url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 68353c1a3..ca4c3ed38 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -50,7 +50,7 @@ struct GitArchiveInputScheme : InputScheme else if (std::regex_match(path[2], refRegex)) ref = path[2]; else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url, path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -63,34 +63,34 @@ struct GitArchiveInputScheme : InputScheme if (std::regex_match(rs, refRegex)) { ref = rs; } else { - throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url, rs); } } else if (size < 2) - throw BadURL("URL '%s' is invalid", url.url); + throw BadURL("URL '%s' is invalid", url); for (auto &[name, value] : url.query) { if (name == "rev") { if (rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); + throw BadURL("URL '%s' contains multiple commit hashes", url); rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + throw BadURL("URL '%s' contains an invalid branch/tag name", url); if (ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + throw BadURL("URL '%s' contains multiple branch/tag names", url); ref = value; } else if (name == "host") { if (!std::regex_match(value, hostRegex)) - throw BadURL("URL '%s' contains an invalid instance host", url.url); + throw BadURL("URL '%s' contains an invalid instance host", url); host_url = value; } // FIXME: barf on unsupported attributes } if (ref && rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev()); Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 2e5cd82c7..0e1b86711 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -26,16 +26,16 @@ struct IndirectInputScheme : InputScheme else if (std::regex_match(path[1], refRegex)) ref = path[1]; else - throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) - throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url, path[1]); ref = path[1]; if (!std::regex_match(path[2], revRegex)) - throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]); rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else - throw BadURL("GitHub URL '%s' is invalid", url.url); + throw BadURL("GitHub URL '%s' is invalid", url); std::string id = path[0]; if (!std::regex_match(id, flakeRegex)) diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2c987f79d..c2fd8139c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -161,7 +161,7 @@ struct MercurialInputScheme : InputScheme { auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; - return {isLocal, isLocal ? url.path : url.base}; + return {isLocal, isLocal ? url.path : url.to_string()}; } StorePath fetchToStore(ref store, Input & input) const diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index ae4cab4c7..8add01c0f 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -4,8 +4,6 @@ project('nix-fetchers', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 246b68c3a..f628e042c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -14,7 +14,7 @@ struct PathInputScheme : InputScheme if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority); Input input{settings}; input.attrs.insert_or_assign("type", "path"); @@ -27,10 +27,10 @@ struct PathInputScheme : InputScheme if (auto n = string2Int(value)) input.attrs.insert_or_assign(name, *n); else - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + throw Error("path URL '%s' has invalid parameter '%s'", url, name); } else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + throw Error("path URL '%s' has unsupported parameter '%s'", url, name); return input; } diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index b7669fe97..85d20644d 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -4,8 +4,6 @@ project('nix-flake-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index d704a26d3..2b1f5124b 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -7,18 +7,60 @@ namespace nix { /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ - /* ---------------------------------------------------------------------------- - * to_string - * --------------------------------------------------------------------------*/ + TEST(parseFlakeRef, path) { + experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); + + fetchers::Settings fetchSettings; + + { + auto s = "/foo/bar"; + auto flakeref = parseFlakeRef(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar"); + } + + { + auto s = "/foo/bar?revCount=123&rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + auto flakeref = parseFlakeRef(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&revCount=123"); + } + + { + auto s = "/foo/bar?xyzzy=123"; + EXPECT_THROW( + parseFlakeRef(fetchSettings, s), + Error); + } + + { + auto s = "/foo/bar#bla"; + EXPECT_THROW( + parseFlakeRef(fetchSettings, s), + Error); + } + + { + auto s = "/foo/bar#bla"; + auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar"); + ASSERT_EQ(fragment, "bla"); + } + + { + auto s = "/foo/bar?revCount=123#bla"; + auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?revCount=123"); + ASSERT_EQ(fragment, "bla"); + } + } TEST(to_string, doesntReencodeUrl) { fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; auto flakeref = parseFlakeRef(fetchSettings, s); - auto parsed = flakeref.to_string(); + auto unparsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; - ASSERT_EQ(parsed, expected); + ASSERT_EQ(unparsed, expected); } } diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 5c3c58e53..1c8765f21 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -4,8 +4,6 @@ project('nix-flake-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index ab882fdab..cdf0a40e8 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -89,23 +89,16 @@ std::pair parsePathFlakeRefWithFragment( bool allowMissing, bool isFlake) { - std::string path = url; - std::string fragment = ""; - std::map query; - auto pathEnd = url.find_first_of("#?"); - auto fragmentStart = pathEnd; - if (pathEnd != std::string::npos && url[pathEnd] == '?') { - fragmentStart = url.find("#"); - } - if (pathEnd != std::string::npos) { - path = url.substr(0, pathEnd); - } - if (fragmentStart != std::string::npos) { - fragment = percentDecode(url.substr(fragmentStart+1)); - } - if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') { - query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); - } + static std::regex pathFlakeRegex( + R"(([^?#]*)(\?([^#]*))?(#(.*))?)", + std::regex::ECMAScript); + + std::smatch match; + auto succeeds = std::regex_match(url, match, pathFlakeRegex); + assert(succeeds); + auto path = match[1].str(); + auto query = decodeQuery(match[3]); + auto fragment = percentDecode(match[5].str()); if (baseDir) { /* Check if 'url' is a path (either absolute or relative @@ -159,11 +152,7 @@ std::pair parsePathFlakeRefWithFragment( while (flakeRoot != "/") { if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, .scheme = "git+file", .authority = "", .path = flakeRoot, @@ -194,11 +183,13 @@ std::pair parsePathFlakeRefWithFragment( path = canonPath(path + "/" + getOr(query, "dir", "")); } - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); + return fromParsedURL(fetchSettings, { + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, isFlake); } /** @@ -220,8 +211,6 @@ static std::optional> parseFlakeIdRef( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ - .url = url, - .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", .path = match[1], diff --git a/src/libflake/meson.build b/src/libflake/meson.build index f9e217729..b757d0d76 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -4,8 +4,6 @@ project('nix-flake', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 5a5684b8d..d875d2c3f 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -4,8 +4,6 @@ project('nix-main-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 87fc8b8d2..00f945f49 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -4,8 +4,6 @@ project('nix-main', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 1ac331ad0..17d18609f 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -4,8 +4,6 @@ project('nix-store-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 2a07e56ac..59d649889 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-store-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index b706fa12c..3ba0795e9 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -4,8 +4,6 @@ project('nix-store-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2ff0ef92f..4d97250d3 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -701,6 +701,7 @@ Goal::Co DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); + outputLocks.unlock(); co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 540097d85..2b3de9de3 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -784,7 +784,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); res.urls.push_back(request.uri); callback(std::move(res)); diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 12a0e6376..79d912497 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -4,8 +4,6 @@ project('nix-store', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 197f5a1c4..29a98d8e2 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -3,7 +3,7 @@ #include "signals.hh" #include "util.hh" -#ifdef WIN32 +#ifdef _WIN32 # include # include # include diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 44cec1afc..ac1297665 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -4,8 +4,6 @@ project('nix-util-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 03ae63f1a..db944cf06 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-util-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 7ef804f34..2c10d4869 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -261,4 +261,18 @@ TEST(pathExists, bogusPathDoesNotExist) { ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } + +/* ---------------------------------------------------------------------------- + * makeParentCanonical + * --------------------------------------------------------------------------*/ + +TEST(makeParentCanonical, noParent) +{ + ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file"))); +} + +TEST(makeParentCanonical, root) +{ + ASSERT_EQ(makeParentCanonical("/"), "/"); +} } diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 83ac79e92..ad2c61711 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -4,8 +4,6 @@ project('nix-util-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 714d5a237..f4fc6e770 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -55,6 +55,10 @@ TEST(filterANSIEscapes, utf8) ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 6), "f🔍bar"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 3), "f🔍"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 2), "f"); + ASSERT_EQ(filterANSIEscapes("foo\u0301", true, 3), "foó"); } TEST(filterANSIEscapes, osc8) diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 7d08f467e..7e1d2aa15 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -20,24 +20,11 @@ namespace nix { } - std::ostream& operator<<(std::ostream& os, const ParsedURL& p) { - return os << "\n" - << "url: " << p.url << "\n" - << "base: " << p.base << "\n" - << "scheme: " << p.scheme << "\n" - << "authority: " << p.authority.value() << "\n" - << "path: " << p.path << "\n" - << "query: " << print_map(p.query) << "\n" - << "fragment: " << p.fragment << "\n"; - } - TEST(parseURL, parsesSimpleHttpUrl) { auto s = "http://www.example.org/file.tar.gz"; auto parsed = parseURL(s); ParsedURL expected { - .url = "http://www.example.org/file.tar.gz", - .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", .path = "/file.tar.gz", @@ -53,8 +40,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "https://www.example.org/file.tar.gz", - .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", .path = "/file.tar.gz", @@ -70,8 +55,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "https://www.example.org/file.tar.gz", - .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", .path = "/file.tar.gz", @@ -87,8 +70,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://www.example.org/file.tar.gz", - .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", .path = "/file.tar.gz", @@ -104,8 +85,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "file+https://www.example.org/video.mp4", - .base = "https://www.example.org/video.mp4", .scheme = "file+https", .authority = "www.example.org", .path = "/video.mp4", @@ -126,8 +105,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://127.0.0.1:8080/file.tar.gz", - .base = "https://127.0.0.1:8080/file.tar.gz", .scheme = "http", .authority = "127.0.0.1:8080", .path = "/file.tar.gz", @@ -143,8 +120,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", - .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .scheme = "http", .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .path = "", @@ -161,8 +136,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", - .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .scheme = "http", .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .path = "", @@ -185,8 +158,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://user:pass@www.example.org/file.tar.gz", - .base = "http://user:pass@www.example.org/file.tar.gz", .scheme = "http", .authority = "user:pass@www.example.org:8080", .path = "/file.tar.gz", @@ -203,8 +174,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "", - .base = "", .scheme = "file", .authority = "", .path = "/none/of//your/business", @@ -228,8 +197,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "ftp://ftp.nixos.org/downloads/nixos.iso", - .base = "ftp://ftp.nixos.org/downloads/nixos.iso", .scheme = "ftp", .authority = "ftp.nixos.org", .path = "/downloads/nixos.iso", diff --git a/src/libutil/config.hh b/src/libutil/config.hh index e98e09bf7..502d2823e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -262,6 +262,7 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } + T & get() { return value; } template bool operator ==(const U & v2) const { return value == v2; } template diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 829700336..923220fd0 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -331,7 +331,7 @@ void syncParent(const Path & path) void recursiveSync(const Path & path) { - /* If it's a file, just fsync and return. */ + /* If it's a file or symlink, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0)); @@ -339,7 +339,8 @@ void recursiveSync(const Path & path) throw SysError("opening file '%1%'", path); fd.fsync(); return; - } + } else if (S_ISLNK(st.st_mode)) + return; /* Otherwise, perform a depth-first traversal of the directory and fsync all the files. */ @@ -384,7 +385,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); + throw SysError("getting status of %1%", path); } if (!S_ISDIR(st.st_mode)) { @@ -416,15 +417,15 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); + throw SysError("chmod %1%", path); } int fd = openat(parentfd, path.c_str(), O_RDONLY); if (fd == -1) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %1%", path); AutoCloseDir dir(fdopendir(fd)); if (!dir) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %1%", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -433,13 +434,13 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b if (childName == "." || childName == "..") continue; _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed); } - if (errno) throw SysError("reading directory '%1%'", path); + if (errno) throw SysError("reading directory %1%", path); } int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; if (unlinkat(parentfd, name.c_str(), flags) == -1) { if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); + throw SysError("cannot unlink %1%", path); } #else // TODO implement @@ -765,4 +766,19 @@ bool isExecutableFileAmbient(const fs::path & exe) { ) == 0; } +std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) +{ + std::filesystem::path path(absPath(rawPath));; + try { + auto parent = path.parent_path(); + if (parent == path) { + // `path` is a root directory => trivially canonical + return parent; + } + return std::filesystem::canonical(parent) / path.filename(); + } catch (fs::filesystem_error & e) { + throw SysError("canonicalising parent path of '%1%'", path); + } } + +} // namespace nix diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 3c49181a0..7fdaba811 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -143,6 +143,23 @@ inline bool symlink_exists(const std::filesystem::path & path) { } // namespace fs +/** + * Canonicalize a path except for the last component. + * + * This is useful for getting the canonical location of a symlink. + * + * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will + * resolve the symlink `l` to its target. + * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target, + * but does ensure that the returned parent part of the path, `foo` is resolved + * to `canonical("foo")`, and can therefore be retrieved without traversing any + * symlinks. + * + * If a relative path is passed, it will be made absolute, so that the parent + * can always be canonicalized. + */ +std::filesystem::path makeParentCanonical(const std::filesystem::path & path); + /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not diff --git a/src/libutil/meson.build b/src/libutil/meson.build index bbe7872cf..ac701d8fd 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -4,8 +4,6 @@ project('nix-util', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', @@ -108,6 +106,8 @@ deps_private += cpuid nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json +cxx = meson.get_compiler('cpp') + config_h = configure_file( configuration : configdata, output : 'config-util.hh', @@ -168,6 +168,10 @@ sources = files( ) include_dirs = [include_directories('.')] +if not cxx.has_header('widechar_width.h', required : false) + # use vendored widechar_width.h + include_dirs += include_directories('./widecharwidth') +endif headers = [config_h] + files( 'abstract-setting-to-json.hh', diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 69ebbf726..0a9f65782 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -29,6 +29,7 @@ mkMesonLibrary (finalAttrs: { ./nix-meson-build-support ../../.version ./.version + ./widecharwidth ./meson.build ./meson.options ./linux/meson.build diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 40f60bb54..5d491e633 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -43,13 +43,25 @@ struct PosixSourceAccessor : virtual SourceAccessor std::optional getPhysicalPath(const CanonPath & path) override; /** - * Create a `PosixSourceAccessor` and `CanonPath` corresponding to + * Create a `PosixSourceAccessor` and `SourcePath` corresponding to * some native path. * * The `PosixSourceAccessor` is rooted as far up the tree as * possible, (e.g. on Windows it could scoped to a drive like * `C:\`). This allows more `..` parent accessing to work. * + * @note When `path` is trusted user input, canonicalize it using + * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc, + * as appropriate for the use case. At least weak canonicalization is + * required for the `SourcePath` to do anything useful at the location it + * points to. + * + * @note A canonicalizing behavior is not built in `createAtRoot` so that + * callers do not accidentally introduce symlink-related security vulnerabilities. + * Furthermore, `createAtRoot` does not know whether the file pointed to by + * `path` should be resolved if it is itself a symlink. In other words, + * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers. + * * See * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) * and diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 4c127ddb0..8a8373f1b 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -11,6 +11,53 @@ # include #endif #include +#include + +namespace { + +inline std::pair charWidthUTF8Helper(std::string_view s) +{ + size_t bytes = 1; + uint32_t ch = s[0]; + uint32_t max = 1U << 7; + if ((ch & 0x80U) == 0U) { + } else if ((ch & 0xe0U) == 0xc0U) { + ch &= 0x1fU; + bytes = 2; + max = 1U << 11; + } else if ((ch & 0xf0U) == 0xe0U) { + ch &= 0x0fU; + bytes = 3; + max = 1U << 16; + } else if ((ch & 0xf8U) == 0xf0U) { + ch &= 0x07U; + bytes = 4; + max = 0x110000U; + } else { + return {bytes, bytes}; // invalid UTF-8 start byte + } + for (size_t i = 1; i < bytes; i++) { + if (i < s.size() && (s[i] & 0xc0) == 0x80) { + ch = (ch << 6) | (s[i] & 0x3f); + } else { + return {i, i}; // invalid UTF-8 encoding; assume one character per byte + } + } + int width = bytes; // in case of overlong encoding + if (ch < max) { + width = widechar_wcwidth(ch); + if (width == widechar_ambiguous) { + width = 1; // just a guess... + } else if (width == widechar_widened_in_9) { + width = 2; + } else if (width < 0) { + width = 0; + } + } + return {width, bytes}; +} + +} namespace nix { @@ -30,7 +77,7 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w size_t w = 0; auto i = s.begin(); - while (w < (size_t) width && i != s.end()) { + while (i != s.end()) { if (*i == '\e') { std::string e; @@ -61,10 +108,12 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w } else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } + do { + if (++w > (size_t) width) + return t; + t += ' '; + } while (w % 8); + i++; } else if (*i == '\r' || *i == '\a') @@ -72,35 +121,18 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w i++; else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; + auto [chWidth, bytes] = charWidthUTF8Helper({i, s.end()}); + w += chWidth; + if (w > (size_t) width) { + break; + } + t += {i, i + bytes}; + i += bytes; } } - return t; } - ////////////////////////////////////////////////////////////////////// static Sync> windowSize{{0, 0}}; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 63b9734ee..8fb1eecfb 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -22,7 +22,6 @@ ParsedURL parseURL(const std::string & url) std::smatch match; if (std::regex_match(url, match, uriRegex)) { - auto & base = match[1]; std::string scheme = match[2]; auto authority = match[3].matched ? std::optional(match[3]) : std::nullopt; @@ -40,8 +39,6 @@ ParsedURL parseURL(const std::string & url) path = "/"; return ParsedURL{ - .url = url, - .base = base, .scheme = scheme, .authority = authority, .path = percentDecode(path), @@ -136,6 +133,12 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } +std::ostream & operator << (std::ostream & os, const ParsedURL & url) +{ + os << url.to_string(); + return os; +} + bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 738ee9f82..2b12f5af2 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -7,9 +7,6 @@ namespace nix { struct ParsedURL { - std::string url; - /// URL without query/fragment - std::string base; std::string scheme; std::optional authority; std::string path; @@ -26,6 +23,8 @@ struct ParsedURL ParsedURL canonicalise(); }; +std::ostream & operator << (std::ostream & os, const ParsedURL & url); + MakeError(BadURL, Error); std::string percentDecode(std::string_view in); diff --git a/src/libutil/widecharwidth/LICENSE b/src/libutil/widecharwidth/LICENSE new file mode 100644 index 000000000..d3b1dd767 --- /dev/null +++ b/src/libutil/widecharwidth/LICENSE @@ -0,0 +1,4 @@ +widecharwidth - wcwidth implementation +Written in 2018 by ridiculous_fish +To the extent possible under law, the author(s) have dedicated all copyright and related and neighboring rights to this software to the public domain worldwide. This software is distributed without any warranty. +You should have received a copy of the CC0 Public Domain Dedication along with this software. If not, see . diff --git a/src/libutil/widecharwidth/widechar_width.h b/src/libutil/widecharwidth/widechar_width.h new file mode 100644 index 000000000..92e63e913 --- /dev/null +++ b/src/libutil/widecharwidth/widechar_width.h @@ -0,0 +1,1559 @@ +/** + * widechar_width.h for Unicode 16.0.0 + * See https://github.com/ridiculousfish/widecharwidth/ + * + * SHA1 file hashes: + * ( + * the hashes for generate.py and the template are git object hashes, + * use `git log --all --find-object=` in the widecharwidth repository + * to see which commit they correspond to, + * or run `git hash-object` on the file to compare. + * The other hashes are simple `sha1sum` style hashes. + * ) + * + * generate.py: 2747bb9402d8eeeca8e566ff947f14308511ecb1 + * template.js: 1249763c5b7c1e308aeb4ca64f1e15bce1fab9b3 + * UnicodeData.txt: 91df83276154240bcedef82a09bde77aa182cf8d + * EastAsianWidth.txt: 0885c0fc1c21eb58954a3bfb785d78559b361d92 + * emoji-data.txt: 1df2f8329dd9f5c238674807de736f316c6b9d87 + */ + +#ifndef WIDECHAR_WIDTH_H +#define WIDECHAR_WIDTH_H + +#include +#include +#include +#include + +namespace { + +/* Special width values */ +enum { + widechar_nonprint = -1, // The character is not printable. + widechar_combining = -2, // The character is a zero-width combiner. + widechar_ambiguous = -3, // The character is East-Asian ambiguous width. + widechar_private_use = -4, // The character is for private use. + widechar_unassigned = -5, // The character is unassigned. + widechar_widened_in_9 = -6, // Width is 1 in Unicode 8, 2 in Unicode 9+. + widechar_non_character = -7 // The character is a noncharacter. +}; + +/* An inclusive range of characters. */ +struct widechar_range { + uint32_t lo; + uint32_t hi; +}; + +/* Simple ASCII characters - used a lot, so we check them first. */ +static const struct widechar_range widechar_ascii_table[] = { + {0x00020, 0x0007E} +}; + +/* Private usage range. */ +static const struct widechar_range widechar_private_table[] = { + {0x0E000, 0x0F8FF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} +}; + +/* Nonprinting characters. */ +static const struct widechar_range widechar_nonprint_table[] = { + {0x00000, 0x0001F}, + {0x0007F, 0x0009F}, + {0x000AD, 0x000AD}, + {0x00600, 0x00605}, + {0x0061C, 0x0061C}, + {0x006DD, 0x006DD}, + {0x0070F, 0x0070F}, + {0x00890, 0x00891}, + {0x008E2, 0x008E2}, + {0x0180E, 0x0180E}, + {0x0200B, 0x0200F}, + {0x02028, 0x0202E}, + {0x02060, 0x02064}, + {0x02066, 0x0206F}, + {0x0D800, 0x0DFFF}, + {0x0FEFF, 0x0FEFF}, + {0x0FFF9, 0x0FFFB}, + {0x110BD, 0x110BD}, + {0x110CD, 0x110CD}, + {0x13430, 0x1343F}, + {0x1BCA0, 0x1BCA3}, + {0x1D173, 0x1D17A}, + {0xE0001, 0xE0001}, + {0xE0020, 0xE007F} +}; + +/* Width 0 combining marks. */ +static const struct widechar_range widechar_combining_table[] = { + {0x00300, 0x0036F}, + {0x00483, 0x00489}, + {0x00591, 0x005BD}, + {0x005BF, 0x005BF}, + {0x005C1, 0x005C2}, + {0x005C4, 0x005C5}, + {0x005C7, 0x005C7}, + {0x00610, 0x0061A}, + {0x0064B, 0x0065F}, + {0x00670, 0x00670}, + {0x006D6, 0x006DC}, + {0x006DF, 0x006E4}, + {0x006E7, 0x006E8}, + {0x006EA, 0x006ED}, + {0x00711, 0x00711}, + {0x00730, 0x0074A}, + {0x007A6, 0x007B0}, + {0x007EB, 0x007F3}, + {0x007FD, 0x007FD}, + {0x00816, 0x00819}, + {0x0081B, 0x00823}, + {0x00825, 0x00827}, + {0x00829, 0x0082D}, + {0x00859, 0x0085B}, + {0x00897, 0x0089F}, + {0x008CA, 0x008E1}, + {0x008E3, 0x00903}, + {0x0093A, 0x0093C}, + {0x0093E, 0x0094F}, + {0x00951, 0x00957}, + {0x00962, 0x00963}, + {0x00981, 0x00983}, + {0x009BC, 0x009BC}, + {0x009BE, 0x009C4}, + {0x009C7, 0x009C8}, + {0x009CB, 0x009CD}, + {0x009D7, 0x009D7}, + {0x009E2, 0x009E3}, + {0x009FE, 0x009FE}, + {0x00A01, 0x00A03}, + {0x00A3C, 0x00A3C}, + {0x00A3E, 0x00A42}, + {0x00A47, 0x00A48}, + {0x00A4B, 0x00A4D}, + {0x00A51, 0x00A51}, + {0x00A70, 0x00A71}, + {0x00A75, 0x00A75}, + {0x00A81, 0x00A83}, + {0x00ABC, 0x00ABC}, + {0x00ABE, 0x00AC5}, + {0x00AC7, 0x00AC9}, + {0x00ACB, 0x00ACD}, + {0x00AE2, 0x00AE3}, + {0x00AFA, 0x00AFF}, + {0x00B01, 0x00B03}, + {0x00B3C, 0x00B3C}, + {0x00B3E, 0x00B44}, + {0x00B47, 0x00B48}, + {0x00B4B, 0x00B4D}, + {0x00B55, 0x00B57}, + {0x00B62, 0x00B63}, + {0x00B82, 0x00B82}, + {0x00BBE, 0x00BC2}, + {0x00BC6, 0x00BC8}, + {0x00BCA, 0x00BCD}, + {0x00BD7, 0x00BD7}, + {0x00C00, 0x00C04}, + {0x00C3C, 0x00C3C}, + {0x00C3E, 0x00C44}, + {0x00C46, 0x00C48}, + {0x00C4A, 0x00C4D}, + {0x00C55, 0x00C56}, + {0x00C62, 0x00C63}, + {0x00C81, 0x00C83}, + {0x00CBC, 0x00CBC}, + {0x00CBE, 0x00CC4}, + {0x00CC6, 0x00CC8}, + {0x00CCA, 0x00CCD}, + {0x00CD5, 0x00CD6}, + {0x00CE2, 0x00CE3}, + {0x00CF3, 0x00CF3}, + {0x00D00, 0x00D03}, + {0x00D3B, 0x00D3C}, + {0x00D3E, 0x00D44}, + {0x00D46, 0x00D48}, + {0x00D4A, 0x00D4D}, + {0x00D57, 0x00D57}, + {0x00D62, 0x00D63}, + {0x00D81, 0x00D83}, + {0x00DCA, 0x00DCA}, + {0x00DCF, 0x00DD4}, + {0x00DD6, 0x00DD6}, + {0x00DD8, 0x00DDF}, + {0x00DF2, 0x00DF3}, + {0x00E31, 0x00E31}, + {0x00E34, 0x00E3A}, + {0x00E47, 0x00E4E}, + {0x00EB1, 0x00EB1}, + {0x00EB4, 0x00EBC}, + {0x00EC8, 0x00ECE}, + {0x00F18, 0x00F19}, + {0x00F35, 0x00F35}, + {0x00F37, 0x00F37}, + {0x00F39, 0x00F39}, + {0x00F3E, 0x00F3F}, + {0x00F71, 0x00F84}, + {0x00F86, 0x00F87}, + {0x00F8D, 0x00F97}, + {0x00F99, 0x00FBC}, + {0x00FC6, 0x00FC6}, + {0x0102B, 0x0103E}, + {0x01056, 0x01059}, + {0x0105E, 0x01060}, + {0x01062, 0x01064}, + {0x01067, 0x0106D}, + {0x01071, 0x01074}, + {0x01082, 0x0108D}, + {0x0108F, 0x0108F}, + {0x0109A, 0x0109D}, + {0x0135D, 0x0135F}, + {0x01712, 0x01715}, + {0x01732, 0x01734}, + {0x01752, 0x01753}, + {0x01772, 0x01773}, + {0x017B4, 0x017D3}, + {0x017DD, 0x017DD}, + {0x0180B, 0x0180D}, + {0x0180F, 0x0180F}, + {0x01885, 0x01886}, + {0x018A9, 0x018A9}, + {0x01920, 0x0192B}, + {0x01930, 0x0193B}, + {0x01A17, 0x01A1B}, + {0x01A55, 0x01A5E}, + {0x01A60, 0x01A7C}, + {0x01A7F, 0x01A7F}, + {0x01AB0, 0x01ACE}, + {0x01B00, 0x01B04}, + {0x01B34, 0x01B44}, + {0x01B6B, 0x01B73}, + {0x01B80, 0x01B82}, + {0x01BA1, 0x01BAD}, + {0x01BE6, 0x01BF3}, + {0x01C24, 0x01C37}, + {0x01CD0, 0x01CD2}, + {0x01CD4, 0x01CE8}, + {0x01CED, 0x01CED}, + {0x01CF4, 0x01CF4}, + {0x01CF7, 0x01CF9}, + {0x01DC0, 0x01DFF}, + {0x020D0, 0x020F0}, + {0x02CEF, 0x02CF1}, + {0x02D7F, 0x02D7F}, + {0x02DE0, 0x02DFF}, + {0x0302A, 0x0302F}, + {0x03099, 0x0309A}, + {0x0A66F, 0x0A672}, + {0x0A674, 0x0A67D}, + {0x0A69E, 0x0A69F}, + {0x0A6F0, 0x0A6F1}, + {0x0A802, 0x0A802}, + {0x0A806, 0x0A806}, + {0x0A80B, 0x0A80B}, + {0x0A823, 0x0A827}, + {0x0A82C, 0x0A82C}, + {0x0A880, 0x0A881}, + {0x0A8B4, 0x0A8C5}, + {0x0A8E0, 0x0A8F1}, + {0x0A8FF, 0x0A8FF}, + {0x0A926, 0x0A92D}, + {0x0A947, 0x0A953}, + {0x0A980, 0x0A983}, + {0x0A9B3, 0x0A9C0}, + {0x0A9E5, 0x0A9E5}, + {0x0AA29, 0x0AA36}, + {0x0AA43, 0x0AA43}, + {0x0AA4C, 0x0AA4D}, + {0x0AA7B, 0x0AA7D}, + {0x0AAB0, 0x0AAB0}, + {0x0AAB2, 0x0AAB4}, + {0x0AAB7, 0x0AAB8}, + {0x0AABE, 0x0AABF}, + {0x0AAC1, 0x0AAC1}, + {0x0AAEB, 0x0AAEF}, + {0x0AAF5, 0x0AAF6}, + {0x0ABE3, 0x0ABEA}, + {0x0ABEC, 0x0ABED}, + {0x0FB1E, 0x0FB1E}, + {0x0FE00, 0x0FE0F}, + {0x0FE20, 0x0FE2F}, + {0x101FD, 0x101FD}, + {0x102E0, 0x102E0}, + {0x10376, 0x1037A}, + {0x10A01, 0x10A03}, + {0x10A05, 0x10A06}, + {0x10A0C, 0x10A0F}, + {0x10A38, 0x10A3A}, + {0x10A3F, 0x10A3F}, + {0x10AE5, 0x10AE6}, + {0x10D24, 0x10D27}, + {0x10D69, 0x10D6D}, + {0x10EAB, 0x10EAC}, + {0x10EFC, 0x10EFF}, + {0x10F46, 0x10F50}, + {0x10F82, 0x10F85}, + {0x11000, 0x11002}, + {0x11038, 0x11046}, + {0x11070, 0x11070}, + {0x11073, 0x11074}, + {0x1107F, 0x11082}, + {0x110B0, 0x110BA}, + {0x110C2, 0x110C2}, + {0x11100, 0x11102}, + {0x11127, 0x11134}, + {0x11145, 0x11146}, + {0x11173, 0x11173}, + {0x11180, 0x11182}, + {0x111B3, 0x111C0}, + {0x111C9, 0x111CC}, + {0x111CE, 0x111CF}, + {0x1122C, 0x11237}, + {0x1123E, 0x1123E}, + {0x11241, 0x11241}, + {0x112DF, 0x112EA}, + {0x11300, 0x11303}, + {0x1133B, 0x1133C}, + {0x1133E, 0x11344}, + {0x11347, 0x11348}, + {0x1134B, 0x1134D}, + {0x11357, 0x11357}, + {0x11362, 0x11363}, + {0x11366, 0x1136C}, + {0x11370, 0x11374}, + {0x113B8, 0x113C0}, + {0x113C2, 0x113C2}, + {0x113C5, 0x113C5}, + {0x113C7, 0x113CA}, + {0x113CC, 0x113D0}, + {0x113D2, 0x113D2}, + {0x113E1, 0x113E2}, + {0x11435, 0x11446}, + {0x1145E, 0x1145E}, + {0x114B0, 0x114C3}, + {0x115AF, 0x115B5}, + {0x115B8, 0x115C0}, + {0x115DC, 0x115DD}, + {0x11630, 0x11640}, + {0x116AB, 0x116B7}, + {0x1171D, 0x1172B}, + {0x1182C, 0x1183A}, + {0x11930, 0x11935}, + {0x11937, 0x11938}, + {0x1193B, 0x1193E}, + {0x11940, 0x11940}, + {0x11942, 0x11943}, + {0x119D1, 0x119D7}, + {0x119DA, 0x119E0}, + {0x119E4, 0x119E4}, + {0x11A01, 0x11A0A}, + {0x11A33, 0x11A39}, + {0x11A3B, 0x11A3E}, + {0x11A47, 0x11A47}, + {0x11A51, 0x11A5B}, + {0x11A8A, 0x11A99}, + {0x11C2F, 0x11C36}, + {0x11C38, 0x11C3F}, + {0x11C92, 0x11CA7}, + {0x11CA9, 0x11CB6}, + {0x11D31, 0x11D36}, + {0x11D3A, 0x11D3A}, + {0x11D3C, 0x11D3D}, + {0x11D3F, 0x11D45}, + {0x11D47, 0x11D47}, + {0x11D8A, 0x11D8E}, + {0x11D90, 0x11D91}, + {0x11D93, 0x11D97}, + {0x11EF3, 0x11EF6}, + {0x11F00, 0x11F01}, + {0x11F03, 0x11F03}, + {0x11F34, 0x11F3A}, + {0x11F3E, 0x11F42}, + {0x11F5A, 0x11F5A}, + {0x13440, 0x13440}, + {0x13447, 0x13455}, + {0x1611E, 0x1612F}, + {0x16AF0, 0x16AF4}, + {0x16B30, 0x16B36}, + {0x16F4F, 0x16F4F}, + {0x16F51, 0x16F87}, + {0x16F8F, 0x16F92}, + {0x16FE4, 0x16FE4}, + {0x16FF0, 0x16FF1}, + {0x1BC9D, 0x1BC9E}, + {0x1CF00, 0x1CF2D}, + {0x1CF30, 0x1CF46}, + {0x1D165, 0x1D169}, + {0x1D16D, 0x1D172}, + {0x1D17B, 0x1D182}, + {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, + {0x1DA00, 0x1DA36}, + {0x1DA3B, 0x1DA6C}, + {0x1DA75, 0x1DA75}, + {0x1DA84, 0x1DA84}, + {0x1DA9B, 0x1DA9F}, + {0x1DAA1, 0x1DAAF}, + {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, + {0x1E01B, 0x1E021}, + {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, + {0x1E08F, 0x1E08F}, + {0x1E130, 0x1E136}, + {0x1E2AE, 0x1E2AE}, + {0x1E2EC, 0x1E2EF}, + {0x1E4EC, 0x1E4EF}, + {0x1E5EE, 0x1E5EF}, + {0x1E8D0, 0x1E8D6}, + {0x1E944, 0x1E94A}, + {0xE0100, 0xE01EF} +}; + +/* Width 0 combining letters. */ +static const struct widechar_range widechar_combiningletters_table[] = { + {0x01160, 0x011FF}, + {0x0D7B0, 0x0D7FF} +}; + +/* Width 2 characters. */ +static const struct widechar_range widechar_doublewide_table[] = { + {0x01100, 0x0115F}, + {0x02329, 0x0232A}, + {0x02630, 0x02637}, + {0x0268A, 0x0268F}, + {0x02E80, 0x02E99}, + {0x02E9B, 0x02EF3}, + {0x02F00, 0x02FD5}, + {0x02FF0, 0x0303E}, + {0x03041, 0x03096}, + {0x03099, 0x030FF}, + {0x03105, 0x0312F}, + {0x03131, 0x0318E}, + {0x03190, 0x031E5}, + {0x031EF, 0x0321E}, + {0x03220, 0x03247}, + {0x03250, 0x0A48C}, + {0x0A490, 0x0A4C6}, + {0x0A960, 0x0A97C}, + {0x0AC00, 0x0D7A3}, + {0x0F900, 0x0FAFF}, + {0x0FE10, 0x0FE19}, + {0x0FE30, 0x0FE52}, + {0x0FE54, 0x0FE66}, + {0x0FE68, 0x0FE6B}, + {0x0FF01, 0x0FF60}, + {0x0FFE0, 0x0FFE6}, + {0x16FE0, 0x16FE4}, + {0x16FF0, 0x16FF1}, + {0x17000, 0x187F7}, + {0x18800, 0x18CD5}, + {0x18CFF, 0x18D08}, + {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, + {0x1B000, 0x1B122}, + {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, + {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, + {0x1B170, 0x1B2FB}, + {0x1D300, 0x1D356}, + {0x1D360, 0x1D376}, + {0x1F200, 0x1F200}, + {0x1F202, 0x1F202}, + {0x1F210, 0x1F219}, + {0x1F21B, 0x1F22E}, + {0x1F230, 0x1F231}, + {0x1F237, 0x1F237}, + {0x1F23B, 0x1F23B}, + {0x1F240, 0x1F248}, + {0x1F260, 0x1F265}, + {0x1F57A, 0x1F57A}, + {0x1F5A4, 0x1F5A4}, + {0x1F6D1, 0x1F6D2}, + {0x1F6D5, 0x1F6D7}, + {0x1F6DC, 0x1F6DF}, + {0x1F6F4, 0x1F6FC}, + {0x1F7E0, 0x1F7EB}, + {0x1F7F0, 0x1F7F0}, + {0x1F90C, 0x1F90F}, + {0x1F919, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1F97F}, + {0x1F985, 0x1F9BF}, + {0x1F9C1, 0x1F9FF}, + {0x1FA70, 0x1FA7C}, + {0x1FA80, 0x1FA89}, + {0x1FA8F, 0x1FAC6}, + {0x1FACE, 0x1FADC}, + {0x1FADF, 0x1FAE9}, + {0x1FAF0, 0x1FAF8}, + {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD} +}; + +/* Ambiguous-width characters. */ +static const struct widechar_range widechar_ambiguous_table[] = { + {0x000A1, 0x000A1}, + {0x000A4, 0x000A4}, + {0x000A7, 0x000A8}, + {0x000AA, 0x000AA}, + {0x000AD, 0x000AE}, + {0x000B0, 0x000B4}, + {0x000B6, 0x000BA}, + {0x000BC, 0x000BF}, + {0x000C6, 0x000C6}, + {0x000D0, 0x000D0}, + {0x000D7, 0x000D8}, + {0x000DE, 0x000E1}, + {0x000E6, 0x000E6}, + {0x000E8, 0x000EA}, + {0x000EC, 0x000ED}, + {0x000F0, 0x000F0}, + {0x000F2, 0x000F3}, + {0x000F7, 0x000FA}, + {0x000FC, 0x000FC}, + {0x000FE, 0x000FE}, + {0x00101, 0x00101}, + {0x00111, 0x00111}, + {0x00113, 0x00113}, + {0x0011B, 0x0011B}, + {0x00126, 0x00127}, + {0x0012B, 0x0012B}, + {0x00131, 0x00133}, + {0x00138, 0x00138}, + {0x0013F, 0x00142}, + {0x00144, 0x00144}, + {0x00148, 0x0014B}, + {0x0014D, 0x0014D}, + {0x00152, 0x00153}, + {0x00166, 0x00167}, + {0x0016B, 0x0016B}, + {0x001CE, 0x001CE}, + {0x001D0, 0x001D0}, + {0x001D2, 0x001D2}, + {0x001D4, 0x001D4}, + {0x001D6, 0x001D6}, + {0x001D8, 0x001D8}, + {0x001DA, 0x001DA}, + {0x001DC, 0x001DC}, + {0x00251, 0x00251}, + {0x00261, 0x00261}, + {0x002C4, 0x002C4}, + {0x002C7, 0x002C7}, + {0x002C9, 0x002CB}, + {0x002CD, 0x002CD}, + {0x002D0, 0x002D0}, + {0x002D8, 0x002DB}, + {0x002DD, 0x002DD}, + {0x002DF, 0x002DF}, + {0x00300, 0x0036F}, + {0x00391, 0x003A1}, + {0x003A3, 0x003A9}, + {0x003B1, 0x003C1}, + {0x003C3, 0x003C9}, + {0x00401, 0x00401}, + {0x00410, 0x0044F}, + {0x00451, 0x00451}, + {0x02010, 0x02010}, + {0x02013, 0x02016}, + {0x02018, 0x02019}, + {0x0201C, 0x0201D}, + {0x02020, 0x02022}, + {0x02024, 0x02027}, + {0x02030, 0x02030}, + {0x02032, 0x02033}, + {0x02035, 0x02035}, + {0x0203B, 0x0203B}, + {0x0203E, 0x0203E}, + {0x02074, 0x02074}, + {0x0207F, 0x0207F}, + {0x02081, 0x02084}, + {0x020AC, 0x020AC}, + {0x02103, 0x02103}, + {0x02105, 0x02105}, + {0x02109, 0x02109}, + {0x02113, 0x02113}, + {0x02116, 0x02116}, + {0x02121, 0x02122}, + {0x02126, 0x02126}, + {0x0212B, 0x0212B}, + {0x02153, 0x02154}, + {0x0215B, 0x0215E}, + {0x02160, 0x0216B}, + {0x02170, 0x02179}, + {0x02189, 0x02189}, + {0x02190, 0x02199}, + {0x021B8, 0x021B9}, + {0x021D2, 0x021D2}, + {0x021D4, 0x021D4}, + {0x021E7, 0x021E7}, + {0x02200, 0x02200}, + {0x02202, 0x02203}, + {0x02207, 0x02208}, + {0x0220B, 0x0220B}, + {0x0220F, 0x0220F}, + {0x02211, 0x02211}, + {0x02215, 0x02215}, + {0x0221A, 0x0221A}, + {0x0221D, 0x02220}, + {0x02223, 0x02223}, + {0x02225, 0x02225}, + {0x02227, 0x0222C}, + {0x0222E, 0x0222E}, + {0x02234, 0x02237}, + {0x0223C, 0x0223D}, + {0x02248, 0x02248}, + {0x0224C, 0x0224C}, + {0x02252, 0x02252}, + {0x02260, 0x02261}, + {0x02264, 0x02267}, + {0x0226A, 0x0226B}, + {0x0226E, 0x0226F}, + {0x02282, 0x02283}, + {0x02286, 0x02287}, + {0x02295, 0x02295}, + {0x02299, 0x02299}, + {0x022A5, 0x022A5}, + {0x022BF, 0x022BF}, + {0x02312, 0x02312}, + {0x02460, 0x024E9}, + {0x024EB, 0x0254B}, + {0x02550, 0x02573}, + {0x02580, 0x0258F}, + {0x02592, 0x02595}, + {0x025A0, 0x025A1}, + {0x025A3, 0x025A9}, + {0x025B2, 0x025B3}, + {0x025B6, 0x025B7}, + {0x025BC, 0x025BD}, + {0x025C0, 0x025C1}, + {0x025C6, 0x025C8}, + {0x025CB, 0x025CB}, + {0x025CE, 0x025D1}, + {0x025E2, 0x025E5}, + {0x025EF, 0x025EF}, + {0x02605, 0x02606}, + {0x02609, 0x02609}, + {0x0260E, 0x0260F}, + {0x0261C, 0x0261C}, + {0x0261E, 0x0261E}, + {0x02640, 0x02640}, + {0x02642, 0x02642}, + {0x02660, 0x02661}, + {0x02663, 0x02665}, + {0x02667, 0x0266A}, + {0x0266C, 0x0266D}, + {0x0266F, 0x0266F}, + {0x0269E, 0x0269F}, + {0x026BF, 0x026BF}, + {0x026C6, 0x026CD}, + {0x026CF, 0x026D3}, + {0x026D5, 0x026E1}, + {0x026E3, 0x026E3}, + {0x026E8, 0x026E9}, + {0x026EB, 0x026F1}, + {0x026F4, 0x026F4}, + {0x026F6, 0x026F9}, + {0x026FB, 0x026FC}, + {0x026FE, 0x026FF}, + {0x0273D, 0x0273D}, + {0x02776, 0x0277F}, + {0x02B56, 0x02B59}, + {0x03248, 0x0324F}, + {0x0E000, 0x0F8FF}, + {0x0FE00, 0x0FE0F}, + {0x0FFFD, 0x0FFFD}, + {0x1F100, 0x1F10A}, + {0x1F110, 0x1F12D}, + {0x1F130, 0x1F169}, + {0x1F170, 0x1F18D}, + {0x1F18F, 0x1F190}, + {0x1F19B, 0x1F1AC}, + {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} +}; + +/* Unassigned characters. */ +static const struct widechar_range widechar_unassigned_table[] = { + {0x00378, 0x00379}, + {0x00380, 0x00383}, + {0x0038B, 0x0038B}, + {0x0038D, 0x0038D}, + {0x003A2, 0x003A2}, + {0x00530, 0x00530}, + {0x00557, 0x00558}, + {0x0058B, 0x0058C}, + {0x00590, 0x00590}, + {0x005C8, 0x005CF}, + {0x005EB, 0x005EE}, + {0x005F5, 0x005FF}, + {0x0070E, 0x0070E}, + {0x0074B, 0x0074C}, + {0x007B2, 0x007BF}, + {0x007FB, 0x007FC}, + {0x0082E, 0x0082F}, + {0x0083F, 0x0083F}, + {0x0085C, 0x0085D}, + {0x0085F, 0x0085F}, + {0x0086B, 0x0086F}, + {0x0088F, 0x0088F}, + {0x00892, 0x00896}, + {0x00984, 0x00984}, + {0x0098D, 0x0098E}, + {0x00991, 0x00992}, + {0x009A9, 0x009A9}, + {0x009B1, 0x009B1}, + {0x009B3, 0x009B5}, + {0x009BA, 0x009BB}, + {0x009C5, 0x009C6}, + {0x009C9, 0x009CA}, + {0x009CF, 0x009D6}, + {0x009D8, 0x009DB}, + {0x009DE, 0x009DE}, + {0x009E4, 0x009E5}, + {0x009FF, 0x00A00}, + {0x00A04, 0x00A04}, + {0x00A0B, 0x00A0E}, + {0x00A11, 0x00A12}, + {0x00A29, 0x00A29}, + {0x00A31, 0x00A31}, + {0x00A34, 0x00A34}, + {0x00A37, 0x00A37}, + {0x00A3A, 0x00A3B}, + {0x00A3D, 0x00A3D}, + {0x00A43, 0x00A46}, + {0x00A49, 0x00A4A}, + {0x00A4E, 0x00A50}, + {0x00A52, 0x00A58}, + {0x00A5D, 0x00A5D}, + {0x00A5F, 0x00A65}, + {0x00A77, 0x00A80}, + {0x00A84, 0x00A84}, + {0x00A8E, 0x00A8E}, + {0x00A92, 0x00A92}, + {0x00AA9, 0x00AA9}, + {0x00AB1, 0x00AB1}, + {0x00AB4, 0x00AB4}, + {0x00ABA, 0x00ABB}, + {0x00AC6, 0x00AC6}, + {0x00ACA, 0x00ACA}, + {0x00ACE, 0x00ACF}, + {0x00AD1, 0x00ADF}, + {0x00AE4, 0x00AE5}, + {0x00AF2, 0x00AF8}, + {0x00B00, 0x00B00}, + {0x00B04, 0x00B04}, + {0x00B0D, 0x00B0E}, + {0x00B11, 0x00B12}, + {0x00B29, 0x00B29}, + {0x00B31, 0x00B31}, + {0x00B34, 0x00B34}, + {0x00B3A, 0x00B3B}, + {0x00B45, 0x00B46}, + {0x00B49, 0x00B4A}, + {0x00B4E, 0x00B54}, + {0x00B58, 0x00B5B}, + {0x00B5E, 0x00B5E}, + {0x00B64, 0x00B65}, + {0x00B78, 0x00B81}, + {0x00B84, 0x00B84}, + {0x00B8B, 0x00B8D}, + {0x00B91, 0x00B91}, + {0x00B96, 0x00B98}, + {0x00B9B, 0x00B9B}, + {0x00B9D, 0x00B9D}, + {0x00BA0, 0x00BA2}, + {0x00BA5, 0x00BA7}, + {0x00BAB, 0x00BAD}, + {0x00BBA, 0x00BBD}, + {0x00BC3, 0x00BC5}, + {0x00BC9, 0x00BC9}, + {0x00BCE, 0x00BCF}, + {0x00BD1, 0x00BD6}, + {0x00BD8, 0x00BE5}, + {0x00BFB, 0x00BFF}, + {0x00C0D, 0x00C0D}, + {0x00C11, 0x00C11}, + {0x00C29, 0x00C29}, + {0x00C3A, 0x00C3B}, + {0x00C45, 0x00C45}, + {0x00C49, 0x00C49}, + {0x00C4E, 0x00C54}, + {0x00C57, 0x00C57}, + {0x00C5B, 0x00C5C}, + {0x00C5E, 0x00C5F}, + {0x00C64, 0x00C65}, + {0x00C70, 0x00C76}, + {0x00C8D, 0x00C8D}, + {0x00C91, 0x00C91}, + {0x00CA9, 0x00CA9}, + {0x00CB4, 0x00CB4}, + {0x00CBA, 0x00CBB}, + {0x00CC5, 0x00CC5}, + {0x00CC9, 0x00CC9}, + {0x00CCE, 0x00CD4}, + {0x00CD7, 0x00CDC}, + {0x00CDF, 0x00CDF}, + {0x00CE4, 0x00CE5}, + {0x00CF0, 0x00CF0}, + {0x00CF4, 0x00CFF}, + {0x00D0D, 0x00D0D}, + {0x00D11, 0x00D11}, + {0x00D45, 0x00D45}, + {0x00D49, 0x00D49}, + {0x00D50, 0x00D53}, + {0x00D64, 0x00D65}, + {0x00D80, 0x00D80}, + {0x00D84, 0x00D84}, + {0x00D97, 0x00D99}, + {0x00DB2, 0x00DB2}, + {0x00DBC, 0x00DBC}, + {0x00DBE, 0x00DBF}, + {0x00DC7, 0x00DC9}, + {0x00DCB, 0x00DCE}, + {0x00DD5, 0x00DD5}, + {0x00DD7, 0x00DD7}, + {0x00DE0, 0x00DE5}, + {0x00DF0, 0x00DF1}, + {0x00DF5, 0x00E00}, + {0x00E3B, 0x00E3E}, + {0x00E5C, 0x00E80}, + {0x00E83, 0x00E83}, + {0x00E85, 0x00E85}, + {0x00E8B, 0x00E8B}, + {0x00EA4, 0x00EA4}, + {0x00EA6, 0x00EA6}, + {0x00EBE, 0x00EBF}, + {0x00EC5, 0x00EC5}, + {0x00EC7, 0x00EC7}, + {0x00ECF, 0x00ECF}, + {0x00EDA, 0x00EDB}, + {0x00EE0, 0x00EFF}, + {0x00F48, 0x00F48}, + {0x00F6D, 0x00F70}, + {0x00F98, 0x00F98}, + {0x00FBD, 0x00FBD}, + {0x00FCD, 0x00FCD}, + {0x00FDB, 0x00FFF}, + {0x010C6, 0x010C6}, + {0x010C8, 0x010CC}, + {0x010CE, 0x010CF}, + {0x01249, 0x01249}, + {0x0124E, 0x0124F}, + {0x01257, 0x01257}, + {0x01259, 0x01259}, + {0x0125E, 0x0125F}, + {0x01289, 0x01289}, + {0x0128E, 0x0128F}, + {0x012B1, 0x012B1}, + {0x012B6, 0x012B7}, + {0x012BF, 0x012BF}, + {0x012C1, 0x012C1}, + {0x012C6, 0x012C7}, + {0x012D7, 0x012D7}, + {0x01311, 0x01311}, + {0x01316, 0x01317}, + {0x0135B, 0x0135C}, + {0x0137D, 0x0137F}, + {0x0139A, 0x0139F}, + {0x013F6, 0x013F7}, + {0x013FE, 0x013FF}, + {0x0169D, 0x0169F}, + {0x016F9, 0x016FF}, + {0x01716, 0x0171E}, + {0x01737, 0x0173F}, + {0x01754, 0x0175F}, + {0x0176D, 0x0176D}, + {0x01771, 0x01771}, + {0x01774, 0x0177F}, + {0x017DE, 0x017DF}, + {0x017EA, 0x017EF}, + {0x017FA, 0x017FF}, + {0x0181A, 0x0181F}, + {0x01879, 0x0187F}, + {0x018AB, 0x018AF}, + {0x018F6, 0x018FF}, + {0x0191F, 0x0191F}, + {0x0192C, 0x0192F}, + {0x0193C, 0x0193F}, + {0x01941, 0x01943}, + {0x0196E, 0x0196F}, + {0x01975, 0x0197F}, + {0x019AC, 0x019AF}, + {0x019CA, 0x019CF}, + {0x019DB, 0x019DD}, + {0x01A1C, 0x01A1D}, + {0x01A5F, 0x01A5F}, + {0x01A7D, 0x01A7E}, + {0x01A8A, 0x01A8F}, + {0x01A9A, 0x01A9F}, + {0x01AAE, 0x01AAF}, + {0x01ACF, 0x01AFF}, + {0x01B4D, 0x01B4D}, + {0x01BF4, 0x01BFB}, + {0x01C38, 0x01C3A}, + {0x01C4A, 0x01C4C}, + {0x01C8B, 0x01C8F}, + {0x01CBB, 0x01CBC}, + {0x01CC8, 0x01CCF}, + {0x01CFB, 0x01CFF}, + {0x01F16, 0x01F17}, + {0x01F1E, 0x01F1F}, + {0x01F46, 0x01F47}, + {0x01F4E, 0x01F4F}, + {0x01F58, 0x01F58}, + {0x01F5A, 0x01F5A}, + {0x01F5C, 0x01F5C}, + {0x01F5E, 0x01F5E}, + {0x01F7E, 0x01F7F}, + {0x01FB5, 0x01FB5}, + {0x01FC5, 0x01FC5}, + {0x01FD4, 0x01FD5}, + {0x01FDC, 0x01FDC}, + {0x01FF0, 0x01FF1}, + {0x01FF5, 0x01FF5}, + {0x01FFF, 0x01FFF}, + {0x02065, 0x02065}, + {0x02072, 0x02073}, + {0x0208F, 0x0208F}, + {0x0209D, 0x0209F}, + {0x020C1, 0x020CF}, + {0x020F1, 0x020FF}, + {0x0218C, 0x0218F}, + {0x0242A, 0x0243F}, + {0x0244B, 0x0245F}, + {0x02B74, 0x02B75}, + {0x02B96, 0x02B96}, + {0x02CF4, 0x02CF8}, + {0x02D26, 0x02D26}, + {0x02D28, 0x02D2C}, + {0x02D2E, 0x02D2F}, + {0x02D68, 0x02D6E}, + {0x02D71, 0x02D7E}, + {0x02D97, 0x02D9F}, + {0x02DA7, 0x02DA7}, + {0x02DAF, 0x02DAF}, + {0x02DB7, 0x02DB7}, + {0x02DBF, 0x02DBF}, + {0x02DC7, 0x02DC7}, + {0x02DCF, 0x02DCF}, + {0x02DD7, 0x02DD7}, + {0x02DDF, 0x02DDF}, + {0x02E5E, 0x02E7F}, + {0x02E9A, 0x02E9A}, + {0x02EF4, 0x02EFF}, + {0x02FD6, 0x02FEF}, + {0x03040, 0x03040}, + {0x03097, 0x03098}, + {0x03100, 0x03104}, + {0x03130, 0x03130}, + {0x0318F, 0x0318F}, + {0x031E6, 0x031EE}, + {0x0321F, 0x0321F}, + {0x03401, 0x04DBE}, + {0x04E01, 0x09FFE}, + {0x0A48D, 0x0A48F}, + {0x0A4C7, 0x0A4CF}, + {0x0A62C, 0x0A63F}, + {0x0A6F8, 0x0A6FF}, + {0x0A7CE, 0x0A7CF}, + {0x0A7D2, 0x0A7D2}, + {0x0A7D4, 0x0A7D4}, + {0x0A7DD, 0x0A7F1}, + {0x0A82D, 0x0A82F}, + {0x0A83A, 0x0A83F}, + {0x0A878, 0x0A87F}, + {0x0A8C6, 0x0A8CD}, + {0x0A8DA, 0x0A8DF}, + {0x0A954, 0x0A95E}, + {0x0A97D, 0x0A97F}, + {0x0A9CE, 0x0A9CE}, + {0x0A9DA, 0x0A9DD}, + {0x0A9FF, 0x0A9FF}, + {0x0AA37, 0x0AA3F}, + {0x0AA4E, 0x0AA4F}, + {0x0AA5A, 0x0AA5B}, + {0x0AAC3, 0x0AADA}, + {0x0AAF7, 0x0AB00}, + {0x0AB07, 0x0AB08}, + {0x0AB0F, 0x0AB10}, + {0x0AB17, 0x0AB1F}, + {0x0AB27, 0x0AB27}, + {0x0AB2F, 0x0AB2F}, + {0x0AB6C, 0x0AB6F}, + {0x0ABEE, 0x0ABEF}, + {0x0ABFA, 0x0ABFF}, + {0x0AC01, 0x0D7A2}, + {0x0D7A4, 0x0D7AF}, + {0x0D7C7, 0x0D7CA}, + {0x0D7FC, 0x0D7FF}, + {0x0FA6E, 0x0FA6F}, + {0x0FADA, 0x0FAFF}, + {0x0FB07, 0x0FB12}, + {0x0FB18, 0x0FB1C}, + {0x0FB37, 0x0FB37}, + {0x0FB3D, 0x0FB3D}, + {0x0FB3F, 0x0FB3F}, + {0x0FB42, 0x0FB42}, + {0x0FB45, 0x0FB45}, + {0x0FBC3, 0x0FBD2}, + {0x0FD90, 0x0FD91}, + {0x0FDC8, 0x0FDCE}, + {0x0FE1A, 0x0FE1F}, + {0x0FE53, 0x0FE53}, + {0x0FE67, 0x0FE67}, + {0x0FE6C, 0x0FE6F}, + {0x0FE75, 0x0FE75}, + {0x0FEFD, 0x0FEFE}, + {0x0FF00, 0x0FF00}, + {0x0FFBF, 0x0FFC1}, + {0x0FFC8, 0x0FFC9}, + {0x0FFD0, 0x0FFD1}, + {0x0FFD8, 0x0FFD9}, + {0x0FFDD, 0x0FFDF}, + {0x0FFE7, 0x0FFE7}, + {0x0FFEF, 0x0FFF8}, + {0x1000C, 0x1000C}, + {0x10027, 0x10027}, + {0x1003B, 0x1003B}, + {0x1003E, 0x1003E}, + {0x1004E, 0x1004F}, + {0x1005E, 0x1007F}, + {0x100FB, 0x100FF}, + {0x10103, 0x10106}, + {0x10134, 0x10136}, + {0x1018F, 0x1018F}, + {0x1019D, 0x1019F}, + {0x101A1, 0x101CF}, + {0x101FE, 0x1027F}, + {0x1029D, 0x1029F}, + {0x102D1, 0x102DF}, + {0x102FC, 0x102FF}, + {0x10324, 0x1032C}, + {0x1034B, 0x1034F}, + {0x1037B, 0x1037F}, + {0x1039E, 0x1039E}, + {0x103C4, 0x103C7}, + {0x103D6, 0x103FF}, + {0x1049E, 0x1049F}, + {0x104AA, 0x104AF}, + {0x104D4, 0x104D7}, + {0x104FC, 0x104FF}, + {0x10528, 0x1052F}, + {0x10564, 0x1056E}, + {0x1057B, 0x1057B}, + {0x1058B, 0x1058B}, + {0x10593, 0x10593}, + {0x10596, 0x10596}, + {0x105A2, 0x105A2}, + {0x105B2, 0x105B2}, + {0x105BA, 0x105BA}, + {0x105BD, 0x105BF}, + {0x105F4, 0x105FF}, + {0x10737, 0x1073F}, + {0x10756, 0x1075F}, + {0x10768, 0x1077F}, + {0x10786, 0x10786}, + {0x107B1, 0x107B1}, + {0x107BB, 0x107FF}, + {0x10806, 0x10807}, + {0x10809, 0x10809}, + {0x10836, 0x10836}, + {0x10839, 0x1083B}, + {0x1083D, 0x1083E}, + {0x10856, 0x10856}, + {0x1089F, 0x108A6}, + {0x108B0, 0x108DF}, + {0x108F3, 0x108F3}, + {0x108F6, 0x108FA}, + {0x1091C, 0x1091E}, + {0x1093A, 0x1093E}, + {0x10940, 0x1097F}, + {0x109B8, 0x109BB}, + {0x109D0, 0x109D1}, + {0x10A04, 0x10A04}, + {0x10A07, 0x10A0B}, + {0x10A14, 0x10A14}, + {0x10A18, 0x10A18}, + {0x10A36, 0x10A37}, + {0x10A3B, 0x10A3E}, + {0x10A49, 0x10A4F}, + {0x10A59, 0x10A5F}, + {0x10AA0, 0x10ABF}, + {0x10AE7, 0x10AEA}, + {0x10AF7, 0x10AFF}, + {0x10B36, 0x10B38}, + {0x10B56, 0x10B57}, + {0x10B73, 0x10B77}, + {0x10B92, 0x10B98}, + {0x10B9D, 0x10BA8}, + {0x10BB0, 0x10BFF}, + {0x10C49, 0x10C7F}, + {0x10CB3, 0x10CBF}, + {0x10CF3, 0x10CF9}, + {0x10D28, 0x10D2F}, + {0x10D3A, 0x10D3F}, + {0x10D66, 0x10D68}, + {0x10D86, 0x10D8D}, + {0x10D90, 0x10E5F}, + {0x10E7F, 0x10E7F}, + {0x10EAA, 0x10EAA}, + {0x10EAE, 0x10EAF}, + {0x10EB2, 0x10EC1}, + {0x10EC5, 0x10EFB}, + {0x10F28, 0x10F2F}, + {0x10F5A, 0x10F6F}, + {0x10F8A, 0x10FAF}, + {0x10FCC, 0x10FDF}, + {0x10FF7, 0x10FFF}, + {0x1104E, 0x11051}, + {0x11076, 0x1107E}, + {0x110C3, 0x110CC}, + {0x110CE, 0x110CF}, + {0x110E9, 0x110EF}, + {0x110FA, 0x110FF}, + {0x11135, 0x11135}, + {0x11148, 0x1114F}, + {0x11177, 0x1117F}, + {0x111E0, 0x111E0}, + {0x111F5, 0x111FF}, + {0x11212, 0x11212}, + {0x11242, 0x1127F}, + {0x11287, 0x11287}, + {0x11289, 0x11289}, + {0x1128E, 0x1128E}, + {0x1129E, 0x1129E}, + {0x112AA, 0x112AF}, + {0x112EB, 0x112EF}, + {0x112FA, 0x112FF}, + {0x11304, 0x11304}, + {0x1130D, 0x1130E}, + {0x11311, 0x11312}, + {0x11329, 0x11329}, + {0x11331, 0x11331}, + {0x11334, 0x11334}, + {0x1133A, 0x1133A}, + {0x11345, 0x11346}, + {0x11349, 0x1134A}, + {0x1134E, 0x1134F}, + {0x11351, 0x11356}, + {0x11358, 0x1135C}, + {0x11364, 0x11365}, + {0x1136D, 0x1136F}, + {0x11375, 0x1137F}, + {0x1138A, 0x1138A}, + {0x1138C, 0x1138D}, + {0x1138F, 0x1138F}, + {0x113B6, 0x113B6}, + {0x113C1, 0x113C1}, + {0x113C3, 0x113C4}, + {0x113C6, 0x113C6}, + {0x113CB, 0x113CB}, + {0x113D6, 0x113D6}, + {0x113D9, 0x113E0}, + {0x113E3, 0x113FF}, + {0x1145C, 0x1145C}, + {0x11462, 0x1147F}, + {0x114C8, 0x114CF}, + {0x114DA, 0x1157F}, + {0x115B6, 0x115B7}, + {0x115DE, 0x115FF}, + {0x11645, 0x1164F}, + {0x1165A, 0x1165F}, + {0x1166D, 0x1167F}, + {0x116BA, 0x116BF}, + {0x116CA, 0x116CF}, + {0x116E4, 0x116FF}, + {0x1171B, 0x1171C}, + {0x1172C, 0x1172F}, + {0x11747, 0x117FF}, + {0x1183C, 0x1189F}, + {0x118F3, 0x118FE}, + {0x11907, 0x11908}, + {0x1190A, 0x1190B}, + {0x11914, 0x11914}, + {0x11917, 0x11917}, + {0x11936, 0x11936}, + {0x11939, 0x1193A}, + {0x11947, 0x1194F}, + {0x1195A, 0x1199F}, + {0x119A8, 0x119A9}, + {0x119D8, 0x119D9}, + {0x119E5, 0x119FF}, + {0x11A48, 0x11A4F}, + {0x11AA3, 0x11AAF}, + {0x11AF9, 0x11AFF}, + {0x11B0A, 0x11BBF}, + {0x11BE2, 0x11BEF}, + {0x11BFA, 0x11BFF}, + {0x11C09, 0x11C09}, + {0x11C37, 0x11C37}, + {0x11C46, 0x11C4F}, + {0x11C6D, 0x11C6F}, + {0x11C90, 0x11C91}, + {0x11CA8, 0x11CA8}, + {0x11CB7, 0x11CFF}, + {0x11D07, 0x11D07}, + {0x11D0A, 0x11D0A}, + {0x11D37, 0x11D39}, + {0x11D3B, 0x11D3B}, + {0x11D3E, 0x11D3E}, + {0x11D48, 0x11D4F}, + {0x11D5A, 0x11D5F}, + {0x11D66, 0x11D66}, + {0x11D69, 0x11D69}, + {0x11D8F, 0x11D8F}, + {0x11D92, 0x11D92}, + {0x11D99, 0x11D9F}, + {0x11DAA, 0x11EDF}, + {0x11EF9, 0x11EFF}, + {0x11F11, 0x11F11}, + {0x11F3B, 0x11F3D}, + {0x11F5B, 0x11FAF}, + {0x11FB1, 0x11FBF}, + {0x11FF2, 0x11FFE}, + {0x1239A, 0x123FF}, + {0x1246F, 0x1246F}, + {0x12475, 0x1247F}, + {0x12544, 0x12F8F}, + {0x12FF3, 0x12FFF}, + {0x13456, 0x1345F}, + {0x143FB, 0x143FF}, + {0x14647, 0x160FF}, + {0x1613A, 0x167FF}, + {0x16A39, 0x16A3F}, + {0x16A5F, 0x16A5F}, + {0x16A6A, 0x16A6D}, + {0x16ABF, 0x16ABF}, + {0x16ACA, 0x16ACF}, + {0x16AEE, 0x16AEF}, + {0x16AF6, 0x16AFF}, + {0x16B46, 0x16B4F}, + {0x16B5A, 0x16B5A}, + {0x16B62, 0x16B62}, + {0x16B78, 0x16B7C}, + {0x16B90, 0x16D3F}, + {0x16D7A, 0x16E3F}, + {0x16E9B, 0x16EFF}, + {0x16F4B, 0x16F4E}, + {0x16F88, 0x16F8E}, + {0x16FA0, 0x16FDF}, + {0x16FE5, 0x16FEF}, + {0x16FF2, 0x16FFF}, + {0x17001, 0x187F6}, + {0x187F8, 0x187FF}, + {0x18CD6, 0x18CFE}, + {0x18D01, 0x18D07}, + {0x18D09, 0x1AFEF}, + {0x1AFF4, 0x1AFF4}, + {0x1AFFC, 0x1AFFC}, + {0x1AFFF, 0x1AFFF}, + {0x1B123, 0x1B131}, + {0x1B133, 0x1B14F}, + {0x1B153, 0x1B154}, + {0x1B156, 0x1B163}, + {0x1B168, 0x1B16F}, + {0x1B2FC, 0x1BBFF}, + {0x1BC6B, 0x1BC6F}, + {0x1BC7D, 0x1BC7F}, + {0x1BC89, 0x1BC8F}, + {0x1BC9A, 0x1BC9B}, + {0x1BCA4, 0x1CBFF}, + {0x1CCFA, 0x1CCFF}, + {0x1CEB4, 0x1CEFF}, + {0x1CF2E, 0x1CF2F}, + {0x1CF47, 0x1CF4F}, + {0x1CFC4, 0x1CFFF}, + {0x1D0F6, 0x1D0FF}, + {0x1D127, 0x1D128}, + {0x1D1EB, 0x1D1FF}, + {0x1D246, 0x1D2BF}, + {0x1D2D4, 0x1D2DF}, + {0x1D2F4, 0x1D2FF}, + {0x1D357, 0x1D35F}, + {0x1D379, 0x1D3FF}, + {0x1D455, 0x1D455}, + {0x1D49D, 0x1D49D}, + {0x1D4A0, 0x1D4A1}, + {0x1D4A3, 0x1D4A4}, + {0x1D4A7, 0x1D4A8}, + {0x1D4AD, 0x1D4AD}, + {0x1D4BA, 0x1D4BA}, + {0x1D4BC, 0x1D4BC}, + {0x1D4C4, 0x1D4C4}, + {0x1D506, 0x1D506}, + {0x1D50B, 0x1D50C}, + {0x1D515, 0x1D515}, + {0x1D51D, 0x1D51D}, + {0x1D53A, 0x1D53A}, + {0x1D53F, 0x1D53F}, + {0x1D545, 0x1D545}, + {0x1D547, 0x1D549}, + {0x1D551, 0x1D551}, + {0x1D6A6, 0x1D6A7}, + {0x1D7CC, 0x1D7CD}, + {0x1DA8C, 0x1DA9A}, + {0x1DAA0, 0x1DAA0}, + {0x1DAB0, 0x1DEFF}, + {0x1DF1F, 0x1DF24}, + {0x1DF2B, 0x1DFFF}, + {0x1E007, 0x1E007}, + {0x1E019, 0x1E01A}, + {0x1E022, 0x1E022}, + {0x1E025, 0x1E025}, + {0x1E02B, 0x1E02F}, + {0x1E06E, 0x1E08E}, + {0x1E090, 0x1E0FF}, + {0x1E12D, 0x1E12F}, + {0x1E13E, 0x1E13F}, + {0x1E14A, 0x1E14D}, + {0x1E150, 0x1E28F}, + {0x1E2AF, 0x1E2BF}, + {0x1E2FA, 0x1E2FE}, + {0x1E300, 0x1E4CF}, + {0x1E4FA, 0x1E5CF}, + {0x1E5FB, 0x1E5FE}, + {0x1E600, 0x1E7DF}, + {0x1E7E7, 0x1E7E7}, + {0x1E7EC, 0x1E7EC}, + {0x1E7EF, 0x1E7EF}, + {0x1E7FF, 0x1E7FF}, + {0x1E8C5, 0x1E8C6}, + {0x1E8D7, 0x1E8FF}, + {0x1E94C, 0x1E94F}, + {0x1E95A, 0x1E95D}, + {0x1E960, 0x1EC70}, + {0x1ECB5, 0x1ED00}, + {0x1ED3E, 0x1EDFF}, + {0x1EE04, 0x1EE04}, + {0x1EE20, 0x1EE20}, + {0x1EE23, 0x1EE23}, + {0x1EE25, 0x1EE26}, + {0x1EE28, 0x1EE28}, + {0x1EE33, 0x1EE33}, + {0x1EE38, 0x1EE38}, + {0x1EE3A, 0x1EE3A}, + {0x1EE3C, 0x1EE41}, + {0x1EE43, 0x1EE46}, + {0x1EE48, 0x1EE48}, + {0x1EE4A, 0x1EE4A}, + {0x1EE4C, 0x1EE4C}, + {0x1EE50, 0x1EE50}, + {0x1EE53, 0x1EE53}, + {0x1EE55, 0x1EE56}, + {0x1EE58, 0x1EE58}, + {0x1EE5A, 0x1EE5A}, + {0x1EE5C, 0x1EE5C}, + {0x1EE5E, 0x1EE5E}, + {0x1EE60, 0x1EE60}, + {0x1EE63, 0x1EE63}, + {0x1EE65, 0x1EE66}, + {0x1EE6B, 0x1EE6B}, + {0x1EE73, 0x1EE73}, + {0x1EE78, 0x1EE78}, + {0x1EE7D, 0x1EE7D}, + {0x1EE7F, 0x1EE7F}, + {0x1EE8A, 0x1EE8A}, + {0x1EE9C, 0x1EEA0}, + {0x1EEA4, 0x1EEA4}, + {0x1EEAA, 0x1EEAA}, + {0x1EEBC, 0x1EEEF}, + {0x1EEF2, 0x1EFFF}, + {0x1F02C, 0x1F02F}, + {0x1F094, 0x1F09F}, + {0x1F0AF, 0x1F0B0}, + {0x1F0C0, 0x1F0C0}, + {0x1F0D0, 0x1F0D0}, + {0x1F0F6, 0x1F0FF}, + {0x1F1AE, 0x1F1E5}, + {0x1F203, 0x1F20F}, + {0x1F23C, 0x1F23F}, + {0x1F249, 0x1F24F}, + {0x1F252, 0x1F25F}, + {0x1F266, 0x1F2FF}, + {0x1F6D8, 0x1F6DB}, + {0x1F6ED, 0x1F6EF}, + {0x1F6FD, 0x1F6FF}, + {0x1F777, 0x1F77A}, + {0x1F7DA, 0x1F7DF}, + {0x1F7EC, 0x1F7EF}, + {0x1F7F1, 0x1F7FF}, + {0x1F80C, 0x1F80F}, + {0x1F848, 0x1F84F}, + {0x1F85A, 0x1F85F}, + {0x1F888, 0x1F88F}, + {0x1F8AE, 0x1F8AF}, + {0x1F8BC, 0x1F8BF}, + {0x1F8C2, 0x1F8FF}, + {0x1FA54, 0x1FA5F}, + {0x1FA6E, 0x1FA6F}, + {0x1FA7D, 0x1FA7F}, + {0x1FA8A, 0x1FA8E}, + {0x1FAC7, 0x1FACD}, + {0x1FADD, 0x1FADE}, + {0x1FAEA, 0x1FAEF}, + {0x1FAF9, 0x1FAFF}, + {0x1FB93, 0x1FB93}, + {0x1FBFA, 0x1FFFD}, + {0x20001, 0x2A6DE}, + {0x2A6E0, 0x2A6FF}, + {0x2A701, 0x2B738}, + {0x2B73A, 0x2B73F}, + {0x2B741, 0x2B81C}, + {0x2B81E, 0x2B81F}, + {0x2B821, 0x2CEA0}, + {0x2CEA2, 0x2CEAF}, + {0x2CEB1, 0x2EBDF}, + {0x2EBE1, 0x2EBEF}, + {0x2EBF1, 0x2EE5C}, + {0x2EE5E, 0x2F7FF}, + {0x2FA1E, 0x2FFFD}, + {0x30001, 0x31349}, + {0x3134B, 0x3134F}, + {0x31351, 0x323AE}, + {0x323B0, 0x3FFFD}, + {0x40000, 0x4FFFD}, + {0x50000, 0x5FFFD}, + {0x60000, 0x6FFFD}, + {0x70000, 0x7FFFD}, + {0x80000, 0x8FFFD}, + {0x90000, 0x9FFFD}, + {0xA0000, 0xAFFFD}, + {0xB0000, 0xBFFFD}, + {0xC0000, 0xCFFFD}, + {0xD0000, 0xDFFFD}, + {0xE0000, 0xE0000}, + {0xE0002, 0xE001F}, + {0xE0080, 0xE00FF}, + {0xE01F0, 0xEFFFD} +}; + +/* Non-characters. */ +static const struct widechar_range widechar_nonchar_table[] = { + {0x0FDD0, 0x0FDEF}, + {0x0FFFE, 0x0FFFF}, + {0x1FFFE, 0x1FFFF}, + {0x2FFFE, 0x2FFFF}, + {0x3FFFE, 0x3FFFF}, + {0x4FFFE, 0x4FFFF}, + {0x5FFFE, 0x5FFFF}, + {0x6FFFE, 0x6FFFF}, + {0x7FFFE, 0x7FFFF}, + {0x8FFFE, 0x8FFFF}, + {0x9FFFE, 0x9FFFF}, + {0xAFFFE, 0xAFFFF}, + {0xBFFFE, 0xBFFFF}, + {0xCFFFE, 0xCFFFF}, + {0xDFFFE, 0xDFFFF}, + {0xEFFFE, 0xEFFFF}, + {0xFFFFE, 0xFFFFF}, + {0x10FFFE, 0x10FFFF} +}; + +/* Characters that were widened from width 1 to 2 in Unicode 9. */ +static const struct widechar_range widechar_widened_table[] = { + {0x0231A, 0x0231B}, + {0x023E9, 0x023EC}, + {0x023F0, 0x023F0}, + {0x023F3, 0x023F3}, + {0x025FD, 0x025FE}, + {0x02614, 0x02615}, + {0x02648, 0x02653}, + {0x0267F, 0x0267F}, + {0x02693, 0x02693}, + {0x026A1, 0x026A1}, + {0x026AA, 0x026AB}, + {0x026BD, 0x026BE}, + {0x026C4, 0x026C5}, + {0x026CE, 0x026CE}, + {0x026D4, 0x026D4}, + {0x026EA, 0x026EA}, + {0x026F2, 0x026F3}, + {0x026F5, 0x026F5}, + {0x026FA, 0x026FA}, + {0x026FD, 0x026FD}, + {0x02705, 0x02705}, + {0x0270A, 0x0270B}, + {0x02728, 0x02728}, + {0x0274C, 0x0274C}, + {0x0274E, 0x0274E}, + {0x02753, 0x02755}, + {0x02757, 0x02757}, + {0x02795, 0x02797}, + {0x027B0, 0x027B0}, + {0x027BF, 0x027BF}, + {0x02B1B, 0x02B1C}, + {0x02B50, 0x02B50}, + {0x02B55, 0x02B55}, + {0x1F004, 0x1F004}, + {0x1F0CF, 0x1F0CF}, + {0x1F18E, 0x1F18E}, + {0x1F191, 0x1F19A}, + {0x1F201, 0x1F201}, + {0x1F21A, 0x1F21A}, + {0x1F22F, 0x1F22F}, + {0x1F232, 0x1F236}, + {0x1F238, 0x1F23A}, + {0x1F250, 0x1F251}, + {0x1F300, 0x1F320}, + {0x1F32D, 0x1F335}, + {0x1F337, 0x1F37C}, + {0x1F37E, 0x1F393}, + {0x1F3A0, 0x1F3CA}, + {0x1F3CF, 0x1F3D3}, + {0x1F3E0, 0x1F3F0}, + {0x1F3F4, 0x1F3F4}, + {0x1F3F8, 0x1F43E}, + {0x1F440, 0x1F440}, + {0x1F442, 0x1F4FC}, + {0x1F4FF, 0x1F53D}, + {0x1F54B, 0x1F54E}, + {0x1F550, 0x1F567}, + {0x1F595, 0x1F596}, + {0x1F5FB, 0x1F64F}, + {0x1F680, 0x1F6C5}, + {0x1F6CC, 0x1F6CC}, + {0x1F6D0, 0x1F6D0}, + {0x1F6EB, 0x1F6EC}, + {0x1F910, 0x1F918}, + {0x1F980, 0x1F984}, + {0x1F9C0, 0x1F9C0} +}; + +template +bool widechar_in_table(const Collection &arr, uint32_t c) { + auto where = std::lower_bound(std::begin(arr), std::end(arr), c, + [](widechar_range p, uint32_t c) { return p.hi < c; }); + return where != std::end(arr) && where->lo <= c; +} + +/* Return the width of character c, or a special negative value. */ +int widechar_wcwidth(uint32_t c) { + if (widechar_in_table(widechar_ascii_table, c)) + return 1; + if (widechar_in_table(widechar_private_table, c)) + return widechar_private_use; + if (widechar_in_table(widechar_nonprint_table, c)) + return widechar_nonprint; + if (widechar_in_table(widechar_nonchar_table, c)) + return widechar_non_character; + if (widechar_in_table(widechar_combining_table, c)) + return widechar_combining; + if (widechar_in_table(widechar_combiningletters_table, c)) + return widechar_combining; + if (widechar_in_table(widechar_doublewide_table, c)) + return 2; + if (widechar_in_table(widechar_ambiguous_table, c)) + return widechar_ambiguous; + if (widechar_in_table(widechar_unassigned_table, c)) + return widechar_unassigned; + if (widechar_in_table(widechar_widened_table, c)) + return widechar_widened_in_9; + return 1; +} + +} // namespace +#endif // WIDECHAR_WIDTH_H diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 308a432e4..d1093597c 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,6 @@ #include "environment-variables.hh" -#ifdef WIN32 +#ifdef _WIN32 # include "processenv.h" namespace nix { diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 71f53ccb8..e2a473a7c 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,7 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" -#ifdef WIN32 +#ifdef _WIN32 #include #include #include diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 53271cef3..7ed1c04a6 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,6 +1,6 @@ #include "file-system.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index af7e987e9..ac2882120 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,4 +1,4 @@ -#ifdef WIN32 +#ifdef _WIN32 # include # include "windows-error.hh" diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 26ad9cba0..b09ef8b90 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,7 +7,7 @@ #include "file-path-impl.hh" #include "util.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix { diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index e69f1ed45..fd4d7c43a 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,7 +23,7 @@ #include #include -#ifdef WIN32 +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index 2780e45f4..438c4221c 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,7 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 13b563510..4e139d5cf 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,7 +1,7 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix::windows { diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 277336ed7..53715e260 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,7 +2,7 @@ ///@file #include "file-descriptor.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix::windows { diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index 4cf4274da..b92f9155f 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,6 +1,6 @@ #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 #include #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 4e48ee859..66c67b43a 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#ifdef WIN32 +#ifdef _WIN32 #include #include "error.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c48549511..09d354832 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -24,7 +24,7 @@ static Path gcRoot; static int rootNr = 0; -enum OutputKind { okPlain, okXML, okJSON }; +enum OutputKind { okPlain, okRaw, okXML, okJSON }; void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, @@ -50,7 +50,11 @@ void processExpr(EvalState & state, const Strings & attrPaths, vRes = v; else state.autoCallFunction(autoArgs, v, vRes); - if (output == okXML) + if (output == okRaw) + std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"); + // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline + // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt. + else if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); @@ -132,6 +136,8 @@ static int main_nix_instantiate(int argc, char * * argv) gcRoot = getArg(*arg, arg, end); else if (*arg == "--indirect") ; + else if (*arg == "--raw") + outputKind = okRaw; else if (*arg == "--xml") outputKind = okXML; else if (*arg == "--json") diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b731b25af..99bb2c726 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -183,9 +183,9 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); cout << fmt("%s\n", store->printStorePath(store->addToStore( - std::string(baseNameOf(i)), {accessor, canonPath}))); + std::string(baseNameOf(i)), sourcePath))); } } @@ -207,10 +207,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( baseNameOf(i), - {accessor, canonPath}, + sourcePath, method, hashAlgo).path)); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5c08f7616..7f15de374 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -37,13 +37,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); auto storePath = dryRun ? store->computeStorePath( - *namePart, {accessor, path2}, caMethod, hashAlgo, {}).first + *namePart, sourcePath, caMethod, hashAlgo, {}).first : store->addToStoreSlow( - *namePart, {accessor, path2}, caMethod, hashAlgo, {}).path; + *namePart, sourcePath, caMethod, hashAlgo, {}).path; logger->cout("%s", store->printStorePath(storePath)); } diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 55aa8971e..7189689cc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -938,7 +938,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - createSymlink(target, to2); + createSymlink(target, os_string_to_string(PathViewNG { to2 })); } else throw Error("file '%s' has unsupported type", from2); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 416cd19b3..eac421d12 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -87,18 +87,35 @@ struct CmdHashBase : Command return std::make_unique(hashAlgo); }; - auto path2 = PosixSourceAccessor::createAtRoot(path); + auto makeSourcePath = [&]() -> SourcePath { + return PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); + }; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: + { + // While usually we could use the some code as for NixArchive, + // the Flat method needs to support FIFOs, such as those + // produced by bash process substitution, e.g.: + // nix hash --mode flat <(echo hi) + // Also symlinks semantics are unambiguous in the flat case, + // so we don't need to go low-level, or reject symlink `path`s. + auto hashSink = makeSink(); + readFile(path, *hashSink); + h = hashSink->finish().first; + break; + } case FileIngestionMethod::NixArchive: { + auto sourcePath = makeSourcePath(); auto hashSink = makeSink(); - dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); + dumpPath(sourcePath, *hashSink, (FileSerialisationMethod) mode); h = hashSink->finish().first; break; } case FileIngestionMethod::Git: { + auto sourcePath = makeSourcePath(); std::function hook; hook = [&](const SourcePath & path) -> git::TreeEntry { auto hashSink = makeSink(); @@ -109,7 +126,7 @@ struct CmdHashBase : Command .hash = hash, }; }; - h = hook(path2).hash; + h = hook(sourcePath).hash; break; } } diff --git a/src/nix/meson.build b/src/nix/meson.build index 1d4840b12..2698cc873 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -4,8 +4,6 @@ project('nix', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 3b37ee7d4..0e6868d8f 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -29,6 +29,47 @@ echo "$hash2" test "$hash1" = "sha256:$hash2" +# The contents can be accessed through a symlink, and this symlink has no effect on the hash +# https://github.com/NixOS/nix/issues/11941 +test_issue_11941() { + local expected actual + mkdir -p "$TEST_ROOT/foo/bar" && ln -s "$TEST_ROOT/foo" "$TEST_ROOT/foo-link" + + # legacy + expected=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo/bar") + actual=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo-link/bar") + [[ "$expected" == "$actual" ]] + actual=$(nix-store --add "$TEST_ROOT/foo-link/bar") + [[ "$expected" == "$actual" ]] + + # nix store add + actual=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/foo/bar") + [[ "$expected" == "$actual" ]] + + # cleanup + rm -r "$TEST_ROOT/foo" "$TEST_ROOT/foo-link" +} +test_issue_11941 + +# A symlink is added to the store as a symlink, not as a copy of the target +test_add_symlink() { + ln -s /bin "$TEST_ROOT/my-bin" + + # legacy + path=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + path=$(nix-store --add "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + + # nix store add + path=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + + # cleanup + rm "$TEST_ROOT/my-bin" +} +test_add_symlink + #### New style commands clearStoreIfPossible diff --git a/tests/functional/eval.nix b/tests/functional/eval.nix index befbd17a9..cabf28c29 100644 --- a/tests/functional/eval.nix +++ b/tests/functional/eval.nix @@ -1,5 +1,5 @@ { int = 123; - str = "foo"; + str = "foo\nbar"; attr.foo = "bar"; } diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 7af49d7fd..ed9c214f5 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -16,8 +16,8 @@ EOF nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval int -f "./eval.nix") == 123 ]] -[[ $(nix eval str -f "./eval.nix") == '"foo"' ]] -[[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]] +[[ $(nix eval str -f "./eval.nix") == '"foo\nbar"' ]] +[[ $(nix eval str --raw -f "./eval.nix") == $'foo\nbar' ]] [[ "$(nix eval attr -f "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix eval int -f - < "./eval.nix") == 123 ]] @@ -28,7 +28,8 @@ nix eval --expr 'assert 1 + 2 == 3; true' nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] -[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]] +[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo\nbar"' ]] +[[ $(nix-instantiate -A str --raw --eval "./eval.nix") == $'foo\nbar' ]] [[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 817f77783..643d729ff 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -63,3 +63,16 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule echo '"foo"' > "$rootRepo"/submodule/sub.nix [[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] + +# Test that `nix flake metadata` parses `submodule` correctly. +cat > "$rootRepo"/flake.nix < "$TEST_ROOT/simple/hi" +ln -s "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it/simple-link" +h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/simple/hi") +[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]] +h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/try/to/mess/with/it/simple-link/hi") +[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]] + +# nix hash --mode nar does not canonicalize a symlink argument. +# Otherwise it can't generate a NAR whose root is a symlink. +# If you want to follow the symlink, pass $(realpath -s ...) instead. +ln -s /non-existent-48cujwe8ndf4as0bne "$TEST_ROOT/symlink-to-nowhere" +h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-nowhere") +[[ 1bl5ry3x1fcbwgr5c2x50bn572iixh4j1p6ax5isxly2ddgn8pbp == "$h" ]] # manually verified hash +if [[ -e /bin ]]; then + ln -s /bin "$TEST_ROOT/symlink-to-bin" + h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-bin") + [[ 0z2mdmkd43l0ijdxfbj1y8vzli15yh9b09n3a3rrygmjshbyypsw == "$h" ]] # manually verified hash +fi diff --git a/tests/functional/lang/eval-fail-string-nul-1.err.exp b/tests/functional/lang/eval-fail-string-nul-1.err.exp new file mode 100644 index 000000000..2dfbea063 Binary files /dev/null and b/tests/functional/lang/eval-fail-string-nul-1.err.exp differ diff --git a/tests/functional/lang/eval-fail-string-nul-1.nix b/tests/functional/lang/eval-fail-string-nul-1.nix new file mode 100644 index 000000000..368940917 Binary files /dev/null and b/tests/functional/lang/eval-fail-string-nul-1.nix differ diff --git a/tests/functional/lang/eval-fail-string-nul-2.err.exp b/tests/functional/lang/eval-fail-string-nul-2.err.exp new file mode 100644 index 000000000..b1cae5325 Binary files /dev/null and b/tests/functional/lang/eval-fail-string-nul-2.err.exp differ diff --git a/tests/functional/lang/eval-fail-string-nul-2.nix b/tests/functional/lang/eval-fail-string-nul-2.nix new file mode 100644 index 000000000..fd6b3258a Binary files /dev/null and b/tests/functional/lang/eval-fail-string-nul-2.nix differ diff --git a/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp b/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp new file mode 100644 index 000000000..129d58bcb --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'toJSON' builtin + at /pwd/lang/eval-fail-toJSON-non-utf-8.nix:1:1: + 1| builtins.toJSON "_invalid UTF-8: _" + | ^ + 2| + + error: JSON serialization error: [json.exception.type_error.316] invalid UTF-8 byte at index 16: 0xFF diff --git a/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix b/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix new file mode 100644 index 000000000..bd1f74de7 --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix @@ -0,0 +1 @@ +builtins.toJSON "_invalid UTF-8: _" diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 933595cd5..83e08c4f5 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -4,8 +4,6 @@ project('nix-functional-tests', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.3', diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index ff1220f35..8e0cb1b22 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -26,6 +26,8 @@ let # Evaluate VMs faster documentation.enable = false; + # this links against nix and might break with our git version. + system.tools.nixos-option.enable = false; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; @@ -157,6 +159,8 @@ in functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + functional_symlinked-home = runNixOSTestFor "x86_64-linux" ./functional/symlinked-home.nix; + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; diff --git a/tests/nixos/functional/symlinked-home.nix b/tests/nixos/functional/symlinked-home.nix new file mode 100644 index 000000000..57c45d5d5 --- /dev/null +++ b/tests/nixos/functional/symlinked-home.nix @@ -0,0 +1,36 @@ +/** + This test runs the functional tests on a NixOS system where the home directory + is symlinked to another location. + + The purpose of this test is to find cases where Nix uses low-level operations + that don't support symlinks on paths that include them. + + It is not a substitute for more intricate, use case-specific tests, but helps + catch common issues. +*/ +# TODO: add symlinked tmpdir +{ ... }: +{ + name = "functional-tests-on-nixos_user_symlinked-home"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + with subtest("prepare symlinked home"): + machine.succeed(""" + ( + set -x + mv /home/alice /home/alice.real + ln -s alice.real /home/alice + ) 1>&2 + """) + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 6c51fcba5..83b85c032 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -10,6 +10,7 @@ let env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + objectThatDoesNotExist = "s3://my-cache/foo-that-does-not-exist?endpoint=http://server:9000®ion=eu-west-1"; in { name = "s3-binary-cache-store"; @@ -20,7 +21,10 @@ in { { virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgs.minio-client ]; - nix.extraOptions = "experimental-features = nix-command"; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; services.minio = { enable = true; region = "eu-west-1"; @@ -35,7 +39,10 @@ in { client = { config, pkgs, ... }: { virtualisation.writableStore = true; - nix.extraOptions = "experimental-features = nix-command"; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; }; }; @@ -54,6 +61,12 @@ in { # Test fetchurl on s3:// URLs while we're at it. client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") + # Test that the format string in the error message is properly setup and won't display `%s` instead of the failed URI + msg = client.fail("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"${objectThatDoesNotExist}\"; }' 2>&1") + if "S3 object '${objectThatDoesNotExist}' does not exist" not in msg: + print(msg) # So that you can see the message that was improperly formatted + raise Exception("Error message formatting didn't work") + # Copy a package from the binary cache. client.fail("nix path-info ${pkgA}")