From cd002ae6dda7be6e339d7e573ee122b5cdc50861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:48:23 +0200 Subject: [PATCH 01/48] flake.nix: switch to nixpkgs 24.11 --- flake.lock | 20 ++++++++++---------- flake.nix | 3 ++- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/flake.lock b/flake.lock index b5d0b881c..10957359a 100644 --- a/flake.lock +++ b/flake.lock @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1719994518, - "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", "type": "github" }, "original": { @@ -48,11 +48,11 @@ ] }, "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "lastModified": 1734279981, + "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", "type": "github" }, "original": { @@ -80,16 +80,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723688146, - "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", + "lastModified": 1734359947, + "narHash": "sha256-1Noao/H+N8nFB4Beoy8fgwrcOQLVm9o4zKW1ODaqK9E=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", + "rev": "48d12d5e70ee91fe8481378e540433a7303dbf6a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-24.05", + "ref": "release-24.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index a5e68609e..ba3cd3b82 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,8 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.11"; + inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 9131905185129034aa12839809ffd8f843107940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:51:29 +0200 Subject: [PATCH 02/48] use libgit2 from nixpkgs --- flake.lock | 24 +++--------------------- flake.nix | 4 +--- packaging/dependencies.nix | 2 -- 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/flake.lock b/flake.lock index 10957359a..ce484a67a 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -61,23 +61,6 @@ "type": "github" } }, - "libgit2": { - "flake": false, - "locked": { - "lastModified": 1715853528, - "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", - "owner": "libgit2", - "repo": "libgit2", - "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", - "type": "github" - }, - "original": { - "owner": "libgit2", - "ref": "v1.8.1", - "repo": "libgit2", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1734359947, @@ -131,7 +114,6 @@ "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", - "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", "nixpkgs-regression": "nixpkgs-regression" diff --git a/flake.nix b/flake.nix index ba3cd3b82..156eb71cd 100644 --- a/flake.nix +++ b/flake.nix @@ -6,7 +6,6 @@ inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; @@ -19,7 +18,7 @@ inputs.git-hooks-nix.inputs.flake-compat.follows = ""; inputs.git-hooks-nix.inputs.gitignore.follows = ""; - outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: + outputs = inputs@{ self, nixpkgs, nixpkgs-regression, ... }: let @@ -164,7 +163,6 @@ if prev.stdenv.hostPlatform.system == "i686-linux" then (prev.pre-commit.override (o: { dotnet-sdk = ""; })).overridePythonAttrs (o: { doCheck = false; }) else prev.pre-commit; - }; in { diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index a8005ce16..78451e549 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -140,8 +140,6 @@ scope: { }); libgit2 = pkgs.libgit2.overrideAttrs (attrs: { - src = inputs.libgit2; - version = inputs.libgit2.lastModifiedDate; cmakeFlags = attrs.cmakeFlags or [] ++ [ "-DUSE_SSH=exec" ]; nativeBuildInputs = attrs.nativeBuildInputs or [] From 1a8bd84f558c44feff234652669e1474a78289fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:53:06 +0200 Subject: [PATCH 03/48] remove upstreamed busybox-sandbox-shell --- packaging/dependencies.nix | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 78451e549..785480921 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -167,30 +167,6 @@ scope: { ]; }); - busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - # TODO change in Nixpkgs, Windows works fine. First commit of # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. toml11 = pkgs.toml11.overrideAttrs (old: { From 81b6b79a5641aaa29244be29bc506e86de51c87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:53:50 +0200 Subject: [PATCH 04/48] remove upstreamed libseccomp --- packaging/dependencies.nix | 8 -------- 1 file changed, 8 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 785480921..531052107 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -114,14 +114,6 @@ scope: { requiredSystemFeatures = [ ]; }; - libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = pkgs.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - boehmgc = pkgs.boehmgc.override { enableLargeConfig = true; }; From b5ad051b6c705fe9a6fe442371d9f19257d68724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 22 Sep 2024 16:56:01 +0200 Subject: [PATCH 05/48] remove upstreamed toml11 package override --- packaging/dependencies.nix | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 531052107..2c7cf701f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -159,12 +159,6 @@ scope: { ]; }); - # TODO change in Nixpkgs, Windows works fine. First commit of - # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. - toml11 = pkgs.toml11.overrideAttrs (old: { - meta.platforms = lib.platforms.all; - }); - inherit resolvePath filesetToSource; mkMesonDerivation = From cf0ba0d20ebdf190141ddc1a114a4a1cf1988bd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Wed, 25 Sep 2024 14:13:33 +0200 Subject: [PATCH 06/48] netbsd: disable cross-compilation Cross-compilation of curl is broken in nixpkgs. Therefore we disable it until nixpkgs fixes the underlying issues. --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 156eb71cd..104cbc34b 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,8 @@ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" "riscv64-unknown-linux-gnu" - "x86_64-unknown-netbsd" + # Disabled because of https://github.com/NixOS/nixpkgs/issues/344423 + # "x86_64-unknown-netbsd" "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; From 2512619cb6b7b5fad7c5024a985996e6c875dfe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 3 Dec 2024 15:51:29 +0100 Subject: [PATCH 07/48] switch to lowdown-unsandboxed package This is needed for macos support as the sandboxed version of lowdown doesn't work in the nix sandbox. --- doc/manual/package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 2e6fcede3..f8133f2e1 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -3,7 +3,7 @@ , meson , ninja -, lowdown +, lowdown-unsandboxed , mdbook , mdbook-linkcheck , jq @@ -42,7 +42,7 @@ mkMesonDerivation (finalAttrs: { passthru.externalNativeBuildInputs = [ meson ninja - (lib.getBin lowdown) + (lib.getBin lowdown-unsandboxed) mdbook mdbook-linkcheck jq From 20ee83fffde3d5892cf012dedeacaf11ce45cf7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 9 Dec 2024 16:03:36 +0100 Subject: [PATCH 08/48] tests/nixos: disable nixos-option --- tests/nixos/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index ff1220f35..a9a438a32 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -26,6 +26,8 @@ let # Evaluate VMs faster documentation.enable = false; + # this links against nix and might break with our git version. + system.tools.nixos-option.enable = false; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; From a8e1b4757e3022dff433693eb8ed3e757dd317c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 19 Dec 2024 18:35:28 +0100 Subject: [PATCH 09/48] filesystem/deletePath: remove unnecessary quotes from error message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paths are already quoted: error: … while fetching the input 'path:/nix/store/rs2s2ca7xs87v82aps54m1p3sqrfz6c8-source' error: chmod '"/nix/store/rs2s2ca7xs87v82aps54m1p3sqrfz6c8-source"': Read-only file system --- src/libutil/file-system.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 829700336..50e90bb57 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -384,7 +384,7 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); + throw SysError("getting status of %1%", path); } if (!S_ISDIR(st.st_mode)) { @@ -416,15 +416,15 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; if ((st.st_mode & PERM_MASK) != PERM_MASK) { if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); + throw SysError("chmod %1%", path); } int fd = openat(parentfd, path.c_str(), O_RDONLY); if (fd == -1) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %1%", path); AutoCloseDir dir(fdopendir(fd)); if (!dir) - throw SysError("opening directory '%1%'", path); + throw SysError("opening directory %1%", path); struct dirent * dirent; while (errno = 0, dirent = readdir(dir.get())) { /* sic */ @@ -433,13 +433,13 @@ static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & b if (childName == "." || childName == "..") continue; _deletePath(dirfd(dir.get()), path + "/" + childName, bytesFreed); } - if (errno) throw SysError("reading directory '%1%'", path); + if (errno) throw SysError("reading directory %1%", path); } int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; if (unlinkat(parentfd, name.c_str(), flags) == -1) { if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); + throw SysError("cannot unlink %1%", path); } #else // TODO implement From 535724fd7953e9b465af419d66536dc810e3e6c2 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 19 Dec 2024 11:14:04 -0800 Subject: [PATCH 10/48] tests/nixos/s3-binary-cache-store: test that "object does not exist" error message is properly formatted --- tests/nixos/s3-binary-cache-store.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 6c51fcba5..45f234645 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -10,6 +10,7 @@ let env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + objectThatDoesNotExist = "s3://my-cache/foo-that-does-not-exist?endpoint=http://server:9000®ion=eu-west-1"; in { name = "s3-binary-cache-store"; @@ -54,6 +55,12 @@ in { # Test fetchurl on s3:// URLs while we're at it. client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") + # Test that the format string in the error message is properly setup and won't display `%s` instead of the failed URI + msg = client.fail("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"${objectThatDoesNotExist}\"; }' 2>&1") + if "S3 object '${objectThatDoesNotExist}' does not exist" not in msg: + print(msg) # So that you can see the message that was improperly formatted + raise Exception("Error message formatting didn't work") + # Copy a package from the binary cache. client.fail("nix path-info ${pkgA}") From f0c1262d2319ff6967139d8f8599ed6176ad1263 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 19 Dec 2024 11:15:25 -0800 Subject: [PATCH 11/48] tests/nixos/s3-binary-cache-store: disable default substituter so it runs faster Since networking is disabled in these VMs, trying to talk to the default cache.nixos.org slows the test down (since it can't resolve it). --- tests/nixos/s3-binary-cache-store.nix | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 45f234645..83b85c032 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -21,7 +21,10 @@ in { { virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgs.minio-client ]; - nix.extraOptions = "experimental-features = nix-command"; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; services.minio = { enable = true; region = "eu-west-1"; @@ -36,7 +39,10 @@ in { client = { config, pkgs, ... }: { virtualisation.writableStore = true; - nix.extraOptions = "experimental-features = nix-command"; + nix.extraOptions = '' + experimental-features = nix-command + substituters = + ''; }; }; From b978fa845022d448c11a1d2e9d0347a57edecbd8 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 19 Dec 2024 10:16:47 -0800 Subject: [PATCH 12/48] libstore: fixup unformatted uri when S3 getObject fails --- src/libstore/filetransfer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 42b93cfe0..8439cc39c 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -767,7 +767,7 @@ struct curlFileTransfer : public FileTransfer auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) - throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); + throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); res.urls.push_back(request.uri); callback(std::move(res)); From 4a91e627a7617cc3d654967358fd29df076ab853 Mon Sep 17 00:00:00 2001 From: Yaroslav Bolyukin Date: Wed, 25 Dec 2024 02:41:20 +0100 Subject: [PATCH 13/48] fix: ignore symlinks in fsync-store-paths Fixes: https://github.com/NixOS/nix/issues/12099 --- src/libutil/file-system.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 50e90bb57..793eb2d9f 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -331,7 +331,7 @@ void syncParent(const Path & path) void recursiveSync(const Path & path) { - /* If it's a file, just fsync and return. */ + /* If it's a file or symlink, just fsync and return. */ auto st = lstat(path); if (S_ISREG(st.st_mode)) { AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0)); @@ -339,7 +339,8 @@ void recursiveSync(const Path & path) throw SysError("opening file '%1%'", path); fd.fsync(); return; - } + } else if (S_ISLNK(st.st_mode)) + return; /* Otherwise, perform a depth-first traversal of the directory and fsync all the files. */ From fe5f02c2c274e8f48a1072807fc9bb3f892584a4 Mon Sep 17 00:00:00 2001 From: NAHO <90870942+trueNAHO@users.noreply.github.com> Date: Fri, 27 Dec 2024 01:56:12 +0100 Subject: [PATCH 14/48] ci: lock Ubuntu runner to ubuntu-22.04 Lock the Ubuntu runner to ubuntu-22.04 to avoid accidental updates [1] and increase reproducibility. [1]: https://github.com/actions/runner-images/issues/10636 --- .github/workflows/ci.yml | 14 +++++++------- .github/workflows/labels.yml | 2 +- .mergify.yml | 4 ++-- doc/manual/source/development/testing.md | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index addafb9f8..2426ab166 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: read-all jobs: eval: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-22.04, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -37,7 +37,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-22.04, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -68,7 +68,7 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-latest' + if: matrix.os == 'ubuntu-22.04' - run: brew install fish if: matrix.os == 'macos-latest' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" @@ -86,7 +86,7 @@ jobs: permissions: contents: none name: Check Docker secrets present for installer tests - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 outputs: docker: ${{ steps.secret.outputs.docker }} steps: @@ -106,7 +106,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && github.ref_name == 'master' - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Check for secrets id: secret diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 34aa4e6bd..9d2ac80a3 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -15,7 +15,7 @@ permissions: jobs: labels: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 if: github.repository_owner == 'NixOS' steps: - uses: actions/labeler@v5 diff --git a/.mergify.yml b/.mergify.yml index 70fccae49..c2e115487 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,9 @@ queue_rules: # all required tests need to go here merge_conditions: - check-success=tests (macos-latest) - - check-success=tests (ubuntu-latest) + - check-success=tests (ubuntu-22.04) - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-latest) + - check-success=installer_test (ubuntu-22.04) - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 30aa7d0d5..7e8762fe0 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-latest` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-22.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup From cf69c99f3ec16deb25ae7a57ee579431828da249 Mon Sep 17 00:00:00 2001 From: Parker Jones <49885263+knotapun@users.noreply.github.com> Date: Fri, 27 Dec 2024 01:11:36 -0500 Subject: [PATCH 15/48] Make `readFileType` doc string consistent The primitive `readFileType p` has a list of acceptable types, and so does `readDir path` This edit makes the formatting of the list consistent between themselves, and other parts of the documentation. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c1f1cf4c1..f19dd473a 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2045,7 +2045,7 @@ static RegisterPrimOp primop_readFileType({ .args = {"p"}, .doc = R"( Determine the directory entry type of a filesystem node, being - one of "directory", "regular", "symlink", or "unknown". + one of `"directory"`, `"regular"`, `"symlink"`, or `"unknown"`. )", .fun = prim_readFileType, }); From 92e30955b9a4acbc01cd6519552ada3385ef3c7e Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sun, 15 Dec 2024 02:29:56 +0100 Subject: [PATCH 16/48] try to calculate character width --- maintainers/flake-module.nix | 1 + src/libutil-tests/terminal.cc | 4 + src/libutil/meson.build | 6 + src/libutil/package.nix | 1 + src/libutil/terminal.cc | 90 +- src/libutil/widecharwidth/LICENSE | 4 + src/libutil/widecharwidth/widechar_width.h | 1559 ++++++++++++++++++++ 7 files changed, 1636 insertions(+), 29 deletions(-) create mode 100644 src/libutil/widecharwidth/LICENSE create mode 100644 src/libutil/widecharwidth/widechar_width.h diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 1a134b91a..6b311c62e 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -356,6 +356,7 @@ ''^src/libutil/util\.cc$'' ''^src/libutil/util\.hh$'' ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/widecharwidth/widechar_width\.h$'' # vendored source ''^src/libutil/windows/file-descriptor\.cc$'' ''^src/libutil/windows/file-path\.cc$'' ''^src/libutil/windows/processes\.cc$'' diff --git a/src/libutil-tests/terminal.cc b/src/libutil-tests/terminal.cc index 714d5a237..f4fc6e770 100644 --- a/src/libutil-tests/terminal.cc +++ b/src/libutil-tests/terminal.cc @@ -55,6 +55,10 @@ TEST(filterANSIEscapes, utf8) ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 6), "f🔍bar"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 3), "f🔍"); + ASSERT_EQ(filterANSIEscapes("f🔍bar", true, 2), "f"); + ASSERT_EQ(filterANSIEscapes("foo\u0301", true, 3), "foó"); } TEST(filterANSIEscapes, osc8) diff --git a/src/libutil/meson.build b/src/libutil/meson.build index bbe7872cf..2c3e3a954 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -108,6 +108,8 @@ deps_private += cpuid nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json +cxx = meson.get_compiler('cpp') + config_h = configure_file( configuration : configdata, output : 'config-util.hh', @@ -168,6 +170,10 @@ sources = files( ) include_dirs = [include_directories('.')] +if not cxx.has_header('widechar_width.h', required : false) + # use vendored widechar_width.h + include_dirs += include_directories('./widecharwidth') +endif headers = [config_h] + files( 'abstract-setting-to-json.hh', diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 69ebbf726..0a9f65782 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -29,6 +29,7 @@ mkMesonLibrary (finalAttrs: { ./nix-meson-build-support ../../.version ./.version + ./widecharwidth ./meson.build ./meson.options ./linux/meson.build diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 4c127ddb0..8a8373f1b 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -11,6 +11,53 @@ # include #endif #include +#include + +namespace { + +inline std::pair charWidthUTF8Helper(std::string_view s) +{ + size_t bytes = 1; + uint32_t ch = s[0]; + uint32_t max = 1U << 7; + if ((ch & 0x80U) == 0U) { + } else if ((ch & 0xe0U) == 0xc0U) { + ch &= 0x1fU; + bytes = 2; + max = 1U << 11; + } else if ((ch & 0xf0U) == 0xe0U) { + ch &= 0x0fU; + bytes = 3; + max = 1U << 16; + } else if ((ch & 0xf8U) == 0xf0U) { + ch &= 0x07U; + bytes = 4; + max = 0x110000U; + } else { + return {bytes, bytes}; // invalid UTF-8 start byte + } + for (size_t i = 1; i < bytes; i++) { + if (i < s.size() && (s[i] & 0xc0) == 0x80) { + ch = (ch << 6) | (s[i] & 0x3f); + } else { + return {i, i}; // invalid UTF-8 encoding; assume one character per byte + } + } + int width = bytes; // in case of overlong encoding + if (ch < max) { + width = widechar_wcwidth(ch); + if (width == widechar_ambiguous) { + width = 1; // just a guess... + } else if (width == widechar_widened_in_9) { + width = 2; + } else if (width < 0) { + width = 0; + } + } + return {width, bytes}; +} + +} namespace nix { @@ -30,7 +77,7 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w size_t w = 0; auto i = s.begin(); - while (w < (size_t) width && i != s.end()) { + while (i != s.end()) { if (*i == '\e') { std::string e; @@ -61,10 +108,12 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w } else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } + do { + if (++w > (size_t) width) + return t; + t += ' '; + } while (w % 8); + i++; } else if (*i == '\r' || *i == '\a') @@ -72,35 +121,18 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w i++; else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; + auto [chWidth, bytes] = charWidthUTF8Helper({i, s.end()}); + w += chWidth; + if (w > (size_t) width) { + break; + } + t += {i, i + bytes}; + i += bytes; } } - return t; } - ////////////////////////////////////////////////////////////////////// static Sync> windowSize{{0, 0}}; diff --git a/src/libutil/widecharwidth/LICENSE b/src/libutil/widecharwidth/LICENSE new file mode 100644 index 000000000..d3b1dd767 --- /dev/null +++ b/src/libutil/widecharwidth/LICENSE @@ -0,0 +1,4 @@ +widecharwidth - wcwidth implementation +Written in 2018 by ridiculous_fish +To the extent possible under law, the author(s) have dedicated all copyright and related and neighboring rights to this software to the public domain worldwide. This software is distributed without any warranty. +You should have received a copy of the CC0 Public Domain Dedication along with this software. If not, see . diff --git a/src/libutil/widecharwidth/widechar_width.h b/src/libutil/widecharwidth/widechar_width.h new file mode 100644 index 000000000..92e63e913 --- /dev/null +++ b/src/libutil/widecharwidth/widechar_width.h @@ -0,0 +1,1559 @@ +/** + * widechar_width.h for Unicode 16.0.0 + * See https://github.com/ridiculousfish/widecharwidth/ + * + * SHA1 file hashes: + * ( + * the hashes for generate.py and the template are git object hashes, + * use `git log --all --find-object=` in the widecharwidth repository + * to see which commit they correspond to, + * or run `git hash-object` on the file to compare. + * The other hashes are simple `sha1sum` style hashes. + * ) + * + * generate.py: 2747bb9402d8eeeca8e566ff947f14308511ecb1 + * template.js: 1249763c5b7c1e308aeb4ca64f1e15bce1fab9b3 + * UnicodeData.txt: 91df83276154240bcedef82a09bde77aa182cf8d + * EastAsianWidth.txt: 0885c0fc1c21eb58954a3bfb785d78559b361d92 + * emoji-data.txt: 1df2f8329dd9f5c238674807de736f316c6b9d87 + */ + +#ifndef WIDECHAR_WIDTH_H +#define WIDECHAR_WIDTH_H + +#include +#include +#include +#include + +namespace { + +/* Special width values */ +enum { + widechar_nonprint = -1, // The character is not printable. + widechar_combining = -2, // The character is a zero-width combiner. + widechar_ambiguous = -3, // The character is East-Asian ambiguous width. + widechar_private_use = -4, // The character is for private use. + widechar_unassigned = -5, // The character is unassigned. + widechar_widened_in_9 = -6, // Width is 1 in Unicode 8, 2 in Unicode 9+. + widechar_non_character = -7 // The character is a noncharacter. +}; + +/* An inclusive range of characters. */ +struct widechar_range { + uint32_t lo; + uint32_t hi; +}; + +/* Simple ASCII characters - used a lot, so we check them first. */ +static const struct widechar_range widechar_ascii_table[] = { + {0x00020, 0x0007E} +}; + +/* Private usage range. */ +static const struct widechar_range widechar_private_table[] = { + {0x0E000, 0x0F8FF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} +}; + +/* Nonprinting characters. */ +static const struct widechar_range widechar_nonprint_table[] = { + {0x00000, 0x0001F}, + {0x0007F, 0x0009F}, + {0x000AD, 0x000AD}, + {0x00600, 0x00605}, + {0x0061C, 0x0061C}, + {0x006DD, 0x006DD}, + {0x0070F, 0x0070F}, + {0x00890, 0x00891}, + {0x008E2, 0x008E2}, + {0x0180E, 0x0180E}, + {0x0200B, 0x0200F}, + {0x02028, 0x0202E}, + {0x02060, 0x02064}, + {0x02066, 0x0206F}, + {0x0D800, 0x0DFFF}, + {0x0FEFF, 0x0FEFF}, + {0x0FFF9, 0x0FFFB}, + {0x110BD, 0x110BD}, + {0x110CD, 0x110CD}, + {0x13430, 0x1343F}, + {0x1BCA0, 0x1BCA3}, + {0x1D173, 0x1D17A}, + {0xE0001, 0xE0001}, + {0xE0020, 0xE007F} +}; + +/* Width 0 combining marks. */ +static const struct widechar_range widechar_combining_table[] = { + {0x00300, 0x0036F}, + {0x00483, 0x00489}, + {0x00591, 0x005BD}, + {0x005BF, 0x005BF}, + {0x005C1, 0x005C2}, + {0x005C4, 0x005C5}, + {0x005C7, 0x005C7}, + {0x00610, 0x0061A}, + {0x0064B, 0x0065F}, + {0x00670, 0x00670}, + {0x006D6, 0x006DC}, + {0x006DF, 0x006E4}, + {0x006E7, 0x006E8}, + {0x006EA, 0x006ED}, + {0x00711, 0x00711}, + {0x00730, 0x0074A}, + {0x007A6, 0x007B0}, + {0x007EB, 0x007F3}, + {0x007FD, 0x007FD}, + {0x00816, 0x00819}, + {0x0081B, 0x00823}, + {0x00825, 0x00827}, + {0x00829, 0x0082D}, + {0x00859, 0x0085B}, + {0x00897, 0x0089F}, + {0x008CA, 0x008E1}, + {0x008E3, 0x00903}, + {0x0093A, 0x0093C}, + {0x0093E, 0x0094F}, + {0x00951, 0x00957}, + {0x00962, 0x00963}, + {0x00981, 0x00983}, + {0x009BC, 0x009BC}, + {0x009BE, 0x009C4}, + {0x009C7, 0x009C8}, + {0x009CB, 0x009CD}, + {0x009D7, 0x009D7}, + {0x009E2, 0x009E3}, + {0x009FE, 0x009FE}, + {0x00A01, 0x00A03}, + {0x00A3C, 0x00A3C}, + {0x00A3E, 0x00A42}, + {0x00A47, 0x00A48}, + {0x00A4B, 0x00A4D}, + {0x00A51, 0x00A51}, + {0x00A70, 0x00A71}, + {0x00A75, 0x00A75}, + {0x00A81, 0x00A83}, + {0x00ABC, 0x00ABC}, + {0x00ABE, 0x00AC5}, + {0x00AC7, 0x00AC9}, + {0x00ACB, 0x00ACD}, + {0x00AE2, 0x00AE3}, + {0x00AFA, 0x00AFF}, + {0x00B01, 0x00B03}, + {0x00B3C, 0x00B3C}, + {0x00B3E, 0x00B44}, + {0x00B47, 0x00B48}, + {0x00B4B, 0x00B4D}, + {0x00B55, 0x00B57}, + {0x00B62, 0x00B63}, + {0x00B82, 0x00B82}, + {0x00BBE, 0x00BC2}, + {0x00BC6, 0x00BC8}, + {0x00BCA, 0x00BCD}, + {0x00BD7, 0x00BD7}, + {0x00C00, 0x00C04}, + {0x00C3C, 0x00C3C}, + {0x00C3E, 0x00C44}, + {0x00C46, 0x00C48}, + {0x00C4A, 0x00C4D}, + {0x00C55, 0x00C56}, + {0x00C62, 0x00C63}, + {0x00C81, 0x00C83}, + {0x00CBC, 0x00CBC}, + {0x00CBE, 0x00CC4}, + {0x00CC6, 0x00CC8}, + {0x00CCA, 0x00CCD}, + {0x00CD5, 0x00CD6}, + {0x00CE2, 0x00CE3}, + {0x00CF3, 0x00CF3}, + {0x00D00, 0x00D03}, + {0x00D3B, 0x00D3C}, + {0x00D3E, 0x00D44}, + {0x00D46, 0x00D48}, + {0x00D4A, 0x00D4D}, + {0x00D57, 0x00D57}, + {0x00D62, 0x00D63}, + {0x00D81, 0x00D83}, + {0x00DCA, 0x00DCA}, + {0x00DCF, 0x00DD4}, + {0x00DD6, 0x00DD6}, + {0x00DD8, 0x00DDF}, + {0x00DF2, 0x00DF3}, + {0x00E31, 0x00E31}, + {0x00E34, 0x00E3A}, + {0x00E47, 0x00E4E}, + {0x00EB1, 0x00EB1}, + {0x00EB4, 0x00EBC}, + {0x00EC8, 0x00ECE}, + {0x00F18, 0x00F19}, + {0x00F35, 0x00F35}, + {0x00F37, 0x00F37}, + {0x00F39, 0x00F39}, + {0x00F3E, 0x00F3F}, + {0x00F71, 0x00F84}, + {0x00F86, 0x00F87}, + {0x00F8D, 0x00F97}, + {0x00F99, 0x00FBC}, + {0x00FC6, 0x00FC6}, + {0x0102B, 0x0103E}, + {0x01056, 0x01059}, + {0x0105E, 0x01060}, + {0x01062, 0x01064}, + {0x01067, 0x0106D}, + {0x01071, 0x01074}, + {0x01082, 0x0108D}, + {0x0108F, 0x0108F}, + {0x0109A, 0x0109D}, + {0x0135D, 0x0135F}, + {0x01712, 0x01715}, + {0x01732, 0x01734}, + {0x01752, 0x01753}, + {0x01772, 0x01773}, + {0x017B4, 0x017D3}, + {0x017DD, 0x017DD}, + {0x0180B, 0x0180D}, + {0x0180F, 0x0180F}, + {0x01885, 0x01886}, + {0x018A9, 0x018A9}, + {0x01920, 0x0192B}, + {0x01930, 0x0193B}, + {0x01A17, 0x01A1B}, + {0x01A55, 0x01A5E}, + {0x01A60, 0x01A7C}, + {0x01A7F, 0x01A7F}, + {0x01AB0, 0x01ACE}, + {0x01B00, 0x01B04}, + {0x01B34, 0x01B44}, + {0x01B6B, 0x01B73}, + {0x01B80, 0x01B82}, + {0x01BA1, 0x01BAD}, + {0x01BE6, 0x01BF3}, + {0x01C24, 0x01C37}, + {0x01CD0, 0x01CD2}, + {0x01CD4, 0x01CE8}, + {0x01CED, 0x01CED}, + {0x01CF4, 0x01CF4}, + {0x01CF7, 0x01CF9}, + {0x01DC0, 0x01DFF}, + {0x020D0, 0x020F0}, + {0x02CEF, 0x02CF1}, + {0x02D7F, 0x02D7F}, + {0x02DE0, 0x02DFF}, + {0x0302A, 0x0302F}, + {0x03099, 0x0309A}, + {0x0A66F, 0x0A672}, + {0x0A674, 0x0A67D}, + {0x0A69E, 0x0A69F}, + {0x0A6F0, 0x0A6F1}, + {0x0A802, 0x0A802}, + {0x0A806, 0x0A806}, + {0x0A80B, 0x0A80B}, + {0x0A823, 0x0A827}, + {0x0A82C, 0x0A82C}, + {0x0A880, 0x0A881}, + {0x0A8B4, 0x0A8C5}, + {0x0A8E0, 0x0A8F1}, + {0x0A8FF, 0x0A8FF}, + {0x0A926, 0x0A92D}, + {0x0A947, 0x0A953}, + {0x0A980, 0x0A983}, + {0x0A9B3, 0x0A9C0}, + {0x0A9E5, 0x0A9E5}, + {0x0AA29, 0x0AA36}, + {0x0AA43, 0x0AA43}, + {0x0AA4C, 0x0AA4D}, + {0x0AA7B, 0x0AA7D}, + {0x0AAB0, 0x0AAB0}, + {0x0AAB2, 0x0AAB4}, + {0x0AAB7, 0x0AAB8}, + {0x0AABE, 0x0AABF}, + {0x0AAC1, 0x0AAC1}, + {0x0AAEB, 0x0AAEF}, + {0x0AAF5, 0x0AAF6}, + {0x0ABE3, 0x0ABEA}, + {0x0ABEC, 0x0ABED}, + {0x0FB1E, 0x0FB1E}, + {0x0FE00, 0x0FE0F}, + {0x0FE20, 0x0FE2F}, + {0x101FD, 0x101FD}, + {0x102E0, 0x102E0}, + {0x10376, 0x1037A}, + {0x10A01, 0x10A03}, + {0x10A05, 0x10A06}, + {0x10A0C, 0x10A0F}, + {0x10A38, 0x10A3A}, + {0x10A3F, 0x10A3F}, + {0x10AE5, 0x10AE6}, + {0x10D24, 0x10D27}, + {0x10D69, 0x10D6D}, + {0x10EAB, 0x10EAC}, + {0x10EFC, 0x10EFF}, + {0x10F46, 0x10F50}, + {0x10F82, 0x10F85}, + {0x11000, 0x11002}, + {0x11038, 0x11046}, + {0x11070, 0x11070}, + {0x11073, 0x11074}, + {0x1107F, 0x11082}, + {0x110B0, 0x110BA}, + {0x110C2, 0x110C2}, + {0x11100, 0x11102}, + {0x11127, 0x11134}, + {0x11145, 0x11146}, + {0x11173, 0x11173}, + {0x11180, 0x11182}, + {0x111B3, 0x111C0}, + {0x111C9, 0x111CC}, + {0x111CE, 0x111CF}, + {0x1122C, 0x11237}, + {0x1123E, 0x1123E}, + {0x11241, 0x11241}, + {0x112DF, 0x112EA}, + {0x11300, 0x11303}, + {0x1133B, 0x1133C}, + {0x1133E, 0x11344}, + {0x11347, 0x11348}, + {0x1134B, 0x1134D}, + {0x11357, 0x11357}, + {0x11362, 0x11363}, + {0x11366, 0x1136C}, + {0x11370, 0x11374}, + {0x113B8, 0x113C0}, + {0x113C2, 0x113C2}, + {0x113C5, 0x113C5}, + {0x113C7, 0x113CA}, + {0x113CC, 0x113D0}, + {0x113D2, 0x113D2}, + {0x113E1, 0x113E2}, + {0x11435, 0x11446}, + {0x1145E, 0x1145E}, + {0x114B0, 0x114C3}, + {0x115AF, 0x115B5}, + {0x115B8, 0x115C0}, + {0x115DC, 0x115DD}, + {0x11630, 0x11640}, + {0x116AB, 0x116B7}, + {0x1171D, 0x1172B}, + {0x1182C, 0x1183A}, + {0x11930, 0x11935}, + {0x11937, 0x11938}, + {0x1193B, 0x1193E}, + {0x11940, 0x11940}, + {0x11942, 0x11943}, + {0x119D1, 0x119D7}, + {0x119DA, 0x119E0}, + {0x119E4, 0x119E4}, + {0x11A01, 0x11A0A}, + {0x11A33, 0x11A39}, + {0x11A3B, 0x11A3E}, + {0x11A47, 0x11A47}, + {0x11A51, 0x11A5B}, + {0x11A8A, 0x11A99}, + {0x11C2F, 0x11C36}, + {0x11C38, 0x11C3F}, + {0x11C92, 0x11CA7}, + {0x11CA9, 0x11CB6}, + {0x11D31, 0x11D36}, + {0x11D3A, 0x11D3A}, + {0x11D3C, 0x11D3D}, + {0x11D3F, 0x11D45}, + {0x11D47, 0x11D47}, + {0x11D8A, 0x11D8E}, + {0x11D90, 0x11D91}, + {0x11D93, 0x11D97}, + {0x11EF3, 0x11EF6}, + {0x11F00, 0x11F01}, + {0x11F03, 0x11F03}, + {0x11F34, 0x11F3A}, + {0x11F3E, 0x11F42}, + {0x11F5A, 0x11F5A}, + {0x13440, 0x13440}, + {0x13447, 0x13455}, + {0x1611E, 0x1612F}, + {0x16AF0, 0x16AF4}, + {0x16B30, 0x16B36}, + {0x16F4F, 0x16F4F}, + {0x16F51, 0x16F87}, + {0x16F8F, 0x16F92}, + {0x16FE4, 0x16FE4}, + {0x16FF0, 0x16FF1}, + {0x1BC9D, 0x1BC9E}, + {0x1CF00, 0x1CF2D}, + {0x1CF30, 0x1CF46}, + {0x1D165, 0x1D169}, + {0x1D16D, 0x1D172}, + {0x1D17B, 0x1D182}, + {0x1D185, 0x1D18B}, + {0x1D1AA, 0x1D1AD}, + {0x1D242, 0x1D244}, + {0x1DA00, 0x1DA36}, + {0x1DA3B, 0x1DA6C}, + {0x1DA75, 0x1DA75}, + {0x1DA84, 0x1DA84}, + {0x1DA9B, 0x1DA9F}, + {0x1DAA1, 0x1DAAF}, + {0x1E000, 0x1E006}, + {0x1E008, 0x1E018}, + {0x1E01B, 0x1E021}, + {0x1E023, 0x1E024}, + {0x1E026, 0x1E02A}, + {0x1E08F, 0x1E08F}, + {0x1E130, 0x1E136}, + {0x1E2AE, 0x1E2AE}, + {0x1E2EC, 0x1E2EF}, + {0x1E4EC, 0x1E4EF}, + {0x1E5EE, 0x1E5EF}, + {0x1E8D0, 0x1E8D6}, + {0x1E944, 0x1E94A}, + {0xE0100, 0xE01EF} +}; + +/* Width 0 combining letters. */ +static const struct widechar_range widechar_combiningletters_table[] = { + {0x01160, 0x011FF}, + {0x0D7B0, 0x0D7FF} +}; + +/* Width 2 characters. */ +static const struct widechar_range widechar_doublewide_table[] = { + {0x01100, 0x0115F}, + {0x02329, 0x0232A}, + {0x02630, 0x02637}, + {0x0268A, 0x0268F}, + {0x02E80, 0x02E99}, + {0x02E9B, 0x02EF3}, + {0x02F00, 0x02FD5}, + {0x02FF0, 0x0303E}, + {0x03041, 0x03096}, + {0x03099, 0x030FF}, + {0x03105, 0x0312F}, + {0x03131, 0x0318E}, + {0x03190, 0x031E5}, + {0x031EF, 0x0321E}, + {0x03220, 0x03247}, + {0x03250, 0x0A48C}, + {0x0A490, 0x0A4C6}, + {0x0A960, 0x0A97C}, + {0x0AC00, 0x0D7A3}, + {0x0F900, 0x0FAFF}, + {0x0FE10, 0x0FE19}, + {0x0FE30, 0x0FE52}, + {0x0FE54, 0x0FE66}, + {0x0FE68, 0x0FE6B}, + {0x0FF01, 0x0FF60}, + {0x0FFE0, 0x0FFE6}, + {0x16FE0, 0x16FE4}, + {0x16FF0, 0x16FF1}, + {0x17000, 0x187F7}, + {0x18800, 0x18CD5}, + {0x18CFF, 0x18D08}, + {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, + {0x1B000, 0x1B122}, + {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, + {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, + {0x1B170, 0x1B2FB}, + {0x1D300, 0x1D356}, + {0x1D360, 0x1D376}, + {0x1F200, 0x1F200}, + {0x1F202, 0x1F202}, + {0x1F210, 0x1F219}, + {0x1F21B, 0x1F22E}, + {0x1F230, 0x1F231}, + {0x1F237, 0x1F237}, + {0x1F23B, 0x1F23B}, + {0x1F240, 0x1F248}, + {0x1F260, 0x1F265}, + {0x1F57A, 0x1F57A}, + {0x1F5A4, 0x1F5A4}, + {0x1F6D1, 0x1F6D2}, + {0x1F6D5, 0x1F6D7}, + {0x1F6DC, 0x1F6DF}, + {0x1F6F4, 0x1F6FC}, + {0x1F7E0, 0x1F7EB}, + {0x1F7F0, 0x1F7F0}, + {0x1F90C, 0x1F90F}, + {0x1F919, 0x1F93A}, + {0x1F93C, 0x1F945}, + {0x1F947, 0x1F97F}, + {0x1F985, 0x1F9BF}, + {0x1F9C1, 0x1F9FF}, + {0x1FA70, 0x1FA7C}, + {0x1FA80, 0x1FA89}, + {0x1FA8F, 0x1FAC6}, + {0x1FACE, 0x1FADC}, + {0x1FADF, 0x1FAE9}, + {0x1FAF0, 0x1FAF8}, + {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD} +}; + +/* Ambiguous-width characters. */ +static const struct widechar_range widechar_ambiguous_table[] = { + {0x000A1, 0x000A1}, + {0x000A4, 0x000A4}, + {0x000A7, 0x000A8}, + {0x000AA, 0x000AA}, + {0x000AD, 0x000AE}, + {0x000B0, 0x000B4}, + {0x000B6, 0x000BA}, + {0x000BC, 0x000BF}, + {0x000C6, 0x000C6}, + {0x000D0, 0x000D0}, + {0x000D7, 0x000D8}, + {0x000DE, 0x000E1}, + {0x000E6, 0x000E6}, + {0x000E8, 0x000EA}, + {0x000EC, 0x000ED}, + {0x000F0, 0x000F0}, + {0x000F2, 0x000F3}, + {0x000F7, 0x000FA}, + {0x000FC, 0x000FC}, + {0x000FE, 0x000FE}, + {0x00101, 0x00101}, + {0x00111, 0x00111}, + {0x00113, 0x00113}, + {0x0011B, 0x0011B}, + {0x00126, 0x00127}, + {0x0012B, 0x0012B}, + {0x00131, 0x00133}, + {0x00138, 0x00138}, + {0x0013F, 0x00142}, + {0x00144, 0x00144}, + {0x00148, 0x0014B}, + {0x0014D, 0x0014D}, + {0x00152, 0x00153}, + {0x00166, 0x00167}, + {0x0016B, 0x0016B}, + {0x001CE, 0x001CE}, + {0x001D0, 0x001D0}, + {0x001D2, 0x001D2}, + {0x001D4, 0x001D4}, + {0x001D6, 0x001D6}, + {0x001D8, 0x001D8}, + {0x001DA, 0x001DA}, + {0x001DC, 0x001DC}, + {0x00251, 0x00251}, + {0x00261, 0x00261}, + {0x002C4, 0x002C4}, + {0x002C7, 0x002C7}, + {0x002C9, 0x002CB}, + {0x002CD, 0x002CD}, + {0x002D0, 0x002D0}, + {0x002D8, 0x002DB}, + {0x002DD, 0x002DD}, + {0x002DF, 0x002DF}, + {0x00300, 0x0036F}, + {0x00391, 0x003A1}, + {0x003A3, 0x003A9}, + {0x003B1, 0x003C1}, + {0x003C3, 0x003C9}, + {0x00401, 0x00401}, + {0x00410, 0x0044F}, + {0x00451, 0x00451}, + {0x02010, 0x02010}, + {0x02013, 0x02016}, + {0x02018, 0x02019}, + {0x0201C, 0x0201D}, + {0x02020, 0x02022}, + {0x02024, 0x02027}, + {0x02030, 0x02030}, + {0x02032, 0x02033}, + {0x02035, 0x02035}, + {0x0203B, 0x0203B}, + {0x0203E, 0x0203E}, + {0x02074, 0x02074}, + {0x0207F, 0x0207F}, + {0x02081, 0x02084}, + {0x020AC, 0x020AC}, + {0x02103, 0x02103}, + {0x02105, 0x02105}, + {0x02109, 0x02109}, + {0x02113, 0x02113}, + {0x02116, 0x02116}, + {0x02121, 0x02122}, + {0x02126, 0x02126}, + {0x0212B, 0x0212B}, + {0x02153, 0x02154}, + {0x0215B, 0x0215E}, + {0x02160, 0x0216B}, + {0x02170, 0x02179}, + {0x02189, 0x02189}, + {0x02190, 0x02199}, + {0x021B8, 0x021B9}, + {0x021D2, 0x021D2}, + {0x021D4, 0x021D4}, + {0x021E7, 0x021E7}, + {0x02200, 0x02200}, + {0x02202, 0x02203}, + {0x02207, 0x02208}, + {0x0220B, 0x0220B}, + {0x0220F, 0x0220F}, + {0x02211, 0x02211}, + {0x02215, 0x02215}, + {0x0221A, 0x0221A}, + {0x0221D, 0x02220}, + {0x02223, 0x02223}, + {0x02225, 0x02225}, + {0x02227, 0x0222C}, + {0x0222E, 0x0222E}, + {0x02234, 0x02237}, + {0x0223C, 0x0223D}, + {0x02248, 0x02248}, + {0x0224C, 0x0224C}, + {0x02252, 0x02252}, + {0x02260, 0x02261}, + {0x02264, 0x02267}, + {0x0226A, 0x0226B}, + {0x0226E, 0x0226F}, + {0x02282, 0x02283}, + {0x02286, 0x02287}, + {0x02295, 0x02295}, + {0x02299, 0x02299}, + {0x022A5, 0x022A5}, + {0x022BF, 0x022BF}, + {0x02312, 0x02312}, + {0x02460, 0x024E9}, + {0x024EB, 0x0254B}, + {0x02550, 0x02573}, + {0x02580, 0x0258F}, + {0x02592, 0x02595}, + {0x025A0, 0x025A1}, + {0x025A3, 0x025A9}, + {0x025B2, 0x025B3}, + {0x025B6, 0x025B7}, + {0x025BC, 0x025BD}, + {0x025C0, 0x025C1}, + {0x025C6, 0x025C8}, + {0x025CB, 0x025CB}, + {0x025CE, 0x025D1}, + {0x025E2, 0x025E5}, + {0x025EF, 0x025EF}, + {0x02605, 0x02606}, + {0x02609, 0x02609}, + {0x0260E, 0x0260F}, + {0x0261C, 0x0261C}, + {0x0261E, 0x0261E}, + {0x02640, 0x02640}, + {0x02642, 0x02642}, + {0x02660, 0x02661}, + {0x02663, 0x02665}, + {0x02667, 0x0266A}, + {0x0266C, 0x0266D}, + {0x0266F, 0x0266F}, + {0x0269E, 0x0269F}, + {0x026BF, 0x026BF}, + {0x026C6, 0x026CD}, + {0x026CF, 0x026D3}, + {0x026D5, 0x026E1}, + {0x026E3, 0x026E3}, + {0x026E8, 0x026E9}, + {0x026EB, 0x026F1}, + {0x026F4, 0x026F4}, + {0x026F6, 0x026F9}, + {0x026FB, 0x026FC}, + {0x026FE, 0x026FF}, + {0x0273D, 0x0273D}, + {0x02776, 0x0277F}, + {0x02B56, 0x02B59}, + {0x03248, 0x0324F}, + {0x0E000, 0x0F8FF}, + {0x0FE00, 0x0FE0F}, + {0x0FFFD, 0x0FFFD}, + {0x1F100, 0x1F10A}, + {0x1F110, 0x1F12D}, + {0x1F130, 0x1F169}, + {0x1F170, 0x1F18D}, + {0x1F18F, 0x1F190}, + {0x1F19B, 0x1F1AC}, + {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, + {0x100000, 0x10FFFD} +}; + +/* Unassigned characters. */ +static const struct widechar_range widechar_unassigned_table[] = { + {0x00378, 0x00379}, + {0x00380, 0x00383}, + {0x0038B, 0x0038B}, + {0x0038D, 0x0038D}, + {0x003A2, 0x003A2}, + {0x00530, 0x00530}, + {0x00557, 0x00558}, + {0x0058B, 0x0058C}, + {0x00590, 0x00590}, + {0x005C8, 0x005CF}, + {0x005EB, 0x005EE}, + {0x005F5, 0x005FF}, + {0x0070E, 0x0070E}, + {0x0074B, 0x0074C}, + {0x007B2, 0x007BF}, + {0x007FB, 0x007FC}, + {0x0082E, 0x0082F}, + {0x0083F, 0x0083F}, + {0x0085C, 0x0085D}, + {0x0085F, 0x0085F}, + {0x0086B, 0x0086F}, + {0x0088F, 0x0088F}, + {0x00892, 0x00896}, + {0x00984, 0x00984}, + {0x0098D, 0x0098E}, + {0x00991, 0x00992}, + {0x009A9, 0x009A9}, + {0x009B1, 0x009B1}, + {0x009B3, 0x009B5}, + {0x009BA, 0x009BB}, + {0x009C5, 0x009C6}, + {0x009C9, 0x009CA}, + {0x009CF, 0x009D6}, + {0x009D8, 0x009DB}, + {0x009DE, 0x009DE}, + {0x009E4, 0x009E5}, + {0x009FF, 0x00A00}, + {0x00A04, 0x00A04}, + {0x00A0B, 0x00A0E}, + {0x00A11, 0x00A12}, + {0x00A29, 0x00A29}, + {0x00A31, 0x00A31}, + {0x00A34, 0x00A34}, + {0x00A37, 0x00A37}, + {0x00A3A, 0x00A3B}, + {0x00A3D, 0x00A3D}, + {0x00A43, 0x00A46}, + {0x00A49, 0x00A4A}, + {0x00A4E, 0x00A50}, + {0x00A52, 0x00A58}, + {0x00A5D, 0x00A5D}, + {0x00A5F, 0x00A65}, + {0x00A77, 0x00A80}, + {0x00A84, 0x00A84}, + {0x00A8E, 0x00A8E}, + {0x00A92, 0x00A92}, + {0x00AA9, 0x00AA9}, + {0x00AB1, 0x00AB1}, + {0x00AB4, 0x00AB4}, + {0x00ABA, 0x00ABB}, + {0x00AC6, 0x00AC6}, + {0x00ACA, 0x00ACA}, + {0x00ACE, 0x00ACF}, + {0x00AD1, 0x00ADF}, + {0x00AE4, 0x00AE5}, + {0x00AF2, 0x00AF8}, + {0x00B00, 0x00B00}, + {0x00B04, 0x00B04}, + {0x00B0D, 0x00B0E}, + {0x00B11, 0x00B12}, + {0x00B29, 0x00B29}, + {0x00B31, 0x00B31}, + {0x00B34, 0x00B34}, + {0x00B3A, 0x00B3B}, + {0x00B45, 0x00B46}, + {0x00B49, 0x00B4A}, + {0x00B4E, 0x00B54}, + {0x00B58, 0x00B5B}, + {0x00B5E, 0x00B5E}, + {0x00B64, 0x00B65}, + {0x00B78, 0x00B81}, + {0x00B84, 0x00B84}, + {0x00B8B, 0x00B8D}, + {0x00B91, 0x00B91}, + {0x00B96, 0x00B98}, + {0x00B9B, 0x00B9B}, + {0x00B9D, 0x00B9D}, + {0x00BA0, 0x00BA2}, + {0x00BA5, 0x00BA7}, + {0x00BAB, 0x00BAD}, + {0x00BBA, 0x00BBD}, + {0x00BC3, 0x00BC5}, + {0x00BC9, 0x00BC9}, + {0x00BCE, 0x00BCF}, + {0x00BD1, 0x00BD6}, + {0x00BD8, 0x00BE5}, + {0x00BFB, 0x00BFF}, + {0x00C0D, 0x00C0D}, + {0x00C11, 0x00C11}, + {0x00C29, 0x00C29}, + {0x00C3A, 0x00C3B}, + {0x00C45, 0x00C45}, + {0x00C49, 0x00C49}, + {0x00C4E, 0x00C54}, + {0x00C57, 0x00C57}, + {0x00C5B, 0x00C5C}, + {0x00C5E, 0x00C5F}, + {0x00C64, 0x00C65}, + {0x00C70, 0x00C76}, + {0x00C8D, 0x00C8D}, + {0x00C91, 0x00C91}, + {0x00CA9, 0x00CA9}, + {0x00CB4, 0x00CB4}, + {0x00CBA, 0x00CBB}, + {0x00CC5, 0x00CC5}, + {0x00CC9, 0x00CC9}, + {0x00CCE, 0x00CD4}, + {0x00CD7, 0x00CDC}, + {0x00CDF, 0x00CDF}, + {0x00CE4, 0x00CE5}, + {0x00CF0, 0x00CF0}, + {0x00CF4, 0x00CFF}, + {0x00D0D, 0x00D0D}, + {0x00D11, 0x00D11}, + {0x00D45, 0x00D45}, + {0x00D49, 0x00D49}, + {0x00D50, 0x00D53}, + {0x00D64, 0x00D65}, + {0x00D80, 0x00D80}, + {0x00D84, 0x00D84}, + {0x00D97, 0x00D99}, + {0x00DB2, 0x00DB2}, + {0x00DBC, 0x00DBC}, + {0x00DBE, 0x00DBF}, + {0x00DC7, 0x00DC9}, + {0x00DCB, 0x00DCE}, + {0x00DD5, 0x00DD5}, + {0x00DD7, 0x00DD7}, + {0x00DE0, 0x00DE5}, + {0x00DF0, 0x00DF1}, + {0x00DF5, 0x00E00}, + {0x00E3B, 0x00E3E}, + {0x00E5C, 0x00E80}, + {0x00E83, 0x00E83}, + {0x00E85, 0x00E85}, + {0x00E8B, 0x00E8B}, + {0x00EA4, 0x00EA4}, + {0x00EA6, 0x00EA6}, + {0x00EBE, 0x00EBF}, + {0x00EC5, 0x00EC5}, + {0x00EC7, 0x00EC7}, + {0x00ECF, 0x00ECF}, + {0x00EDA, 0x00EDB}, + {0x00EE0, 0x00EFF}, + {0x00F48, 0x00F48}, + {0x00F6D, 0x00F70}, + {0x00F98, 0x00F98}, + {0x00FBD, 0x00FBD}, + {0x00FCD, 0x00FCD}, + {0x00FDB, 0x00FFF}, + {0x010C6, 0x010C6}, + {0x010C8, 0x010CC}, + {0x010CE, 0x010CF}, + {0x01249, 0x01249}, + {0x0124E, 0x0124F}, + {0x01257, 0x01257}, + {0x01259, 0x01259}, + {0x0125E, 0x0125F}, + {0x01289, 0x01289}, + {0x0128E, 0x0128F}, + {0x012B1, 0x012B1}, + {0x012B6, 0x012B7}, + {0x012BF, 0x012BF}, + {0x012C1, 0x012C1}, + {0x012C6, 0x012C7}, + {0x012D7, 0x012D7}, + {0x01311, 0x01311}, + {0x01316, 0x01317}, + {0x0135B, 0x0135C}, + {0x0137D, 0x0137F}, + {0x0139A, 0x0139F}, + {0x013F6, 0x013F7}, + {0x013FE, 0x013FF}, + {0x0169D, 0x0169F}, + {0x016F9, 0x016FF}, + {0x01716, 0x0171E}, + {0x01737, 0x0173F}, + {0x01754, 0x0175F}, + {0x0176D, 0x0176D}, + {0x01771, 0x01771}, + {0x01774, 0x0177F}, + {0x017DE, 0x017DF}, + {0x017EA, 0x017EF}, + {0x017FA, 0x017FF}, + {0x0181A, 0x0181F}, + {0x01879, 0x0187F}, + {0x018AB, 0x018AF}, + {0x018F6, 0x018FF}, + {0x0191F, 0x0191F}, + {0x0192C, 0x0192F}, + {0x0193C, 0x0193F}, + {0x01941, 0x01943}, + {0x0196E, 0x0196F}, + {0x01975, 0x0197F}, + {0x019AC, 0x019AF}, + {0x019CA, 0x019CF}, + {0x019DB, 0x019DD}, + {0x01A1C, 0x01A1D}, + {0x01A5F, 0x01A5F}, + {0x01A7D, 0x01A7E}, + {0x01A8A, 0x01A8F}, + {0x01A9A, 0x01A9F}, + {0x01AAE, 0x01AAF}, + {0x01ACF, 0x01AFF}, + {0x01B4D, 0x01B4D}, + {0x01BF4, 0x01BFB}, + {0x01C38, 0x01C3A}, + {0x01C4A, 0x01C4C}, + {0x01C8B, 0x01C8F}, + {0x01CBB, 0x01CBC}, + {0x01CC8, 0x01CCF}, + {0x01CFB, 0x01CFF}, + {0x01F16, 0x01F17}, + {0x01F1E, 0x01F1F}, + {0x01F46, 0x01F47}, + {0x01F4E, 0x01F4F}, + {0x01F58, 0x01F58}, + {0x01F5A, 0x01F5A}, + {0x01F5C, 0x01F5C}, + {0x01F5E, 0x01F5E}, + {0x01F7E, 0x01F7F}, + {0x01FB5, 0x01FB5}, + {0x01FC5, 0x01FC5}, + {0x01FD4, 0x01FD5}, + {0x01FDC, 0x01FDC}, + {0x01FF0, 0x01FF1}, + {0x01FF5, 0x01FF5}, + {0x01FFF, 0x01FFF}, + {0x02065, 0x02065}, + {0x02072, 0x02073}, + {0x0208F, 0x0208F}, + {0x0209D, 0x0209F}, + {0x020C1, 0x020CF}, + {0x020F1, 0x020FF}, + {0x0218C, 0x0218F}, + {0x0242A, 0x0243F}, + {0x0244B, 0x0245F}, + {0x02B74, 0x02B75}, + {0x02B96, 0x02B96}, + {0x02CF4, 0x02CF8}, + {0x02D26, 0x02D26}, + {0x02D28, 0x02D2C}, + {0x02D2E, 0x02D2F}, + {0x02D68, 0x02D6E}, + {0x02D71, 0x02D7E}, + {0x02D97, 0x02D9F}, + {0x02DA7, 0x02DA7}, + {0x02DAF, 0x02DAF}, + {0x02DB7, 0x02DB7}, + {0x02DBF, 0x02DBF}, + {0x02DC7, 0x02DC7}, + {0x02DCF, 0x02DCF}, + {0x02DD7, 0x02DD7}, + {0x02DDF, 0x02DDF}, + {0x02E5E, 0x02E7F}, + {0x02E9A, 0x02E9A}, + {0x02EF4, 0x02EFF}, + {0x02FD6, 0x02FEF}, + {0x03040, 0x03040}, + {0x03097, 0x03098}, + {0x03100, 0x03104}, + {0x03130, 0x03130}, + {0x0318F, 0x0318F}, + {0x031E6, 0x031EE}, + {0x0321F, 0x0321F}, + {0x03401, 0x04DBE}, + {0x04E01, 0x09FFE}, + {0x0A48D, 0x0A48F}, + {0x0A4C7, 0x0A4CF}, + {0x0A62C, 0x0A63F}, + {0x0A6F8, 0x0A6FF}, + {0x0A7CE, 0x0A7CF}, + {0x0A7D2, 0x0A7D2}, + {0x0A7D4, 0x0A7D4}, + {0x0A7DD, 0x0A7F1}, + {0x0A82D, 0x0A82F}, + {0x0A83A, 0x0A83F}, + {0x0A878, 0x0A87F}, + {0x0A8C6, 0x0A8CD}, + {0x0A8DA, 0x0A8DF}, + {0x0A954, 0x0A95E}, + {0x0A97D, 0x0A97F}, + {0x0A9CE, 0x0A9CE}, + {0x0A9DA, 0x0A9DD}, + {0x0A9FF, 0x0A9FF}, + {0x0AA37, 0x0AA3F}, + {0x0AA4E, 0x0AA4F}, + {0x0AA5A, 0x0AA5B}, + {0x0AAC3, 0x0AADA}, + {0x0AAF7, 0x0AB00}, + {0x0AB07, 0x0AB08}, + {0x0AB0F, 0x0AB10}, + {0x0AB17, 0x0AB1F}, + {0x0AB27, 0x0AB27}, + {0x0AB2F, 0x0AB2F}, + {0x0AB6C, 0x0AB6F}, + {0x0ABEE, 0x0ABEF}, + {0x0ABFA, 0x0ABFF}, + {0x0AC01, 0x0D7A2}, + {0x0D7A4, 0x0D7AF}, + {0x0D7C7, 0x0D7CA}, + {0x0D7FC, 0x0D7FF}, + {0x0FA6E, 0x0FA6F}, + {0x0FADA, 0x0FAFF}, + {0x0FB07, 0x0FB12}, + {0x0FB18, 0x0FB1C}, + {0x0FB37, 0x0FB37}, + {0x0FB3D, 0x0FB3D}, + {0x0FB3F, 0x0FB3F}, + {0x0FB42, 0x0FB42}, + {0x0FB45, 0x0FB45}, + {0x0FBC3, 0x0FBD2}, + {0x0FD90, 0x0FD91}, + {0x0FDC8, 0x0FDCE}, + {0x0FE1A, 0x0FE1F}, + {0x0FE53, 0x0FE53}, + {0x0FE67, 0x0FE67}, + {0x0FE6C, 0x0FE6F}, + {0x0FE75, 0x0FE75}, + {0x0FEFD, 0x0FEFE}, + {0x0FF00, 0x0FF00}, + {0x0FFBF, 0x0FFC1}, + {0x0FFC8, 0x0FFC9}, + {0x0FFD0, 0x0FFD1}, + {0x0FFD8, 0x0FFD9}, + {0x0FFDD, 0x0FFDF}, + {0x0FFE7, 0x0FFE7}, + {0x0FFEF, 0x0FFF8}, + {0x1000C, 0x1000C}, + {0x10027, 0x10027}, + {0x1003B, 0x1003B}, + {0x1003E, 0x1003E}, + {0x1004E, 0x1004F}, + {0x1005E, 0x1007F}, + {0x100FB, 0x100FF}, + {0x10103, 0x10106}, + {0x10134, 0x10136}, + {0x1018F, 0x1018F}, + {0x1019D, 0x1019F}, + {0x101A1, 0x101CF}, + {0x101FE, 0x1027F}, + {0x1029D, 0x1029F}, + {0x102D1, 0x102DF}, + {0x102FC, 0x102FF}, + {0x10324, 0x1032C}, + {0x1034B, 0x1034F}, + {0x1037B, 0x1037F}, + {0x1039E, 0x1039E}, + {0x103C4, 0x103C7}, + {0x103D6, 0x103FF}, + {0x1049E, 0x1049F}, + {0x104AA, 0x104AF}, + {0x104D4, 0x104D7}, + {0x104FC, 0x104FF}, + {0x10528, 0x1052F}, + {0x10564, 0x1056E}, + {0x1057B, 0x1057B}, + {0x1058B, 0x1058B}, + {0x10593, 0x10593}, + {0x10596, 0x10596}, + {0x105A2, 0x105A2}, + {0x105B2, 0x105B2}, + {0x105BA, 0x105BA}, + {0x105BD, 0x105BF}, + {0x105F4, 0x105FF}, + {0x10737, 0x1073F}, + {0x10756, 0x1075F}, + {0x10768, 0x1077F}, + {0x10786, 0x10786}, + {0x107B1, 0x107B1}, + {0x107BB, 0x107FF}, + {0x10806, 0x10807}, + {0x10809, 0x10809}, + {0x10836, 0x10836}, + {0x10839, 0x1083B}, + {0x1083D, 0x1083E}, + {0x10856, 0x10856}, + {0x1089F, 0x108A6}, + {0x108B0, 0x108DF}, + {0x108F3, 0x108F3}, + {0x108F6, 0x108FA}, + {0x1091C, 0x1091E}, + {0x1093A, 0x1093E}, + {0x10940, 0x1097F}, + {0x109B8, 0x109BB}, + {0x109D0, 0x109D1}, + {0x10A04, 0x10A04}, + {0x10A07, 0x10A0B}, + {0x10A14, 0x10A14}, + {0x10A18, 0x10A18}, + {0x10A36, 0x10A37}, + {0x10A3B, 0x10A3E}, + {0x10A49, 0x10A4F}, + {0x10A59, 0x10A5F}, + {0x10AA0, 0x10ABF}, + {0x10AE7, 0x10AEA}, + {0x10AF7, 0x10AFF}, + {0x10B36, 0x10B38}, + {0x10B56, 0x10B57}, + {0x10B73, 0x10B77}, + {0x10B92, 0x10B98}, + {0x10B9D, 0x10BA8}, + {0x10BB0, 0x10BFF}, + {0x10C49, 0x10C7F}, + {0x10CB3, 0x10CBF}, + {0x10CF3, 0x10CF9}, + {0x10D28, 0x10D2F}, + {0x10D3A, 0x10D3F}, + {0x10D66, 0x10D68}, + {0x10D86, 0x10D8D}, + {0x10D90, 0x10E5F}, + {0x10E7F, 0x10E7F}, + {0x10EAA, 0x10EAA}, + {0x10EAE, 0x10EAF}, + {0x10EB2, 0x10EC1}, + {0x10EC5, 0x10EFB}, + {0x10F28, 0x10F2F}, + {0x10F5A, 0x10F6F}, + {0x10F8A, 0x10FAF}, + {0x10FCC, 0x10FDF}, + {0x10FF7, 0x10FFF}, + {0x1104E, 0x11051}, + {0x11076, 0x1107E}, + {0x110C3, 0x110CC}, + {0x110CE, 0x110CF}, + {0x110E9, 0x110EF}, + {0x110FA, 0x110FF}, + {0x11135, 0x11135}, + {0x11148, 0x1114F}, + {0x11177, 0x1117F}, + {0x111E0, 0x111E0}, + {0x111F5, 0x111FF}, + {0x11212, 0x11212}, + {0x11242, 0x1127F}, + {0x11287, 0x11287}, + {0x11289, 0x11289}, + {0x1128E, 0x1128E}, + {0x1129E, 0x1129E}, + {0x112AA, 0x112AF}, + {0x112EB, 0x112EF}, + {0x112FA, 0x112FF}, + {0x11304, 0x11304}, + {0x1130D, 0x1130E}, + {0x11311, 0x11312}, + {0x11329, 0x11329}, + {0x11331, 0x11331}, + {0x11334, 0x11334}, + {0x1133A, 0x1133A}, + {0x11345, 0x11346}, + {0x11349, 0x1134A}, + {0x1134E, 0x1134F}, + {0x11351, 0x11356}, + {0x11358, 0x1135C}, + {0x11364, 0x11365}, + {0x1136D, 0x1136F}, + {0x11375, 0x1137F}, + {0x1138A, 0x1138A}, + {0x1138C, 0x1138D}, + {0x1138F, 0x1138F}, + {0x113B6, 0x113B6}, + {0x113C1, 0x113C1}, + {0x113C3, 0x113C4}, + {0x113C6, 0x113C6}, + {0x113CB, 0x113CB}, + {0x113D6, 0x113D6}, + {0x113D9, 0x113E0}, + {0x113E3, 0x113FF}, + {0x1145C, 0x1145C}, + {0x11462, 0x1147F}, + {0x114C8, 0x114CF}, + {0x114DA, 0x1157F}, + {0x115B6, 0x115B7}, + {0x115DE, 0x115FF}, + {0x11645, 0x1164F}, + {0x1165A, 0x1165F}, + {0x1166D, 0x1167F}, + {0x116BA, 0x116BF}, + {0x116CA, 0x116CF}, + {0x116E4, 0x116FF}, + {0x1171B, 0x1171C}, + {0x1172C, 0x1172F}, + {0x11747, 0x117FF}, + {0x1183C, 0x1189F}, + {0x118F3, 0x118FE}, + {0x11907, 0x11908}, + {0x1190A, 0x1190B}, + {0x11914, 0x11914}, + {0x11917, 0x11917}, + {0x11936, 0x11936}, + {0x11939, 0x1193A}, + {0x11947, 0x1194F}, + {0x1195A, 0x1199F}, + {0x119A8, 0x119A9}, + {0x119D8, 0x119D9}, + {0x119E5, 0x119FF}, + {0x11A48, 0x11A4F}, + {0x11AA3, 0x11AAF}, + {0x11AF9, 0x11AFF}, + {0x11B0A, 0x11BBF}, + {0x11BE2, 0x11BEF}, + {0x11BFA, 0x11BFF}, + {0x11C09, 0x11C09}, + {0x11C37, 0x11C37}, + {0x11C46, 0x11C4F}, + {0x11C6D, 0x11C6F}, + {0x11C90, 0x11C91}, + {0x11CA8, 0x11CA8}, + {0x11CB7, 0x11CFF}, + {0x11D07, 0x11D07}, + {0x11D0A, 0x11D0A}, + {0x11D37, 0x11D39}, + {0x11D3B, 0x11D3B}, + {0x11D3E, 0x11D3E}, + {0x11D48, 0x11D4F}, + {0x11D5A, 0x11D5F}, + {0x11D66, 0x11D66}, + {0x11D69, 0x11D69}, + {0x11D8F, 0x11D8F}, + {0x11D92, 0x11D92}, + {0x11D99, 0x11D9F}, + {0x11DAA, 0x11EDF}, + {0x11EF9, 0x11EFF}, + {0x11F11, 0x11F11}, + {0x11F3B, 0x11F3D}, + {0x11F5B, 0x11FAF}, + {0x11FB1, 0x11FBF}, + {0x11FF2, 0x11FFE}, + {0x1239A, 0x123FF}, + {0x1246F, 0x1246F}, + {0x12475, 0x1247F}, + {0x12544, 0x12F8F}, + {0x12FF3, 0x12FFF}, + {0x13456, 0x1345F}, + {0x143FB, 0x143FF}, + {0x14647, 0x160FF}, + {0x1613A, 0x167FF}, + {0x16A39, 0x16A3F}, + {0x16A5F, 0x16A5F}, + {0x16A6A, 0x16A6D}, + {0x16ABF, 0x16ABF}, + {0x16ACA, 0x16ACF}, + {0x16AEE, 0x16AEF}, + {0x16AF6, 0x16AFF}, + {0x16B46, 0x16B4F}, + {0x16B5A, 0x16B5A}, + {0x16B62, 0x16B62}, + {0x16B78, 0x16B7C}, + {0x16B90, 0x16D3F}, + {0x16D7A, 0x16E3F}, + {0x16E9B, 0x16EFF}, + {0x16F4B, 0x16F4E}, + {0x16F88, 0x16F8E}, + {0x16FA0, 0x16FDF}, + {0x16FE5, 0x16FEF}, + {0x16FF2, 0x16FFF}, + {0x17001, 0x187F6}, + {0x187F8, 0x187FF}, + {0x18CD6, 0x18CFE}, + {0x18D01, 0x18D07}, + {0x18D09, 0x1AFEF}, + {0x1AFF4, 0x1AFF4}, + {0x1AFFC, 0x1AFFC}, + {0x1AFFF, 0x1AFFF}, + {0x1B123, 0x1B131}, + {0x1B133, 0x1B14F}, + {0x1B153, 0x1B154}, + {0x1B156, 0x1B163}, + {0x1B168, 0x1B16F}, + {0x1B2FC, 0x1BBFF}, + {0x1BC6B, 0x1BC6F}, + {0x1BC7D, 0x1BC7F}, + {0x1BC89, 0x1BC8F}, + {0x1BC9A, 0x1BC9B}, + {0x1BCA4, 0x1CBFF}, + {0x1CCFA, 0x1CCFF}, + {0x1CEB4, 0x1CEFF}, + {0x1CF2E, 0x1CF2F}, + {0x1CF47, 0x1CF4F}, + {0x1CFC4, 0x1CFFF}, + {0x1D0F6, 0x1D0FF}, + {0x1D127, 0x1D128}, + {0x1D1EB, 0x1D1FF}, + {0x1D246, 0x1D2BF}, + {0x1D2D4, 0x1D2DF}, + {0x1D2F4, 0x1D2FF}, + {0x1D357, 0x1D35F}, + {0x1D379, 0x1D3FF}, + {0x1D455, 0x1D455}, + {0x1D49D, 0x1D49D}, + {0x1D4A0, 0x1D4A1}, + {0x1D4A3, 0x1D4A4}, + {0x1D4A7, 0x1D4A8}, + {0x1D4AD, 0x1D4AD}, + {0x1D4BA, 0x1D4BA}, + {0x1D4BC, 0x1D4BC}, + {0x1D4C4, 0x1D4C4}, + {0x1D506, 0x1D506}, + {0x1D50B, 0x1D50C}, + {0x1D515, 0x1D515}, + {0x1D51D, 0x1D51D}, + {0x1D53A, 0x1D53A}, + {0x1D53F, 0x1D53F}, + {0x1D545, 0x1D545}, + {0x1D547, 0x1D549}, + {0x1D551, 0x1D551}, + {0x1D6A6, 0x1D6A7}, + {0x1D7CC, 0x1D7CD}, + {0x1DA8C, 0x1DA9A}, + {0x1DAA0, 0x1DAA0}, + {0x1DAB0, 0x1DEFF}, + {0x1DF1F, 0x1DF24}, + {0x1DF2B, 0x1DFFF}, + {0x1E007, 0x1E007}, + {0x1E019, 0x1E01A}, + {0x1E022, 0x1E022}, + {0x1E025, 0x1E025}, + {0x1E02B, 0x1E02F}, + {0x1E06E, 0x1E08E}, + {0x1E090, 0x1E0FF}, + {0x1E12D, 0x1E12F}, + {0x1E13E, 0x1E13F}, + {0x1E14A, 0x1E14D}, + {0x1E150, 0x1E28F}, + {0x1E2AF, 0x1E2BF}, + {0x1E2FA, 0x1E2FE}, + {0x1E300, 0x1E4CF}, + {0x1E4FA, 0x1E5CF}, + {0x1E5FB, 0x1E5FE}, + {0x1E600, 0x1E7DF}, + {0x1E7E7, 0x1E7E7}, + {0x1E7EC, 0x1E7EC}, + {0x1E7EF, 0x1E7EF}, + {0x1E7FF, 0x1E7FF}, + {0x1E8C5, 0x1E8C6}, + {0x1E8D7, 0x1E8FF}, + {0x1E94C, 0x1E94F}, + {0x1E95A, 0x1E95D}, + {0x1E960, 0x1EC70}, + {0x1ECB5, 0x1ED00}, + {0x1ED3E, 0x1EDFF}, + {0x1EE04, 0x1EE04}, + {0x1EE20, 0x1EE20}, + {0x1EE23, 0x1EE23}, + {0x1EE25, 0x1EE26}, + {0x1EE28, 0x1EE28}, + {0x1EE33, 0x1EE33}, + {0x1EE38, 0x1EE38}, + {0x1EE3A, 0x1EE3A}, + {0x1EE3C, 0x1EE41}, + {0x1EE43, 0x1EE46}, + {0x1EE48, 0x1EE48}, + {0x1EE4A, 0x1EE4A}, + {0x1EE4C, 0x1EE4C}, + {0x1EE50, 0x1EE50}, + {0x1EE53, 0x1EE53}, + {0x1EE55, 0x1EE56}, + {0x1EE58, 0x1EE58}, + {0x1EE5A, 0x1EE5A}, + {0x1EE5C, 0x1EE5C}, + {0x1EE5E, 0x1EE5E}, + {0x1EE60, 0x1EE60}, + {0x1EE63, 0x1EE63}, + {0x1EE65, 0x1EE66}, + {0x1EE6B, 0x1EE6B}, + {0x1EE73, 0x1EE73}, + {0x1EE78, 0x1EE78}, + {0x1EE7D, 0x1EE7D}, + {0x1EE7F, 0x1EE7F}, + {0x1EE8A, 0x1EE8A}, + {0x1EE9C, 0x1EEA0}, + {0x1EEA4, 0x1EEA4}, + {0x1EEAA, 0x1EEAA}, + {0x1EEBC, 0x1EEEF}, + {0x1EEF2, 0x1EFFF}, + {0x1F02C, 0x1F02F}, + {0x1F094, 0x1F09F}, + {0x1F0AF, 0x1F0B0}, + {0x1F0C0, 0x1F0C0}, + {0x1F0D0, 0x1F0D0}, + {0x1F0F6, 0x1F0FF}, + {0x1F1AE, 0x1F1E5}, + {0x1F203, 0x1F20F}, + {0x1F23C, 0x1F23F}, + {0x1F249, 0x1F24F}, + {0x1F252, 0x1F25F}, + {0x1F266, 0x1F2FF}, + {0x1F6D8, 0x1F6DB}, + {0x1F6ED, 0x1F6EF}, + {0x1F6FD, 0x1F6FF}, + {0x1F777, 0x1F77A}, + {0x1F7DA, 0x1F7DF}, + {0x1F7EC, 0x1F7EF}, + {0x1F7F1, 0x1F7FF}, + {0x1F80C, 0x1F80F}, + {0x1F848, 0x1F84F}, + {0x1F85A, 0x1F85F}, + {0x1F888, 0x1F88F}, + {0x1F8AE, 0x1F8AF}, + {0x1F8BC, 0x1F8BF}, + {0x1F8C2, 0x1F8FF}, + {0x1FA54, 0x1FA5F}, + {0x1FA6E, 0x1FA6F}, + {0x1FA7D, 0x1FA7F}, + {0x1FA8A, 0x1FA8E}, + {0x1FAC7, 0x1FACD}, + {0x1FADD, 0x1FADE}, + {0x1FAEA, 0x1FAEF}, + {0x1FAF9, 0x1FAFF}, + {0x1FB93, 0x1FB93}, + {0x1FBFA, 0x1FFFD}, + {0x20001, 0x2A6DE}, + {0x2A6E0, 0x2A6FF}, + {0x2A701, 0x2B738}, + {0x2B73A, 0x2B73F}, + {0x2B741, 0x2B81C}, + {0x2B81E, 0x2B81F}, + {0x2B821, 0x2CEA0}, + {0x2CEA2, 0x2CEAF}, + {0x2CEB1, 0x2EBDF}, + {0x2EBE1, 0x2EBEF}, + {0x2EBF1, 0x2EE5C}, + {0x2EE5E, 0x2F7FF}, + {0x2FA1E, 0x2FFFD}, + {0x30001, 0x31349}, + {0x3134B, 0x3134F}, + {0x31351, 0x323AE}, + {0x323B0, 0x3FFFD}, + {0x40000, 0x4FFFD}, + {0x50000, 0x5FFFD}, + {0x60000, 0x6FFFD}, + {0x70000, 0x7FFFD}, + {0x80000, 0x8FFFD}, + {0x90000, 0x9FFFD}, + {0xA0000, 0xAFFFD}, + {0xB0000, 0xBFFFD}, + {0xC0000, 0xCFFFD}, + {0xD0000, 0xDFFFD}, + {0xE0000, 0xE0000}, + {0xE0002, 0xE001F}, + {0xE0080, 0xE00FF}, + {0xE01F0, 0xEFFFD} +}; + +/* Non-characters. */ +static const struct widechar_range widechar_nonchar_table[] = { + {0x0FDD0, 0x0FDEF}, + {0x0FFFE, 0x0FFFF}, + {0x1FFFE, 0x1FFFF}, + {0x2FFFE, 0x2FFFF}, + {0x3FFFE, 0x3FFFF}, + {0x4FFFE, 0x4FFFF}, + {0x5FFFE, 0x5FFFF}, + {0x6FFFE, 0x6FFFF}, + {0x7FFFE, 0x7FFFF}, + {0x8FFFE, 0x8FFFF}, + {0x9FFFE, 0x9FFFF}, + {0xAFFFE, 0xAFFFF}, + {0xBFFFE, 0xBFFFF}, + {0xCFFFE, 0xCFFFF}, + {0xDFFFE, 0xDFFFF}, + {0xEFFFE, 0xEFFFF}, + {0xFFFFE, 0xFFFFF}, + {0x10FFFE, 0x10FFFF} +}; + +/* Characters that were widened from width 1 to 2 in Unicode 9. */ +static const struct widechar_range widechar_widened_table[] = { + {0x0231A, 0x0231B}, + {0x023E9, 0x023EC}, + {0x023F0, 0x023F0}, + {0x023F3, 0x023F3}, + {0x025FD, 0x025FE}, + {0x02614, 0x02615}, + {0x02648, 0x02653}, + {0x0267F, 0x0267F}, + {0x02693, 0x02693}, + {0x026A1, 0x026A1}, + {0x026AA, 0x026AB}, + {0x026BD, 0x026BE}, + {0x026C4, 0x026C5}, + {0x026CE, 0x026CE}, + {0x026D4, 0x026D4}, + {0x026EA, 0x026EA}, + {0x026F2, 0x026F3}, + {0x026F5, 0x026F5}, + {0x026FA, 0x026FA}, + {0x026FD, 0x026FD}, + {0x02705, 0x02705}, + {0x0270A, 0x0270B}, + {0x02728, 0x02728}, + {0x0274C, 0x0274C}, + {0x0274E, 0x0274E}, + {0x02753, 0x02755}, + {0x02757, 0x02757}, + {0x02795, 0x02797}, + {0x027B0, 0x027B0}, + {0x027BF, 0x027BF}, + {0x02B1B, 0x02B1C}, + {0x02B50, 0x02B50}, + {0x02B55, 0x02B55}, + {0x1F004, 0x1F004}, + {0x1F0CF, 0x1F0CF}, + {0x1F18E, 0x1F18E}, + {0x1F191, 0x1F19A}, + {0x1F201, 0x1F201}, + {0x1F21A, 0x1F21A}, + {0x1F22F, 0x1F22F}, + {0x1F232, 0x1F236}, + {0x1F238, 0x1F23A}, + {0x1F250, 0x1F251}, + {0x1F300, 0x1F320}, + {0x1F32D, 0x1F335}, + {0x1F337, 0x1F37C}, + {0x1F37E, 0x1F393}, + {0x1F3A0, 0x1F3CA}, + {0x1F3CF, 0x1F3D3}, + {0x1F3E0, 0x1F3F0}, + {0x1F3F4, 0x1F3F4}, + {0x1F3F8, 0x1F43E}, + {0x1F440, 0x1F440}, + {0x1F442, 0x1F4FC}, + {0x1F4FF, 0x1F53D}, + {0x1F54B, 0x1F54E}, + {0x1F550, 0x1F567}, + {0x1F595, 0x1F596}, + {0x1F5FB, 0x1F64F}, + {0x1F680, 0x1F6C5}, + {0x1F6CC, 0x1F6CC}, + {0x1F6D0, 0x1F6D0}, + {0x1F6EB, 0x1F6EC}, + {0x1F910, 0x1F918}, + {0x1F980, 0x1F984}, + {0x1F9C0, 0x1F9C0} +}; + +template +bool widechar_in_table(const Collection &arr, uint32_t c) { + auto where = std::lower_bound(std::begin(arr), std::end(arr), c, + [](widechar_range p, uint32_t c) { return p.hi < c; }); + return where != std::end(arr) && where->lo <= c; +} + +/* Return the width of character c, or a special negative value. */ +int widechar_wcwidth(uint32_t c) { + if (widechar_in_table(widechar_ascii_table, c)) + return 1; + if (widechar_in_table(widechar_private_table, c)) + return widechar_private_use; + if (widechar_in_table(widechar_nonprint_table, c)) + return widechar_nonprint; + if (widechar_in_table(widechar_nonchar_table, c)) + return widechar_non_character; + if (widechar_in_table(widechar_combining_table, c)) + return widechar_combining; + if (widechar_in_table(widechar_combiningletters_table, c)) + return widechar_combining; + if (widechar_in_table(widechar_doublewide_table, c)) + return 2; + if (widechar_in_table(widechar_ambiguous_table, c)) + return widechar_ambiguous; + if (widechar_in_table(widechar_unassigned_table, c)) + return widechar_unassigned; + if (widechar_in_table(widechar_widened_table, c)) + return widechar_widened_in_9; + return 1; +} + +} // namespace +#endif // WIDECHAR_WIDTH_H From 1e2cace5f1dbd55ed76afbad7519100b76af5d88 Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 28 Dec 2024 23:40:25 +0100 Subject: [PATCH 17/48] fix documentation of substring --- src/libexpr/primops.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index f19dd473a..a0e2753b5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4059,7 +4059,7 @@ static RegisterPrimOp primop_toString({ }); /* `substring start len str' returns the substring of `str' starting - at character position `min(start, stringLength str)' inclusive and + at byte position `min(start, stringLength str)' inclusive and ending at `min(start + len, stringLength str)'. `start' must be non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -4098,7 +4098,7 @@ static RegisterPrimOp primop_substring({ .name = "__substring", .args = {"start", "len", "s"}, .doc = R"( - Return the substring of *s* from character position *start* + Return the substring of *s* from byte position *start* (zero-based) up to but not including *start + len*. If *start* is greater than the length of the string, an empty string is returned. If *start + len* lies beyond the end of the string or *len* is `-1`, From b5f10655ede1c5f8986f622de7ad73daeda7d5fc Mon Sep 17 00:00:00 2001 From: NAHO <90870942+trueNAHO@users.noreply.github.com> Date: Fri, 27 Dec 2024 01:56:53 +0100 Subject: [PATCH 18/48] ci: update Ubuntu runner to ubuntu-24.04 Link: https://github.com/actions/runner-images/issues/10636 --- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/labels.yml | 2 +- .mergify.yml | 4 ++-- doc/manual/source/development/testing.md | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2426ab166..7592f60b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ permissions: read-all jobs: eval: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-latest] + os: [ubuntu-24.04, macos-latest] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -37,7 +37,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, macos-latest] + os: [ubuntu-24.04, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -68,7 +68,7 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-24.04' - run: brew install fish if: matrix.os == 'macos-latest' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" @@ -86,7 +86,7 @@ jobs: permissions: contents: none name: Check Docker secrets present for installer tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: docker: ${{ steps.secret.outputs.docker }} steps: @@ -106,7 +106,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' && github.event_name == 'push' && github.ref_name == 'master' - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Check for secrets id: secret @@ -158,7 +158,7 @@ jobs: docker push $IMAGE_ID:master vm_tests: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main @@ -173,7 +173,7 @@ jobs: flake_regressions: needs: vm_tests - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Checkout nix uses: actions/checkout@v4 diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 9d2ac80a3..23a5d9e51 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -15,7 +15,7 @@ permissions: jobs: labels: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: github.repository_owner == 'NixOS' steps: - uses: actions/labeler@v5 diff --git a/.mergify.yml b/.mergify.yml index c2e115487..827e97290 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -3,9 +3,9 @@ queue_rules: # all required tests need to go here merge_conditions: - check-success=tests (macos-latest) - - check-success=tests (ubuntu-22.04) + - check-success=tests (ubuntu-24.04) - check-success=installer_test (macos-latest) - - check-success=installer_test (ubuntu-22.04) + - check-success=installer_test (ubuntu-24.04) - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 7e8762fe0..082863bcc 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-22.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup From ce1e9ba85a1937ba695aab4da34bf310c5a8ecc9 Mon Sep 17 00:00:00 2001 From: NAHO <90870942+trueNAHO@users.noreply.github.com> Date: Fri, 27 Dec 2024 02:14:55 +0100 Subject: [PATCH 19/48] ci: lock macOS runner to macos-14 --- .github/workflows/ci.yml | 6 +++--- .mergify.yml | 4 ++-- doc/manual/source/development/testing.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7592f60b9..cd567cd63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-latest] + os: [ubuntu-24.04, macos-14] runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-latest] + os: [ubuntu-24.04, macos-14] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -70,7 +70,7 @@ jobs: - run: sudo apt install fish zsh if: matrix.os == 'ubuntu-24.04' - run: brew install fish - if: matrix.os == 'macos-latest' + if: matrix.os == 'macos-14' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" diff --git a/.mergify.yml b/.mergify.yml index 827e97290..13d2002ea 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,9 +2,9 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=tests (macos-latest) + - check-success=tests (macos-14) - check-success=tests (ubuntu-24.04) - - check-success=installer_test (macos-latest) + - check-success=installer_test (macos-14) - check-success=installer_test (ubuntu-24.04) - check-success=vm_tests batch_size: 5 diff --git a/doc/manual/source/development/testing.md b/doc/manual/source/development/testing.md index 082863bcc..d582ce4b4 100644 --- a/doc/manual/source/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -297,7 +297,7 @@ Creating a Cachix cache for your installer tests and adding its authorisation to - `armv7l-linux` - `x86_64-darwin` -- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-latest`) will try to install Nix with the cached installer and run a trivial Nix command. +- The `installer_test` job (which runs on `ubuntu-24.04` and `macos-14`) will try to install Nix with the cached installer and run a trivial Nix command. ### One-time setup From 5c968be16288b7ad54b8c6d94cd0566be811ba50 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sun, 29 Dec 2024 16:05:16 +0000 Subject: [PATCH 20/48] m4/gcc_bug_80431.m4: drop unused file As autotools-based build system is gone the file is not used anymore. --- m4/gcc_bug_80431.m4 | 66 --------------------------------------------- 1 file changed, 66 deletions(-) delete mode 100644 m4/gcc_bug_80431.m4 diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 deleted file mode 100644 index cdc4ddb40..000000000 --- a/m4/gcc_bug_80431.m4 +++ /dev/null @@ -1,66 +0,0 @@ -# Ensure that this bug is not present in the C++ toolchain we are using. -# -# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 -# -# The test program is from that issue, with only a slight modification -# to set an exit status instead of printing strings. -AC_DEFUN([ENSURE_NO_GCC_BUG_80431], -[ - AC_MSG_CHECKING([that GCC bug 80431 is fixed]) - AC_LANG_PUSH(C++) - AC_RUN_IFELSE( - [AC_LANG_PROGRAM( - [[ - #include - - static bool a = true; - static bool b = true; - - struct Options { }; - - struct Option - { - Option(Options * options) - { - a = false; - } - - ~Option() - { - b = false; - } - }; - - struct MyOptions : Options { }; - - struct MyOptions2 : virtual MyOptions - { - Option foo{this}; - }; - ]], - [[ - { - MyOptions2 opts; - } - return (a << 1) | b; - ]])], - [status_80431=0], - [status_80431=$?], - [status_80431='']) - AC_LANG_POP(C++) - AS_CASE([$status_80431], - [''],[ - AC_MSG_RESULT(cannot check because cross compiling) - AC_MSG_NOTICE(assume we are bug free) - ], - [0],[ - AC_MSG_RESULT(yes) - ], - [2],[ - AC_MSG_RESULT(no) - AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) - ], - [ - AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) - ]) -]) From edbfe863ce4ae4b89e554f29807e62674055f251 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Sun, 29 Dec 2024 15:54:53 +0000 Subject: [PATCH 21/48] libcmd: update to support lowdown-1.4 API Upstream change https://github.com/kristapsdz/lowdown/commit/bab1d75079ac3866fc14e1fa4825ae64a94a7c0e moved a few fields from `lowdown_opts` toa new `lowdown_opts_term` struct. As a result the build started failing as: nix-cmd> [2/17] Compiling C++ object libnixcmd.so.p/markdown.cc.o nix-cmd> FAILED: libnixcmd.so.p/markdown.cc.o nix-cmd> g++ -Ilibnixcmd.so.p -I. -I.. -I/nix/store/b0bnrk5lacxbpgxgnc28r8q3wcazrgxj-nix-util-2.26.0pre-dev/include/nix -I/nix/store/cxnynq9ykyj4xxv6wf6dw7r0aw5x6n9k-libarchive-3.7.7-dev/include -I/nix/store/bfgjwkcb8snkizx578rzdahi75m8zyh4-nlohmann_json-3.11.3/include -I/nix/store/3sx8bq3sip6j2nv1m5xx4gbdp33v7iy6-nix-store-2.26.0pre-dev/include/nix -I/nix/store/sih2dgqzvsbv7p510lkfmas7s7wbsl4j-nix-fetchers-2.26.0pre-dev/include/nix -I/nix/store/68p8s20fsiiakj7nys7grbaixfnhsdzs-nix-expr-2.26.0pre-dev/include/nix -I/nix/store/gw7wknhzhfzzj9zww2kyi5xrzgf1ndki-boehm-gc-8.2.8-dev/include -I/nix/store/3jwb9j4vnsk5saq3wfyyp9il3mhs41l9-nix-flake-2.26.0pre-dev/include/nix -I/nix/store/8nwjvmq7m48v8g646jrxkikv6x47bc3m-nix-main-2.26.0pre-dev/include/nix -I/nix/store/rb0hzsw5wc1a7daizhpj824mbxlvijrq-lowdown-1.4.0-dev/include -I/nix/store/m388ywpk53fsp8r98brfd7nf1f5sskv0-editline-1.17.1-dev/include -fdiagnostics-color=always -D_GLIBCXX_ASSERTIONS=1 -D_FILE_OFFSET_BITS=64 -Wall -Winvalid-pch -std=c++2a -include config-util.hh -include config-store.hh -include config-expr.hh -include config-main.hh -include config-cmd.hh -Wdeprecated-copy -Werror=suggest-override -Werror=switch -Werror=switch-enum -Werror=unused-result -Wignored-qualifiers -Wimplicit-fallthrough -Wno-deprecated-declarations -O3 -fPIC -pthread -std=c++2a -std=c++2a -std=c++2a -std=c++2a -std=c++2a -std=c++2a -MD -MQ libnixcmd.so.p/markdown.cc.o -MF libnixcmd.so.p/markdown.cc.o.d -o libnixcmd.so.p/markdown.cc.o -c ../markdown.cc nix-cmd> ../markdown.cc: In function 'std::string nix::doRenderMarkdownToTerminal(std::string_view)': nix-cmd> ../markdown.cc:28:5: error: 'lowdown_opts' has no non-static data member named 'cols' nix-cmd> 28 | }; nix-cmd> | ^ The change adds version-based conditional to support both pre-1.4 and 1.4 forms of the initialization. Closes: https://github.com/NixOS/nix/issues/12113 --- src/libcmd/markdown.cc | 20 ++++++++++++++++---- src/libcmd/meson.build | 2 ++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 6a0d05d9f..4566e6ba6 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -16,13 +16,25 @@ static std::string doRenderMarkdownToTerminal(std::string_view markdown) { int windowWidth = getWindowSize().second; - struct lowdown_opts opts - { - .type = LOWDOWN_TERM, - .maxdepth = 20, +#if HAVE_LOWDOWN_1_4 + struct lowdown_opts_term opts_term { .cols = (size_t) std::max(windowWidth - 5, 60), .hmargin = 0, .vmargin = 0, + }; +#endif + struct lowdown_opts opts + { + .type = LOWDOWN_TERM, +#if HAVE_LOWDOWN_1_4 + .term = opts_term, +#endif + .maxdepth = 20, +#if !HAVE_LOWDOWN_1_4 + .cols = (size_t) std::max(windowWidth - 5, 60), + .hmargin = 0, + .vmargin = 0, +#endif .feat = LOWDOWN_COMMONMARK | LOWDOWN_FENCED | LOWDOWN_DEFLIST | LOWDOWN_TABLES, .oflags = LOWDOWN_TERM_NOLINK, }; diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 222817c81..914db0108 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -36,6 +36,8 @@ deps_public += nlohmann_json lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) deps_private += lowdown configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) +# The API changed slightly around terminal initialization. +configdata.set('HAVE_LOWDOWN_1_4', lowdown.version().version_compare('>= 1.4.0').to_int()) readline_flavor = get_option('readline-flavor') if readline_flavor == 'editline' From 1a402e0c53609df25539ae852ca7db6696e6f174 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 31 Dec 2024 13:52:18 +0100 Subject: [PATCH 22/48] test: test eval of newlines with raw output --- tests/functional/eval.nix | 2 +- tests/functional/eval.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/functional/eval.nix b/tests/functional/eval.nix index befbd17a9..cabf28c29 100644 --- a/tests/functional/eval.nix +++ b/tests/functional/eval.nix @@ -1,5 +1,5 @@ { int = 123; - str = "foo"; + str = "foo\nbar"; attr.foo = "bar"; } diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 7af49d7fd..18f8589df 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -16,8 +16,8 @@ EOF nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval int -f "./eval.nix") == 123 ]] -[[ $(nix eval str -f "./eval.nix") == '"foo"' ]] -[[ $(nix eval str --raw -f "./eval.nix") == 'foo' ]] +[[ $(nix eval str -f "./eval.nix") == '"foo\nbar"' ]] +[[ $(nix eval str --raw -f "./eval.nix") == $'foo\nbar' ]] [[ "$(nix eval attr -f "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix eval attr --json -f "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix eval int -f - < "./eval.nix") == 123 ]] @@ -28,7 +28,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] -[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo"' ]] +[[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo\nbar"' ]] [[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] From 7a8a28629c61c75af010ff0a5a88c16c4ce536c7 Mon Sep 17 00:00:00 2001 From: Martin Fischer Date: Tue, 31 Dec 2024 14:04:50 +0100 Subject: [PATCH 23/48] feat(nix-instantiate): add --raw flag The experimental `nix eval` command already supports a `--raw` flag. This commit implements the same flag for the stable nix-instantiate command. Until now instructions and scripts that didn't want to rely on experimental features had to use workarounds such as: nix-instantiate --eval | tr -d \" (which also undesirably also removes double quotation marks within the string), or nix-instantiate --eval | jq -j (which undesirably depends on another package). Co-authored-by: Silvan Mosberger --- doc/manual/rl-next/nix-instantiate-raw.md | 8 ++++++++ doc/manual/source/command-ref/nix-copy-closure.md | 2 +- doc/manual/source/command-ref/nix-instantiate.md | 7 ++++++- src/nix-instantiate/nix-instantiate.cc | 10 ++++++++-- tests/functional/eval.sh | 1 + 5 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 doc/manual/rl-next/nix-instantiate-raw.md diff --git a/doc/manual/rl-next/nix-instantiate-raw.md b/doc/manual/rl-next/nix-instantiate-raw.md new file mode 100644 index 000000000..fb4a72b88 --- /dev/null +++ b/doc/manual/rl-next/nix-instantiate-raw.md @@ -0,0 +1,8 @@ +--- +synopsis: "`nix-instantiate --eval` now supports `--raw`" +prs: [12119] +--- + +The `nix-instantiate --eval` command now supports a `--raw` flag, when used +the evaluation result must be a string, which is printed verbatim without +quotation marks or escaping. diff --git a/doc/manual/source/command-ref/nix-copy-closure.md b/doc/manual/source/command-ref/nix-copy-closure.md index 8cfd6ebad..b7e31d93b 100644 --- a/doc/manual/source/command-ref/nix-copy-closure.md +++ b/doc/manual/source/command-ref/nix-copy-closure.md @@ -84,7 +84,7 @@ When using public key authentication, you can avoid typing the passphrase with ` > Copy GNU Hello from a remote machine using a known store path, and run it: > > ```shell-session -> $ storePath="$(nix-instantiate --eval '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath | tr -d '"')" +> $ storePath="$(nix-instantiate --eval --raw '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath)" > $ nix-copy-closure --from alice@itchy.example.org "$storePath" > $ "$storePath"/bin/hello > Hello, world! diff --git a/doc/manual/source/command-ref/nix-instantiate.md b/doc/manual/source/command-ref/nix-instantiate.md index 6f6fcdc1f..487ef8f10 100644 --- a/doc/manual/source/command-ref/nix-instantiate.md +++ b/doc/manual/source/command-ref/nix-instantiate.md @@ -5,7 +5,7 @@ # Synopsis `nix-instantiate` - [`--parse` | `--eval` [`--strict`] [`--json`] [`--xml`] ] + [`--parse` | `--eval` [`--strict`] [`--raw` | `--json` | `--xml`] ] [`--read-write-mode`] [`--arg` *name* *value*] [{`--attr`| `-A`} *attrPath*] @@ -102,6 +102,11 @@ standard input. > This option can cause non-termination, because lazy data > structures can be infinitely large. +- `--raw` + + When used with `--eval`, the evaluation result must be a string, + which is printed verbatim, without quoting, escaping or trailing newline. + - `--json` When used with `--eval`, print the resulting value as an JSON diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c48549511..09d354832 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -24,7 +24,7 @@ static Path gcRoot; static int rootNr = 0; -enum OutputKind { okPlain, okXML, okJSON }; +enum OutputKind { okPlain, okRaw, okXML, okJSON }; void processExpr(EvalState & state, const Strings & attrPaths, bool parseOnly, bool strict, Bindings & autoArgs, @@ -50,7 +50,11 @@ void processExpr(EvalState & state, const Strings & attrPaths, vRes = v; else state.autoCallFunction(autoArgs, v, vRes); - if (output == okXML) + if (output == okRaw) + std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"); + // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline + // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt. + else if (output == okXML) printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); else if (output == okJSON) { printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); @@ -132,6 +136,8 @@ static int main_nix_instantiate(int argc, char * * argv) gcRoot = getArg(*arg, arg, end); else if (*arg == "--indirect") ; + else if (*arg == "--raw") + outputKind = okRaw; else if (*arg == "--xml") outputKind = okXML; else if (*arg == "--json") diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 18f8589df..ed9c214f5 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -29,6 +29,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ $(nix-instantiate -A int --eval "./eval.nix") == 123 ]] [[ $(nix-instantiate -A str --eval "./eval.nix") == '"foo\nbar"' ]] +[[ $(nix-instantiate -A str --raw --eval "./eval.nix") == $'foo\nbar' ]] [[ "$(nix-instantiate -A attr --eval "./eval.nix")" == '{ foo = "bar"; }' ]] [[ $(nix-instantiate -A attr --eval --json "./eval.nix") == '{"foo":"bar"}' ]] [[ $(nix-instantiate -A int --eval - < "./eval.nix") == 123 ]] From 5e21bdc623939a5b44f679f1436cb9e9e5d6eef5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 31 Dec 2024 17:24:36 +0100 Subject: [PATCH 24/48] .github/ci: Use fixed names This lets us update "runs-on" without creating a mismatch with the required checks that are configured for the repo in the github ui. --- .github/workflows/ci.yml | 28 +++++++++++++++++++++------- .mergify.yml | 8 ++++---- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd567cd63..6169c0924 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,8 +20,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-14] - runs-on: ${{ matrix.os }} + include: + - scenario: on ubuntu + runs-on: ubuntu-24.04 + os: linux + - scenario: on macos + runs-on: macos-14 + os: darwin + name: tests ${{ matrix.scenario }} + runs-on: ${{ matrix.runs-on }} timeout-minutes: 60 steps: - uses: actions/checkout@v4 @@ -37,7 +44,7 @@ jobs: # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'ubuntu-24.04' + if: matrix.os == 'linux' - run: scripts/build-checks - run: scripts/prepare-installer-for-github-actions - name: Upload installer tarball @@ -51,8 +58,15 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04, macos-14] - runs-on: ${{ matrix.os }} + include: + - scenario: on ubuntu + runs-on: ubuntu-24.04 + os: linux + - scenario: on macos + runs-on: macos-14 + os: darwin + name: installer test ${{ matrix.scenario }} + runs-on: ${{ matrix.runs-on }} steps: - uses: actions/checkout@v4 - name: Download installer tarball @@ -68,9 +82,9 @@ jobs: install_url: 'http://localhost:8126/install' install_options: "--tarball-url-prefix http://localhost:8126/" - run: sudo apt install fish zsh - if: matrix.os == 'ubuntu-24.04' + if: matrix.os == 'linux' - run: brew install fish - if: matrix.os == 'macos-14' + if: matrix.os == 'darwin' - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" diff --git a/.mergify.yml b/.mergify.yml index 13d2002ea..bed17f516 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -2,10 +2,10 @@ queue_rules: - name: default # all required tests need to go here merge_conditions: - - check-success=tests (macos-14) - - check-success=tests (ubuntu-24.04) - - check-success=installer_test (macos-14) - - check-success=installer_test (ubuntu-24.04) + - check-success=tests on macos + - check-success=tests on ubuntu + - check-success=installer_test on macos + - check-success=installer_test on ubuntu - check-success=vm_tests batch_size: 5 From 4a2310a3a09a4e9bde51b003c2a562a4a49cfabf Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Tue, 31 Dec 2024 17:52:06 +0100 Subject: [PATCH 25/48] toJSON: re-throw serialization exception --- src/libexpr/value-to-json.cc | 6 +++++- src/libexpr/value-to-json.hh | 3 +++ tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp | 8 ++++++++ tests/functional/lang/eval-fail-toJSON-non-utf-8.nix | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp create mode 100644 tests/functional/lang/eval-fail-toJSON-non-utf-8.nix diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 8044fe347..5aa4fe4fd 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -108,7 +108,11 @@ json printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { - str << printValueAsJSON(state, strict, v, pos, context, copyToStore); + try { + str << printValueAsJSON(state, strict, v, pos, context, copyToStore); + } catch (nlohmann::json::exception & e) { + throw JSONSerializationError("JSON serialization error: %s", e.what()); + } } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 47ac90313..867c4e3a8 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -16,4 +16,7 @@ nlohmann::json printValueAsJSON(EvalState & state, bool strict, void printValueAsJSON(EvalState & state, bool strict, Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); + +MakeError(JSONSerializationError, Error); + } diff --git a/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp b/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp new file mode 100644 index 000000000..129d58bcb --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON-non-utf-8.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'toJSON' builtin + at /pwd/lang/eval-fail-toJSON-non-utf-8.nix:1:1: + 1| builtins.toJSON "_invalid UTF-8: _" + | ^ + 2| + + error: JSON serialization error: [json.exception.type_error.316] invalid UTF-8 byte at index 16: 0xFF diff --git a/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix b/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix new file mode 100644 index 000000000..bd1f74de7 --- /dev/null +++ b/tests/functional/lang/eval-fail-toJSON-non-utf-8.nix @@ -0,0 +1 @@ +builtins.toJSON "_invalid UTF-8: _" From 359a0840e26fed25a2e978f6b3035436948430e6 Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Sat, 21 Dec 2024 04:22:27 +0000 Subject: [PATCH 26/48] packaging: use optimization level 3 and LTO by default --- nix-meson-build-support/common/meson.build | 4 ---- packaging/dependencies.nix | 15 +++++++++++++++ src/libcmd/meson.build | 2 -- src/libexpr-c/meson.build | 2 -- src/libexpr-test-support/meson.build | 2 -- src/libexpr-tests/meson.build | 2 -- src/libexpr/meson.build | 2 -- src/libfetchers-tests/meson.build | 2 -- src/libfetchers/meson.build | 2 -- src/libflake-c/meson.build | 2 -- src/libflake-tests/meson.build | 2 -- src/libflake/meson.build | 2 -- src/libmain-c/meson.build | 2 -- src/libmain/meson.build | 2 -- src/libstore-c/meson.build | 2 -- src/libstore-test-support/meson.build | 2 -- src/libstore-tests/meson.build | 2 -- src/libstore/meson.build | 2 -- src/libutil-c/meson.build | 2 -- src/libutil-test-support/meson.build | 2 -- src/libutil-tests/meson.build | 2 -- src/libutil/meson.build | 2 -- src/nix/meson.build | 2 -- tests/functional/meson.build | 2 -- 24 files changed, 15 insertions(+), 48 deletions(-) diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index f0322183e..67b6658f5 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -16,7 +16,3 @@ add_project_arguments( '-Wno-deprecated-declarations', language : 'cpp', ) - -if get_option('buildtype') not in ['debug'] - add_project_arguments('-O3', language : 'cpp') -endif diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 2c7cf701f..4bc7495e7 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -66,6 +66,21 @@ let mesonLayer = finalAttrs: prevAttrs: { + # NOTE: + # As of https://github.com/NixOS/nixpkgs/blob/8baf8241cea0c7b30e0b8ae73474cb3de83c1a30/pkgs/by-name/me/meson/setup-hook.sh#L26, + # `mesonBuildType` defaults to `plain` if not specified. We want our Nix-built binaries to be optimized by default. + # More on build types here: https://mesonbuild.com/Builtin-options.html#details-for-buildtype. + mesonBuildType = "release"; + # NOTE: + # Users who are debugging Nix builds are expected to set the environment variable `mesonBuildType`, per the + # guidance in https://github.com/NixOS/nix/blob/8a3fc27f1b63a08ac983ee46435a56cf49ebaf4a/doc/manual/source/development/debugging.md?plain=1#L10. + # For this reason, we don't want to refer to `finalAttrs.mesonBuildType` here, but rather use the environment variable. + preConfigure = prevAttrs.preConfigure or "" + '' + case "$mesonBuildType" in + release|minsize) appendToVar mesonFlags "-Db_lto=true" ;; + *) appendToVar mesonFlags "-Db_lto=false" ;; + esac + ''; nativeBuildInputs = [ pkgs.buildPackages.meson pkgs.buildPackages.ninja diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 914db0108..4145f408a 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -4,8 +4,6 @@ project('nix-cmd', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 1556dae51..9487132cf 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -4,8 +4,6 @@ project('nix-expr-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index 64d4fe218..56e814cd1 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-expr-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index f37e85e57..667a0d7b7 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -4,8 +4,6 @@ project('nix-expr-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index b3c559ba7..b33aebc86 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -4,8 +4,6 @@ project('nix-expr', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 3e82c6111..739435501 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -4,8 +4,6 @@ project('nix-fetchers-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index b4408e943..58afbb7d0 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -4,8 +4,6 @@ project('nix-fetchers', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index b7669fe97..85d20644d 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -4,8 +4,6 @@ project('nix-flake-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 5c3c58e53..1c8765f21 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -4,8 +4,6 @@ project('nix-flake-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libflake/meson.build b/src/libflake/meson.build index f9e217729..b757d0d76 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -4,8 +4,6 @@ project('nix-flake', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 5a5684b8d..d875d2c3f 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -4,8 +4,6 @@ project('nix-main-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 87fc8b8d2..00f945f49 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -4,8 +4,6 @@ project('nix-main', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index 1ac331ad0..17d18609f 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -4,8 +4,6 @@ project('nix-store-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index 2a07e56ac..59d649889 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-store-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index b706fa12c..3ba0795e9 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -4,8 +4,6 @@ project('nix-store-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 12a0e6376..79d912497 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -4,8 +4,6 @@ project('nix-store', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 44cec1afc..ac1297665 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -4,8 +4,6 @@ project('nix-util-c', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 03ae63f1a..db944cf06 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -4,8 +4,6 @@ project('nix-util-test-support', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 83ac79e92..ad2c61711 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -4,8 +4,6 @@ project('nix-util-tests', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 2c3e3a954..ac701d8fd 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -4,8 +4,6 @@ project('nix-util', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.1', diff --git a/src/nix/meson.build b/src/nix/meson.build index 1d4840b12..2698cc873 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -4,8 +4,6 @@ project('nix', 'cpp', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail 'localstatedir=/nix/var', ], diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 933595cd5..83e08c4f5 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -4,8 +4,6 @@ project('nix-functional-tests', 'cpp_std=c++2a', # TODO(Qyriad): increase the warning level 'warning_level=1', - 'debug=true', - 'optimization=2', 'errorlogs=true', # Please print logs for tests that fail ], meson_version : '>= 1.3', From a44e9dd1ea664a9616d7dabb548095bb3f375f7e Mon Sep 17 00:00:00 2001 From: Philipp Otterbein Date: Sat, 4 Jan 2025 16:14:06 +0100 Subject: [PATCH 27/48] correctly parse strings with null bytes and throw error --- src/libexpr/eval.cc | 8 ++++-- src/libexpr/lexer.l | 25 +++++++++++------- src/libexpr/value.hh | 2 +- .../lang/eval-fail-string-nul-1.err.exp | Bin 0 -> 209 bytes .../lang/eval-fail-string-nul-1.nix | Bin 0 -> 10 bytes .../lang/eval-fail-string-nul-2.err.exp | Bin 0 -> 256 bytes .../lang/eval-fail-string-nul-2.nix | Bin 0 -> 22 bytes 7 files changed, 22 insertions(+), 13 deletions(-) create mode 100644 tests/functional/lang/eval-fail-string-nul-1.err.exp create mode 100644 tests/functional/lang/eval-fail-string-nul-1.nix create mode 100644 tests/functional/lang/eval-fail-string-nul-2.err.exp create mode 100644 tests/functional/lang/eval-fail-string-nul-2.nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index fe5f05ab8..21dd5a294 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3185,12 +3185,16 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { return v.print(str); } -void forceNoNullByte(std::string_view s) +void forceNoNullByte(std::string_view s, std::function pos) { if (s.find('\0') != s.npos) { using namespace std::string_view_literals; auto str = replaceStrings(std::string(s), "\0"sv, "␀"sv); - throw Error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + Error error("input string '%s' cannot be represented as Nix string because it contains null bytes", str); + if (pos) { + error.atPos(pos()); + } + throw error; } } diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index a7e44cb72..067f86e01 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -41,16 +41,18 @@ namespace nix { // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. -static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) +// getting the position is expensive and thus it is implemented lazily. +static StringToken unescapeStr(char * const s, size_t length, std::function && pos) { - char * result = s; + bool noNullByte = true; char * t = s; - char c; // the input string is terminated with *two* NULs, so we can safely take // *one* character after the one being checked against. - while ((c = *s++)) { + for (size_t i = 0; i < length; t++) { + char c = s[i++]; + noNullByte &= c != '\0'; if (c == '\\') { - c = *s++; + c = s[i++]; if (c == 'n') *t = '\n'; else if (c == 'r') *t = '\r'; else if (c == 't') *t = '\t'; @@ -59,12 +61,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) else if (c == '\r') { /* Normalise CR and CR/LF into LF. */ *t = '\n'; - if (*s == '\n') s++; /* cr/lf */ + if (s[i] == '\n') i++; /* cr/lf */ } else *t = c; - t++; } - return {result, size_t(t - result)}; + if (!noNullByte) { + forceNoNullByte({s, size_t(t - s)}, std::move(pos)); + } + return {s, size_t(t - s)}; } static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) @@ -175,7 +179,7 @@ or { return OR_KW; } /* It is impossible to match strings ending with '$' with one regex because trailing contexts are only valid at the end of a rule. (A sane but undocumented limitation.) */ - yylval->str = unescapeStr(state->symbols, yytext, yyleng); + yylval->str = unescapeStr(yytext, yyleng, [&]() { return state->positions[CUR_POS]; }); return STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } @@ -191,6 +195,7 @@ or { return OR_KW; } \'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } ([^\$\']|\$[^\{\']|\'[^\'\$])+ { yylval->str = {yytext, (size_t) yyleng, true}; + forceNoNullByte(yylval->str, [&]() { return state->positions[CUR_POS]; }); return IND_STR; } \'\'\$ | @@ -203,7 +208,7 @@ or { return OR_KW; } return IND_STR; } \'\'\\{ANY} { - yylval->str = unescapeStr(state->symbols, yytext + 2, yyleng - 2); + yylval->str = unescapeStr(yytext + 2, yyleng - 2, [&]() { return state->positions[CUR_POS]; }); return IND_STR; } \$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 88fcae986..8925693e3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -510,6 +510,6 @@ typedef std::shared_ptr RootValue; RootValue allocRootValue(Value * v); -void forceNoNullByte(std::string_view s); +void forceNoNullByte(std::string_view s, std::function = nullptr); } diff --git a/tests/functional/lang/eval-fail-string-nul-1.err.exp b/tests/functional/lang/eval-fail-string-nul-1.err.exp new file mode 100644 index 0000000000000000000000000000000000000000..2dfbea0635c8c2ea159249879c4d89523c2a7366 GIT binary patch literal 209 zcmY+8F%E+;6hk|63X6p)MKIDq_H+{VbbPipj#u zG%KVo2L&^b4`$yq8*iGc_{BE1wMXaQY*v)pqs#WEJOx;R`^jn7;s?AsKQI6Q literal 0 HcmV?d00001 diff --git a/tests/functional/lang/eval-fail-string-nul-1.nix b/tests/functional/lang/eval-fail-string-nul-1.nix new file mode 100644 index 0000000000000000000000000000000000000000..3689409171139a7d310eac52df0be4315b6ba786 GIT binary patch literal 10 RcmY#N%g<*>N-R?10ss()0>}UW literal 0 HcmV?d00001 diff --git a/tests/functional/lang/eval-fail-string-nul-2.err.exp b/tests/functional/lang/eval-fail-string-nul-2.err.exp new file mode 100644 index 0000000000000000000000000000000000000000..b1cae5325d90a2c05f7304db89bca0826c2f98f0 GIT binary patch literal 256 zcmY+9K?=e!5Jg>kioeS$meySk;Sq#MJ1v1QDM_joy6^~|(UaIJK{~rX^XBtE#~7l4 zlX1#OIXW4jfIfsH%Di9CzpxaGP-sacWLQSzF>+$b+<_7t7IF*eHqEjgb_F(Y!F5NS IH|lWw0QY`GT>t<8 literal 0 HcmV?d00001 diff --git a/tests/functional/lang/eval-fail-string-nul-2.nix b/tests/functional/lang/eval-fail-string-nul-2.nix new file mode 100644 index 0000000000000000000000000000000000000000..fd6b3258a5e414e6e4812113aed4ece422aa76bc GIT binary patch literal 22 acmY#a=TcBe%g+ar3_zTeSj45S&IJH2l>`U? literal 0 HcmV?d00001 From 98d75de1ea00b31cdff9a22e881ac01be5ac2ae9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 5 Jan 2025 19:02:28 +0000 Subject: [PATCH 28/48] windows: fix conditional compilation variable --- src/libstore/windows/pathlocks.cc | 2 +- src/libutil/windows/environment-variables.cc | 2 +- src/libutil/windows/file-descriptor.cc | 2 +- src/libutil/windows/file-system.cc | 2 +- src/libutil/windows/muxable-pipe.cc | 2 +- src/libutil/windows/os-string.cc | 2 +- src/libutil/windows/processes.cc | 2 +- src/libutil/windows/users.cc | 2 +- src/libutil/windows/windows-async-pipe.cc | 2 +- src/libutil/windows/windows-async-pipe.hh | 2 +- src/libutil/windows/windows-error.cc | 2 +- src/libutil/windows/windows-error.hh | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 197f5a1c4..29a98d8e2 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -3,7 +3,7 @@ #include "signals.hh" #include "util.hh" -#ifdef WIN32 +#ifdef _WIN32 # include # include # include diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 308a432e4..d1093597c 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -1,6 +1,6 @@ #include "environment-variables.hh" -#ifdef WIN32 +#ifdef _WIN32 # include "processenv.h" namespace nix { diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 71f53ccb8..e2a473a7c 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -5,7 +5,7 @@ #include "windows-error.hh" #include "file-path.hh" -#ifdef WIN32 +#ifdef _WIN32 #include #include #include diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 53271cef3..7ed1c04a6 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -1,6 +1,6 @@ #include "file-system.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix { Descriptor openDirectory(const std::filesystem::path & path) diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc index af7e987e9..ac2882120 100644 --- a/src/libutil/windows/muxable-pipe.cc +++ b/src/libutil/windows/muxable-pipe.cc @@ -1,4 +1,4 @@ -#ifdef WIN32 +#ifdef _WIN32 # include # include "windows-error.hh" diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc index 26ad9cba0..b09ef8b90 100644 --- a/src/libutil/windows/os-string.cc +++ b/src/libutil/windows/os-string.cc @@ -7,7 +7,7 @@ #include "file-path-impl.hh" #include "util.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix { diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index e69f1ed45..fd4d7c43a 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -23,7 +23,7 @@ #include #include -#ifdef WIN32 +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index 2780e45f4..438c4221c 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -4,7 +4,7 @@ #include "file-system.hh" #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc index 13b563510..4e139d5cf 100644 --- a/src/libutil/windows/windows-async-pipe.cc +++ b/src/libutil/windows/windows-async-pipe.cc @@ -1,7 +1,7 @@ #include "windows-async-pipe.hh" #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix::windows { diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh index 277336ed7..53715e260 100644 --- a/src/libutil/windows/windows-async-pipe.hh +++ b/src/libutil/windows/windows-async-pipe.hh @@ -2,7 +2,7 @@ ///@file #include "file-descriptor.hh" -#ifdef WIN32 +#ifdef _WIN32 namespace nix::windows { diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index 4cf4274da..b92f9155f 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -1,6 +1,6 @@ #include "windows-error.hh" -#ifdef WIN32 +#ifdef _WIN32 #include #define WIN32_LEAN_AND_MEAN #include diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index 4e48ee859..66c67b43a 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#ifdef WIN32 +#ifdef _WIN32 #include #include "error.hh" From 1eba904b7994985996f6ee1571ef70c6e9a91ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 5 Jan 2025 19:14:29 +0000 Subject: [PATCH 29/48] nix/flake: fix build on windows --- src/nix/flake.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 55aa8971e..7189689cc 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -938,7 +938,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } continue; } else - createSymlink(target, to2); + createSymlink(target, os_string_to_string(PathViewNG { to2 })); } else throw Error("file '%s' has unsupported type", from2); From 438a20427f40e8ec458444fb156e984f8137ec13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 6 Jan 2025 22:53:39 +0100 Subject: [PATCH 30/48] mergify: fix installer test name --- .mergify.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index bed17f516..5d2bf8520 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -4,8 +4,8 @@ queue_rules: merge_conditions: - check-success=tests on macos - check-success=tests on ubuntu - - check-success=installer_test on macos - - check-success=installer_test on ubuntu + - check-success=installer test on macos + - check-success=installer test on ubuntu - check-success=vm_tests batch_size: 5 From 69853c067c74e9421cfa41e48bf8af04266206d3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:37:25 +0100 Subject: [PATCH 31/48] Add makeParentCanonical() --- src/libutil/file-system.cc | 15 +++++++++++++++ src/libutil/file-system.hh | 17 +++++++++++++++++ src/libutil/posix-source-accessor.hh | 12 ++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 793eb2d9f..923220fd0 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -766,4 +766,19 @@ bool isExecutableFileAmbient(const fs::path & exe) { ) == 0; } +std::filesystem::path makeParentCanonical(const std::filesystem::path & rawPath) +{ + std::filesystem::path path(absPath(rawPath));; + try { + auto parent = path.parent_path(); + if (parent == path) { + // `path` is a root directory => trivially canonical + return parent; + } + return std::filesystem::canonical(parent) / path.filename(); + } catch (fs::filesystem_error & e) { + throw SysError("canonicalising parent path of '%1%'", path); + } } + +} // namespace nix diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 3c49181a0..7fdaba811 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -143,6 +143,23 @@ inline bool symlink_exists(const std::filesystem::path & path) { } // namespace fs +/** + * Canonicalize a path except for the last component. + * + * This is useful for getting the canonical location of a symlink. + * + * Consider the case where `foo/l` is a symlink. `canonical("foo/l")` will + * resolve the symlink `l` to its target. + * `makeParentCanonical("foo/l")` will not resolve the symlink `l` to its target, + * but does ensure that the returned parent part of the path, `foo` is resolved + * to `canonical("foo")`, and can therefore be retrieved without traversing any + * symlinks. + * + * If a relative path is passed, it will be made absolute, so that the parent + * can always be canonicalized. + */ +std::filesystem::path makeParentCanonical(const std::filesystem::path & path); + /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 40f60bb54..d60abdd9a 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -50,6 +50,18 @@ struct PosixSourceAccessor : virtual SourceAccessor * possible, (e.g. on Windows it could scoped to a drive like * `C:\`). This allows more `..` parent accessing to work. * + * @note When `path` is trusted user input, canonicalize it using + * `std::filesystem::canonical`, `makeParentCanonical`, `std::filesystem::weakly_canonical`, etc, + * as appropriate for the use case. At least weak canonicalization is + * required for the `SourcePath` to do anything useful at the location it + * points to. + * + * @note A canonicalizing behavior is not built in `createAtRoot` so that + * callers do not accidentally introduce symlink-related security vulnerabilities. + * Furthermore, `createAtRoot` does not know whether the file pointed to by + * `path` should be resolved if it is itself a symlink. In other words, + * `createAtRoot` can not decide between aforementioned `canonical`, `makeParentCanonical`, etc. for its callers. + * * See * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) * and From 36563c69a48a47d8b58340f22d82d542f6b9f025 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 17:21:56 +0100 Subject: [PATCH 32/48] fix: Handle symlinks and FIFOs in `nix hash` where possible Fixes https://github.com/NixOS/nix/issues/11756 Fixes https://github.com/NixOS/nix/issues/11681 --- src/libutil/posix-source-accessor.hh | 2 +- src/nix/hash.cc | 23 ++++++++++++++++++++--- tests/functional/hash-path.sh | 24 ++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index d60abdd9a..5d491e633 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -43,7 +43,7 @@ struct PosixSourceAccessor : virtual SourceAccessor std::optional getPhysicalPath(const CanonPath & path) override; /** - * Create a `PosixSourceAccessor` and `CanonPath` corresponding to + * Create a `PosixSourceAccessor` and `SourcePath` corresponding to * some native path. * * The `PosixSourceAccessor` is rooted as far up the tree as diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 416cd19b3..eac421d12 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -87,18 +87,35 @@ struct CmdHashBase : Command return std::make_unique(hashAlgo); }; - auto path2 = PosixSourceAccessor::createAtRoot(path); + auto makeSourcePath = [&]() -> SourcePath { + return PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); + }; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: + { + // While usually we could use the some code as for NixArchive, + // the Flat method needs to support FIFOs, such as those + // produced by bash process substitution, e.g.: + // nix hash --mode flat <(echo hi) + // Also symlinks semantics are unambiguous in the flat case, + // so we don't need to go low-level, or reject symlink `path`s. + auto hashSink = makeSink(); + readFile(path, *hashSink); + h = hashSink->finish().first; + break; + } case FileIngestionMethod::NixArchive: { + auto sourcePath = makeSourcePath(); auto hashSink = makeSink(); - dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); + dumpPath(sourcePath, *hashSink, (FileSerialisationMethod) mode); h = hashSink->finish().first; break; } case FileIngestionMethod::Git: { + auto sourcePath = makeSourcePath(); std::function hook; hook = [&](const SourcePath & path) -> git::TreeEntry { auto hashSink = makeSink(); @@ -109,7 +126,7 @@ struct CmdHashBase : Command .hash = hash, }; }; - h = hook(path2).hash; + h = hook(sourcePath).hash; break; } } diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh index 86d782a95..74e319504 100755 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -92,3 +92,27 @@ try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" rm "$TEST_ROOT/hash-path/hello" ln -s x "$TEST_ROOT/hash-path/hello" try2 md5 "f78b733a68f5edbdf9413899339eaa4a" + +# Flat mode supports process substitution +h=$(nix hash path --mode flat --type sha256 --base32 <(printf "SMASH THE STATE")) +[[ 0d9n3r2i4m1zgy0wpqbsyabsfzgs952066bfp8gwvcg4mkr4r5g8 == "$h" ]] + +# Flat mode supports process substitution (hash file) +h=$(nix hash file --type sha256 --base32 <(printf "SMASH THE STATE")) +[[ 0d9n3r2i4m1zgy0wpqbsyabsfzgs952066bfp8gwvcg4mkr4r5g8 == "$h" ]] + +# Symlinks in the ancestry are ok and don't affect the result +mkdir -p "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it" +echo hi > "$TEST_ROOT/simple/hi" +ln -s "$TEST_ROOT/simple" "$TEST_ROOT/try/to/mess/with/it/simple-link" +h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/simple/hi") +[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]] +h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/try/to/mess/with/it/simple-link/hi") +[[ 1xmr8jicvzszfzpz46g37mlpvbzjl2wpwvl2b05psipssyp1sm8h == "$h" ]] + +# nix hash --mode nar does not canonicalize a symlink argument. +# Otherwise it can't generate a NAR whose root is a symlink. +# If you want to follow the symlink, pass $(realpath -s ...) instead. +ln -s /non-existent-48cujwe8ndf4as0bne "$TEST_ROOT/symlink-to-nowhere" +h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-nowhere") +[[ 1bl5ry3x1fcbwgr5c2x50bn572iixh4j1p6ax5isxly2ddgn8pbp == "$h" ]] # manually verified hash From 91e91f62fa2187f49073851f9a3dbff740092859 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:38:47 +0100 Subject: [PATCH 33/48] doc: Document `nix-store --add-fixed` symlink behavior Tested with nix run nix/2.3-maintenance#nix-store -- --add some_symlink nix run nix/2.3-maintenance#nix-store -- --add-fixed sha256 --recursive some_symlink --- doc/manual/source/command-ref/nix-store/add-fixed.md | 3 +++ doc/manual/source/command-ref/nix-store/add.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/doc/manual/source/command-ref/nix-store/add-fixed.md b/doc/manual/source/command-ref/nix-store/add-fixed.md index bebf15026..2ea90a135 100644 --- a/doc/manual/source/command-ref/nix-store/add-fixed.md +++ b/doc/manual/source/command-ref/nix-store/add-fixed.md @@ -21,6 +21,9 @@ This operation has the following options: Use recursive instead of flat hashing mode, used when adding directories to the store. + *paths* that refer to symlinks are not dereferenced, but added to the store + as symlinks with the same target. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} diff --git a/doc/manual/source/command-ref/nix-store/add.md b/doc/manual/source/command-ref/nix-store/add.md index 87d504cd3..ab4740723 100644 --- a/doc/manual/source/command-ref/nix-store/add.md +++ b/doc/manual/source/command-ref/nix-store/add.md @@ -11,6 +11,9 @@ The operation `--add` adds the specified paths to the Nix store. It prints the resulting paths in the Nix store on standard output. +*paths* that refer to symlinks are not dereferenced, but added to the store +as symlinks with the same target. + {{#include ./opt-common.md}} {{#include ../opt-common.md}} From c0b64f3377b37c036c574b731c3aeffb3fddc614 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:46:47 +0100 Subject: [PATCH 34/48] refactor: Don't re-construct SourcePath unnecessarily --- src/nix-store/nix-store.cc | 8 ++++---- src/nix/add-to-store.cc | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b731b25af..659a237f9 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -183,9 +183,9 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(i); cout << fmt("%s\n", store->printStorePath(store->addToStore( - std::string(baseNameOf(i)), {accessor, canonPath}))); + std::string(baseNameOf(i)), sourcePath))); } } @@ -207,10 +207,10 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(i); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( baseNameOf(i), - {accessor, canonPath}, + sourcePath, method, hashAlgo).path)); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 5c08f7616..45563cb2b 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -37,13 +37,13 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path); + auto sourcePath = PosixSourceAccessor::createAtRoot(path); auto storePath = dryRun ? store->computeStorePath( - *namePart, {accessor, path2}, caMethod, hashAlgo, {}).first + *namePart, sourcePath, caMethod, hashAlgo, {}).first : store->addToStoreSlow( - *namePart, {accessor, path2}, caMethod, hashAlgo, {}).path; + *namePart, sourcePath, caMethod, hashAlgo, {}).path; logger->cout("%s", store->printStorePath(storePath)); } From ddbbf537676441ff5c09bd64310fec0f2619a820 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 13 Dec 2024 18:54:36 +0100 Subject: [PATCH 35/48] fix: Resolve CLI parent symlinks before adding to store Fixes https://github.com/NixOS/nix/issues/11941 --- src/nix-store/nix-store.cc | 4 ++-- src/nix/add-to-store.cc | 2 +- tests/functional/add.sh | 41 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 659a237f9..99bb2c726 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -183,7 +183,7 @@ static void opAdd(Strings opFlags, Strings opArgs) if (!opFlags.empty()) throw UsageError("unknown flag"); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); cout << fmt("%s\n", store->printStorePath(store->addToStore( std::string(baseNameOf(i)), sourcePath))); } @@ -207,7 +207,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) opArgs.pop_front(); for (auto & i : opArgs) { - auto sourcePath = PosixSourceAccessor::createAtRoot(i); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(i)); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( baseNameOf(i), sourcePath, diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 45563cb2b..7f15de374 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -37,7 +37,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { if (!namePart) namePart = baseNameOf(path); - auto sourcePath = PosixSourceAccessor::createAtRoot(path); + auto sourcePath = PosixSourceAccessor::createAtRoot(makeParentCanonical(path)); auto storePath = dryRun ? store->computeStorePath( diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 3b37ee7d4..0e6868d8f 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -29,6 +29,47 @@ echo "$hash2" test "$hash1" = "sha256:$hash2" +# The contents can be accessed through a symlink, and this symlink has no effect on the hash +# https://github.com/NixOS/nix/issues/11941 +test_issue_11941() { + local expected actual + mkdir -p "$TEST_ROOT/foo/bar" && ln -s "$TEST_ROOT/foo" "$TEST_ROOT/foo-link" + + # legacy + expected=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo/bar") + actual=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/foo-link/bar") + [[ "$expected" == "$actual" ]] + actual=$(nix-store --add "$TEST_ROOT/foo-link/bar") + [[ "$expected" == "$actual" ]] + + # nix store add + actual=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/foo/bar") + [[ "$expected" == "$actual" ]] + + # cleanup + rm -r "$TEST_ROOT/foo" "$TEST_ROOT/foo-link" +} +test_issue_11941 + +# A symlink is added to the store as a symlink, not as a copy of the target +test_add_symlink() { + ln -s /bin "$TEST_ROOT/my-bin" + + # legacy + path=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + path=$(nix-store --add "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + + # nix store add + path=$(nix store add --hash-algo sha256 --mode nar "$TEST_ROOT/my-bin") + [[ "$(readlink "$path")" == /bin ]] + + # cleanup + rm "$TEST_ROOT/my-bin" +} +test_add_symlink + #### New style commands clearStoreIfPossible From 628c11d237dc38bf4e35c498d418026c215de5cf Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 16 Dec 2024 09:53:36 +0100 Subject: [PATCH 36/48] test: Add hydraJobs.tests.functional_symlinked-home --- tests/nixos/default.nix | 2 ++ tests/nixos/functional/symlinked-home.nix | 36 +++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/nixos/functional/symlinked-home.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index a9a438a32..8e0cb1b22 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -159,6 +159,8 @@ in functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + functional_symlinked-home = runNixOSTestFor "x86_64-linux" ./functional/symlinked-home.nix; + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; diff --git a/tests/nixos/functional/symlinked-home.nix b/tests/nixos/functional/symlinked-home.nix new file mode 100644 index 000000000..57c45d5d5 --- /dev/null +++ b/tests/nixos/functional/symlinked-home.nix @@ -0,0 +1,36 @@ +/** + This test runs the functional tests on a NixOS system where the home directory + is symlinked to another location. + + The purpose of this test is to find cases where Nix uses low-level operations + that don't support symlinks on paths that include them. + + It is not a substitute for more intricate, use case-specific tests, but helps + catch common issues. +*/ +# TODO: add symlinked tmpdir +{ ... }: +{ + name = "functional-tests-on-nixos_user_symlinked-home"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + with subtest("prepare symlinked home"): + machine.succeed(""" + ( + set -x + mv /home/alice /home/alice.real + ln -s alice.real /home/alice + ) 1>&2 + """) + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} From 5a5a86949a2fca18f61d1117ab29cc87281c48d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 15 Dec 2024 19:03:14 +0100 Subject: [PATCH 37/48] makeParentCanonical: test case where parent is empty --- src/libutil-tests/file-system.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libutil-tests/file-system.cc b/src/libutil-tests/file-system.cc index 7ef804f34..2c10d4869 100644 --- a/src/libutil-tests/file-system.cc +++ b/src/libutil-tests/file-system.cc @@ -261,4 +261,18 @@ TEST(pathExists, bogusPathDoesNotExist) { ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); } + +/* ---------------------------------------------------------------------------- + * makeParentCanonical + * --------------------------------------------------------------------------*/ + +TEST(makeParentCanonical, noParent) +{ + ASSERT_EQ(makeParentCanonical("file"), absPath(std::filesystem::path("file"))); +} + +TEST(makeParentCanonical, root) +{ + ASSERT_EQ(makeParentCanonical("/"), "/"); +} } From 4c74d679b6b4db9eccea94afc285176871253967 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Dec 2024 16:55:27 +0100 Subject: [PATCH 38/48] test: Avoid regressing accidental use of weakly_canonical instead of makeParentCanonical I'd messed up a rebase in my previous iteration, causing `weakly_canonical` to reappear, but not trigger a test failure. These two functions behave similarly when the argument is a path that points to a broken symlink. `weakly_canonical` would not resolve it because the target doesn't exist, and `makeParentCanonical` would not resolve it, because it never resolves the final path element. This new test case now also tests a valid symlink, "differentiating" the two. --- tests/functional/hash-path.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh index 74e319504..4894ae391 100755 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -116,3 +116,8 @@ h=$(nix hash path --type sha256 --base32 "$TEST_ROOT/try/to/mess/with/it/simple- ln -s /non-existent-48cujwe8ndf4as0bne "$TEST_ROOT/symlink-to-nowhere" h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-nowhere") [[ 1bl5ry3x1fcbwgr5c2x50bn572iixh4j1p6ax5isxly2ddgn8pbp == "$h" ]] # manually verified hash +if [[ -e /bin ]]; then + ln -s /bin "$TEST_ROOT/symlink-to-bin" + h=$(nix hash path --mode nar --type sha256 --base32 "$TEST_ROOT/symlink-to-bin") + [[ 0z2mdmkd43l0ijdxfbj1y8vzli15yh9b09n3a3rrygmjshbyypsw == "$h" ]] # manually verified hash +fi From f705ce7f9a95bc37df236d88eef1dbd9e2ae5f31 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:45:58 +0100 Subject: [PATCH 39/48] ParsedURL: Remove url field This prevents a 'url' field that is out of sync with the other fields. You can use to_string() to get the full URL. --- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/github.cc | 16 ++++++++-------- src/libfetchers/indirect.cc | 8 ++++---- src/libfetchers/path.cc | 6 +++--- src/libflake/flake/flakeref.cc | 2 -- src/libutil-tests/url.cc | 22 ---------------------- src/libutil/url.cc | 7 ++++++- src/libutil/url.hh | 5 +++-- 8 files changed, 25 insertions(+), 43 deletions(-) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index b105c252a..87bf0dd71 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -66,7 +66,7 @@ Input Input::fromURL( } } - throw Error("input '%s' is unsupported", url.url); + throw Error("input '%s' is unsupported", url); } Input Input::fromAttrs(const Settings & settings, Attrs && attrs) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 308cff33a..185941988 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -50,7 +50,7 @@ struct GitArchiveInputScheme : InputScheme else if (std::regex_match(path[2], refRegex)) ref = path[2]; else - throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); + throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url, path[2]); } else if (size > 3) { std::string rs; for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { @@ -63,34 +63,34 @@ struct GitArchiveInputScheme : InputScheme if (std::regex_match(rs, refRegex)) { ref = rs; } else { - throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url, rs); } } else if (size < 2) - throw BadURL("URL '%s' is invalid", url.url); + throw BadURL("URL '%s' is invalid", url); for (auto &[name, value] : url.query) { if (name == "rev") { if (rev) - throw BadURL("URL '%s' contains multiple commit hashes", url.url); + throw BadURL("URL '%s' contains multiple commit hashes", url); rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) - throw BadURL("URL '%s' contains an invalid branch/tag name", url.url); + throw BadURL("URL '%s' contains an invalid branch/tag name", url); if (ref) - throw BadURL("URL '%s' contains multiple branch/tag names", url.url); + throw BadURL("URL '%s' contains multiple branch/tag names", url); ref = value; } else if (name == "host") { if (!std::regex_match(value, hostRegex)) - throw BadURL("URL '%s' contains an invalid instance host", url.url); + throw BadURL("URL '%s' contains an invalid instance host", url); host_url = value; } // FIXME: barf on unsupported attributes } if (ref && rev) - throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); + throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url, *ref, rev->gitRev()); Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 2e5cd82c7..0e1b86711 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -26,16 +26,16 @@ struct IndirectInputScheme : InputScheme else if (std::regex_match(path[1], refRegex)) ref = path[1]; else - throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url, path[1]); } else if (path.size() == 3) { if (!std::regex_match(path[1], refRegex)) - throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url, path[1]); ref = path[1]; if (!std::regex_match(path[2], revRegex)) - throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url, path[2]); rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else - throw BadURL("GitHub URL '%s' is invalid", url.url); + throw BadURL("GitHub URL '%s' is invalid", url); std::string id = path[0]; if (!std::regex_match(id, flakeRegex)) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 246b68c3a..f628e042c 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -14,7 +14,7 @@ struct PathInputScheme : InputScheme if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") - throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); + throw Error("path URL '%s' should not have an authority ('%s')", url, *url.authority); Input input{settings}; input.attrs.insert_or_assign("type", "path"); @@ -27,10 +27,10 @@ struct PathInputScheme : InputScheme if (auto n = string2Int(value)) input.attrs.insert_or_assign(name, *n); else - throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); + throw Error("path URL '%s' has invalid parameter '%s'", url, name); } else - throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); + throw Error("path URL '%s' has unsupported parameter '%s'", url, name); return input; } diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index ab882fdab..957495795 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -162,7 +162,6 @@ std::pair parsePathFlakeRefWithFragment( auto base = std::string("git+file://") + flakeRoot; auto parsedURL = ParsedURL{ - .url = base, // FIXME .base = base, .scheme = "git+file", .authority = "", @@ -220,7 +219,6 @@ static std::optional> parseFlakeIdRef( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ - .url = url, .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 7d08f467e..6ffae4f3e 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -20,23 +20,11 @@ namespace nix { } - std::ostream& operator<<(std::ostream& os, const ParsedURL& p) { - return os << "\n" - << "url: " << p.url << "\n" - << "base: " << p.base << "\n" - << "scheme: " << p.scheme << "\n" - << "authority: " << p.authority.value() << "\n" - << "path: " << p.path << "\n" - << "query: " << print_map(p.query) << "\n" - << "fragment: " << p.fragment << "\n"; - } - TEST(parseURL, parsesSimpleHttpUrl) { auto s = "http://www.example.org/file.tar.gz"; auto parsed = parseURL(s); ParsedURL expected { - .url = "http://www.example.org/file.tar.gz", .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", @@ -53,7 +41,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "https://www.example.org/file.tar.gz", .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", @@ -70,7 +57,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "https://www.example.org/file.tar.gz", .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", @@ -87,7 +73,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://www.example.org/file.tar.gz", .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", @@ -104,7 +89,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "file+https://www.example.org/video.mp4", .base = "https://www.example.org/video.mp4", .scheme = "file+https", .authority = "www.example.org", @@ -126,7 +110,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://127.0.0.1:8080/file.tar.gz", .base = "https://127.0.0.1:8080/file.tar.gz", .scheme = "http", .authority = "127.0.0.1:8080", @@ -143,7 +126,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .scheme = "http", .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", @@ -161,7 +143,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .scheme = "http", .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", @@ -185,7 +166,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "http://user:pass@www.example.org/file.tar.gz", .base = "http://user:pass@www.example.org/file.tar.gz", .scheme = "http", .authority = "user:pass@www.example.org:8080", @@ -203,7 +183,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "", .base = "", .scheme = "file", .authority = "", @@ -228,7 +207,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .url = "ftp://ftp.nixos.org/downloads/nixos.iso", .base = "ftp://ftp.nixos.org/downloads/nixos.iso", .scheme = "ftp", .authority = "ftp.nixos.org", diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 63b9734ee..86ca035b9 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -40,7 +40,6 @@ ParsedURL parseURL(const std::string & url) path = "/"; return ParsedURL{ - .url = url, .base = base, .scheme = scheme, .authority = authority, @@ -136,6 +135,12 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } +std::ostream & operator << (std::ostream & os, const ParsedURL & url) +{ + os << url.to_string(); + return os; +} + bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 738ee9f82..d85001185 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -7,9 +7,8 @@ namespace nix { struct ParsedURL { - std::string url; /// URL without query/fragment - std::string base; + std::string base; // FIXME: remove std::string scheme; std::optional authority; std::string path; @@ -26,6 +25,8 @@ struct ParsedURL ParsedURL canonicalise(); }; +std::ostream & operator << (std::ostream & os, const ParsedURL & url); + MakeError(BadURL, Error); std::string percentDecode(std::string_view in); From 4077aa43a803db33108ce39e799089608707dcbb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Jan 2025 14:52:00 +0100 Subject: [PATCH 40/48] ParsedURL: Remove base field --- src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libflake/flake/flakeref.cc | 4 ---- src/libutil-tests/url.cc | 11 ----------- src/libutil/url.cc | 1 - src/libutil/url.hh | 2 -- 6 files changed, 2 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index c73f53765..d894550c0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -426,7 +426,7 @@ struct GitInputScheme : InputScheme auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - repoInfo.url = repoInfo.isLocal ? url.path : url.base; + repoInfo.url = repoInfo.isLocal ? url.path : url.to_string(); // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 2c987f79d..c2fd8139c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -161,7 +161,7 @@ struct MercurialInputScheme : InputScheme { auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; - return {isLocal, isLocal ? url.path : url.base}; + return {isLocal, isLocal ? url.path : url.to_string()}; } StorePath fetchToStore(ref store, Input & input) const diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 957495795..60efe1612 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -159,10 +159,7 @@ std::pair parsePathFlakeRefWithFragment( while (flakeRoot != "/") { if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - auto parsedURL = ParsedURL{ - .base = base, .scheme = "git+file", .authority = "", .path = flakeRoot, @@ -219,7 +216,6 @@ static std::optional> parseFlakeIdRef( if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ - .base = "flake:" + match.str(1), .scheme = "flake", .authority = "", .path = match[1], diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index 6ffae4f3e..7e1d2aa15 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -25,7 +25,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", .path = "/file.tar.gz", @@ -41,7 +40,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", .path = "/file.tar.gz", @@ -57,7 +55,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://www.example.org/file.tar.gz", .scheme = "https", .authority = "www.example.org", .path = "/file.tar.gz", @@ -73,7 +70,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://www.example.org/file.tar.gz", .scheme = "http", .authority = "www.example.org", .path = "/file.tar.gz", @@ -89,7 +85,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://www.example.org/video.mp4", .scheme = "file+https", .authority = "www.example.org", .path = "/video.mp4", @@ -110,7 +105,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "https://127.0.0.1:8080/file.tar.gz", .scheme = "http", .authority = "127.0.0.1:8080", .path = "/file.tar.gz", @@ -126,7 +120,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .scheme = "http", .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080", .path = "", @@ -143,7 +136,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .scheme = "http", .authority = "[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", .path = "", @@ -166,7 +158,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "http://user:pass@www.example.org/file.tar.gz", .scheme = "http", .authority = "user:pass@www.example.org:8080", .path = "/file.tar.gz", @@ -183,7 +174,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "", .scheme = "file", .authority = "", .path = "/none/of//your/business", @@ -207,7 +197,6 @@ namespace nix { auto parsed = parseURL(s); ParsedURL expected { - .base = "ftp://ftp.nixos.org/downloads/nixos.iso", .scheme = "ftp", .authority = "ftp.nixos.org", .path = "/downloads/nixos.iso", diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 86ca035b9..4d4d983b8 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -40,7 +40,6 @@ ParsedURL parseURL(const std::string & url) path = "/"; return ParsedURL{ - .base = base, .scheme = scheme, .authority = authority, .path = percentDecode(path), diff --git a/src/libutil/url.hh b/src/libutil/url.hh index d85001185..2b12f5af2 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -7,8 +7,6 @@ namespace nix { struct ParsedURL { - /// URL without query/fragment - std::string base; // FIXME: remove std::string scheme; std::optional authority; std::string path; From 3a5fccc41881a20c183038c02dea7f4995d4eabb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domagoj=20Mi=C5=A1kovi=C4=87?= <157494086+allrealmsoflife@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:42:26 +0100 Subject: [PATCH 41/48] outdated building instructions, update documentation.md The current instructions for building the Nix manual include a command that doesn't work as described. Specifically: ``` nix build .#nix^doc ``` Running this command results in the error: ``` error: derivation '/nix/store/hddqxzfqgx2fhj8q66ss3idym7pk7aj1-nix-2.26.0pre20250107_383ab87.drv' does not have wanted outputs 'doc' ``` However, this command works if you specify the Nix version explicitly, such as: ``` nix build nix/2.24.11#nix^doc ``` Additionally, these commands are run within the Nix root directory. However, the nix build .#nix^doc command does work when run from the nixpkgs directory and generates the NixOS manual. I'm not sure if I'm missing something. Is the `nix^doc` supposed to be added somehow to flake outputs? The incremental build section does not work since as make has been decommissioned in favor of Meson. Should this be simply deleted? --- doc/manual/source/development/documentation.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/source/development/documentation.md b/doc/manual/source/development/documentation.md index 2e188f232..30cc8adc4 100644 --- a/doc/manual/source/development/documentation.md +++ b/doc/manual/source/development/documentation.md @@ -19,10 +19,11 @@ nix-build -E '(import ./.).packages.${builtins.currentSystem}.nix.doc' or ```console -nix build .#nix^doc +nix build .#nix-manual ``` -and open `./result-doc/share/doc/nix/manual/index.html`. +and open `./result/share/doc/nix/manual/index.html`. + To build the manual incrementally, [enter the development shell](./building.md) and run: From 5230d3ecc4cd3a3d965902a56b5a21bcc99821c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domagoj=20Mi=C5=A1kovi=C4=87?= <157494086+allrealmsoflife@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:20:44 +0100 Subject: [PATCH 42/48] Document `--max-freed` for `nix-collect-garbage` (#12155) * Update nix-collect-garbage.md Referencing issue at: https://github.com/NixOS/nix/issues/12132 Copied the description of `--max-freed` option from https://github.com/NixOS/nix/blob/442a2623e48357ff72c77bb11cf2cf06d94d2f90/doc/manual/source/command-ref/nix-store/gc.md?plain=1#L39-L44 --- doc/manual/source/command-ref/nix-collect-garbage.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/manual/source/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md index bd05f2816..763179b8e 100644 --- a/doc/manual/source/command-ref/nix-collect-garbage.md +++ b/doc/manual/source/command-ref/nix-collect-garbage.md @@ -62,6 +62,15 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-time) on each found profile. See the documentation of that command for additional information about the *period* argument. + - [`--max-freed`](#opt-max-freed) *bytes* + + + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. + {{#include ./opt-common.md}} {{#include ./env-common.md}} From 28caa35a97ed4837a62cb054f66cf02041292cc5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Jan 2025 18:38:53 +0100 Subject: [PATCH 43/48] parsePathFlakeRefWithFragment(): Handle 'path?query' without a fragment Commands like `nix flake metadata '.?submodules=1'` ignored the query part of the URL, while `nix build '.?submodules=1#foo'` did work correctly because of the presence of the fragment part. --- src/libflake/flake/flakeref.cc | 27 +++++++------------ tests/functional/flakes/flake-in-submodule.sh | 13 +++++++++ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 60efe1612..58cd45632 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -89,23 +89,16 @@ std::pair parsePathFlakeRefWithFragment( bool allowMissing, bool isFlake) { - std::string path = url; - std::string fragment = ""; - std::map query; - auto pathEnd = url.find_first_of("#?"); - auto fragmentStart = pathEnd; - if (pathEnd != std::string::npos && url[pathEnd] == '?') { - fragmentStart = url.find("#"); - } - if (pathEnd != std::string::npos) { - path = url.substr(0, pathEnd); - } - if (fragmentStart != std::string::npos) { - fragment = percentDecode(url.substr(fragmentStart+1)); - } - if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') { - query = decodeQuery(url.substr(pathEnd + 1, fragmentStart - pathEnd - 1)); - } + static std::regex pathFlakeRegex( + R"(([^?#]*)(\?([^#]*))?(#(.*))?)", + std::regex::ECMAScript); + + std::smatch match; + auto succeeds = std::regex_match(url, match, pathFlakeRegex); + assert(succeeds); + auto path = match[1].str(); + auto query = decodeQuery(match[3]); + auto fragment = percentDecode(match[5].str()); if (baseDir) { /* Check if 'url' is a path (either absolute or relative diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 817f77783..643d729ff 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -63,3 +63,16 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule echo '"foo"' > "$rootRepo"/submodule/sub.nix [[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] + +# Test that `nix flake metadata` parses `submodule` correctly. +cat > "$rootRepo"/flake.nix < Date: Wed, 8 Jan 2025 22:15:45 +0100 Subject: [PATCH 44/48] derivation-goal: unlock output lock to avoid deadlock guix discovered in their code base. Maybe we should do the same. --- src/libstore/build/derivation-goal.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 2ff0ef92f..4d97250d3 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -701,6 +701,7 @@ Goal::Co DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); + outputLocks.unlock(); co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } From 83ff523865f87f61b0b2046bd3bb2d793e09911d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 12:17:09 +0100 Subject: [PATCH 45/48] parsePathFlakeRefWithFragment(): Handle query params in the non-git case Backported from lazy-trees. --- src/libflake/flake/flakeref.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/libflake/flake/flakeref.cc b/src/libflake/flake/flakeref.cc index 58cd45632..cdf0a40e8 100644 --- a/src/libflake/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -183,11 +183,13 @@ std::pair parsePathFlakeRefWithFragment( path = canonPath(path + "/" + getOr(query, "dir", "")); } - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); + return fromParsedURL(fetchSettings, { + .scheme = "path", + .authority = "", + .path = path, + .query = query, + .fragment = fragment + }, isFlake); } /** From 5f7b535b819fb2ad5f7f05bf73fee7a2e7f7c9c8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 12:18:16 +0100 Subject: [PATCH 46/48] parsePathFlakeRefWithFragment(): Add unit tests --- src/libflake-tests/flakeref.cc | 50 ++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index d704a26d3..c7007a7af 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -7,18 +7,58 @@ namespace nix { /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ - /* ---------------------------------------------------------------------------- - * to_string - * --------------------------------------------------------------------------*/ + TEST(parseFlakeRef, path) { + fetchers::Settings fetchSettings; + + { + auto s = "/foo/bar"; + auto flakeref = parseFlakeRef(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar"); + } + + { + auto s = "/foo/bar?revCount=123&rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + auto flakeref = parseFlakeRef(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?rev=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa&revCount=123"); + } + + { + auto s = "/foo/bar?xyzzy=123"; + EXPECT_THROW( + parseFlakeRef(fetchSettings, s), + Error); + } + + { + auto s = "/foo/bar#bla"; + EXPECT_THROW( + parseFlakeRef(fetchSettings, s), + Error); + } + + { + auto s = "/foo/bar#bla"; + auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar"); + ASSERT_EQ(fragment, "bla"); + } + + { + auto s = "/foo/bar?revCount=123#bla"; + auto [flakeref, fragment] = parseFlakeRefWithFragment(fetchSettings, s); + ASSERT_EQ(flakeref.to_string(), "path:/foo/bar?revCount=123"); + ASSERT_EQ(fragment, "bla"); + } + } TEST(to_string, doesntReencodeUrl) { fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; auto flakeref = parseFlakeRef(fetchSettings, s); - auto parsed = flakeref.to_string(); + auto unparsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; - ASSERT_EQ(parsed, expected); + ASSERT_EQ(unparsed, expected); } } From 1a38e62a094f332776ab8b60bd3274e762f7b2cb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 16:38:33 +0100 Subject: [PATCH 47/48] Remove unused variable --- src/libutil/url.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 4d4d983b8..8fb1eecfb 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -22,7 +22,6 @@ ParsedURL parseURL(const std::string & url) std::smatch match; if (std::regex_match(url, match, uriRegex)) { - auto & base = match[1]; std::string scheme = match[2]; auto authority = match[3].matched ? std::optional(match[3]) : std::nullopt; From 3ad0f45e79bc1abc96a7c05d5979552e5f936d47 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 9 Jan 2025 16:42:37 +0100 Subject: [PATCH 48/48] Attempt to make the FlakeRef test succeed on macOS --- src/libflake-tests/flakeref.cc | 2 ++ src/libutil/config.hh | 1 + 2 files changed, 3 insertions(+) diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index c7007a7af..2b1f5124b 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -8,6 +8,8 @@ namespace nix { /* ----------- tests for flake/flakeref.hh --------------------------------------------------*/ TEST(parseFlakeRef, path) { + experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); + fetchers::Settings fetchSettings; { diff --git a/src/libutil/config.hh b/src/libutil/config.hh index e98e09bf7..502d2823e 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -262,6 +262,7 @@ public: operator const T &() const { return value; } operator T &() { return value; } const T & get() const { return value; } + T & get() { return value; } template bool operator ==(const U & v2) const { return value == v2; } template