From d9abce4ad4b6888183271c0a4051981dee5fffe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Mon, 12 Sep 2022 22:50:18 +0200 Subject: [PATCH 1/5] libfetchers: avoid api.github.com ratelimit if no github token is set If we don't have any github token, we won't be able to fetch private repos, but we are also more likely to run into API limits since we don't have a token. To mitigate this only ever use the github api if we actually have a token. --- src/libfetchers/github.cc | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index a491d82a6..2115ce2f5 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -262,17 +262,20 @@ struct GitHubInputScheme : GitArchiveInputScheme DownloadUrl getDownloadUrl(const Input & input) const override { - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com"); - auto url = fmt( - host == "github.com" - ? "https://api.%s/repos/%s/%s/tarball/%s" - : "https://%s/api/v3/repos/%s/%s/tarball/%s", - host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), + Headers headers = makeHeadersWithAuthTokens(host); + // If we have no auth headers then we default to the public archive + // urls so we do not run into rate limits. + const auto urlFmt = + host != "github.com" + ? "https://%s/api/v3/repos/%s/%s/tarball/%s" + : headers.empty() + ? "https://%s/%s/%s/archive/%s.tar.gz" + : "https://api.%s/repos/%s/%s/tarball/%s"; + + const auto url = fmt(urlFmt, host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); return DownloadUrl { url, headers }; } From cd86eeb693342c79b01ff38ca723088dc1e42291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Fri, 28 Oct 2022 12:19:37 +0200 Subject: [PATCH 2/5] Move savedArgv into libmain `savedArgv` is not accessible by plugins when defined in main binary. Moving it into one of the nix lib fix the problem. --- src/libmain/shared.cc | 1 + src/nix/main.cc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index c1cf38565..a58428762 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -33,6 +33,7 @@ namespace nix { +char * * savedArgv; static bool gcWarning = true; diff --git a/src/nix/main.cc b/src/nix/main.cc index d78312944..956730276 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -53,7 +53,6 @@ static bool haveInternet() } std::string programPath; -char * * savedArgv; struct HelpRequested { }; From 9f1dd0df5b54a7dc75b618034482ed42ce34383d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= Date: Sat, 29 Oct 2022 21:51:29 +0200 Subject: [PATCH 3/5] Update test after api.github.com ratelimit avoidance --- tests/github-flakes.nix | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index fc481c7e3..1b60a9f9a 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -7,7 +7,7 @@ with import (nixpkgs + "/nixos/lib/testing-python.nix") { let - # Generate a fake root CA and a fake api.github.com / channels.nixos.org certificate. + # Generate a fake root CA and a fake api.github.com / github.com / channels.nixos.org certificate. cert = pkgs.runCommand "cert" { buildInputs = [ pkgs.openssl ]; } '' mkdir -p $out @@ -18,7 +18,7 @@ let openssl req -newkey rsa:2048 -nodes -keyout $out/server.key \ -subj "/C=CN/ST=Denial/L=Springfield/O=Dis/CN=github.com" -out server.csr - openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:channels.nixos.org") \ + openssl x509 -req -extfile <(printf "subjectAltName=DNS:api.github.com,DNS:github.com,DNS:channels.nixos.org") \ -days 36500 -in server.csr -CA $out/ca.crt -CAkey ca.key -CAcreateserial -out $out/server.crt ''; @@ -47,18 +47,20 @@ let api = pkgs.runCommand "nixpkgs-flake" {} '' - mkdir -p $out/tarball + mkdir -p $out/commits + echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD + ''; + + archive = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir -p $out/archive dir=NixOS-nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- - tar cfz $out/tarball/${nixpkgs.rev} $dir --hard-dereference - - mkdir -p $out/commits - echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD + tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference ''; - in makeTest ( @@ -97,6 +99,16 @@ makeTest ( } ]; }; + services.httpd.virtualHosts."github.com" = + { forceSSL = true; + sslServerKey = "${cert}/server.key"; + sslServerCert = "${cert}/server.crt"; + servedDirs = + [ { urlPath = "/NixOS/nixpkgs"; + dir = archive; + } + ]; + }; }; client = @@ -109,7 +121,7 @@ makeTest ( nix.extraOptions = "experimental-features = nix-command flakes"; environment.systemPackages = [ pkgs.jq ]; networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = - [ "channels.nixos.org" "api.github.com" ]; + [ "channels.nixos.org" "api.github.com" "github.com" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; }; }; @@ -123,7 +135,7 @@ makeTest ( github.wait_for_unit("httpd.service") - client.succeed("curl -v https://api.github.com/ >&2") + client.succeed("curl -v https://github.com/ >&2") client.succeed("nix registry list | grep nixpkgs") rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") From e00761af73f68fa7e6b833b4800a17e93f715097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Oct 2022 07:10:58 +0100 Subject: [PATCH 4/5] Also test github flakes when access tokens are provided --- tests/github-flakes.nix | 66 +++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 9 deletions(-) diff --git a/tests/github-flakes.nix b/tests/github-flakes.nix index 1b60a9f9a..43a4f1432 100644 --- a/tests/github-flakes.nix +++ b/tests/github-flakes.nix @@ -37,6 +37,17 @@ let "owner": "NixOS", "repo": "nixpkgs" } + }, + { + "from": { + "type": "indirect", + "id": "private-flake" + }, + "to": { + "type": "github", + "owner": "fancy-enterprise", + "repo": "private-flake" + } } ], "version": 2 @@ -45,9 +56,27 @@ let destination = "/flake-registry.json"; }; - api = pkgs.runCommand "nixpkgs-flake" {} + private-flake-rev = "9f1dd0df5b54a7dc75b618034482ed42ce34383d"; + + private-flake-api = pkgs.runCommand "private-flake" {} + '' + mkdir -p $out/{commits,tarball} + + # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit + echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD + + # Setup tarball download via API + dir=private-flake + mkdir $dir + echo '{ outputs = {...}: {}; }' > $dir/flake.nix + tar cfz $out/tarball/${private-flake-rev} $dir --hard-dereference + ''; + + nixpkgs-api = pkgs.runCommand "nixpkgs-flake" {} '' mkdir -p $out/commits + + # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD ''; @@ -95,7 +124,10 @@ makeTest ( sslServerCert = "${cert}/server.crt"; servedDirs = [ { urlPath = "/repos/NixOS/nixpkgs"; - dir = api; + dir = nixpkgs-api; + } + { urlPath = "/repos/fancy-enterprise/private-flake"; + dir = private-flake-api; } ]; }; @@ -119,7 +151,6 @@ makeTest ( virtualisation.memorySize = 4096; nix.binaryCaches = lib.mkForce [ ]; nix.extraOptions = "experimental-features = nix-command flakes"; - environment.systemPackages = [ pkgs.jq ]; networking.hosts.${(builtins.head nodes.github.config.networking.interfaces.eth1.ipv4.addresses).address} = [ "channels.nixos.org" "api.github.com" "github.com" ]; security.pki.certificateFiles = [ "${cert}/ca.crt" ]; @@ -133,22 +164,39 @@ makeTest ( start_all() + def cat_log(): + github.succeed("cat /var/log/httpd/*.log >&2") + github.wait_for_unit("httpd.service") client.succeed("curl -v https://github.com/ >&2") - client.succeed("nix registry list | grep nixpkgs") + out = client.succeed("nix registry list") + print(out) + assert "github:NixOS/nixpkgs" in out, "nixpkgs flake not found" + assert "github:fancy-enterprise/private-flake" in out, "private flake not found" + cat_log() - rev = client.succeed("nix flake info nixpkgs --json | jq -r .revision") - assert rev.strip() == "${nixpkgs.rev}", "revision mismatch" + # If no github access token is provided, nix should use the public archive url... + out = client.succeed("nix flake metadata nixpkgs --json") + print(out) + info = json.loads(out) + assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}" + cat_log() + + # ... otherwise it should use the API + out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0") + print(out) + info = json.loads(out) + assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}" + cat_log() client.succeed("nix registry pin nixpkgs") - - client.succeed("nix flake info nixpkgs --tarball-ttl 0 >&2") + client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") - info = json.loads(client.succeed("nix flake info nixpkgs --json")) + info = json.loads(client.succeed("nix flake metadata nixpkgs --json")) date = time.strftime("%Y%m%d%H%M%S", time.gmtime(info['lastModified'])) assert date == "${nixpkgs.lastModifiedDate}", "time mismatch" From 06a6a7959eaa859dab591bb5386031c0d3d1b803 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 22:08:02 +0000 Subject: [PATCH 5/5] Bump cachix/cachix-action from 11 to 12 Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 11 to 12. - [Release notes](https://github.com/cachix/cachix-action/releases) - [Commits](https://github.com/cachix/cachix-action/compare/v11...v12) --- updated-dependencies: - dependency-name: cachix/cachix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7efb90913..dafba6d85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - uses: cachix/install-nix-action@v18 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v11 + - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -59,7 +59,7 @@ jobs: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v18 - - uses: cachix/cachix-action@v11 + - uses: cachix/cachix-action@v12 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -105,7 +105,7 @@ jobs: - uses: cachix/install-nix-action@v18 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v11 + - uses: cachix/cachix-action@v12 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}'