From 69c7b42d2850d947eb818638029abefe83e4cb95 Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Wed, 12 Feb 2025 15:46:28 -0500 Subject: [PATCH 1/6] feat: access tokens per repo --- src/libfetchers/github.cc | 43 +++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ec469df7c..87b7c0ad3 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -172,9 +172,22 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const { auto tokens = settings.accessTokens.get(); + std::string answer; + size_t answer_match_len = 0; + if(! url.empty()) { + for (auto & token : tokens) { + auto match_len = url.find(token.first); + if (match_len != std::string::npos && token.first.length() > answer_match_len) { + answer = token.second; + answer_match_len = token.first.length(); + } + } + if (!answer.empty()) + return answer; + } if (auto token = get(tokens, host)) return *token; return {}; @@ -182,10 +195,22 @@ struct GitArchiveInputScheme : InputScheme Headers makeHeadersWithAuthTokens( const fetchers::Settings & settings, - const std::string & host) const + const std::string & host, + const Input & input) const + { + auto owner = getStrAttr(input.attrs, "owner"); + auto repo = getStrAttr(input.attrs, "repo"); + auto urlGen = fmt( "%s/%s/%s", host, owner, repo); + return makeHeadersWithAuthTokens(settings, host, urlGen); + } + + Headers makeHeadersWithAuthTokens( + const fetchers::Settings & settings, + const std::string & host, + const std::string & url) const { Headers headers; - auto accessToken = getAccessToken(settings, host); + auto accessToken = getAccessToken(settings, host, url); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -366,7 +391,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getOwner(input), getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(*input.settings, host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); auto json = nlohmann::json::parse( readFile( @@ -383,7 +408,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(*input.settings, host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -440,7 +465,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(*input.settings, host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); auto json = nlohmann::json::parse( readFile( @@ -470,7 +495,7 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(*input.settings, host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); return DownloadUrl { url, headers }; } @@ -510,7 +535,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(*input.settings, host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); std::string refUri; if (ref == "HEAD") { @@ -557,7 +582,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(*input.settings, host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); return DownloadUrl { url, headers }; } From a9f4d73d3ed82e91268af305e27f9281909a7826 Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Thu, 13 Feb 2025 06:05:54 -0500 Subject: [PATCH 2/6] feat: test and document access-token prefix support --- src/libfetchers-tests/access-tokens.cc | 79 ++++++++++++++++++++++++++ src/libfetchers-tests/meson.build | 1 + src/libfetchers/fetch-settings.hh | 8 ++- src/libfetchers/fetchers.hh | 3 + src/libfetchers/github.cc | 2 +- 5 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 src/libfetchers-tests/access-tokens.cc diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc new file mode 100644 index 000000000..02ca082d4 --- /dev/null +++ b/src/libfetchers-tests/access-tokens.cc @@ -0,0 +1,79 @@ +#include +#include "fetchers.hh" +#include "fetch-settings.hh" +#include "json-utils.hh" +#include +#include "tests/characterization.hh" + +namespace nix::fetchers { + +using nlohmann::json; + +class AccessKeysTest : public ::testing::Test +{ +protected: + +public: + void SetUp() override { + experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); + } + void TearDown() override { } +}; + +TEST_F(AccessKeysTest, singleGitHub) +{ + fetchers::Settings fetchSettings = fetchers::Settings{}; + fetchSettings.accessTokens.get().insert({"github.com","token"}); + auto i = Input::fromURL(fetchSettings, "github:a/b"); + + auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); + ASSERT_EQ(token,"token"); +} + +TEST_F(AccessKeysTest, repoGitHub) +{ + fetchers::Settings fetchSettings = fetchers::Settings{}; + fetchSettings.accessTokens.get().insert({"github.com","token"}); + fetchSettings.accessTokens.get().insert({"github.com/a/b","another_token"}); + fetchSettings.accessTokens.get().insert({"github.com/a/c","yet_another_token"}); + auto i = Input::fromURL(fetchSettings, "github:a/a"); + + auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/a"); + ASSERT_EQ(token,"token"); + + token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); + ASSERT_EQ(token,"another_token"); + + token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/c"); + ASSERT_EQ(token,"yet_another_token"); +} + +TEST_F(AccessKeysTest, multipleGitLab) +{ + fetchers::Settings fetchSettings = fetchers::Settings{}; + fetchSettings.accessTokens.get().insert({"gitlab.com","token"}); + fetchSettings.accessTokens.get().insert({"gitlab.com/a/b","another_token"}); + auto i = Input::fromURL(fetchSettings, "gitlab:a/b"); + + auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/b"); + ASSERT_EQ(token,"another_token"); + + token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/c"); + ASSERT_EQ(token,"token"); +} + +TEST_F(AccessKeysTest, multipleSourceHut) +{ + fetchers::Settings fetchSettings = fetchers::Settings{}; + fetchSettings.accessTokens.get().insert({"git.sr.ht","token"}); + fetchSettings.accessTokens.get().insert({"git.sr.ht/~a/b","another_token"}); + auto i = Input::fromURL(fetchSettings, "sourcehut:a/b"); + + auto token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/b"); + ASSERT_EQ(token,"another_token"); + + token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/c"); + ASSERT_EQ(token,"token"); +} + +} diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index 739435501..243bbab80 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -44,6 +44,7 @@ subdir('nix-meson-build-support/common') sources = files( 'public-key.cc', + 'access-tokens.cc', ) include_dirs = [include_directories('.')] diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 2ad8aa327..c6c3ca7a7 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -23,9 +23,11 @@ struct Settings : public Config Access tokens are specified as a string made up of space-separated `host=token` values. The specific token used is selected by matching the `host` portion against the - "host" specification of the input. The actual use of the - `token` value is determined by the type of resource being - accessed: + "host" specification of the input. The `host` portion may + contain a path element which will match against the prefix + URL for the input. (eg: `github.com/org=token`). The actual use + of the `token` value is determined by the type of resource + being accessed: * Github: the token value is the OAUTH-TOKEN string obtained as the Personal Access Token from the Github server (see diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 37de1f507..01354a6e3 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -264,6 +264,9 @@ struct InputScheme virtual std::optional isRelative(const Input & input) const { return std::nullopt; } + + virtual std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const + { return {};} }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 87b7c0ad3..c34ed844b 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -172,7 +172,7 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const override { auto tokens = settings.accessTokens.get(); std::string answer; From 269efa01b31432995f2e8700990bad410ffae77d Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Thu, 13 Feb 2025 12:45:37 -0500 Subject: [PATCH 3/6] fix: ensure access-token matches are complete --- src/libfetchers-tests/access-tokens.cc | 24 ++++++++++++++++++++++-- src/libfetchers/github.cc | 11 +++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 02ca082d4..43f623970 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -20,16 +20,36 @@ public: void TearDown() override { } }; -TEST_F(AccessKeysTest, singleGitHub) +TEST_F(AccessKeysTest, singleOrgGitHub) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"github.com","token"}); + fetchSettings.accessTokens.get().insert({"github.com/a","token"}); auto i = Input::fromURL(fetchSettings, "github:a/b"); auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); ASSERT_EQ(token,"token"); } +TEST_F(AccessKeysTest, nonMatches) +{ + fetchers::Settings fetchSettings = fetchers::Settings{}; + fetchSettings.accessTokens.get().insert({"github.com","token"}); + auto i = Input::fromURL(fetchSettings, "gitlab:github.com/evil"); + + auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/github.com/evil"); + ASSERT_EQ(token,std::nullopt); +} + +TEST_F(AccessKeysTest, noPartialMatches) +{ + fetchers::Settings fetchSettings = fetchers::Settings{}; + fetchSettings.accessTokens.get().insert({"github.com/partial","token"}); + auto i = Input::fromURL(fetchSettings, "github:partial-match/repo"); + + auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/partial-match"); + ASSERT_EQ(token,std::nullopt); +} + TEST_F(AccessKeysTest, repoGitHub) { fetchers::Settings fetchSettings = fetchers::Settings{}; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index c34ed844b..3c8a587c2 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -179,8 +179,15 @@ struct GitArchiveInputScheme : InputScheme size_t answer_match_len = 0; if(! url.empty()) { for (auto & token : tokens) { - auto match_len = url.find(token.first); - if (match_len != std::string::npos && token.first.length() > answer_match_len) { + auto first = url.find(token.first); + if ( + first != std::string::npos + && token.first.length() > answer_match_len + && first == 0 + && url.substr(0,token.first.length()) == token.first + && (url.length() == token.first.length() || url[token.first.length()] == '/') + ) + { answer = token.second; answer_match_len = token.first.length(); } From 753f00c351108ee8a91d068da9d1943a67c063ec Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Thu, 13 Feb 2025 12:47:09 -0500 Subject: [PATCH 4/6] fix: add comment about longest-possible match --- src/libfetchers/github.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 3c8a587c2..8747498be 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -172,6 +172,7 @@ struct GitArchiveInputScheme : InputScheme return input; } + // Search for the longest possible match starting from the begining and ending at either the end or a path segment. std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const override { auto tokens = settings.accessTokens.get(); From 3b5514e0c6605f057dbf945ebe83ce2b60d39230 Mon Sep 17 00:00:00 2001 From: Thomas Bereknyei Date: Thu, 13 Feb 2025 13:04:38 -0500 Subject: [PATCH 5/6] fix: linting --- src/libfetchers-tests/access-tokens.cc | 47 +++++++++++++------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 43f623970..5f4ceedaa 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -14,86 +14,87 @@ class AccessKeysTest : public ::testing::Test protected: public: - void SetUp() override { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); + void SetUp() override + { + experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); } - void TearDown() override { } + void TearDown() override {} }; TEST_F(AccessKeysTest, singleOrgGitHub) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"github.com/a","token"}); + fetchSettings.accessTokens.get().insert({"github.com/a", "token"}); auto i = Input::fromURL(fetchSettings, "github:a/b"); auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); - ASSERT_EQ(token,"token"); + ASSERT_EQ(token, "token"); } TEST_F(AccessKeysTest, nonMatches) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"github.com","token"}); + fetchSettings.accessTokens.get().insert({"github.com", "token"}); auto i = Input::fromURL(fetchSettings, "gitlab:github.com/evil"); auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/github.com/evil"); - ASSERT_EQ(token,std::nullopt); + ASSERT_EQ(token, std::nullopt); } TEST_F(AccessKeysTest, noPartialMatches) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"github.com/partial","token"}); + fetchSettings.accessTokens.get().insert({"github.com/partial", "token"}); auto i = Input::fromURL(fetchSettings, "github:partial-match/repo"); auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/partial-match"); - ASSERT_EQ(token,std::nullopt); + ASSERT_EQ(token, std::nullopt); } TEST_F(AccessKeysTest, repoGitHub) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"github.com","token"}); - fetchSettings.accessTokens.get().insert({"github.com/a/b","another_token"}); - fetchSettings.accessTokens.get().insert({"github.com/a/c","yet_another_token"}); + fetchSettings.accessTokens.get().insert({"github.com", "token"}); + fetchSettings.accessTokens.get().insert({"github.com/a/b", "another_token"}); + fetchSettings.accessTokens.get().insert({"github.com/a/c", "yet_another_token"}); auto i = Input::fromURL(fetchSettings, "github:a/a"); auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/a"); - ASSERT_EQ(token,"token"); + ASSERT_EQ(token, "token"); token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b"); - ASSERT_EQ(token,"another_token"); + ASSERT_EQ(token, "another_token"); token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/c"); - ASSERT_EQ(token,"yet_another_token"); + ASSERT_EQ(token, "yet_another_token"); } TEST_F(AccessKeysTest, multipleGitLab) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"gitlab.com","token"}); - fetchSettings.accessTokens.get().insert({"gitlab.com/a/b","another_token"}); + fetchSettings.accessTokens.get().insert({"gitlab.com", "token"}); + fetchSettings.accessTokens.get().insert({"gitlab.com/a/b", "another_token"}); auto i = Input::fromURL(fetchSettings, "gitlab:a/b"); auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/b"); - ASSERT_EQ(token,"another_token"); + ASSERT_EQ(token, "another_token"); token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/c"); - ASSERT_EQ(token,"token"); + ASSERT_EQ(token, "token"); } TEST_F(AccessKeysTest, multipleSourceHut) { fetchers::Settings fetchSettings = fetchers::Settings{}; - fetchSettings.accessTokens.get().insert({"git.sr.ht","token"}); - fetchSettings.accessTokens.get().insert({"git.sr.ht/~a/b","another_token"}); + fetchSettings.accessTokens.get().insert({"git.sr.ht", "token"}); + fetchSettings.accessTokens.get().insert({"git.sr.ht/~a/b", "another_token"}); auto i = Input::fromURL(fetchSettings, "sourcehut:a/b"); auto token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/b"); - ASSERT_EQ(token,"another_token"); + ASSERT_EQ(token, "another_token"); token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/c"); - ASSERT_EQ(token,"token"); + ASSERT_EQ(token, "token"); } } From c07172220cc105a66555cda3be1a46929a913fc5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 25 Feb 2025 15:10:16 +0100 Subject: [PATCH 6/6] refact: Rename url -> hostAndPath https://github.com/NixOS/nix/pull/12465/files#r1955286197 > Perhaps that is a misnomer. --- src/libfetchers/github.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8747498be..9cddd8571 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -208,17 +208,17 @@ struct GitArchiveInputScheme : InputScheme { auto owner = getStrAttr(input.attrs, "owner"); auto repo = getStrAttr(input.attrs, "repo"); - auto urlGen = fmt( "%s/%s/%s", host, owner, repo); - return makeHeadersWithAuthTokens(settings, host, urlGen); + auto hostAndPath = fmt( "%s/%s/%s", host, owner, repo); + return makeHeadersWithAuthTokens(settings, host, hostAndPath); } Headers makeHeadersWithAuthTokens( const fetchers::Settings & settings, const std::string & host, - const std::string & url) const + const std::string & hostAndPath) const { Headers headers; - auto accessToken = getAccessToken(settings, host, url); + auto accessToken = getAccessToken(settings, host, hostAndPath); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr)