From a48632f2e00ccab40c94d498b10591beb76ec6bf Mon Sep 17 00:00:00 2001 From: Justin Bailey Date: Fri, 11 Jul 2025 17:30:03 -0700 Subject: [PATCH 1/3] Better Handling for Expired Credentials When AWS credentials expired, in some scenarios they led to the nix process aborting with an error similar to ' Unable to parse ExceptionName: ExpiredToken'. This change updates the S3 handling code such that those errors are treated like 403s or 404s. Closes #13459 --- src/libstore/s3-binary-cache-store.cc | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 618112d1c..b1cba3358 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -37,10 +37,11 @@ namespace nix { struct S3Error : public Error { Aws::S3::S3Errors err; + Aws::String exceptionName; template - S3Error(Aws::S3::S3Errors err, const Args & ... args) - : Error(args...), err(err) { }; + S3Error(Aws::S3::S3Errors err, Aws::String exceptionName, const Args & ... args) + : Error(args...), err(err), exceptionName(exceptionName) { }; }; /* Helper: given an Outcome, return R in case of success, or @@ -51,6 +52,7 @@ R && checkAws(std::string_view s, Aws::Utils::Outcome && outcome) if (!outcome.IsSuccess()) throw S3Error( outcome.GetError().GetErrorType(), + outcome.GetError().GetExceptionName(), fmt( "%s: %s (request id: %s)", s, @@ -226,7 +228,13 @@ S3Helper::FileTransferResult S3Helper::getObject( } catch (S3Error & e) { if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) && - (e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw; + (e.err != Aws::S3::S3Errors::ACCESS_DENIED) && + // Expired tokens are not really an error, more of a caching problem. Should be treated same as 403. + // + // AWS unwilling to provide a specific error type for the situation (https://github.com/aws/aws-sdk-cpp/issues/1843) + // so use this hack + (e.exceptionName != "ExpiredToken") + ) throw; } auto now2 = std::chrono::steady_clock::now(); @@ -325,15 +333,22 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { stats.head++; + // error: AWS error fetching 'vjgpmfn7s6vkynymnk8jfx2fcxnsbd6b.narinfo': Unable to parse ExceptionName: ExpiredToken Message: The provided token has expired. auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() .WithBucket(config->bucketName) .WithKey(path)); + printError("Checking for file"); + if (!res.IsSuccess()) { auto & error = res.GetError(); if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY + // Expired tokens are not really an error, more of a caching problem. Should be treated same as 403. + // AWS unwilling to provide a specific error type for the situation (https://github.com/aws/aws-sdk-cpp/issues/1843) + // so use this hack + || (error.GetErrorType() == Aws::S3::S3Errors::UNKNOWN && error.GetExceptionName() == "ExpiredToken") // If bucket listing is disabled, 404s turn into 403s || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) return false; From 22d6969d669d492d4948967f76432c1c6a74efd5 Mon Sep 17 00:00:00 2001 From: m4dc4p Date: Sat, 12 Jul 2025 08:05:52 -0700 Subject: [PATCH 2/3] Update src/libstore/s3-binary-cache-store.cc Co-authored-by: Eelco Dolstra --- src/libstore/s3-binary-cache-store.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index b1cba3358..f3d029c5e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -333,7 +333,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { stats.head++; - // error: AWS error fetching 'vjgpmfn7s6vkynymnk8jfx2fcxnsbd6b.narinfo': Unable to parse ExceptionName: ExpiredToken Message: The provided token has expired. auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() .WithBucket(config->bucketName) From f786c0b8d1ae75a4ecdd83332ec1f8d6da45b0f3 Mon Sep 17 00:00:00 2001 From: m4dc4p Date: Sat, 12 Jul 2025 08:06:09 -0700 Subject: [PATCH 3/3] Update src/libstore/s3-binary-cache-store.cc Co-authored-by: Eelco Dolstra --- src/libstore/s3-binary-cache-store.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index f3d029c5e..cbb47c063 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -338,8 +338,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore .WithBucket(config->bucketName) .WithKey(path)); - printError("Checking for file"); - if (!res.IsSuccess()) { auto & error = res.GetError(); if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND