diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a917188d9..8fc4f14f2 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -33,6 +33,9 @@ using namespace std::string_literals; namespace nix { +const unsigned int RETRY_TIME_MS_DEFAULT = 250; +const unsigned int RETRY_TIME_MS_TOO_MANY_REQUESTS = 60000; + FileTransferSettings fileTransferSettings; static GlobalConfig::Register rFileTransferSettings(&fileTransferSettings); @@ -399,6 +402,8 @@ struct curlFileTransfer : public FileTransfer { auto finishTime = std::chrono::steady_clock::now(); + auto retryTimeMs = request.baseRetryTimeMs; + auto httpStatus = getHTTPStatus(); debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", @@ -449,10 +454,12 @@ struct curlFileTransfer : public FileTransfer } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { // Don't retry on authentication/authorization failures err = Forbidden; - } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { + } else if (httpStatus == 429) { + // 429 means too many requests, so we retry (with a substantially longer delay) + retryTimeMs = RETRY_TIME_MS_TOO_MANY_REQUESTS; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408) { // Most 4xx errors are client errors and are probably not worth retrying: // * 408 means the server timed out waiting for us, so we try again - // * 429 means too many requests, so we retry (with a delay) err = Misc; } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { // Let's treat most 5xx (server) errors as transient, except for a handful: @@ -518,7 +525,7 @@ struct curlFileTransfer : public FileTransfer || writtenToSink == 0 || (acceptRanges && encoding.empty()))) { - int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); + int ms = retryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937)); if (writtenToSink) warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); else diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index f9b1f620f..10c3ec7ef 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -58,6 +58,8 @@ struct FileTransferSettings : Config extern FileTransferSettings fileTransferSettings; +extern const unsigned int RETRY_TIME_MS_DEFAULT; + struct FileTransferRequest { std::string uri; @@ -67,7 +69,7 @@ struct FileTransferRequest bool head = false; bool post = false; size_t tries = fileTransferSettings.tries; - unsigned int baseRetryTimeMs = 250; + unsigned int baseRetryTimeMs = RETRY_TIME_MS_DEFAULT; ActivityId parentAct; bool decompress = true; std::optional data;