From 3d5bdaeb77a2ef2e0d88c0c00fd0bd25a11e6a60 Mon Sep 17 00:00:00 2001 From: Alex Dale Date: Fri, 18 Apr 2025 12:24:53 -0700 Subject: [PATCH] Retry provisioning and license requests on 502 errors [ Merge of http://go/wvgerrit/219370 ] Work around the server problem. Bug: 345333545 Change-Id: Ie1b0021f545c77d61bb2363494239f4fe000f3d5 --- .../cdm/core/test/license_holder.cpp | 6 +- .../cdm/core/test/provisioning_holder.cpp | 2 +- libwvdrmengine/cdm/core/test/url_request.cpp | 85 ++++++++++++------- libwvdrmengine/cdm/core/test/url_request.h | 5 +- 4 files changed, 64 insertions(+), 34 deletions(-) diff --git a/libwvdrmengine/cdm/core/test/license_holder.cpp b/libwvdrmengine/cdm/core/test/license_holder.cpp index de131c62..d3748c88 100644 --- a/libwvdrmengine/cdm/core/test/license_holder.cpp +++ b/libwvdrmengine/cdm/core/test/license_holder.cpp @@ -103,7 +103,7 @@ void LicenseHolder::GenerateAndPostRenewalRequest( void LicenseHolder::FetchRenewal() { ASSERT_NE(request_in_flight_, nullptr) << "Failed for " << content_id(); ASSERT_NO_FATAL_FAILURE( - request_in_flight_->AssertOkResponse(&request_response_)) + request_in_flight_->AssertOkResponseWithRetry(&request_response_)) << "Renewal failed for " << content_id(); } @@ -162,7 +162,7 @@ void LicenseHolder::GenerateAndPostReleaseRequest( void LicenseHolder::FetchRelease() { ASSERT_NE(request_in_flight_, nullptr) << "Failed for " << content_id(); ASSERT_NO_FATAL_FAILURE( - request_in_flight_->AssertOkResponse(&request_response_)) + request_in_flight_->AssertOkResponseWithRetry(&request_response_)) << "Renewal failed for " << content_id(); } @@ -310,7 +310,7 @@ void LicenseHolder::GetKeyResponse(const CdmKeyRequest& key_request) { std::string http_response; url_request.PostRequest(key_request.message); - ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponse(&http_response)) + ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponseWithRetry(&http_response)) << "Failed for " << content_id(); LicenseRequest license_request; license_request.GetDrmMessage(http_response, key_response_); diff --git a/libwvdrmengine/cdm/core/test/provisioning_holder.cpp b/libwvdrmengine/cdm/core/test/provisioning_holder.cpp index 697beb97..39074b06 100644 --- a/libwvdrmengine/cdm/core/test/provisioning_holder.cpp +++ b/libwvdrmengine/cdm/core/test/provisioning_holder.cpp @@ -69,7 +69,7 @@ void ProvisioningHolder::Provision(CdmCertificateType cert_type, url_request.PostCertRequestInQueryString(request); // Receive and parse response. - ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponse(&response_)) + ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponseWithRetry(&response_)) << "Failed to fetch provisioning response. " << DumpProvAttempt(request, response_, cert_type); diff --git a/libwvdrmengine/cdm/core/test/url_request.cpp b/libwvdrmengine/cdm/core/test/url_request.cpp index b7f4a7c1..0c37e3b7 100644 --- a/libwvdrmengine/cdm/core/test/url_request.cpp +++ b/libwvdrmengine/cdm/core/test/url_request.cpp @@ -5,9 +5,12 @@ #include "url_request.h" #include +#include + +#include +#include #include -#include #include "http_socket.h" #include "log.h" @@ -23,11 +26,15 @@ const int kConnectTimeoutMs = 15000; const int kWriteTimeoutMs = 12000; const int kReadTimeoutMs = 12000; constexpr int kHttpOk = 200; +const std::vector kRetryCodes = {502, 504}; const std::string kGoogleHeaderUpper("X-Google"); const std::string kGoogleHeaderLower("x-google"); const std::string kCrLf("\r\n"); +constexpr unsigned kRetryCount = 3; +constexpr unsigned kRetryIntervalSeconds = 1; + // Concatenate all chunks into one blob and returns the response with // header information. void ConcatenateChunkedResponse(const std::string http_response, @@ -126,13 +133,34 @@ bool UrlRequest::GetResponse(std::string* message) { return true; } -void UrlRequest::AssertOkResponse(std::string* message) { +void UrlRequest::AssertOkResponseWithRetry(std::string* message) { ASSERT_TRUE(message); - ASSERT_TRUE(GetResponse(message)); - const int status_code = GetStatusCode(*message); - ASSERT_EQ(kHttpOk, status_code) << "HTTP response from " << socket_.url() - << ": (" << message->size() << ") :\n" - << *message; + int status_code = 0; + for (unsigned i = 0; i < kRetryCount; i++) { + *message = ""; + ASSERT_TRUE(GetResponse(message)) << "For attempt " << (i + 1); + status_code = GetStatusCode(*message); + // If we didn't get a retry status, then we're done. + if (std::find(kRetryCodes.begin(), kRetryCodes.end(), status_code) == + kRetryCodes.end()) { + ASSERT_EQ(kHttpOk, status_code) << "HTTP response from " << socket_.url() + << ": (" << message->size() << ") :\n" + << *message; + return; + } + std::cerr << "Temporary failure HTTP response from " << socket_.url() + << ": (" << message->size() << ") :\n" + << *message << "\n" + << "Attempt " << (i + 1) << "\n"; + socket_.CloseSocket(); + is_connected_ = false; + sleep(kRetryIntervalSeconds << i); + Reconnect(); + SendRequestOnce(); + } + GTEST_FAIL() << "HTTP response from " << socket_.url() << ": (" + << message->size() << ") :\n" + << *message; } // static @@ -189,36 +217,35 @@ bool UrlRequest::GetDebugHeaderFields( bool UrlRequest::PostRequestWithPath(const std::string& path, const std::string& data) { - std::string request; + request_.clear(); - request.append("POST "); - request.append(path); - request.append(" HTTP/1.1\r\n"); + request_.append("POST "); + request_.append(path); + request_.append(" HTTP/1.1\r\n"); - request.append("Host: "); - request.append(socket_.domain_name()); - request.append("\r\n"); + request_.append("Host: "); + request_.append(socket_.domain_name()); + request_.append("\r\n"); - request.append("Connection: close\r\n"); - request.append("User-Agent: Widevine CDM v1.0\r\n"); - request.append("X-Return-Encrypted-Headers: request_and_response\r\n"); + request_.append("Connection: close\r\n"); + request_.append("User-Agent: Widevine CDM v1.0\r\n"); + request_.append("X-Return-Encrypted-Headers: request_and_response\r\n"); - // buffer to store length of data as a string - char data_size_buffer[32] = {0}; - snprintf(data_size_buffer, sizeof(data_size_buffer), "%zu", data.size()); + request_.append("Content-Length: "); + request_.append(std::to_string(data.size())); + request_.append("\r\n"); - request.append("Content-Length: "); - request.append(data_size_buffer); // appends size of data - request.append("\r\n"); + request_.append("\r\n"); // empty line to terminate headers - request.append("\r\n"); // empty line to terminate headers - - request.append(data); + request_.append(data); + return SendRequestOnce(); +} +bool UrlRequest::SendRequestOnce() { const int ret = socket_.WriteAndLogErrors( - request.c_str(), static_cast(request.size()), kWriteTimeoutMs); - LOGV("HTTP request: (%zu): %s", request.size(), request.c_str()); - LOGV("HTTP request hex: %s", wvutil::b2a_hex(request).c_str()); + request_.c_str(), static_cast(request_.size()), kWriteTimeoutMs); + LOGV("HTTP request: (%zu): %s", request_.size(), request_.c_str()); + LOGV("HTTP request hex: %s", wvutil::b2a_hex(request_).c_str()); return ret != -1; } diff --git a/libwvdrmengine/cdm/core/test/url_request.h b/libwvdrmengine/cdm/core/test/url_request.h index c2d0d35e..68dc593e 100644 --- a/libwvdrmengine/cdm/core/test/url_request.h +++ b/libwvdrmengine/cdm/core/test/url_request.h @@ -29,7 +29,8 @@ class UrlRequest { bool GetResponse(std::string* message); static int GetStatusCode(const std::string& response); // Get the response, and expect the status is OK. - void AssertOkResponse(std::string* message); + // It will retry if the response code is in the 500 range. + void AssertOkResponseWithRetry(std::string* message); static bool GetDebugHeaderFields( const std::string& response, @@ -37,9 +38,11 @@ class UrlRequest { private: bool PostRequestWithPath(const std::string& path, const std::string& data); + bool SendRequestOnce(); bool is_connected_; HttpSocket socket_; + std::string request_; CORE_DISALLOW_COPY_AND_ASSIGN(UrlRequest); };