Retry provisioning and license requests on 502 errors

[ Merge of http://go/wvgerrit/219370 ]

Work around the server problem.

Bug: 345333545
Change-Id: Ie1b0021f545c77d61bb2363494239f4fe000f3d5

libwvdrmengine/cdm/core/test/license_holder.cpp

@@ -103,7 +103,7 @@ void LicenseHolder::GenerateAndPostRenewalRequest(
 void LicenseHolder::FetchRenewal() {
   ASSERT_NE(request_in_flight_, nullptr) << "Failed for " << content_id();
   ASSERT_NO_FATAL_FAILURE(
-      request_in_flight_->AssertOkResponse(&request_response_))
+      request_in_flight_->AssertOkResponseWithRetry(&request_response_))
       << "Renewal failed for " << content_id();
 }
 
@@ -162,7 +162,7 @@ void LicenseHolder::GenerateAndPostReleaseRequest(
 void LicenseHolder::FetchRelease() {
   ASSERT_NE(request_in_flight_, nullptr) << "Failed for " << content_id();
   ASSERT_NO_FATAL_FAILURE(
-      request_in_flight_->AssertOkResponse(&request_response_))
+      request_in_flight_->AssertOkResponseWithRetry(&request_response_))
       << "Renewal failed for " << content_id();
 }
 
@@ -310,7 +310,7 @@ void LicenseHolder::GetKeyResponse(const CdmKeyRequest& key_request) {
 
   std::string http_response;
   url_request.PostRequest(key_request.message);
-  ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponse(&http_response))
+  ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponseWithRetry(&http_response))
       << "Failed for " << content_id();
   LicenseRequest license_request;
   license_request.GetDrmMessage(http_response, key_response_);

libwvdrmengine/cdm/core/test/provisioning_holder.cpp

@@ -69,7 +69,7 @@ void ProvisioningHolder::Provision(CdmCertificateType cert_type,
   url_request.PostCertRequestInQueryString(request);
 
   // Receive and parse response.
-  ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponse(&response_))
+  ASSERT_NO_FATAL_FAILURE(url_request.AssertOkResponseWithRetry(&response_))
       << "Failed to fetch provisioning response. "
       << DumpProvAttempt(request, response_, cert_type);
 

libwvdrmengine/cdm/core/test/url_request.cpp

@@ -5,9 +5,12 @@
 #include "url_request.h"
 
 #include <errno.h>
+#include <unistd.h>
+
+#include <iostream>
+#include <sstream>
 
 #include <gtest/gtest.h>
-#include <sstream>
 
 #include "http_socket.h"
 #include "log.h"
@@ -23,11 +26,15 @@ const int kConnectTimeoutMs = 15000;
 const int kWriteTimeoutMs = 12000;
 const int kReadTimeoutMs = 12000;
 constexpr int kHttpOk = 200;
+const std::vector<int> kRetryCodes = {502, 504};
 
 const std::string kGoogleHeaderUpper("X-Google");
 const std::string kGoogleHeaderLower("x-google");
 const std::string kCrLf("\r\n");
 
+constexpr unsigned kRetryCount = 3;
+constexpr unsigned kRetryIntervalSeconds = 1;
+
 // Concatenate all chunks into one blob and returns the response with
 // header information.
 void ConcatenateChunkedResponse(const std::string http_response,
@@ -126,13 +133,34 @@ bool UrlRequest::GetResponse(std::string* message) {
   return true;
 }
 
-void UrlRequest::AssertOkResponse(std::string* message) {
+void UrlRequest::AssertOkResponseWithRetry(std::string* message) {
   ASSERT_TRUE(message);
-  ASSERT_TRUE(GetResponse(message));
-  const int status_code = GetStatusCode(*message);
-  ASSERT_EQ(kHttpOk, status_code) << "HTTP response from " << socket_.url()
-                                  << ": (" << message->size() << ") :\n"
-                                  << *message;
+  int status_code = 0;
+  for (unsigned i = 0; i < kRetryCount; i++) {
+    *message = "";
+    ASSERT_TRUE(GetResponse(message)) << "For attempt " << (i + 1);
+    status_code = GetStatusCode(*message);
+    // If we didn't get a retry status, then we're done.
+    if (std::find(kRetryCodes.begin(), kRetryCodes.end(), status_code) ==
+        kRetryCodes.end()) {
+      ASSERT_EQ(kHttpOk, status_code) << "HTTP response from " << socket_.url()
+                                      << ": (" << message->size() << ") :\n"
+                                      << *message;
+      return;
+    }
+    std::cerr << "Temporary failure HTTP response from " << socket_.url()
+              << ": (" << message->size() << ") :\n"
+              << *message << "\n"
+              << "Attempt " << (i + 1) << "\n";
+    socket_.CloseSocket();
+    is_connected_ = false;
+    sleep(kRetryIntervalSeconds << i);
+    Reconnect();
+    SendRequestOnce();
+  }
+  GTEST_FAIL() << "HTTP response from " << socket_.url() << ": ("
+               << message->size() << ") :\n"
+               << *message;
 }
 
 // static
@@ -189,36 +217,35 @@ bool UrlRequest::GetDebugHeaderFields(
 
 bool UrlRequest::PostRequestWithPath(const std::string& path,
                                      const std::string& data) {
-  std::string request;
+  request_.clear();
 
-  request.append("POST ");
-  request.append(path);
-  request.append(" HTTP/1.1\r\n");
+  request_.append("POST ");
+  request_.append(path);
+  request_.append(" HTTP/1.1\r\n");
 
-  request.append("Host: ");
-  request.append(socket_.domain_name());
-  request.append("\r\n");
+  request_.append("Host: ");
+  request_.append(socket_.domain_name());
+  request_.append("\r\n");
 
-  request.append("Connection: close\r\n");
-  request.append("User-Agent: Widevine CDM v1.0\r\n");
-  request.append("X-Return-Encrypted-Headers: request_and_response\r\n");
+  request_.append("Connection: close\r\n");
+  request_.append("User-Agent: Widevine CDM v1.0\r\n");
+  request_.append("X-Return-Encrypted-Headers: request_and_response\r\n");
 
-  // buffer to store length of data as a string
-  char data_size_buffer[32] = {0};
-  snprintf(data_size_buffer, sizeof(data_size_buffer), "%zu", data.size());
+  request_.append("Content-Length: ");
+  request_.append(std::to_string(data.size()));
+  request_.append("\r\n");
 
-  request.append("Content-Length: ");
-  request.append(data_size_buffer);  // appends size of data
-  request.append("\r\n");
+  request_.append("\r\n");  // empty line to terminate headers
 
-  request.append("\r\n");  // empty line to terminate headers
-
-  request.append(data);
+  request_.append(data);
+  return SendRequestOnce();
+}
 
+bool UrlRequest::SendRequestOnce() {
   const int ret = socket_.WriteAndLogErrors(
-      request.c_str(), static_cast<int>(request.size()), kWriteTimeoutMs);
-  LOGV("HTTP request: (%zu): %s", request.size(), request.c_str());
-  LOGV("HTTP request hex: %s", wvutil::b2a_hex(request).c_str());
+      request_.c_str(), static_cast<int>(request_.size()), kWriteTimeoutMs);
+  LOGV("HTTP request: (%zu): %s", request_.size(), request_.c_str());
+  LOGV("HTTP request hex: %s", wvutil::b2a_hex(request_).c_str());
   return ret != -1;
 }
 

libwvdrmengine/cdm/core/test/url_request.h

@@ -29,7 +29,8 @@ class UrlRequest {
   bool GetResponse(std::string* message);
   static int GetStatusCode(const std::string& response);
   // Get the response, and expect the status is OK.
-  void AssertOkResponse(std::string* message);
+  // It will retry if the response code is in the 500 range.
+  void AssertOkResponseWithRetry(std::string* message);
 
   static bool GetDebugHeaderFields(
       const std::string& response,
@@ -37,9 +38,11 @@ class UrlRequest {
 
  private:
   bool PostRequestWithPath(const std::string& path, const std::string& data);
+  bool SendRequestOnce();
 
   bool is_connected_;
   HttpSocket socket_;
+  std::string request_;
 
   CORE_DISALLOW_COPY_AND_ASSIGN(UrlRequest);
 };