Add support for Widevine ECM v3

Widevine ECM v3 is redesigned mainly based on protobuf, and supports new features including carrying fingerprinting and service blocking information. Existing clients must upgrade the Widevine CAS plugin to use the new ECM v3.

common/rsa_key.cc

@@ -27,6 +27,7 @@
 
 #include "glog/logging.h"
 #include "openssl/bn.h"
+#include "openssl/digest.h"
 #include "openssl/err.h"
 #include "openssl/evp.h"
 #include "openssl/rsa.h"
@@ -67,6 +68,32 @@ std::string GetMessageDigest(const std::string& message,
   return "";
 }
 
+const EVP_MD* GetHashMd(widevine::HashAlgorithm hash_algorithm) {
+  switch (hash_algorithm) {
+    case widevine::HashAlgorithm::kUnspecified:
+    case widevine::HashAlgorithm::kSha1:
+      return EVP_sha1();
+    case widevine::HashAlgorithm::kSha256:
+      return EVP_sha256();
+  }
+  LOG(FATAL) << "Unexpected hash algorithm: "
+             << static_cast<int>(hash_algorithm);
+  return nullptr;
+}
+
+bool IsMessageTooSmall(const std::string& message) {
+  DCHECK(!message.empty());
+  // The most significant byte is encoded first in the message. See
+  // https://tools.ietf.org/html/rfc8017. To make sure the big number is greater
+  // than 2^64, we need to make sure there is at least a non-zero number in the
+  // first "LENGTH_IN_BITS - 64" bits of the message,
+  // i.e. "LENGTH_IN_BYTES - 8" bytes of the message.
+  const int kMinimumSizeInBytes = 8;
+  for (int i = 0; i < message.length() - kMinimumSizeInBytes; i++) {
+    if (message[i] != 0) return false;
+  }
+  return true;
+}
 }  // namespace
 
 namespace widevine {
@@ -117,7 +144,6 @@ bool RsaPrivateKey::Decrypt(const std::string& encrypted_message,
   return true;
 }
 
-
 bool RsaPrivateKey::GenerateSignature(const std::string& message,
                                       HashAlgorithm hash_algorithm,
                                       std::string* signature) const {
@@ -134,12 +160,18 @@ bool RsaPrivateKey::GenerateSignature(const std::string& message,
     return false;
   }
 
+  const EVP_MD* hash = GetHashMd(hash_algorithm);
+  if (hash == nullptr) {
+    LOG(ERROR) << "No hash md";
+    return false;
+  }
+
   // Add PSS padding.
   size_t rsa_size = RSA_size(key_);
   std::string padded_digest(rsa_size, 0);
   if (!RSA_padding_add_PKCS1_PSS_mgf1(
           key_, reinterpret_cast<unsigned char*>(&padded_digest[0]),
-          reinterpret_cast<unsigned char*>(&message_digest[0]), EVP_sha1(),
+          reinterpret_cast<unsigned char*>(&message_digest[0]), hash,
           EVP_sha1(), kPssSaltLength)) {
     LOG(ERROR) << "RSA padding failure: "
                << OpenSSLErrorString(ERR_get_error());
@@ -222,17 +254,22 @@ bool RsaPublicKey::Encrypt(const std::string& clear_message,
   }
   size_t rsa_size = RSA_size(key_);
   encrypted_message->assign(rsa_size, 0);
-  if (RSA_public_encrypt(
-          clear_message.size(),
-          const_cast<unsigned char*>(
-              reinterpret_cast<const unsigned char*>(clear_message.data())),
-          reinterpret_cast<unsigned char*>(&(*encrypted_message)[0]), key_,
-          RSA_PKCS1_OAEP_PADDING) != static_cast<int>(rsa_size)) {
-    LOG(ERROR) << "RSA public encrypt failure: "
-               << OpenSSLErrorString(ERR_get_error());
-    return false;
+  const int kRetryAttempt = 1;
+  for (int i = 0; i < 1 + kRetryAttempt; i++) {
+    if (RSA_public_encrypt(
+            clear_message.size(),
+            const_cast<unsigned char*>(
+                reinterpret_cast<const unsigned char*>(clear_message.data())),
+            reinterpret_cast<unsigned char*>(&(*encrypted_message)[0]), key_,
+            RSA_PKCS1_OAEP_PADDING) != static_cast<int>(rsa_size)) {
+      LOG(ERROR) << "RSA public encrypt failure: "
+                 << OpenSSLErrorString(ERR_get_error());
+      return false;
+    }
+    if (!IsMessageTooSmall(*encrypted_message)) return true;
   }
-  return true;
+  LOG(ERROR) << "RSA public encryption randomness error";
+  return false;
 }
 
 bool RsaPublicKey::VerifySignature(const std::string& message,
@@ -261,18 +298,23 @@ bool RsaPublicKey::VerifySignature(const std::string& message,
     return false;
   }
 
-  // Hash the message using SHA1 using corresponding hash algorithm.
+  // Hash the message using the corresponding hash algorithm.
   std::string message_digest = GetMessageDigest(message, hash_algorithm);
   if (message_digest.empty()) {
     LOG(ERROR) << "Empty message digest";
     return false;
   }
 
+  const EVP_MD* hash = GetHashMd(hash_algorithm);
+  if (hash == nullptr) {
+    LOG(ERROR) << "No hash md";
+    return false;
+  }
+
   // Verify PSS padding.
   if (RSA_verify_PKCS1_PSS_mgf1(
-          key_, reinterpret_cast<unsigned char*>(&message_digest[0]),
-          EVP_sha1(), EVP_sha1(),
-          reinterpret_cast<unsigned char*>(&padded_digest[0]),
+          key_, reinterpret_cast<unsigned char*>(&message_digest[0]), hash,
+          EVP_sha1(), reinterpret_cast<unsigned char*>(&padded_digest[0]),
           kPssSaltLength) == 0) {
     LOG(ERROR) << "RSA Verify PSS padding failure: "
                << OpenSSLErrorString(ERR_get_error());