whitebox/crypto_utils/rsa_util.cc

////////////////////////////////////////////////////////////////////////////////
// Copyright 2016 Google LLC.
//
// This software is licensed under the terms defined in the Widevine Master
// License Agreement. For a copy of this agreement, please contact
// widevine-licensing@google.com.
////////////////////////////////////////////////////////////////////////////////
//
// Description:
//   RSA utility functions for serializing and deserializing RSA keys,
//   encryption, and signing.

#include "crypto_utils/rsa_util.h"

#include <limits.h>

#include <cstring>
#include <memory>

#include "base/logging.h"
#include "crypto_utils/private_key_util.h"
#include "third_party/boringssl/src/include/openssl/pem.h"
#include "third_party/boringssl/src/include/openssl/x509.h"

namespace {
int BigNumGreaterThanPow2(const BIGNUM* b, int n) {
  if (BN_is_negative(b) || n == INT_MAX) {
    return 0;
  }
  int b_bits = BN_num_bits(b);
  return b_bits > n + 1 || (b_bits == n + 1 && !BN_is_pow2(b));
}
}  // anonymous namespace

namespace widevine {
namespace rsa_util {

bool SerializeRsaPrivateKey(const RSA* private_key,
                            std::string* serialized_private_key) {
  return private_key_util::SerializeKey<RSA>(private_key, i2d_RSAPrivateKey_bio,
                                             serialized_private_key);
}

bool DeserializeRsaPrivateKey(const std::string& serialized_private_key,
                              RSA** private_key) {
  return private_key_util::DeserializeKey<RSA>(
      serialized_private_key, d2i_RSAPrivateKey_bio, private_key);
}

bool SerializeRsaPublicKey(const RSA* public_key,
                           std::string* serialized_public_key) {
  return private_key_util::SerializeKey<RSA>(public_key, i2d_RSAPublicKey_bio,
                                             serialized_public_key);
}

bool DeserializeRsaPublicKey(const std::string& serialized_public_key,
                             RSA** public_key) {
  return private_key_util::DeserializeKey<RSA>(
      serialized_public_key, d2i_RSAPublicKey_bio, public_key);
}

bool SerializePrivateKeyInfo(const RSA* private_key,
                             std::string* serialized_private_key) {
  if (private_key == nullptr) {
    LOG(ERROR) << "Private RSA key is nullptr.";
    return false;
  }
  if (serialized_private_key == nullptr) {
    LOG(ERROR) << "Pointer to hold serialized PrivateKeyInfo is nullptr.";
    return false;
  }
  // The following method of serializing a PKCS#8 PrivateKeyInfo object
  // was obtained from analyzing the openssl utility code, as the official
  // mechanism via i2d_PKCS8PrivateKey_bio is broken in the current openssl
  // version (1.0.0c). Please refer to b/8560683.
  EVP_PKEY* evp = EVP_PKEY_new();
  if (evp == nullptr) {
    LOG(ERROR) << "EVP_PKEY_new returned nullptr.";
    return false;
  }
  bool success = false;
  PKCS8_PRIV_KEY_INFO* pkcs8_pki = nullptr;
  BIO* bio = nullptr;
  if (EVP_PKEY_set1_RSA(evp, const_cast<RSA*>(private_key)) == 0) {
    LOG(ERROR) << "EVP_PKEY_set1_RSA failed.";
    goto cleanup;
  }
  pkcs8_pki = EVP_PKEY2PKCS8(evp);
  if (pkcs8_pki == nullptr) {
    LOG(ERROR) << "EVP_PKEY2PKCS8 returned nullptr.";
    goto cleanup;
  }
  bio = BIO_new(BIO_s_mem());
  if (bio == nullptr) {
    LOG(ERROR) << "BIO_new returned nullptr.";
    goto cleanup;
  }
  if (i2d_PKCS8_PRIV_KEY_INFO_bio(bio, pkcs8_pki) == 0) {
    LOG(ERROR) << "i2d_PKCS8_PRIV_KEY_INFO_bio failed.";
    goto cleanup;
  }
  {
    int serialized_size = BIO_pending(bio);
    serialized_private_key->assign(serialized_size, 0);
    if (BIO_read(bio, &(*serialized_private_key)[0], serialized_size) !=
        serialized_size) {
      LOG(ERROR) << "BIO_read failed.";
      goto cleanup;
    }
  }
  success = true;

cleanup:
  if (bio != nullptr) {
    BIO_free(bio);
  }
  if (pkcs8_pki != nullptr) {
    PKCS8_PRIV_KEY_INFO_free(pkcs8_pki);
  }
  EVP_PKEY_free(evp);
  return success;
}

bool DeserializePrivateKeyInfo(const std::string& serialized_private_key,
                               RSA** private_key) {
  if (serialized_private_key.empty()) {
    LOG(ERROR) << "Serialized PrivateKeyInfo is empty.";
    return false;
  }
  if (private_key == nullptr) {
    LOG(ERROR) << "Pointer to hold new RSA private key is nullptr.";
    return false;
  }
  // The following method of deserializing a PKCS#8 PrivateKeyInfo object
  // was obtained from analyzing the openssl utility code, as the official
  // mechanism via d2i_PKCS8PrivateKey_bio is broken in the current openssl
  // version (1.0.0c). Please refer to b/8560683.
  BIO* bio = BIO_new_mem_buf(const_cast<char*>(serialized_private_key.data()),
                             serialized_private_key.size());
  if (bio == nullptr) {
    LOG(ERROR) << "BIO_new_mem_buf returned nullptr";
    return false;
  }
  bool success = false;
  EVP_PKEY* evp = nullptr;
  PKCS8_PRIV_KEY_INFO* pkcs8_pki = d2i_PKCS8_PRIV_KEY_INFO_bio(bio, nullptr);
  if (pkcs8_pki == nullptr) {
    LOG(ERROR) << "d2i_PKCS8_PRIV_KEY_INFO_bio returned nullptr.";
    goto cleanup;
  }
  evp = EVP_PKCS82PKEY(pkcs8_pki);
  if (evp == nullptr) {
    LOG(ERROR) << "EVP_PKCS82PKEY returned nullptr.";
    goto cleanup;
  }
  *private_key = EVP_PKEY_get1_RSA(evp);
  if (*private_key == nullptr) {
    LOG(ERROR) << "PrivateKeyInfo did not contain an RSA key.";
    goto cleanup;
  }
  success = true;

cleanup:
  if (evp != nullptr) {
    EVP_PKEY_free(evp);
  }
  if (pkcs8_pki != nullptr) {
    PKCS8_PRIV_KEY_INFO_free(pkcs8_pki);
  }
  BIO_free(bio);
  return success;
}

bool RsaPrivateKeyToPrivateKeyInfo(const std::string& rsa_private_key,
                                   std::string* private_key_info) {
  RSA* key = nullptr;
  if (DeserializeRsaPrivateKey(rsa_private_key, &key)) {
    bool success = SerializePrivateKeyInfo(key, private_key_info);
    RSA_free(key);
    return success;
  }
  return false;
}

bool PrivateKeyInfoToRsaPrivateKey(const std::string& private_key_info,
                                   std::string* rsa_private_key) {
  RSA* key = nullptr;
  if (DeserializePrivateKeyInfo(private_key_info, &key)) {
    bool success = SerializeRsaPrivateKey(key, rsa_private_key);
    RSA_free(key);
    return success;
  }
  return false;
}

bool SerializeEncryptedPrivateKeyInfo(const RSA* private_key,
                                      const std::string& passphrase,
                                      std::string* serialized_private_key) {
  if (private_key == nullptr) {
    LOG(ERROR) << "Private RSA key is nullptr.";
    return false;
  }
  if (passphrase.empty()) {
    LOG(ERROR) << "Passphrase for RSA key encryption is empty.";
    return false;
  }
  if (serialized_private_key == nullptr) {
    LOG(ERROR)
        << "Pointer to hold serialized EncryptedPrivateKeyInfo is nullptr.";
    return false;
  }
  EVP_PKEY* evp = EVP_PKEY_new();
  if (evp == nullptr) {
    LOG(ERROR) << "EVP_PKEY_new returned nullptr.";
    return false;
  }
  bool success = false;
  BIO* bio = nullptr;
  if (EVP_PKEY_set1_RSA(evp, const_cast<RSA*>(private_key)) == 0) {
    LOG(ERROR) << "EVP_PKEY_set1_RSA failed.";
    goto cleanup;
  }
  bio = BIO_new(BIO_s_mem());
  if (bio == nullptr) {
    LOG(ERROR) << "BIO_new returned nullptr.";
    goto cleanup;
  }
  if (i2d_PKCS8PrivateKey_bio(bio, evp, EVP_aes_256_cbc(),
                              const_cast<char*>(passphrase.data()),
                              passphrase.size(), nullptr, nullptr) == 0) {
    LOG(ERROR) << "i2d_PKCS8PrivateKey_bio failed.";
    goto cleanup;
  }
  {
    int serialized_size = BIO_pending(bio);
    serialized_private_key->assign(serialized_size, 0);
    if (BIO_read(bio, &(*serialized_private_key)[0], serialized_size) !=
        serialized_size) {
      LOG(ERROR) << "BIO_read failed.";
      goto cleanup;
    }
  }
  success = true;

cleanup:
  if (bio != nullptr) {
    BIO_free(bio);
  }
  EVP_PKEY_free(evp);
  return success;
}

namespace {
// Password retrieval function used by DeserializeEncryptedPrivateKeyInfo below.
int get_password(char* buf, int size, int rwflag, void* u) {
  CHECK(buf);
  CHECK(u);
  const std::string* pass(static_cast<const std::string*>(u));
  if (!pass->empty() && size >= static_cast<int>(pass->size())) {
    memcpy(buf, pass->data(), pass->size());
    return pass->size();
  }
  return 0;
}
}  // namespace

bool DeserializeEncryptedPrivateKeyInfo(
    const std::string& serialized_private_key,
    const std::string& passphrase,
    RSA** private_key) {
  if (serialized_private_key.empty()) {
    LOG(ERROR) << "Serialized RSAEncryptedPrivateKeyInfo is empty.";
    return false;
  }
  if (passphrase.empty()) {
    LOG(ERROR) << "Passphrase for RSA key decryption is empty.";
    return false;
  }
  if (private_key == nullptr) {
    LOG(ERROR) << "Pointer to hold new RSA private key is nullptr.";
    return false;
  }
  BIO* bio = BIO_new_mem_buf(const_cast<char*>(serialized_private_key.data()),
                             serialized_private_key.size());
  if (bio == nullptr) {
    LOG(ERROR) << "BIO_new_mem_buf returned nullptr";
    return false;
  }
  bool success = false;
  EVP_PKEY* evp = d2i_PKCS8PrivateKey_bio(
      bio, nullptr, get_password, const_cast<std::string*>(&passphrase));
  if (evp == nullptr) {
    LOG(ERROR) << "d2i_PKCS8PrivateKey_bio returned nullptr.";
    goto cleanup;
  }
  *private_key = EVP_PKEY_get1_RSA(evp);
  if (*private_key == nullptr) {
    LOG(ERROR) << "EncryptedPrivateKeyInfo did not contain an RSA key.";
    goto cleanup;
  }
  success = true;

cleanup:
  if (evp != nullptr) {
    EVP_PKEY_free(evp);
  }
  BIO_free(bio);
  return success;
}

bool RsaPrivateKeyToEncryptedPrivateKeyInfo(const std::string& rsa_private_key,
                                            const std::string& passphrase,
                                            std::string* private_key_info) {
  RSA* key = nullptr;
  if (DeserializeRsaPrivateKey(rsa_private_key, &key)) {
    bool success =
        SerializeEncryptedPrivateKeyInfo(key, passphrase, private_key_info);
    RSA_free(key);
    return success;
  }
  return false;
}

bool EncryptedPrivateKeyInfoToRsaPrivateKey(const std::string& private_key_info,
                                            const std::string& passphrase,
                                            std::string* rsa_private_key) {
  RSA* key = nullptr;
  if (DeserializeEncryptedPrivateKeyInfo(private_key_info, passphrase, &key)) {
    bool success = SerializeRsaPrivateKey(key, rsa_private_key);
    RSA_free(key);
    return success;
  }
  return false;
}

bool ConvertToEulerTotient(RSA* rsa) {
  // RSA key generation requires computing e (public exponent) and d (private
  // exponent), such that m^(e * d) = 1 (mod n) for all m coprime to n.
  // BoringSSL previously computed this by taking the inverse of e modulo the
  // Euler totient, (p - 1) * (q - 1). However, it now uses the Carmichael
  // totient, lcm(p - 1, q - 1). These two methods produce equivalent RSA keys.
  //
  // This breaks some vendors' RSA code which use a custom RSA format (rather
  // than the standard one in RFC 8017) which omits most of the required
  // parameters. They then attempt to recover those parameters, but their
  // recovery algorithm breaks when using the Carmichael totient. To work around
  // this bug, re-compute the private exponent against the Euler totient.
  const BIGNUM *e, *p, *q;
  RSA_get0_key(rsa, nullptr /* n */, &e, nullptr /* d */);
  RSA_get0_factors(rsa, &p, &q);

  bssl::UniquePtr<BN_CTX> ctx(BN_CTX_new());
  bssl::UniquePtr<BIGNUM> pm1(BN_new());
  bssl::UniquePtr<BIGNUM> qm1(BN_new());
  bssl::UniquePtr<BIGNUM> totient(BN_new());
  bssl::UniquePtr<BIGNUM> d(BN_new());
  if (!ctx || !pm1 || !qm1 || !totient || !d ||
      !BN_sub(pm1.get(), p, BN_value_one()) ||
      !BN_sub(qm1.get(), q, BN_value_one()) ||
      !BN_mul(totient.get(), pm1.get(), qm1.get(), ctx.get()) ||
      !BN_mod_inverse(d.get(), e, totient.get(), ctx.get())) {
    return false;
  }

  // Perform a sanity check that d is still valid after conversion.
  // d > 2 ^ (nlen / 2) per Appendix B 3.1 in FIPS 186/4.
  int prime_bits = (8 * RSA_size(rsa)) / 2;
  if (!BigNumGreaterThanPow2(d.get(), prime_bits)) {
    return false;
  }

  if (!RSA_set0_key(rsa, nullptr /* n */, nullptr /* e */, d.get())) {
    return false;
  }
  d.release();  // RSA_set0_key takes ownership on success.

  if (!RSA_check_key(rsa)) {
    return false;
  }

  return true;
}

bool ConvertToEulerTotient(const std::string& private_key,
                           std::string* euler_private_key) {
  CHECK(euler_private_key);
  RSA* rsa_ptr;
  if (!rsa_util::DeserializeRsaPrivateKey(private_key, &rsa_ptr)) {
    return false;
  }
  bssl::UniquePtr<RSA> rsa(rsa_ptr);
  if (!rsa_util::ConvertToEulerTotient(rsa.get()) ||
      !rsa_util::SerializeRsaPrivateKey(rsa.get(), euler_private_key)) {
    return false;
  }

  return true;
}

bool ConvertToCarmichaelTotient(RSA* rsa) {
  bssl::UniquePtr<BN_CTX> ctx(BN_CTX_new());
  bssl::UniquePtr<BIGNUM> pm1(BN_new());
  bssl::UniquePtr<BIGNUM> qm1(BN_new());
  bssl::UniquePtr<BIGNUM> gcd(BN_new());
  bssl::UniquePtr<BIGNUM> totient(BN_new());
  bssl::UniquePtr<BIGNUM> d(BN_new());
  // This calculates d = e^-1 (mod lcm(p-1, q-1)).
  // This is equivalent to what is used in RSA_generate_key in BoringSSL.
  if (!BN_sub(pm1.get(), rsa->p, BN_value_one()) ||
      !BN_sub(qm1.get(), rsa->q, BN_value_one()) ||
      !BN_mul(totient.get(), pm1.get(), qm1.get(), ctx.get()) ||
      !BN_gcd(gcd.get(), pm1.get(), qm1.get(), ctx.get()) ||
      !BN_div(totient.get(), nullptr, totient.get(), gcd.get(), ctx.get()) ||
      !BN_mod_inverse(d.get(), rsa->e, totient.get(), ctx.get())) {
    return false;
  }

  // Perform a sanity check that d is still valid after conversion.
  // d > 2 ^ (nlen / 2) per Appendix B 3.1 in FIPS 186/4.
  int prime_bits = (8 * RSA_size(rsa)) / 2;
  if (!BigNumGreaterThanPow2(d.get(), prime_bits)) {
    return false;
  }

  // TODO(user): Replace this with |RSA_set0_key| once BoringSSL has
  // finished transitioning to the OpenSSL 1.1.0 API.
  BN_free(rsa->d);
  rsa->d = d.release();

  if (!RSA_check_key(rsa)) {
    return false;
  }

  return true;
}

bool ConvertToCarmichaelTotient(const std::string& private_key,
                                std::string* carmichael_private_key) {
  CHECK(carmichael_private_key);
  RSA* rsa_ptr;
  if (!rsa_util::DeserializeRsaPrivateKey(private_key, &rsa_ptr)) {
    return false;
  }
  bssl::UniquePtr<RSA> rsa(rsa_ptr);
  if (!rsa_util::ConvertToCarmichaelTotient(rsa.get()) ||
      !rsa_util::SerializeRsaPrivateKey(rsa.get(), carmichael_private_key)) {
    return false;
  }

  return true;
}

}  // namespace rsa_util
}  // namespace widevine