Add detail about L3 initialization

Bug: b/70299597
Merge of http://go/wvgerrit/67304
Test: Android, CE CDM, and Linux tests

There's a few different things that can go wrong in the L3
initialization, with seeding and device key failures among others. They
should be recorded in metrics to track. Along the same lines, since
multiple errors can happen in conjunction, metrics needs to change to
add more fields for errors. This CL also adds the
hidl_metrics_adapter_unittest to the Android test scripts.

Change-Id: Ie5bcf81bbe294a1136c58410f90087a13b3d911d

libwvdrmengine/build_and_run_all_unit_tests.sh

@@ -94,6 +94,7 @@ try_adb_push event_metric_unittest
 try_adb_push file_store_unittest
 try_adb_push file_utils_unittest
 try_adb_push generic_crypto_unittest
+try_adb_push hidl_metrics_adapter_unittest
 try_adb_push http_socket_test
 try_adb_push initialization_data_unittest
 try_adb_push libwvdrmdrmplugin_hidl_test

libwvdrmengine/cdm/core/src/oemcrypto_adapter_dynamic.cpp

@@ -437,8 +437,9 @@ class WatchDog {
       if (size == size_read && flag) {
         LOGE("Previous L3 Init failed.");
         if (metrics == NULL) return;
-        metrics->OemCryptoDynamicAdapterMetrics::SetInitializationMode(
-            wvcdm::metrics::OEMCrypto_INITIALIZED_L3_INITIALIZATION_FAILED);
+        metrics
+            ->OemCryptoDynamicAdapterMetrics::SetPreviousInitializationFailure(
+                wvcdm::metrics::OEMCrypto_INITIALIZED_L3_INITIALIZATION_FAILED);
       }
     }
   }
@@ -480,7 +481,8 @@ class WatchDog {
   }
 
   // Called by main thread to wait for worker thread.
-  OEMCryptoResult WaitForStatusAndCleanUp() {
+  OEMCryptoResult WaitForStatusAndCleanUp(
+      wvcdm::metrics::OemCryptoDynamicAdapterMetrics* metrics) {
     bool should_delete;
     OEMCryptoResult status;
     {
@@ -492,15 +494,51 @@ class WatchDog {
         gave_up_ = true;
         status_ = OEMCrypto_ERROR_INIT_FAILED;
         LOGE("XXX WATCH DOG ERROR XXX");
-        // HACK: this normally just returns an error.  However, we are using it
-        // as a signal to dump debugging information.
-        Level3_GetOEMPublicCertificate(0, NULL, NULL);
+        Level3_OutputErrorLogs();
         SaveFailureInformation();
         // This is controversial.  The argument for an abort here is that if we
         // do not abort, we will suck all the life out of the user's battery.
         // By saving information to the file system, above, we can still track
         // metrics.
         abort();
+      } else if (metrics != NULL) {
+        // Even if we succeeded in calling initialize, there might be failures
+        // in the initialize process that we want to record.
+        wvoec3::Level3InitializationState state =
+            Level3_GetInitializationState();
+        switch (state) {
+          case wvoec3::LEVEL3_INITIALIZATION_UNKNOWN_FAILURE:
+            metrics
+                ->OemCryptoDynamicAdapterMetrics::SetLevel3InitializationError(
+                    wvcdm::metrics::
+                        OEMCrypto_INITIALIZED_L3_INITIALIZATION_FAILED);
+            break;
+          case wvoec3::LEVEL3_SEED_FAILURE:
+            metrics
+                ->OemCryptoDynamicAdapterMetrics::SetLevel3InitializationError(
+                    wvcdm::metrics::OEMCrypto_INITIALIZED_L3_RNG_FAILED);
+            break;
+          case wvoec3::LEVEL3_SAVE_DEVICE_KEYS_FAILURE:
+            metrics
+                ->OemCryptoDynamicAdapterMetrics::SetLevel3InitializationError(
+                    wvcdm::metrics::
+                        OEMCrypto_INITIALIZED_L3_SAVE_DEVICE_KEYS_FAILED);
+            break;
+          case wvoec3::LEVEL3_READ_DEVICE_KEYS_FAILURE:
+            metrics
+                ->OemCryptoDynamicAdapterMetrics::SetLevel3InitializationError(
+                    wvcdm::metrics::
+                        OEMCrypto_INITIALIZED_L3_READ_DEVICE_KEYS_FAILED);
+            break;
+          case wvoec3::LEVEL3_VERIFY_DEVICE_KEYS_FAILURE:
+            metrics
+                ->OemCryptoDynamicAdapterMetrics::SetLevel3InitializationError(
+                    wvcdm::metrics::
+                        OEMCrypto_INITIALIZED_L3_VERIFY_DEVICE_KEYS_FAILED);
+            break;
+          default:
+            break;
+        }
       }
       // If we gave up waiting for init thread, we should not delete the mutex
       // out from under it.
@@ -600,7 +638,7 @@ class Adapter {
       LOGI("Level 3 Build Info (v%d): %s", level3_.version,
            level3_.BuildInformation());
     }
-    OEMCryptoResult result = watcher->WaitForStatusAndCleanUp();
+    OEMCryptoResult result = watcher->WaitForStatusAndCleanUp(&metrics);
     if (Level3_IsInApp()) {
       metrics.SetInitializationMode(
           wvcdm::metrics::OEMCrypto_INITIALIZED_USING_IN_APP);

libwvdrmengine/cdm/metrics/include/metrics_collections.h

@@ -107,10 +107,13 @@ typedef enum OEMCryptoInitializationMode {
   OEMCrypto_INITIALIZED_USING_L1_INSTALLED_KEYBOX = 13,
   OEMCrypto_INITIALIZED_USING_L3_INVALID_L1 = 14,
   OEMCrypto_INITIALIZED_USING_L1_WITH_PROVISIONING_3_0 = 15,
-  OEMCrypto_INITIALIZED_L3_INITIALIZATION_FAILED = 16
+  OEMCrypto_INITIALIZED_L3_INITIALIZATION_FAILED = 16,
+  OEMCrypto_INITIALIZED_L3_RNG_FAILED = 17,
+  OEMCrypto_INITIALIZED_L3_SAVE_DEVICE_KEYS_FAILED = 18,
+  OEMCrypto_INITIALIZED_L3_READ_DEVICE_KEYS_FAILED = 19,
+  OEMCrypto_INITIALIZED_L3_VERIFY_DEVICE_KEYS_FAILED = 20,
 } OEMCryptoInitializationMode;
 
-
 // This class contains metrics for Crypto Session and OEM Crypto.
 class CryptoMetrics {
  public:
@@ -302,6 +305,8 @@ class OemCryptoDynamicAdapterMetrics {
 
   // Set methods for OEMCrypto metrics.
   void SetInitializationMode(OEMCryptoInitializationMode mode);
+  void SetLevel3InitializationError(OEMCryptoInitializationMode mode);
+  void SetPreviousInitializationFailure(OEMCryptoInitializationMode mode);
   void SetL1ApiVersion(uint32_t version);
   void SetL1MinApiVersion(uint32_t version);
 
@@ -315,7 +320,11 @@ class OemCryptoDynamicAdapterMetrics {
 
  private:
   mutable std::mutex adapter_lock_;
+  ValueMetric<OEMCryptoInitializationMode>
+      level3_oemcrypto_initialization_error_;
   ValueMetric<OEMCryptoInitializationMode> oemcrypto_initialization_mode_;
+  ValueMetric<OEMCryptoInitializationMode>
+      previous_oemcrypto_initialization_failure_;
   ValueMetric<uint32_t> oemcrypto_l1_api_version_;
   ValueMetric<uint32_t> oemcrypto_l1_min_api_version_;
 };

libwvdrmengine/cdm/metrics/src/metrics.proto

@@ -168,7 +168,7 @@ message WvCdmMetrics {
 
   // These are metrics recorded at the Engine level. This includes CryptoSession
   // metrics that were captured in the context of the engine.
-  // next id: 29
+  // next id: 31
   message EngineMetrics {
     optional CryptoMetrics crypto_metrics = 1;
 
@@ -200,6 +200,9 @@ message WvCdmMetrics {
     repeated CounterMetric cdm_engine_remove_usage_info = 26;
     repeated DistributionMetric cdm_engine_restore_key_time_us = 27;
     repeated CounterMetric cdm_engine_unprovision = 28;
+    // OEMCrypto Initialize Metrics.
+    optional ValueMetric level3_oemcrypto_initialization_error = 29;
+    optional ValueMetric previous_oemcrypto_initialization_failure = 30;
   }
 
   optional EngineMetrics engine_metrics = 1;

libwvdrmengine/cdm/metrics/src/metrics_collections.cpp

@@ -180,7 +180,9 @@ void SessionMetrics::SerializeSessionMetrics(
 }
 
 OemCryptoDynamicAdapterMetrics::OemCryptoDynamicAdapterMetrics()
-    : oemcrypto_initialization_mode_(),
+    : level3_oemcrypto_initialization_error_(),
+      oemcrypto_initialization_mode_(),
+      previous_oemcrypto_initialization_failure_(),
       oemcrypto_l1_api_version_(),
       oemcrypto_l1_min_api_version_() {}
 
@@ -190,6 +192,18 @@ void OemCryptoDynamicAdapterMetrics::SetInitializationMode(
   oemcrypto_initialization_mode_.Record(mode);
 }
 
+void OemCryptoDynamicAdapterMetrics::SetLevel3InitializationError(
+    OEMCryptoInitializationMode mode) {
+  std::unique_lock<std::mutex> lock(adapter_lock_);
+  level3_oemcrypto_initialization_error_.Record(mode);
+}
+
+void OemCryptoDynamicAdapterMetrics::SetPreviousInitializationFailure(
+    OEMCryptoInitializationMode mode) {
+  std::unique_lock<std::mutex> lock(adapter_lock_);
+  previous_oemcrypto_initialization_failure_.Record(mode);
+}
+
 void OemCryptoDynamicAdapterMetrics::SetL1ApiVersion(uint32_t version) {
   std::unique_lock<std::mutex> lock(adapter_lock_);
   oemcrypto_l1_api_version_.Record(version);
@@ -204,8 +218,12 @@ void OemCryptoDynamicAdapterMetrics::Serialize(
     WvCdmMetrics::EngineMetrics *engine_metrics) const {
   std::unique_lock<std::mutex> lock(adapter_lock_);
 
+  engine_metrics->set_allocated_level3_oemcrypto_initialization_error(
+      oemcrypto_initialization_mode_.ToProto());
   engine_metrics->set_allocated_oemcrypto_initialization_mode(
       oemcrypto_initialization_mode_.ToProto());
+  engine_metrics->set_allocated_previous_oemcrypto_initialization_failure(
+      oemcrypto_initialization_mode_.ToProto());
   engine_metrics->set_allocated_oemcrypto_l1_api_version(
       oemcrypto_l1_api_version_.ToProto());
   engine_metrics->set_allocated_oemcrypto_l1_min_api_version(
@@ -215,7 +233,9 @@ void OemCryptoDynamicAdapterMetrics::Serialize(
 void OemCryptoDynamicAdapterMetrics::Clear() {
   std::unique_lock<std::mutex> lock(adapter_lock_);
 
+  level3_oemcrypto_initialization_error_.Clear();
   oemcrypto_initialization_mode_.Clear();
+  previous_oemcrypto_initialization_failure_.Clear();
   oemcrypto_l1_api_version_.Clear();
   oemcrypto_l1_min_api_version_.Clear();
 }

libwvdrmengine/cdm/test/wv_cdm_metrics_test.cpp

@@ -62,12 +62,24 @@ TEST_F(WvContentDecryptionModuleMetricsTest, EngineOnlyMetrics) {
             decryptor_.GetMetrics(kDefaultCdmIdentifier, &metrics));
 
   // 100 is an arbitrary high value that shouldn't ever occur.
+  EXPECT_THAT(
+      metrics.engine_metrics().level3_oemcrypto_initialization_error()
+      .int_value(), Lt(100));
+  EXPECT_THAT(
+      metrics.engine_metrics().level3_oemcrypto_initialization_error()
+      .int_value(), Ge(0));
   EXPECT_THAT(
       metrics.engine_metrics().oemcrypto_initialization_mode().int_value(),
       Lt(100));
   EXPECT_THAT(
       metrics.engine_metrics().oemcrypto_initialization_mode().int_value(),
       Ge(0));
+  EXPECT_THAT(
+      metrics.engine_metrics().previous_oemcrypto_initialization_failure()
+      .int_value(), Lt(100));
+  EXPECT_THAT(
+      metrics.engine_metrics().previous_oemcrypto_initialization_failure()
+      .int_value(), Ge(0));
   ASSERT_THAT(metrics.engine_metrics()
                   .cdm_engine_get_provisioning_request_time_us().size(), Eq(1));
   EXPECT_THAT(metrics.engine_metrics()
@@ -95,7 +107,11 @@ TEST_F(WvContentDecryptionModuleMetricsTest, EngineAndSessionMetrics) {
 
   // Spot check some metric values.
   // Validate engine-level metrics.
+  EXPECT_TRUE(metrics.engine_metrics()
+      .has_level3_oemcrypto_initialization_error());
   EXPECT_TRUE(metrics.engine_metrics().has_oemcrypto_initialization_mode());
+  EXPECT_TRUE(metrics.engine_metrics()
+      .has_previous_oemcrypto_initialization_failure());
   ASSERT_THAT(metrics.engine_metrics().cdm_engine_open_session().size(), Eq(1));
   EXPECT_THAT(metrics.engine_metrics().cdm_engine_open_session(0).count(),
               Eq(1));

libwvdrmengine/mediadrm/src_hidl/hidl_metrics_adapter.cpp

@@ -349,6 +349,16 @@ void HidlMetricsAdapter::AddEngineMetrics(
   group_builder.AddCounters(
       "cdm_engine_unprovision",
       proto_metrics.cdm_engine_unprovision());
+  if (proto_metrics.has_level3_oemcrypto_initialization_error()) {
+    group_builder.AddValue(
+        "level3_oemcrypto_initialization_error",
+        proto_metrics.level3_oemcrypto_initialization_error());
+  }
+  if (proto_metrics.has_previous_oemcrypto_initialization_failure()) {
+    group_builder.AddValue(
+        "previous_oemcrypto_initialization_failure",
+        proto_metrics.previous_oemcrypto_initialization_failure());
+  }
 
   group_vector_.emplace_back(group_builder.Build());
 }

libwvdrmengine/mediadrm/test/hidl_metrics_adapter_unittest.cpp

@@ -398,7 +398,11 @@ TEST(HidlMetricsAdapterTest, EngineAndSessionAllMetrics) {
   drm_metrics::WvCdmMetrics::EngineMetrics engine_metrics;
   *(engine_metrics.mutable_crypto_metrics()) = crypto_metrics;
   // OEMCrypto Initialize Metrics.
+  engine_metrics.mutable_level3_oemcrypto_initialization_error()
+      ->set_int_value(1);
   engine_metrics.mutable_oemcrypto_initialization_mode()->set_int_value(1);
+  engine_metrics.mutable_previous_oemcrypto_initialization_failure()
+      ->set_int_value(1);
   engine_metrics.mutable_oemcrypto_l1_api_version()->set_int_value(1);
   engine_metrics.mutable_oemcrypto_l1_min_api_version()->set_int_value(1);
   // CdmEngine Metrics.
@@ -435,7 +439,7 @@ TEST(HidlMetricsAdapterTest, EngineAndSessionAllMetrics) {
   hidl_vec<DrmMetricGroup> hidl_metrics;
   HidlMetricsAdapter::ToHidlMetrics(metrics_proto, &hidl_metrics);
   ASSERT_EQ(2U, hidl_metrics.size());
-  EXPECT_EQ(83U, hidl_metrics[0].metrics.size());
+  EXPECT_EQ(85U, hidl_metrics[0].metrics.size());
   EXPECT_EQ(63U, hidl_metrics[1].metrics.size());
 }
 

libwvdrmengine/oemcrypto/include/level3.h

@@ -379,6 +379,19 @@ OEMCryptoResult Level3_CopyBuffer(OEMCrypto_SESSION session,
                                   size_t data_length,
                                   OEMCrypto_DestBufferDesc* out_buffer,
                                   uint8_t subsample_flags);
+
+// The following are specific to Google's Level 3 implementation and are not
+// required.
+
+enum Level3InitializationState {
+  LEVEL3_INITIALIZATION_SUCCESS = 0,
+  LEVEL3_INITIALIZATION_UNKNOWN_FAILURE = 1,
+  LEVEL3_SEED_FAILURE = 2,
+  LEVEL3_SAVE_DEVICE_KEYS_FAILURE = 3,
+  LEVEL3_READ_DEVICE_KEYS_FAILURE = 4,
+  LEVEL3_VERIFY_DEVICE_KEYS_FAILURE = 5,
+};
+
 /*
  * Level3_GetInitializationState
  *
@@ -386,7 +399,7 @@ OEMCryptoResult Level3_CopyBuffer(OEMCrypto_SESSION session,
  *   Return any warning or error condition which occurred during
  *   initialization.  On some platforms, this value will be logged and metrics
  *   will be gathered on production devices.  This is an optional feature, and
- *   OEMCrypto may always return 0, even if Level3_Initialize failed.  This
+ *   OEMCrypto may always return 0, even if Level3_Initialize failed. This
  *   function may be called whether Level3_Initialize succeeded or not.
  *
  * Parameters:
@@ -396,17 +409,44 @@ OEMCryptoResult Level3_CopyBuffer(OEMCrypto_SESSION session,
  *   No other function calls will be made while this function is running.
  *
  * Returns:
- *   0 - no warnings or errors during initialization
+ *   LEVEL3_INITIALIZATION_SUCCESS - no warnings or errors during initialization
+ *   LEVEL3_SEED_FAILURE - error in seeding the software RNG
+ *   LEVEL3_SAVE_DEVICE_KEYS_FAILURE - failed to save device keys to file system
+ *   LEVEL3_READ_DEVICE_KEYS_FAILURE - failed to read device keys from file
+ *   system
+ *   LEVEL3_VERIFY_DEVICE_KEYS_FAILURE - failed to verify decrypted device keys
  *
  * Version:
  *   This method is new in API version 14.
  */
-OEMCryptoResult Level3_GetInitializationState(void);
+Level3InitializationState Level3_GetInitializationState(void);
+
+/*
+ * Level3_OutputErrorLogs
+ *
+ * Description:
+ *   Call to output any errors in the Level 3 execution if the Level 3 has
+ *   failed. This method should only be called if the Level 3 has failed in
+ *   an unrecoverable state, and needs to be reinitialized.
+ *
+ * Parameters:
+ *   N/A
+ *
+ * Threading:
+ *   No other function calls will be made while this function is running.
+ *
+ * Returns:
+ *   N/A
+ *
+ * Version:
+ *   This method is new in API version 15.
+ */
+void Level3_OutputErrorLogs();
 
 }  // extern "C"
 
-// The following are interfaces needed for Level3 OEMCrypto specifically, which
-// partners are expected to implement.
+// The following are interfaces needed for Google's Level 3 OEMCrypto
+// specifically, which partners are expected to implement.
 
 // Returns a stable, unique identifier for the device. This could be a
 // serial number or any other character sequence representing that device.

libwvdrmengine/run_all_unit_tests.sh

@@ -96,12 +96,13 @@ adb_shell_run cdm_engine_test
 adb_shell_run cdm_session_unittest
 adb_shell_run counter_metric_unittest
 adb_shell_run crypto_session_unittest
-adb_shell_run generic_crypto_unittest
 adb_shell_run device_files_unittest
 adb_shell_run distribution_unittest
 adb_shell_run event_metric_unittest
 adb_shell_run file_store_unittest
 adb_shell_run file_utils_unittest
+adb_shell_run generic_crypto_unittest
+adb_shell_run hidl_metrics_adapter_unittest
 adb_shell_run http_socket_test
 adb_shell_run initialization_data_unittest
 adb_shell_run libwvdrmdrmplugin_hidl_test