Source release 17.1.0
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,5 @@
|
||||
// Copyright 2019 Google LLC. All rights reserved. This file and proprietary
|
||||
// source code may only be used and distributed under the Widevine Master
|
||||
// source code may only be used and distributed under the Widevine
|
||||
// License Agreement.
|
||||
|
||||
/*********************************************************************
|
||||
@@ -61,7 +61,7 @@ typedef enum OEMCryptoResult {
|
||||
OEMCrypto_ERROR_INVALID_NONCE = 32,
|
||||
OEMCrypto_ERROR_TOO_MANY_KEYS = 33,
|
||||
OEMCrypto_ERROR_DEVICE_NOT_RSA_PROVISIONED = 34,
|
||||
OEMCrypto_ERROR_INVALID_RSA_KEY = 35,
|
||||
OEMCrypto_ERROR_INVALID_RSA_KEY = 35, /* deprecated */
|
||||
OEMCrypto_ERROR_KEY_EXPIRED = 36,
|
||||
OEMCrypto_ERROR_INSUFFICIENT_RESOURCES = 37,
|
||||
OEMCrypto_ERROR_INSUFFICIENT_HDCP = 38,
|
||||
@@ -87,6 +87,12 @@ typedef enum OEMCryptoResult {
|
||||
OEMCrypto_ERROR_LICENSE_RELOAD = 57,
|
||||
OEMCrypto_ERROR_MULTIPLE_USAGE_ENTRIES = 58,
|
||||
OEMCrypto_WARNING_MIXED_OUTPUT_PROTECTION = 59,
|
||||
OEMCrypto_ERROR_INVALID_ENTITLED_KEY_SESSION = 60,
|
||||
OEMCrypto_ERROR_NEEDS_KEYBOX_PROVISIONING = 61,
|
||||
OEMCrypto_ERROR_UNSUPPORTED_CIPHER = 62,
|
||||
OEMCrypto_ERROR_DVR_FORBIDDEN = 63,
|
||||
OEMCrypto_ERROR_INSUFFICIENT_PRIVILEGE = 64,
|
||||
OEMCrypto_ERROR_INVALID_KEY = 65,
|
||||
/* ODK return values */
|
||||
ODK_ERROR_BASE = 1000,
|
||||
ODK_ERROR_CORE_MESSAGE = ODK_ERROR_BASE,
|
||||
@@ -95,6 +101,11 @@ typedef enum OEMCryptoResult {
|
||||
ODK_TIMER_EXPIRED = ODK_ERROR_BASE + 3,
|
||||
ODK_UNSUPPORTED_API = ODK_ERROR_BASE + 4,
|
||||
ODK_STALE_RENEWAL = ODK_ERROR_BASE + 5,
|
||||
/* OPK return values */
|
||||
OPK_ERROR_BASE = 2000,
|
||||
OPK_ERROR_REMOTE_CALL = OPK_ERROR_BASE,
|
||||
OPK_ERROR_INCOMPATIBLE_VERSION = OPK_ERROR_BASE + 1,
|
||||
OPK_ERROR_NO_PERSISTENT_DATA = OPK_ERROR_BASE + 2,
|
||||
} OEMCryptoResult;
|
||||
/* clang-format on */
|
||||
|
||||
@@ -109,6 +120,11 @@ typedef enum OEMCrypto_Usage_Entry_Status {
|
||||
kInactiveUnused = 4,
|
||||
} OEMCrypto_Usage_Entry_Status;
|
||||
|
||||
typedef enum OEMCrypto_ProvisioningRenewalType {
|
||||
OEMCrypto_NoRenewal = 0,
|
||||
OEMCrypto_RenewalACert = 1,
|
||||
} OEMCrypto_ProvisioningRenewalType;
|
||||
|
||||
/**
|
||||
* OEMCrypto_LicenseType is used in the license message to indicate if the key
|
||||
* objects are for content keys, or for entitlement keys.
|
||||
@@ -116,7 +132,7 @@ typedef enum OEMCrypto_Usage_Entry_Status {
|
||||
typedef enum OEMCrypto_LicenseType {
|
||||
OEMCrypto_ContentLicense = 0,
|
||||
OEMCrypto_EntitlementLicense = 1,
|
||||
OEMCrypto_LicenstType_MaxValue = OEMCrypto_EntitlementLicense,
|
||||
OEMCrypto_LicenseType_MaxValue = OEMCrypto_EntitlementLicense,
|
||||
} OEMCrypto_LicenseType;
|
||||
|
||||
/* Private key type used in the provisioning response. */
|
||||
@@ -135,6 +151,62 @@ typedef struct {
|
||||
size_t length;
|
||||
} OEMCrypto_Substring;
|
||||
|
||||
/**
|
||||
* Used to specify information about CMI Descriptor 0.
|
||||
* @param id: ID value of CMI Descriptor assigned by DTLA.
|
||||
* @param length: byte length of the usage rules field.
|
||||
* @param data: usage rules data.
|
||||
*/
|
||||
typedef struct {
|
||||
uint8_t id; // 0x00
|
||||
uint8_t extension; // 0x00
|
||||
uint16_t length; // 0x01
|
||||
uint8_t data;
|
||||
} OEMCrypto_DTCP2_CMI_Descriptor_0;
|
||||
|
||||
/**
|
||||
* Used to specify information about CMI Descriptor 1.
|
||||
* @param id: ID value of CMI Descriptor assigned by DTLA.
|
||||
* @param extension: specified by the CMI descriptor
|
||||
* @param length: byte length of the usage rules field.
|
||||
* @param data: usage rules data.
|
||||
*/
|
||||
typedef struct {
|
||||
uint8_t id; // 0x01
|
||||
uint8_t extension; // 0x00
|
||||
uint16_t length; // 0x03
|
||||
uint8_t data[3];
|
||||
} OEMCrypto_DTCP2_CMI_Descriptor_1;
|
||||
|
||||
/**
|
||||
* Used to specify information about CMI Descriptor 2.
|
||||
* @param id: ID value of CMI Descriptor assigned by DTLA.
|
||||
* @param extension: specified by the CMI descriptor
|
||||
* @param length: byte length of the usage rules field.
|
||||
* @param data: usage rules data.
|
||||
*/
|
||||
typedef struct {
|
||||
uint8_t id; // 0x02
|
||||
uint8_t extension; // 0x00
|
||||
uint16_t length; // 0x03
|
||||
uint8_t data[3];
|
||||
} OEMCrypto_DTCP2_CMI_Descriptor_2;
|
||||
|
||||
/**
|
||||
* Used to specify the required DTCP2 level. If dtcp2_required is 0, there are
|
||||
* no requirements on any of the keys. If dtcp2_required is 1, any key with the
|
||||
* kControlHDCPRequired bit set requires DTCP2 in its output.
|
||||
* @param dtcp2_required: specifies whether dtcp2 is required. 0 = not required,
|
||||
* 1 = DTCP2 required.
|
||||
* @param cmi_descriptor_1: three bytes of CMI descriptor 1
|
||||
*/
|
||||
typedef struct {
|
||||
uint8_t dtcp2_required; // 0 = not required. 1 = DTCP2 v1 required.
|
||||
OEMCrypto_DTCP2_CMI_Descriptor_0 cmi_descriptor_0;
|
||||
OEMCrypto_DTCP2_CMI_Descriptor_1 cmi_descriptor_1;
|
||||
OEMCrypto_DTCP2_CMI_Descriptor_2 cmi_descriptor_2;
|
||||
} OEMCrypto_DTCP2_CMI_Packet;
|
||||
|
||||
/**
|
||||
* Points to the relevant fields for a content key. The fields are extracted
|
||||
* from the License Response message offered to OEMCrypto_LoadKeys(). Each
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2018 Google LLC. All Rights Reserved. This file and proprietary
|
||||
// source code may only be used and distributed under the Widevine Master
|
||||
// source code may only be used and distributed under the Widevine
|
||||
// License Agreement.
|
||||
|
||||
#ifndef WV_OEMCRYPTO_TYPES_H_
|
||||
@@ -23,6 +23,14 @@ typedef struct WidevineKeybox { // 128 bytes total.
|
||||
uint8_t crc_[4];
|
||||
} WidevineKeybox;
|
||||
|
||||
// This is the format for a key control block.
|
||||
typedef struct {
|
||||
uint8_t verification[4];
|
||||
uint32_t duration;
|
||||
uint32_t nonce;
|
||||
uint32_t control_bits;
|
||||
} KeyControlBlock;
|
||||
|
||||
/*
|
||||
* SRM_Restriction_Data
|
||||
*
|
||||
@@ -39,6 +47,12 @@ const uint32_t kControlObserveDataPath = (1u << 31);
|
||||
const uint32_t kControlObserveHDCP = (1u << 30);
|
||||
const uint32_t kControlObserveCGMS = (1u << 29);
|
||||
const uint32_t kControlRequireAntiRollbackHardware = (1u << 28);
|
||||
// The two bits kControlWhiteboxSecurityLevelMask are not used in
|
||||
// OEMCrypto; they are only used for whitebox testing.
|
||||
const uint32_t kControlWhiteboxSecurityLevelShift = 26;
|
||||
const uint32_t kControlWhiteboxSecurityLevelMask =
|
||||
(0x03u << kControlWhiteboxSecurityLevelShift);
|
||||
const uint32_t kControlAllowDVRRecording = (1u << 25);
|
||||
const uint32_t kControlAllowHashVerification = (1u << 24);
|
||||
const uint32_t kSharedLicense = (1u << 23);
|
||||
const uint32_t kControlSRMVersionRequired = (1u << 22);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Copyright 2018 Google LLC. All Rights Reserved. This file and proprietary
|
||||
// source code may only be used and distributed under the Widevine Master
|
||||
// source code may only be used and distributed under the Widevine
|
||||
// License Agreement.
|
||||
|
||||
/*********************************************************************
|
||||
@@ -18,7 +18,7 @@
|
||||
#include "OEMCryptoCENC.h"
|
||||
#include "string_conversions.h" // needed for htonll64.
|
||||
|
||||
namespace wvcdm {
|
||||
namespace wvutil {
|
||||
|
||||
class Unpacked_PST_Report {
|
||||
public:
|
||||
@@ -84,12 +84,12 @@ class Unpacked_PST_Report {
|
||||
int64_t time;
|
||||
memcpy(&time, buffer_ + kseconds_since_license_received_offset,
|
||||
sizeof(int64_t));
|
||||
return ntohll64(time);
|
||||
return wvutil::ntohll64(time);
|
||||
}
|
||||
|
||||
// Parameter time is in host byte order.
|
||||
void set_seconds_since_license_received(int64_t time) const {
|
||||
time = ntohll64(time);
|
||||
time = wvutil::ntohll64(time);
|
||||
memcpy(buffer_ + kseconds_since_license_received_offset, &time,
|
||||
sizeof(int64_t));
|
||||
}
|
||||
@@ -143,6 +143,6 @@ class Unpacked_PST_Report {
|
||||
static const size_t kseconds_since_last_decrypt_offset = 40;
|
||||
static const size_t kpst_offset = 48;
|
||||
};
|
||||
} // namespace wvcdm
|
||||
} // namespace wvutil
|
||||
|
||||
#endif // PST_REPORT_H_
|
||||
|
||||
125
oemcrypto/test/README.md
Normal file
125
oemcrypto/test/README.md
Normal file
@@ -0,0 +1,125 @@
|
||||
# OEMCrypto Unit Tests
|
||||
|
||||
## Basic Functionality Tests
|
||||
|
||||
Most unit tests in this category verify that the basic functionality of opening
|
||||
sessions, initializing and terminating the system, and reporting status work
|
||||
correctly.
|
||||
|
||||
## Decrypt Tests
|
||||
|
||||
The decrypt tests verify that encrypted data is correctly decrypted with the
|
||||
desired key. These tests cover a large variety of patterns, sample sizes, and
|
||||
subsample sizes.
|
||||
|
||||
## Secure Buffers
|
||||
|
||||
If OEMCrypto implements the function `OEMCrypto_AllocateSecureBuffer`, then all
|
||||
of the decrypt tests will also run with the output buffer being a secure
|
||||
buffer. If the function `OEMCrypto_SupportsDecryptHash` returns
|
||||
`OEMCrypto_CRC_Clear_Buffer`, then the secure buffer decryption will be verified
|
||||
with the CRC32 hash of the input data.
|
||||
|
||||
## Usage Table Tests
|
||||
|
||||
Usage table tests verify that the usage table is correctly procesed. The usage
|
||||
table is used to control reloading keys for offline playback, and for reporting
|
||||
secure stops for online playback.
|
||||
|
||||
## Duration Tests
|
||||
|
||||
Duration tests verify that license durations are enforced correctly. Most of
|
||||
this functionality can be met by keeping an accurate system time, and calling
|
||||
the ODK functions as described in the document "License Duration and Renewal".
|
||||
|
||||
## OEMCrypto Memory Unit Tests
|
||||
|
||||
### Objective
|
||||
|
||||
* Add OEMCrypto buffer overflow unit tests (indirect way of fuzzing) to verify
|
||||
OEMCrypto API behavior when the parameters passed to the API are out of
|
||||
range or not reasonable. The API can return an error code, but shouldn't
|
||||
crash.
|
||||
|
||||
* A lot of OEMCrypto APIs take buffers and their length as inputs to the APIs
|
||||
and we have added unit tests with buffers of varying lengths (small
|
||||
to huge) to verify API behavior which is an indirect and simplest way of
|
||||
fuzz testing to detect buffer overflows.
|
||||
|
||||
* Add the tests for OEMCrypto APIs with prefix `OEMCryptoMemory` in the
|
||||
following format. Huge length is set at 100 MB as of now.
|
||||
|
||||
```cpp
|
||||
for (size_t length=small_length; length<huge_length; length=length * 2) {
|
||||
Create buffer of size length.
|
||||
Call api expecting it not to crash or segfault.
|
||||
}
|
||||
```
|
||||
|
||||
* Add tests for OEMCrypto APIs with out of range values for length and offsets
|
||||
of OEMCryptoSubstring struct. This length and offset fields are used to read
|
||||
values from an input buffer in most of the APIs. This can cause buffer
|
||||
overflows if the length and offset fields are not validated against the
|
||||
input buffer.
|
||||
|
||||
### Background
|
||||
|
||||
* Security is the top priority for Widevine. We came up with a simple approach
|
||||
to catch most common issues with widevine's implementations. A simplest
|
||||
approach is to add OEMCrypto unit tests to verify OEMCrypto API behavior
|
||||
when the parameters are out of range, meaning for an unreasonable length
|
||||
which can cause buffer overflows. Most of the implementation either does not
|
||||
validate input length parameters or copies data to secure buffers out of TA
|
||||
space causing memory corruptions, buffer overflows. Partners who implement
|
||||
OEMCrypto implementations will run OEMCrypto unit tests as part of the
|
||||
process.
|
||||
|
||||
* We have added unit tests with parameters that can cause buffer overflows if
|
||||
the parameters are not validated. This way partners can catch issues
|
||||
earlier in the process when they run OEMCrypto unit tests. All the unit
|
||||
tests with prefix `OEMCryptoMemory` are added to test the above scenario.
|
||||
|
||||
### What to expect from these tests
|
||||
|
||||
* `OEMCryptoMemory*` tests are designed to fail if the API doesn't have
|
||||
enough validations around input buffer lengths, parameters or the
|
||||
OEMCryptoSubstring struct. If the API doesn't have validations which might
|
||||
lead to a crash, the test fails with a segfault or an appropriate crash
|
||||
message based on the API implementation.
|
||||
|
||||
* Find out for what buffer length, the API is crashing and then debugging the
|
||||
test against the OEMCrypto implementation should be able to provide
|
||||
information about the error.
|
||||
|
||||
* Another way to debug would be to compile the tests with sanitizer flags,
|
||||
which will be able to provide detailed information about the crash.
|
||||
|
||||
* Partners are expected to fix issues with the API so that the tests don't
|
||||
fail.
|
||||
|
||||
* As these tests run for varying lengths from small to huge buffer lengths,
|
||||
some of the tests might take longer to run(~3 minutes).
|
||||
|
||||
* `OEMCryptoMemoryInstallKeyboxForHugeKeyboxBuffer*` tests which tries to
|
||||
call install keybox API with varying buffer lengths. This test by default
|
||||
is not compiled as it overwrites the keybox on the device. Uncomment,
|
||||
compile and run the tests only if you have ability to recover the keybox
|
||||
on device where the test is ran.
|
||||
|
||||
## Filtering out tests
|
||||
|
||||
The source code will check for functionality of OEMCrypto and filter out tests
|
||||
that are not required. For example, if a device uses a keybox, then Provisioning
|
||||
3.0 tests are skipped, and vice versa.
|
||||
|
||||
If you wish to skip slow tests because you only want to verify basic
|
||||
functionality, then you can set the environment variable `GTEST_FILTER`,
|
||||
as documented
|
||||
[here](https://github.com/google/googletest/blob/master/docs/advanced.md#running-a-subset-of-the-tests).
|
||||
|
||||
For example, to skip the duration tests, buffer overflow tests and long running
|
||||
stress tests, you would set
|
||||
|
||||
```
|
||||
GTEST_FILTER="*-*Duration*:*TimingTest*:*Memory*:*Huge*:*NonceFlood*:*ManyUsageEntries*:*Defrag*"
|
||||
```
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
(c020:0d112d7ea200;
|
||||
@@ -1 +0,0 @@
|
||||
(e2!0;u
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
0a4c08001248000000020000101907d9ffde13aa95c122678053362136bdf8408f8276e4c2d87ec52b61aa1b9f646e58734930acebe899b3e464189a14a87202fb02570640bd22ef44b2d7e3912250a230a14080112100915007caa9b5931b76a3a85f046523e10011a09393837363534333231180120002a0c313838363738373430350000
|
||||
@@ -1 +0,0 @@
|
||||
(2dea200;u
|
||||
@@ -1 +0,0 @@
|
||||
0a4c020:0d1190d79fef02570640bd22ef44b2d7e3912250a200
|
||||
@@ -1 +0,0 @@
|
||||
e2!0;u
|
||||
@@ -1 +0,0 @@
|
||||
e
|
||||
@@ -1 +0,0 @@
|
||||
(ea200;u
|
||||
@@ -1 +0,0 @@
|
||||
0a(c020:0d112d7ea200;
|
||||
@@ -1 +0,0 @@
|
||||
0a4c000000200:0101907d9ffde02570640bd22ef44b2d7e3912250a230a1407363534333231180120002a0c313838363738373430350000
|
||||
Binary file not shown.
Binary file not shown.
@@ -1 +0,0 @@
|
||||
0a4c000000220:01019dd79fef02570640bd22ef44b2d7e3912250a200
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
e;
|
||||
@@ -1 +0,0 @@
|
||||
0a4c000000200:010197d9ffde02570640bd22ef44b2d7e3912250a230a1407363534333231180120002a0c313838363738373430350000
|
||||
@@ -1 +0,0 @@
|
||||
0a4c00000020000101907d9ffde02570640bd22ef44b2d7e3912250a230a14080112100915007caa9b5931b76a3a85f046523e10011a09393837363534333231180120002a0c313838363738373430350000
|
||||
@@ -1 +0,0 @@
|
||||
0a4c020:0d112d7e3912250a200;
|
||||
@@ -1 +0,0 @@
|
||||
0a4c08001248000000020000101907d9ffde02570640bd22ef44b2d7e3912250a230a14080112100915007caa9b5931b76a3a85f046523e10011a09393837363534333231180120002a0c313838363738373430350000
|
||||
@@ -1 +0,0 @@
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
e2;
|
||||
Binary file not shown.
@@ -1 +0,0 @@
|
||||
0a4c08001248000000020000101907d9ffde13aa95c122678053362136bdf8408f8276e4c2d87ec52b61aa1b9f646e58734930acebe899b3e464189a14a87202fb02574e70640bd22ef44b2d7e3912250a230a14080112100915007caa9b5931b76a3a85f046523e10011a09393837363534333231180120002a0c313838363738373430350000
|
||||
@@ -1,13 +1,17 @@
|
||||
# OEMCRYPTO Fuzzing
|
||||
|
||||
Refer to [Setting up Clusterfuzz](build_clusterfuzz.md) if you are interested
|
||||
in setting up a local instance of cluster fuzz to run fuzzing on your own
|
||||
OEMCrypto implementations on linux.
|
||||
|
||||
## Objective
|
||||
|
||||
* Run fuzzing on OEMCrypto public APIs on linux using google supported
|
||||
clusterfuzz infrastructure to find security vulnerabilities.
|
||||
|
||||
|
||||
Design Document - https://docs.google.com/document/d/1mdSV2irJZz5Y9uYb5DmSIddBjrAIZU9q8G5Q_BGpA4I/edit?usp=sharing
|
||||
|
||||
Fuzzing at google -
|
||||
|
||||
Fuzzing at google -
|
||||
[go/fuzzing](https://g3doc.corp.google.com/security/fuzzing/g3doc/fuzzing_resources.md?cl=head)
|
||||
## Monitoring
|
||||
### Cluster fuzz statistics
|
||||
@@ -18,7 +22,7 @@
|
||||
|
||||
The options to select are `Job type: libfuzzer_asan_oemcrypto` and `Fuzzer:
|
||||
fuzzer name you are looking for`
|
||||
|
||||
|
||||
Example: [load_license_fuzz](https://clusterfuzz.corp.google.com/fuzzer-stats?group_by=by-day&date_start=2020-07-11&date_end=2020-07-17&fuzzer=libFuzzer_oemcrypto_load_license_fuzz&job=libfuzzer_asan_oemcrypto)
|
||||
|
||||
### Issues filed by clusterfuzz - Fixing those issues
|
||||
@@ -76,13 +80,31 @@
|
||||
$ export PATH_TO_CDM_DIR=..
|
||||
$ gyp --format=ninja --depth=$(pwd) oemcrypto/oemcrypto_unittests.gyp
|
||||
$ ninja -C out/Default/
|
||||
$ ./out/Default/oemcrypto_unittests --generate_corpus
|
||||
$ mkdir oemcrypto/test/fuzz_tests/corpus/<fuzzername>_seed_corpus
|
||||
# Generate corpus by excluding buffer overflow tests.
|
||||
$ ./out/Default/oemcrypto_unittests --generate_corpus \
|
||||
--gtest_filter=-"*Huge*"
|
||||
```
|
||||
|
||||
* To avoid uploading huge binary files to git repository, the corpus files
|
||||
will be saved in fuzzername_seed_corpus.zip format in blockbuster project's
|
||||
oemcrypto_fuzzing_corpus GCS bucket using gsutil. If you need permissions
|
||||
for blockbuster project, contact widevine-engprod@google.com.
|
||||
* There can be lot of duplicate corpus files that are generated from unit
|
||||
tests. We can minimize the corpus files to only a subset of files that
|
||||
cover unique paths within the API when run using fuzzer. Run following
|
||||
command to minimize corpus.
|
||||
|
||||
```shell
|
||||
$ cd /path/to/cdm/repo
|
||||
# build fuzzer binaries
|
||||
$ ./oemcrypto/test/fuzz_tests/build_oemcrypto_fuzztests
|
||||
$ mkdir /tmp/minimized_corpus
|
||||
# minimize corpus
|
||||
$ ./out/Default/<fuzz_target_binary> -merge=1 /tmp/minimized_corpus \
|
||||
<FULL_CORPUS_DIR>
|
||||
```
|
||||
|
||||
* To avoid uploading huge binary files to git repository, the minimized corpus
|
||||
files will be saved in fuzzername_seed_corpus.zip format in blockbuster
|
||||
project's oemcrypto_fuzzing_corpus GCS bucket using gsutil. If you need
|
||||
permissions for blockbuster project, contact widevine-engprod@google.com.
|
||||
|
||||
```shell
|
||||
$ gsutil cp gs://oemcrypto_fuzzing_corpus/<fuzzername_seed_corpus.zip> \
|
||||
@@ -104,17 +126,13 @@
|
||||
oemcrypto_fuzztests.gypi cflags_cc in order to generate additional debug
|
||||
information locally.
|
||||
|
||||
* Build and test fuzz scripts locally using:
|
||||
* Build and test fuzz scripts locally using following commands. The build
|
||||
script builds fuzz binaries for both oemcrypto reference implementation
|
||||
as well as opk implementation.
|
||||
|
||||
```shell
|
||||
$ export CXX=clang++
|
||||
$ export CC=clang
|
||||
$ export GYP_DEFINES="clang=1"
|
||||
$ cd /path/to/cdm/repo
|
||||
$ export PATH_TO_CDM_DIR=.
|
||||
$ gyp --format=ninja --depth=$(pwd) \
|
||||
oemcrypto/test/fuzz_tests/oemcrypto_fuzztests.gyp
|
||||
$ ninja -C out/Default/
|
||||
$ cd PATH_TO_CDM_DIR
|
||||
$ ./oemcrypto/test/fuzz_tests/build_oemcrypto_fuzztests
|
||||
$ mkdir /tmp/new_interesting_corpus
|
||||
$ ./out/Default/fuzzer_binary /tmp/new_interesting_corpus \
|
||||
/path/to/fuzz/seed/corpus/folder
|
||||
@@ -127,8 +145,9 @@
|
||||
$ ./out/Default/fuzzer_binary crash_input_file
|
||||
```
|
||||
## Adding a new OEMCrypto fuzz script
|
||||
* In order to fuzz a new OEMCrypto API in future, a fuzz script can be added to
|
||||
oemcrypto/test/fuzz_tests folder which ends with _fuzz.cc.
|
||||
* In order to fuzz a new OEMCrypto API in future, a fuzz script can be added
|
||||
to oemcrypto/test/fuzz_tests folder which starts with oemcrypto and ends
|
||||
with fuzz.cc(GCB build script for oemcrypto fuzzers expects the format).
|
||||
|
||||
* In the program, define the function LLVMFuzzerTestOneInput with the following signature:
|
||||
```
|
||||
@@ -164,50 +183,14 @@
|
||||
|
||||
### Adding a new fuzz script to the build script:
|
||||
|
||||
* In order to update build script such as adding a new fuzzer to build script,
|
||||
we need to update the build script in docker image from cloud repository.
|
||||
[Build script.](https://widevine-internal.googlesource.com/cloud/+/refs/heads/master/docker
|
||||
/cloud_build/oemcrypto/release/ubuntu/fuzz/build.sh)
|
||||
* As long as a new fuzz script is added which starts with oemcrypto and ends
|
||||
with fuzz, the build command can be added to build_oemcrypto_fuzztests.
|
||||
GCB script uses build_oemcrypto_fuzztests script to build fuzz binaries
|
||||
and make them available for clusterfuzz to run continuously.
|
||||
|
||||
Add the new fuzz script name to fuzzers variable and follow steps in README
|
||||
to upload new docker image. Make sure you update the tag to be higher than
|
||||
latest version in GCR.
|
||||
|
||||
Run the following command from your machine to update the docker image tag
|
||||
in the git trigger.
|
||||
|
||||
```shell
|
||||
stubby call --rpc_creds_file=/tmp/mint.txt \
|
||||
blade:alphasource-ci-proctor-metadata-service-prod \
|
||||
ProctorMetadataService.UpdateTrigger --proto2 <<EOF
|
||||
trigger {
|
||||
cloud_project_number: 257246079067
|
||||
name: "cdm-git-trigger"
|
||||
id: "e8939c9a-d971-4c05-91b5-e0544abf872b"
|
||||
state: LIVE
|
||||
git_trigger {
|
||||
url: "https://widevine-internal.googlesource.com/cdm"
|
||||
branch_name: "master"
|
||||
}
|
||||
build_configs {
|
||||
build {
|
||||
steps {
|
||||
name: "gcr.io/google.com/blockbuster-1154/
|
||||
cloud-build-oemcrypto-release-ubuntu-fuzz:LATEST_TAG_VERSION"
|
||||
}
|
||||
}
|
||||
}
|
||||
result_config {
|
||||
email_config {
|
||||
notify_condition {
|
||||
condition: ON_FAILURE
|
||||
}
|
||||
to_address: "wideving-engprod@google.com"
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
```
|
||||
* If the new fuzzer cannot follow the naming convention OR GCB script needs
|
||||
to be updated for any other reason, refer to [this section](https://docs.google.com/document/d/1mdSV2irJZz5Y9uYb5DmSIddBjrAIZU9q8G5Q_BGpA4I/edit#heading=h.bu9yfftdonkg)
|
||||
section.
|
||||
|
||||
## Generate code coverage reports locally
|
||||
|
||||
@@ -244,4 +227,4 @@
|
||||
|
||||
* Once the build job is successful, latest code coverage reports can be
|
||||
downloaded from [GCS](https://pantheon.corp.google.com/storage/browser/oemcrypto_fuzzing_code_coverage_reports;tab=objects?forceOnBucketsSortingFiltering=false&project=google.com:blockbuster-1154&prefix=).
|
||||
The coverage report folder uploaded to GCS is appended with timestamp.
|
||||
The coverage report folder uploaded to GCS is appended with timestamp.
|
||||
|
||||
174
oemcrypto/test/fuzz_tests/build_clusterfuzz.md
Normal file
174
oemcrypto/test/fuzz_tests/build_clusterfuzz.md
Normal file
@@ -0,0 +1,174 @@
|
||||
# OEMCRYPTO Fuzzing - Build clustefuzz and run fuzzing
|
||||
|
||||
## Objective
|
||||
|
||||
* Run fuzzing on OEMCrypto public APIs on linux by building open sourced
|
||||
clusterfuzz source code in order to find security vulnerabilities.
|
||||
|
||||
[Clusterfuzz][1]
|
||||
|
||||
* Partners who implement OEMCrypto can follow these instructions to build
|
||||
clusterfuzz, the fuzzing framework and run fuzzing using fuzzer scripts
|
||||
provided by the Widevine team at Google.
|
||||
|
||||
## Glossary
|
||||
|
||||
* Fuzzing - Fuzzing is a methodology where random, interesting, unexpected
|
||||
inputs are fed to APIs in order to crash those, thereby catching any
|
||||
security vulnerabilities with the code.
|
||||
|
||||
* Fuzzing engines - [libfuzzer][4], afl, honggfuzz are the actual fuzzing
|
||||
engines that get the coverage information from API, use that to generate
|
||||
more interesting inputs which can be passed to fuzzer.
|
||||
|
||||
* Seed corpus - Fuzzing engine trying to generate interesting inputs from an
|
||||
empty file is not efficient. Seed corpus is the initial input that a fuzzer
|
||||
can accept and call the API with that. Fuzzing engine can then mutate this
|
||||
seed corpus to generate more inputs to fuzzer.
|
||||
|
||||
* Clusterfuzz - ClusterFuzz is a scalable fuzzing infrastructure that finds
|
||||
security and stability issues in software. Google uses ClusterFuzz to fuzz
|
||||
all Google products. Clusterfuzz provides us with the capability, tools to
|
||||
upload fuzz binaries and make use of the fuzzing engines to run fuzzing,
|
||||
find crashes and organizes the information. Clusterfuzz framework is open
|
||||
sourced, the source code can be downloaded and framework can be built
|
||||
locally or by using google cloud.
|
||||
|
||||
* Fuzzing output - Fuzzing is used to pass random inputs to API in order to
|
||||
ensure that API is crash resistant. We are not testing functionality via
|
||||
fuzzing. Fuzz scripts run continuously until they find a crash with the API
|
||||
under test.
|
||||
|
||||
## Building fuzz scripts
|
||||
|
||||
This section outlines the steps to build fuzz binaries that can be run
|
||||
continuously using clusterfuzz.
|
||||
|
||||
> **Note:** All the directories mentioned below are relative to cdm repository
|
||||
> root directory.
|
||||
|
||||
1. Fuzz scripts for OEMCrypto APIs are provided by the Widevine team at Google
|
||||
located under `oemcrypto/test/fuzz_tests` directory.
|
||||
|
||||
> **Note:** Prerequisites to run the following step are [here][10]. We also need
|
||||
> to install ninja.
|
||||
|
||||
2. Build a static library of your OEMCrypto implementation.
|
||||
* Compile and link your OEMCrypto implementation source with
|
||||
`-fsanitize=address,fuzzer` flag as per these [instructions][9] when
|
||||
building a static library.
|
||||
|
||||
* Run `./oemcrypto/test/fuzz_tests/build_partner_oemcrypto_fuzztests
|
||||
<oemcrypto_static_library_path>` script from cdm repository root
|
||||
directory.
|
||||
|
||||
* This will generate fuzz binaries under the `out/Default` directory.
|
||||
|
||||
|
||||
|
||||
> **Note:** Alternatively, you can use your own build systems, for which you
|
||||
> will need to define your own build files with the OEMCrypto fuzz source files
|
||||
> included. You can find the the fuzz source files in
|
||||
> `oemcrypto/test/fuzz_tests/partner_oemcrypto_fuzztests.gyp` and
|
||||
> `oemcrypto/test/fuzz_tests/partner_oemcrypto_fuzztests.gypi`.
|
||||
|
||||
3. Seed corpus for each fuzz script can be found under
|
||||
`oemcrypto/test/fuzz_tests/corpus` directory. Some fuzzers are simple and do
|
||||
not have seed corpus associated with them.
|
||||
|
||||
4. Create a zip file `oemcrypto_fuzzers_yyyymmddhhmmss.zip` with fuzz binaries
|
||||
and respective seed corpus zip files. Structure of a sample zip file with
|
||||
fuzzer binaries and seed corpus would look like following:
|
||||
|
||||
```
|
||||
* fuzzerA
|
||||
* fuzzerA_seed_corpus.zip
|
||||
* fuzzerB
|
||||
* fuzzerB_seed_corpus.zip
|
||||
* fuzzerC (fuzzerC doesn't have seed corpus associated with it)
|
||||
```
|
||||
|
||||
## Building clusterfuzz
|
||||
|
||||
* OEMCrypto implementation can be fuzzed by building clusterfuzz code which is
|
||||
open sourced and using it to run fuzzing. Use a Linux VM to build
|
||||
clusterfuzz.
|
||||
|
||||
> **Note:** You may see some issues with python modules missing, please install
|
||||
> those modules if you see errors. If you have multiple versions of python on
|
||||
> the VM, then use `python<version> -m pipenv shell` when you are at [this][3]
|
||||
> step.
|
||||
|
||||
* Follow these [instructions][2] in order to download clusterfuzz repository,
|
||||
build it locally or create a continuous fuzz infrastructure setup using
|
||||
google cloud.
|
||||
|
||||
## Running fuzzers on local clusterfuzz instance
|
||||
|
||||
* If you prefer to run fuzzing on a local machine instead of having a
|
||||
production setup using google cloud, then follow these [instructions][6] to
|
||||
add a job to the local clusterfuzz instance.
|
||||
|
||||
> **Note:** Job name should have a fuzzing engine and sanitizer as part of it. A
|
||||
> libfuzzer and asan jobs should have libfuzzer_asan in the job name.
|
||||
|
||||
* Create a job e:g:`libfuzzer_asan_oemcrypto` and upload previously created
|
||||
`oemcrypto_fuzzers_yyyymmddhhmmss.zip` as a custom build. Future uploads of
|
||||
zip file should have a name greater than current name. Following the above
|
||||
naming standard will ensure zip file names are always in ascending order.
|
||||
|
||||
* Once the job is added and clusterfuzz bot is running, fuzzing should be up
|
||||
and running. Results can be monitored as mentioned [here][6].
|
||||
|
||||
* On a local clusterfuzz instance, only one fuzzer is being fuzzed at a time.
|
||||
|
||||
> **Note:** Fuzzing is time consuming. Finding issues as well as clusterfuzz
|
||||
> regressing and fixing the issues can take time. We need fuzzing to run at
|
||||
> least for a couple of weeks to have good coverage.
|
||||
|
||||
## Finding fuzz crashes
|
||||
|
||||
Once the clusterfuzz finds an issue, it logs crash information such as the
|
||||
build, test case and stack trace for the crash.
|
||||
|
||||
* Test cases tab should show the fuzz crash and test case that caused the
|
||||
crash. Run `./fuzz_binary <test_case>` in order to debug the crash locally.
|
||||
|
||||
More information about different types of logs is as below:
|
||||
|
||||
* [Bot logs][7] will show information related to fuzzing, number of crashes
|
||||
that a particular fuzzer finds, number of new crashes, number of known
|
||||
crashes etc.
|
||||
|
||||
* [Local GCS][8] in your clusterfuzz checkout folder will store the fuzz
|
||||
binaries that are being fuzzed, seed corpus etc.
|
||||
|
||||
* `local_gcs/test-fuzz-logs-bucket` will store information related to fuzz
|
||||
crashes if any were found by the fuzzing engine. It will store crash
|
||||
information categorized by fuzzer and by each day. It will also store test
|
||||
case that caused the crash.
|
||||
|
||||
* `/path/to/my-bot/clusterfuzz/log.txt` will have any log information from
|
||||
fuzzer script and OEMCrypto implementation.
|
||||
|
||||
## Fixing issues
|
||||
|
||||
* Once you are able to debug using the crash test case, apply fix to the
|
||||
implementation, create `oemcrypto_fuzzers_yyyymmddhhmmss.zip` with latest
|
||||
fuzz binaries.
|
||||
|
||||
* Upload the latest fuzz binary to the fuzz job that was created earlier.
|
||||
Fuzzer will recognize the fix and mark the crash as fixed in test cases tab
|
||||
once the regression finishes. You do not need to update crashes as fixed,
|
||||
clusterfuzz will do that.
|
||||
|
||||
[1]: https://google.github.io/clusterfuzz/
|
||||
[2]: https://google.github.io/clusterfuzz/getting-started/
|
||||
[3]: https://google.github.io/clusterfuzz/getting-started/prerequisites/#loading-pipenv
|
||||
[4]: https://llvm.org/docs/LibFuzzer.html
|
||||
[5]: https://google.github.io/clusterfuzz/setting-up-fuzzing/libfuzzer-and-afl/
|
||||
[6]: https://google.github.io/clusterfuzz/setting-up-fuzzing/libfuzzer-and-afl/#checking-results
|
||||
[7]: https://google.github.io/clusterfuzz/getting-started/local-instance/#viewing-logs
|
||||
[8]: https://google.github.io/clusterfuzz/getting-started/local-instance/#local-google-cloud-storage
|
||||
[9]: https://google.github.io/clusterfuzz/setting-up-fuzzing/libfuzzer-and-afl/#libfuzzer
|
||||
[10]: https://google.github.io/clusterfuzz/setting-up-fuzzing/libfuzzer-and-afl/#prerequisites
|
||||
24
oemcrypto/test/fuzz_tests/build_oemcrypto_fuzztests
Executable file
24
oemcrypto/test/fuzz_tests/build_oemcrypto_fuzztests
Executable file
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Also, if you are fixing this script, it should probably be moved to the
|
||||
# jenkins directory, so that it is next to all the other scripts that LUCI
|
||||
# runs.
|
||||
|
||||
set -ex
|
||||
|
||||
export CXX=clang++
|
||||
export CC=clang
|
||||
export GYP_DEFINES="$GYP_DEFINES clang=1"
|
||||
export PATH_TO_CDM_DIR=.
|
||||
export PYTHONPATH="$PYTHONPATH:$PATH_TO_CDM_DIR/third_party"
|
||||
|
||||
python3 $PATH_TO_CDM_DIR/third_party/gyp/__init__.py --format=ninja \
|
||||
--depth=$(pwd) oemcrypto/test/fuzz_tests/oemcrypto_fuzztests.gyp
|
||||
ninja -C out/Default
|
||||
# oemcrypto_opk_fuzztests.gypi has flags to instrument all the gyp targets
|
||||
# with fuzzer flags.
|
||||
python3 $PATH_TO_CDM_DIR/third_party/gyp/__init__.py --format=ninja \
|
||||
--depth=$(pwd) \
|
||||
--include=oemcrypto/test/fuzz_tests/oemcrypto_opk_fuzztests.gypi \
|
||||
oemcrypto/test/fuzz_tests/oemcrypto_opk_fuzztests.gyp
|
||||
ninja -C out/Default
|
||||
15
oemcrypto/test/fuzz_tests/build_partner_oemcrypto_fuzztests
Executable file
15
oemcrypto/test/fuzz_tests/build_partner_oemcrypto_fuzztests
Executable file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
# For use by partners to generate fuzz binaries for their OEMCrypto
|
||||
# implementation on linux.
|
||||
|
||||
export CXX=clang++
|
||||
export CC=clang
|
||||
export GYP_DEFINES="$GYP_DEFINES clang=1 oemcrypto_static_library=$1"
|
||||
export PATH_TO_CDM_DIR=.
|
||||
export PYTHONPATH="$PYTHONPATH:$PATH_TO_CDM_DIR/third_party"
|
||||
|
||||
python3 $PATH_TO_CDM_DIR/third_party/gyp/__init__.py --format=ninja \
|
||||
--depth=$(pwd) oemcrypto/test/fuzz_tests/partner_oemcrypto_fuzztests.gyp
|
||||
ninja -C out/Default
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user