ce_cdm/build.py

#!/usr/bin/python3 -B
# Copyright 2018 Google LLC. All Rights Reserved. This file and proprietary
# source code may only be used and distributed under the Widevine License
# Agreement.

# b/230527132: Even though this usually uses Python3, we still need to support
# Python2 for Cobalt builds.

"""Generates build files and builds the CDM source release."""

# Lint as: python2, python3
from __future__ import print_function

import argparse
import json
import math
import os
import subprocess
import sys

import build_utils

# Absolute path to Widevine CDM project repository.
CDM_TOP_PATH = os.path.abspath(os.path.dirname(__file__))
COBALT_TOP_PATH = os.path.join(CDM_TOP_PATH, '..', '..')

# NOTE: Use relative paths for most of this because the Xcode generator tends
# to ignore the output directory if you use absolute paths.
PLATFORMS_DIR_PATH = 'platforms'

# Exit status for script if failure arises.
EXIT_FAILURE = 1


def IsNinjaInstalled():
  """Determine if ninja is installed."""
  try:
    if hasattr(subprocess, 'DEVNULL'):
      # Provides better compatibility for Windows, not available in Python 2.x
      dev_null = subprocess.DEVNULL
    else:
      dev_null = open(os.devnull, 'w')
    subprocess.check_call(['ninja', '--version'],
                          stdout=dev_null,
                          stderr=dev_null)
    return True
  except subprocess.CalledProcessError:
    # Error code returned, probably not the ninja we're looking for.
    return False
  except OSError:
    # No such command found.
    return False


def PlatformExists(platform, print_details=False):
  """Determine if specified platform exists.

  Args:
    platform: Name of the platform (ex. "x86-64")
    print_details: An optional flag to enable printing of the reason the
      platform was determined to not exist.  Does not print anything if the
      platform is valid.

  Returns:
    True if the there exists configuration files for the specified |platform|,
    False otherwise.
  """
  target_path = os.path.join(CDM_TOP_PATH, PLATFORMS_DIR_PATH, platform)
  platform_gypi_path = os.path.join(target_path, 'settings.gypi')
  platform_environment_path = os.path.join(target_path, 'environment.py')

  def vprint(msg):
    if print_details:
      print(msg, file=sys.stderr)

  if not os.path.isdir(target_path):
    vprint(' Target path does not exist: platform = {}'.format(platform))
    return False

  if not os.path.isfile(platform_gypi_path):
    vprint(' Target platform is missing settings.gypi file: settings_path = {}'
           .format(platform_gypi_path))
    return False
  if not os.path.isfile(platform_environment_path):
    vprint(
        ' Target platform is missing environment.py file: env_path = {}'.format(
            platform_environment_path))
    return False
  return True


def RetrieveListOfPlatforms():
  """Retrieves a list of names of the support platform.

  Returns:
    A list of strings containing the name of the platform
  """
  path = os.path.join(CDM_TOP_PATH, PLATFORMS_DIR_PATH)
  if not os.path.isdir(path):
    print(
        'Cannot find platforms directory: expected_path = {}'.format(
            path),
        file=sys.stderr)
    return []
  return sorted(filter(PlatformExists, os.listdir(path)))


def VerboseSubprocess(args):
  """Print sub-process command and execute."""
  print(' Running: ' + ' '.join(args))
  return subprocess.call(args)


def RunMake(unused_output_path, options):
  """Run Make as build system."""
  os.environ['BUILDTYPE'] = options.build_config

  if options.jobs is not None:
    if math.isinf(options.jobs):
      job_args = ['-j']
    else:
      job_args = ['-j', str(options.jobs)]
  else:
    job_args = []

  if options.verbose:
    job_args.append('-v')
  job_args += options.target
  return VerboseSubprocess(['make', '-C', CDM_TOP_PATH] + job_args)


def RunNinja(output_path, options):
  """Run Ninja as build system."""
  build_path = os.path.join(output_path, options.build_config)

  if options.jobs is not None:
    if math.isinf(options.jobs):
      print('Ninja cannot run an infinite number of jobs', file=sys.stderr)
      print('Running at most 1000 jobs')
      options.jobs = 1000
    job_args = ['-j', str(options.jobs)]
  else:
    job_args = []

  if options.verbose:
    job_args += ['-v']
  job_args += options.target
  return VerboseSubprocess(['ninja', '-C', build_path] + job_args)


def RunXcode(output_path, options):
  """Run Xcode as a build system."""
  # Xcode generates a separate project for each GYP file and requires telling
  # which project to use for a target.  So we need to generate a mapping of
  # target name to the project that defined it.
  projects = (['cdm/cdm_unittests.xcodeproj'] +
              [val.replace('.gyp', '.xcodeproj') for val in options.extra_gyp])
  target_map = {}
  if 'all' in options.target or 'All' in options.target:
    target_map['All'] = os.path.join(output_path, projects[0])
    targets = ['All']
  else:
    targets = options.target
    for name in projects:
      # List every target in a project.
      project = os.path.join(output_path, name)
      cmd = ['xcodebuild', 'build', '-project', project, '-list', '-json']
      data = json.loads(subprocess.check_output(cmd).decode('utf8'))
      for scheme in data['project']['schemes'] + data['project']['targets']:
        # 'All' will appear in each project, but it is handled above.
        target_map[scheme] = project

  for scheme in targets:
    cmd = [
        'xcodebuild', 'test' if options.xcode_test else 'build',
        '-project', target_map[scheme],
        '-scheme', scheme,
        '-configuration', options.build_config,
        '-derivedDataPath', os.path.join(output_path, 'DerivedData'),
    ]
    if options.xcode_test:
      cmd += ['-only-testing:' + scheme]
    if options.ios_device_name:
      if options.ios:
        cmd += ['-destination', 'platform=iOS,name=' + options.ios_device_name]
      else:
        cmd += [
            '-destination',
            'platform=iOS Simulator,name=' + options.ios_device_name,
        ]
    elif options.ios:
      cmd += ['-destination', 'generic/platform=iOS']
    elif options.ios_sim:
      cmd += ['-destination', 'generic/platform=iOS Simulator']
    else:
      cmd += ['-destination', 'platform=macOS,arch=x86_64']

    if options.jobs is not None:
      cmd += ['-jobs', str(options.jobs)]
    if not options.verbose:
      cmd += ['-quiet']
    ret = VerboseSubprocess(cmd)
    if ret != 0:
      return ret
  return 0


# Map from generator name to generator invocation function.
BUILDERS = {
    'make': RunMake,
    'ninja': RunNinja,
    'xcode': RunXcode,
}


def GetBuilder(generator):
  return BUILDERS.get(generator)


def ImportPlatform(platform, is_cobalt, skip_deps, gyp_args):
  """Handles platform-specific setup for the named platform.

  Computes platform-specific paths, sets gyp arguments for platform-specific
  gypis and output paths, imports a platform-specific module, and exports
  platform-specific environment variables.

  Args:
    platform: The name of the platform.
    skip_deps: Whether to skip updating submodules.
    gyp_args: An array of gyp arguments to which this function will append.

  Returns:
    The path to the root of the build output.
  """

  print(' Target Platform: ' + platform)
  assert PlatformExists(platform)

  target_path = os.path.join(CDM_TOP_PATH, PLATFORMS_DIR_PATH, platform)
  platform_gypi_path = os.path.join(target_path, 'settings.gypi')
  # Use an absolute path so the Ninja generator finds the correct path; this
  # still works with the Xcode generator, which doesn't like absolute paths.
  if is_cobalt:
    output_path = os.path.join('out')
  else:
    output_path = os.path.join(CDM_TOP_PATH, 'out', platform)
    gyp_args.append('--generator-output=' + output_path)

  gyp_args.append('--include=' + platform_gypi_path)
  gyp_args.append('-Goutput_dir=' + output_path)

  target = build_utils.LoadPlatform(platform)

  if not skip_deps and 'submodules' in target:
    for path in target['submodules']:
      print(' Updating submodule:', path)
      if subprocess.call(['git', '-C', CDM_TOP_PATH, 'submodule', 'update',
                          '--init', path]) != 0:
        return None

  if 'gyp_args' in target:
    gyp_args.extend(target['gyp_args'])

  if 'export_variables' in target:
    for variable, value in target['export_variables'].items():
      existing_value = os.environ.get(variable)
      if not existing_value:
        if '"' not in value and ('/' in value or '\\' in value):
          # Use absolute paths since the output directory will be different.  If
          # "value" is already absolute, os.path.join will only use that part.
          value = os.path.normpath(os.path.join(CDM_TOP_PATH, target_path,
                                                value))
        os.environ[variable] = value
        print('* Set {} to "{}"'.format(variable, value))
      else:
        print('* Did not set {} to "{}"'.format(variable, value))
        print('    {} is already set to "{}"'.format(variable, existing_value))

  return output_path


def main(args):
  if sys.platform == 'darwin':
    print('Host platform is Mac - use xcode for default generator')
    default_generator = 'xcode'
  elif IsNinjaInstalled():
    print('ninja is installed - use ninja for default generator')
    default_generator = 'ninja'
  else:
    print('ninja is not installed - use make for default generator')
    default_generator = 'make'

  parser = argparse.ArgumentParser()
  parser.add_argument(
      'platform',
      nargs='?',
      help=('The platform to target. To add a new platform, create a new '
            'platform directory with "environment.py" and "settings.gypi" '
            'files under platforms/.'),
      choices=RetrieveListOfPlatforms())

  build_config_group = parser.add_mutually_exclusive_group(required=True)
  build_config_group.add_argument(
      '-d',
      '--debug',
      dest='build_config',
      action='store_const',
      const='debug',
      help='Build a debug build. This is shorthand for "--config debug".')
  build_config_group.add_argument(
      '-r',
      '--release',
      dest='build_config',
      action='store_const',
      const='release',
      help='Build a release build. This is shorthand for "--config release".')
  build_config_group.add_argument(
      '-c',
      '--config',
      dest='build_config',
      help=('Select a build configuration to use. Any configuration defined in '
            'the chosen platform\'s "settings.gypi" file may be used.'))

  ios_group = parser.add_mutually_exclusive_group()
  ios_group.add_argument(
      '--ios',
      action='store_true',
      help=('On Mac, build for iOS device instead.'))
  ios_group.add_argument(
      '--ios_sim',
      action='store_true',
      help=('On Mac, build for iOS simulator instead.'))

  parser.add_argument(
      '-g',
      '--generator',
      default=default_generator,
      help='Which build system to use. Defaults to {}.'.format(
          default_generator),
      choices=BUILDERS.keys())
  parser.add_argument(
      '-t',
      '--target',
      nargs='+',
      default=['all', 'widevine_ce_cdm_shared'],
      help='Which target(s) to build.  Can specify multiple values.')
  parser.add_argument(
      '-j',
      '--jobs',
      nargs='?',
      const=float('inf'),
      type=int,
      help=('When building, run up to this many jobs in parallel. The default '
            'is the default for your chosen generator. If this flag is '
            'specified without a value, jobs will spawn without limit.'))
  parser.add_argument(
      '-D',
      '--define',
      action='append',
      default=[],
      help=('Pass variable definitions to GYP. (May be specified multiple '
            'times.)'))
  parser.add_argument(
      '-e',
      '--extra_gyp',
      action='append',
      default=[],
      help=('External GYP file that is processed after the standard GYP files. '
            '(May be specified multiple times.)'))
  parser.add_argument(
      '--xcode_test',
      action='store_true',
      help='When using Xcode, run tests in addition to building.')
  parser.add_argument(
      '--ios_device_name',
      help='Use the given iOS device name for builds/tests.')
  parser.add_argument(
      '-v',
      '--verbose',
      action='store_true',
      help=('Print verbose build output, including verbose output from the '
            'generator.'))
  parser.add_argument(
      '--skip-deps',
      action='store_true',
      help="Don't download/update submodules.")
  parser.add_argument(
      '--cobalt',
      metavar='CONFIG',
      help=('Build using Cobalt tools using the given Cobalt config; project '
            'must be checked out within the Cobalt source tree.'))

  options = parser.parse_args(args)

  if options.cobalt:
    if sys.version_info.major != 2:
      parser.error('Must use python2 with --cobalt')
    if options.ios or options.ios_sim:
      parser.error('Cannot use --cobalt with --ios/--ios_sim')
    if not options.platform:
      options.platform = 'cobalt'
  if not options.platform:
    parser.error('Must specify platform or use --cobalt')

  # pylint: disable=C6204
  # If gyp has been installed locally in third_party, this will find it.
  # Irrelevant if gyp has been installed globally.
  if options.cobalt:
    os.chdir(COBALT_TOP_PATH)
    sys.path.insert(1, os.path.join(COBALT_TOP_PATH, 'tools', 'gyp', 'pylib'))
    sys.path.append(COBALT_TOP_PATH)
  else:
    os.chdir(CDM_TOP_PATH)
    sys.path.insert(1, os.path.join(CDM_TOP_PATH, 'third_party'))
  import gyp
  # pylint: enable=C6204

  if not PlatformExists(options.platform, print_details=True):
    platforms = RetrieveListOfPlatforms()
    if platforms:
      print(' Available platforms: ' + ', '.join(platforms), file=sys.stderr)
    return EXIT_FAILURE

  gyp_args = [
      '-D', 'generator=' + options.generator,
      # On iOS, we can't easily pass environment variables, so we define a
      # compiler flag to store the filter.
      '-D', 'gtest_filter=' + os.environ.get('GTEST_FILTER', '*'),
      '--depth', '.',
  ]
  if options.cobalt:
    os.environ['COBALT_CONFIG'] = options.cobalt
    options.build_config = options.cobalt + '_' + options.build_config
    gyp_args += [
        '-f', options.generator + '-' + options.cobalt,
        '--toplevel-dir=.',
        '-DOS=starboard',
        '-Gconfig=' + options.build_config,
    ]
  else:
    gyp_args += [
        '-f', options.generator,
    ]

  if options.ios or options.ios_sim:
    gyp_args += ['-DOS=ios']
  for var in options.define:
    gyp_args.append('-D' + var)

  for var in options.extra_gyp:
    gyp_args.append(var)

  output_path = ImportPlatform(options.platform, options.cobalt,
                               options.skip_deps, gyp_args)
  if not output_path:
    return 1

  if options.cobalt:
    gyp_args.append(os.path.join('third_party', 'cdm', 'cdm',
                                 'cdm_unittests.gyp'))
  else:
    gyp_args.append(os.path.join('cdm', 'cdm_unittests.gyp'))

  print(' Running: {}'.format(' '.join(['gyp'] + gyp_args)))
  retval = gyp.main(gyp_args)
  if retval != 0:
    return retval

  # The gyp argument --build=xyz only works on newer versions of gyp and
  # ignores the generator flag output_dir (as of 2014-05-28 with ninja).
  # So instead of using --build, we simply invoke the build system ourselves.
  builder = GetBuilder(options.generator)
  if builder is None:
    print(' Cannot automatically build with this generator', file=sys.stderr)
    print(' Please start the build manually', file=sys.stderr)
    return EXIT_FAILURE
  return builder(output_path, options)


if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))