update-prebuilts.py

#!/usr/bin/env python
#
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=not-callable, relative-import

"""Update the prebuilt clang from the build server."""

import argparse
import inspect
import logging
import os
import shutil
import subprocess
import sys
import utils


BRANCH = 'aosp-llvm-toolchain'


def logger():
    """Returns the module level logger."""
    return logging.getLogger(__name__)


def unchecked_call(cmd, *args, **kwargs):
    """subprocess.call with logging."""
    logger().info('unchecked_call: %s', subprocess.list2cmdline(cmd))
    return subprocess.call(cmd, *args, **kwargs)


def check_call(cmd, *args, **kwargs):
    """subprocess.check_call with logging."""
    logger().info('check_call: %s', subprocess.list2cmdline(cmd))
    subprocess.check_call(cmd, *args, **kwargs)


class ArgParser(argparse.ArgumentParser):
    def __init__(self):
        super(ArgParser, self).__init__(
            description=inspect.getdoc(sys.modules[__name__]))

        self.add_argument(
            'build', metavar='BUILD',
            help='Build number to pull from the build server.')

        self.add_argument(
            '-b', '--bug', help='Bug to reference in commit message.')

        self.add_argument(
            '--use-current-branch', action='store_true',
            help='Do not repo start a new branch for the update.')

        self.add_argument(
            '--skip-fetch',
            '-sf',
            action='store_true',
            default=False,
            help='Skip the fetch, and only do the extraction step')

        self.add_argument(
            '--skip-cleanup',
            '-sc',
            action='store_true',
            default=False,
            help='Skip the cleanup, and leave intermediate files')


def fetch_artifact(branch, target, build, pattern):
    fetch_artifact_path = '/google/data/ro/projects/android/fetch_artifact'
    cmd = [fetch_artifact_path, '--branch={}'.format(branch),
           '--target={}'.format(target), '--bid={}'.format(build), pattern]
    check_call(cmd)


def extract_package(package, install_dir):
    cmd = ['tar', 'xf', package, '-C', install_dir]
    check_call(cmd)


def extract_clang_info(clang_dir):
    version_file_path = os.path.join(clang_dir, 'AndroidVersion.txt')
    with open(version_file_path) as version_file:
        # e.g. for contents: ['7.0.1', 'based on r326829']
        contents = [l.strip() for l in version_file.readlines()]
        version = contents[0]
        revision = contents[1].split()[-1]
        return version, revision


def symlink_to_linux_resource_dir(install_dir):
    # Assume we're in a Darwin (non-linux) prebuilt dir.  Find the Clang version
    # string.  Pick the longest string, if there's more than one.
    version_dirs = os.listdir(os.path.join(install_dir, 'lib64', 'clang'))
    if len(version_dirs) != 0:
        version_dirs.sort(key=len)
    version_dir = version_dirs[-1]

    symlink_dir = os.path.join(install_dir, 'lib64', 'clang', version_dir, 'lib')
    link_src = os.path.join('/'.join(['..'] * 6), 'linux-x86', symlink_dir, 'linux')
    link_dst = 'linux'

    # 'cd' to symlink_dir and create a symlink from link_dst to link_src
    prebuilt_dir = os.getcwd()
    os.chdir(symlink_dir)
    os.symlink(link_src, link_dst)
    os.chdir(prebuilt_dir)


def format_bug(bug):
    """Formats a bug for use in a commit message.

    Bugs might be a number, in which case they're a buganizer reference to be
    formatted. If not, assume the user knows what they're doing and just return
    the string as-is.
    """
    try:
        return 'http://b/{}'.format(int(bug))
    except ValueError:
        return bug


def update_clang(host, build_number, use_current_branch, download_dir, bug,
                 manifest):
    prebuilt_dir = utils.android_path('prebuilts/clang/host', host)
    os.chdir(prebuilt_dir)

    if not use_current_branch:
        branch_name = 'update-clang-{}'.format(build_number)
        unchecked_call(
            ['repo', 'abandon', branch_name, '.'])
        check_call(
            ['repo', 'start', branch_name, '.'])

    host_filename = host
    # Get the correct host file for Windows.  The git project name is
    # different than the package created by build.py.
    if host == 'windows-x86':
        host_filename = 'windows-x86-64'

    package = '{}/clang-{}-{}.tar.bz2'.format(
        download_dir, build_number, host_filename)
    manifest_file = '{}/{}'.format(download_dir, manifest)

    extract_package(package, prebuilt_dir)

    extract_subdir = 'clang-' + build_number
    clang_version, svn_revision = extract_clang_info(extract_subdir)

    # Install into clang-<svn_revision>.  Suffixes ('a', 'b', 'c' etc.), if any,
    # are included in the svn_revision.
    install_subdir = 'clang-' + svn_revision
    os.rename(extract_subdir, install_subdir)

    # Some platform tests (e.g. system/bt/profile/sdp) build directly with
    # coverage instrumentation and rely on the driver to pick the correct
    # profile runtime.  Symlink the Linux resource dir from the Linux toolchain
    # into the Darwin toolchain so the runtime is found by the Darwin Clang
    # driver.
    if host == 'darwin-x86':
        symlink_to_linux_resource_dir(install_subdir)

    shutil.copy(manifest_file, prebuilt_dir + '/' + install_subdir)

    check_call(['git', 'add', install_subdir])

    # If there is no difference with the new files, we are already done.
    diff = unchecked_call(['git', 'diff', '--cached', '--quiet'])
    if diff == 0:
        logger().info('Bypassed commit with no diff')
        return

    message_lines = [
        'Update prebuilt Clang to {}.'.format(svn_revision),
        '', 'clang {} (based on {}) from build {}.'.format(
            clang_version, svn_revision, build_number)
    ]
    if bug is not None:
        message_lines.append('')
        message_lines.append('Bug: {}'.format(format_bug(bug)))
    message_lines.append('Test: N/A')
    message = '\n'.join(message_lines)
    check_call(['git', 'commit', '-m', message])


def main():
    args = ArgParser().parse_args()
    logging.basicConfig(level=logging.INFO)

    do_fetch = not args.skip_fetch
    do_cleanup = not args.skip_cleanup

    download_dir = os.path.realpath('.download')
    if do_fetch:
        if os.path.isdir(download_dir):
            shutil.rmtree(download_dir)
        os.makedirs(download_dir)

    os.chdir(download_dir)

    targets = ['darwin_mac', 'linux', 'windows_x86_64']
    hosts = ['darwin-x86', 'linux-x86', 'windows-x86']
    clang_pattern = 'clang-*.tar.bz2'
    manifest = 'manifest_{}.xml'.format(args.build)
    branch = 'aosp-llvm-toolchain'

    try:
        if do_fetch:
            fetch_artifact(branch, targets[0], args.build, manifest)
            for target in targets:
                fetch_artifact(branch, target, args.build, clang_pattern)

        for host in hosts:
            update_clang(host, args.build, args.use_current_branch,
                         download_dir, args.bug, manifest)
    finally:
        if do_cleanup:
            shutil.rmtree(download_dir)

    return 0


if __name__ == '__main__':
    main()