Automatically generate RS prebuilts using python script.
Bug: 22377128
Test: manual
- Use build_rs.py to make all RenderScript related host/device
prebuilts, and package them appropriately into archives for all the
supported host OS.
- The build server can then be configured to build RS prebuilts
per build, automatically.
- A new PHONY target rs-prebuilts-all is added.
- update-prebuilts.py will pull the artifacts/archives from build
server and generate CLs to update the prebuilts in the tree. The
build number and build branch information will be automatically
included in the commit message.
Change-Id: I35a6553c9d0ab399978c31fcd1d881f900752e09
diff --git a/build_rs.py b/build_rs.py
new file mode 100755
index 0000000..923b5cb
--- /dev/null
+++ b/build_rs.py
@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import print_function
+
+import argparse
+import glob
+import multiprocessing
+import os
+import shutil
+import subprocess
+import sys
+import re
+
+
+THIS_DIR = os.path.realpath(os.path.dirname(__file__))
+ORIG_ENV = dict(os.environ)
+
+
+def android_path(*args):
+ out_dir = os.path.realpath(os.path.join(THIS_DIR, '../..', *args))
+ return out_dir
+
+
+def build_path(*args):
+ # Our multistage build directories will be placed under OUT_DIR if it is in
+ # the environment. By default they will be placed under
+ # $ANDROID_BUILD_TOP/out.
+ top_out = ORIG_ENV.get('OUT_DIR', android_path('out'))
+ if not os.path.isabs(top_out):
+ top_out = os.path.realpath(top_out)
+ out_dir = os.path.join(top_out, *args)
+ return out_dir
+
+
+def install_file(src, dst):
+ print('Copying ' + src)
+ shutil.copy2(src, dst)
+
+
+def install_directory(src, dst):
+ print('Copying ' + src)
+ shutil.copytree(src, dst)
+
+
+def build(out_dir):
+ products = (
+ 'aosp_arm',
+ 'aosp_arm64',
+ 'aosp_mips',
+ # 'aosp_mips64',
+ 'aosp_x86',
+ 'aosp_x86_64',
+ )
+ for product in products:
+ build_product(out_dir, product)
+
+
+def build_product(out_dir, product):
+ env = dict(ORIG_ENV)
+ env['ANDROID_USE_BUILDCACHE'] = 'false'
+ env['FORCE_BUILD_LLVM_COMPONENTS'] = 'true'
+ env['FORCE_BUILD_RS_COMPAT'] = 'true'
+ env['OUT_DIR'] = out_dir
+ env['SKIP_LLVM_TESTS'] = 'true'
+ env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
+ env['TARGET_BUILD_VARIANT'] = 'userdebug'
+ env['TARGET_PRODUCT'] = product
+
+ jobs_arg = '-j{}'.format(multiprocessing.cpu_count())
+ targets = [
+ # PHONY target specified in frameworks/rs/Android.mk.
+ 'rs-prebuilts-full',
+ # We have to explicitly specify the jar for JACK to build.
+ android_path('out/target/common/obj/JAVA_LIBRARIES/' +
+ 'android-support-v8-renderscript_intermediates/classes.jar')
+ ]
+ subprocess.check_call(
+ ['make', jobs_arg] + targets, cwd=android_path(), env=env)
+
+
+def package_toolchain(build_dir, build_name, host, dist_dir):
+ package_name = 'renderscript-' + build_name
+ install_host_dir = build_path('install', host)
+ install_dir = os.path.join(install_host_dir, package_name)
+
+ # Remove any previously installed toolchain so it doesn't pollute the
+ # build.
+ if os.path.exists(install_host_dir):
+ shutil.rmtree(install_host_dir)
+
+ install_toolchain(build_dir, install_dir, host)
+
+ tarball_name = package_name + '-' + host
+ package_path = os.path.join(dist_dir, tarball_name) + '.tar.bz2'
+ print('Packaging ' + package_path)
+ args = [
+ 'tar', '-cjC', install_host_dir, '-f', package_path, package_name
+ ]
+ subprocess.check_call(args)
+
+
+def install_toolchain(build_dir, install_dir, host):
+ install_built_host_files(build_dir, install_dir, host)
+ install_clang_headers(build_dir, install_dir, host)
+ install_built_device_files(build_dir, install_dir, host)
+ install_license_files(install_dir)
+ install_repo_prop(install_dir)
+
+
+def install_built_host_files(build_dir, install_dir, host):
+ is_windows = host.startswith('windows')
+ is_darwin = host.startswith('darwin-x86')
+ bin_ext = '.exe' if is_windows else ''
+
+ if is_windows:
+ lib_ext = '.dll'
+ elif is_darwin:
+ lib_ext = '.dylib'
+ else:
+ lib_ext = '.so'
+
+ built_files = [
+ 'bin/llvm-rs-cc' + bin_ext,
+ 'bin/bcc_compat' + bin_ext,
+ ]
+
+ if is_windows:
+ built_files.extend([
+ 'lib/libbcc' + lib_ext,
+ 'lib/libbcinfo' + lib_ext,
+ 'lib/libclang' + lib_ext,
+ 'lib/libLLVM' + lib_ext,
+ ])
+ else:
+ built_files.extend([
+ 'lib64/libbcc' + lib_ext,
+ 'lib64/libbcinfo' + lib_ext,
+ 'lib64/libclang' + lib_ext,
+ 'lib64/libLLVM' + lib_ext,
+ 'lib64/libc++' + lib_ext,
+ ])
+
+ for built_file in built_files:
+ dirname = os.path.dirname(built_file)
+ install_path = os.path.join(install_dir, dirname)
+ if not os.path.exists(install_path):
+ os.makedirs(install_path)
+
+ built_path = os.path.join(build_dir, 'host', host, built_file)
+ install_file(built_path, install_path)
+
+ file_name = os.path.basename(built_file)
+
+ # Only strip bin files (not libs) on darwin.
+ if not is_darwin or built_file.startswith('bin/'):
+ subprocess.check_call(
+ ['strip', os.path.join(install_path, file_name)])
+
+
+def install_clang_headers(build_dir, install_dir, host):
+ def should_copy(path):
+ if os.path.basename(path) in ('Makefile', 'CMakeLists.txt'):
+ return False
+ _, ext = os.path.splitext(path)
+ if ext == '.mk':
+ return False
+ return True
+
+ headers_src = android_path('external/clang/lib/Headers')
+ headers_dst = os.path.join(
+ install_dir, 'clang-include')
+ os.makedirs(headers_dst)
+ for header in os.listdir(headers_src):
+ if not should_copy(header):
+ continue
+ install_file(os.path.join(headers_src, header), headers_dst)
+
+ install_file(android_path('bionic/libc/include/stdatomic.h'), headers_dst)
+
+
+def install_built_device_files(build_dir, install_dir, host):
+ product_to_arch = {
+ 'generic': 'arm',
+ 'generic_arm64': 'arm64',
+ 'generic_mips': 'mips',
+ # 'generic_mips64': 'mips64el',
+ 'generic_x86': 'x86',
+ 'generic_x86_64': 'x86_64',
+ }
+
+ bc_lib = 'librsrt'
+
+ static_libs = {
+ 'libRScpp_static',
+ 'libcompiler_rt'
+ }
+
+ shared_libs = {
+ 'libRSSupport.so',
+ 'libRSSupportIO.so',
+ 'libblasV8.so',
+ }
+
+ for product, arch in product_to_arch.items():
+ lib_dir = os.path.join(install_dir, 'platform', arch)
+ os.makedirs(lib_dir)
+
+ # Copy librsrt_ARCH.bc.
+ if not host.startswith('windows'):
+ lib_name = bc_lib + '_' + arch + '.bc'
+ built_lib = os.path.join(build_dir, 'host', host, 'lib64', lib_name)
+ install_file(built_lib, os.path.join(lib_dir, bc_lib + '.bc'))
+
+ # Copy static libs and share libs.
+ product_dir = os.path.join(build_dir, 'target/product', product)
+ static_lib_dir = os.path.join(product_dir, 'obj/STATIC_LIBRARIES')
+ shared_lib_dir = os.path.join(product_dir, 'obj/lib')
+ for static_lib in static_libs:
+ built_lib = os.path.join(
+ static_lib_dir, static_lib + '_intermediates/' + static_lib + '.a')
+ lib_name = static_lib + '.a'
+ install_file(built_lib, os.path.join(lib_dir, lib_name))
+ for shared_lib in shared_libs:
+ built_lib = os.path.join(shared_lib_dir, shared_lib)
+ lib_name = shared_lib
+ install_file(built_lib, os.path.join(lib_dir, lib_name))
+
+ # Copy renderscript-v8.jar.
+ lib_dir = os.path.join(install_dir, 'platform')
+ jar_dir = os.path.join(build_dir, 'target/common/obj/JAVA_LIBRARIES/'
+ 'android-support-v8-renderscript_intermediates/classes.jar')
+ install_file(jar_dir, os.path.join(lib_dir, 'renderscript-v8.jar'))
+
+ # Copy RS runtime headers.
+ headers_dst_base = os.path.join(install_dir, 'platform', 'rs')
+
+ headers_src = android_path('frameworks/rs/scriptc')
+ headers_dst = os.path.join(headers_dst_base, 'scriptc')
+ install_directory(headers_src, headers_dst)
+
+ # Copy RS C++ API headers.
+ headers_src = android_path('frameworks/rs/cpp/util')
+ headers_dst = os.path.join(headers_dst_base, 'cpp/util')
+ install_directory(headers_src, headers_dst)
+ install_file(android_path('frameworks/rs/rsDefines.h'), headers_dst_base)
+ install_file(android_path('frameworks/rs/cpp/RenderScript.h'), os.path.join(headers_dst_base, 'cpp'))
+ install_file(android_path('frameworks/rs/cpp/rsCppStructs.h'), os.path.join(headers_dst_base, 'cpp'))
+
+
+def install_license_files(install_dir):
+ projects = (
+ 'external/clang',
+ 'external/compiler-rt',
+ 'external/llvm',
+ 'frameworks/compile/slang',
+ 'frameworks/compile/libbcc',
+ # 'frameworks/rs', # No notice license file found.
+ )
+
+ notices = []
+ for project in projects:
+ project_path = android_path(project)
+ license_pattern = os.path.join(project_path, 'MODULE_LICENSE_*')
+ for license_file in glob.glob(license_pattern):
+ install_file(license_file, install_dir)
+ with open(os.path.join(project_path, 'NOTICE')) as notice_file:
+ notices.append(notice_file.read())
+ with open(os.path.join(install_dir, 'NOTICE'), 'w') as notice_file:
+ notice_file.write('\n'.join(notices))
+
+
+def install_repo_prop(install_dir):
+ file_name = 'repo.prop'
+
+ dist_dir = os.environ.get('DIST_DIR')
+ if dist_dir is not None:
+ dist_repo_prop = os.path.join(dist_dir, file_name)
+ shutil.copy(dist_repo_prop, install_dir)
+ else:
+ out_file = os.path.join(install_dir, file_name)
+ with open(out_file, 'w') as prop_file:
+ cmd = [
+ 'repo', 'forall', '-c',
+ 'echo $REPO_PROJECT $(git rev-parse HEAD)',
+ ]
+ subprocess.check_call(cmd, stdout=prop_file)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ '--build-name', default='dev', help='Release name for the package.')
+
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+
+ if sys.platform.startswith('linux'):
+ hosts = ['linux-x86', 'windows-x86']
+ elif sys.platform == 'darwin':
+ hosts = ['darwin-x86']
+ else:
+ raise RuntimeError('Unsupported host: {}'.format(sys.platform))
+
+ out_dir = build_path()
+ build(out_dir=out_dir)
+
+ dist_dir = ORIG_ENV.get('DIST_DIR', out_dir)
+ for host in hosts:
+ package_toolchain(out_dir, args.build_name, host, dist_dir)
+
+
+if __name__ == '__main__':
+ main()
