blob: e1f34cc86b417556e41e62243342425d8afd3282 [file] [log] [blame]
#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This tool translates a collection of BUILD.gn files into a mostly equivalent
# Android.bp file for the Android Soong build system. The input to the tool is a
# JSON description of the GN build definition generated with the following
# command:
#
# gn desc out --format=json --all-toolchains "//*" > desc.json
#
# The tool is then given a list of GN labels for which to generate Android.bp
# build rules. The dependencies for the GN labels are squashed to the generated
# Android.bp target, except for actions which get their own genrule. Some
# libraries are also mapped to their Android equivalents -- see |builtin_deps|.
import argparse
import json
import os
import re
import sys
from typing import Dict
from typing import List
from typing import Optional
import gn_utils
from gn_utils import GnParser
from compat import itervalues
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Arguments for the GN output directory.
gn_args = ' '.join([
'is_debug=false',
'is_perfetto_build_generator=true',
'perfetto_build_with_android=true',
'target_cpu="arm"',
'target_os="android"',
])
# Default targets to translate to the blueprint file.
default_targets = [
'//:libperfetto_client_experimental',
'//:libperfetto',
'//:perfetto_integrationtests',
'//:perfetto_unittests',
'//protos/perfetto/trace:perfetto_trace_protos',
'//protos/perfetto/trace/android:perfetto_winscope_extensions_zero',
'//src/android_internal:libperfetto_android_internal',
'//src/base:perfetto_base_default_platform',
'//src/shared_lib:libperfetto_c',
'//src/perfetto_cmd:perfetto',
'//src/perfetto_cmd:trigger_perfetto',
'//src/profiling/memory:heapprofd_client',
'//src/profiling/memory:heapprofd_client_api',
'//src/profiling/memory:heapprofd_api_noop',
'//src/profiling/memory:heapprofd',
'//src/profiling/memory:heapprofd_standalone_client',
'//src/profiling/perf:traced_perf',
'//src/traced/probes:traced_probes',
'//src/traced/service:traced',
'//src/traced_relay:traced_relay',
'//src/trace_processor:trace_processor_shell',
'//src/trace_redaction:trace_redactor',
'//src/java_sdk/main:perfetto_java_sdk_app',
'//test/cts:perfetto_cts_deps',
'//test/cts:perfetto_cts_jni_deps',
'//test:perfetto_gtest_logcat_printer',
'//test:perfetto_end_to_end_integrationtests',
'//test/vts:perfetto_vts_deps',
]
# Host targets
ipc_plugin = '//src/ipc/protoc_plugin:ipc_plugin(%s)' % gn_utils.HOST_TOOLCHAIN
protozero_plugin = '//src/protozero/protoc_plugin:protozero_plugin(%s)' % (
gn_utils.HOST_TOOLCHAIN)
cppgen_plugin = '//src/protozero/protoc_plugin:cppgen_plugin(%s)' % (
gn_utils.HOST_TOOLCHAIN)
default_targets += [
'//src/traceconv:traceconv(%s)' % gn_utils.HOST_TOOLCHAIN,
protozero_plugin,
ipc_plugin,
]
# Defines a custom init_rc argument to be applied to the corresponding output
# blueprint target.
target_initrc = {
'//src/traced/service:traced': {'perfetto.rc'},
'//src/profiling/memory:heapprofd': {'heapprofd.rc'},
'//src/profiling/perf:traced_perf': {'traced_perf.rc'},
}
target_host_supported = [
'//:libperfetto',
'//:libperfetto_client_experimental',
'//protos/perfetto/trace:perfetto_trace_protos',
'//protos/perfetto/trace/android:perfetto_winscope_extensions_zero',
'//src/shared_lib:libperfetto_c',
'//src/trace_processor:demangle',
'//src/trace_processor:trace_processor_shell',
'//src/traced/probes:traced_probes',
'//src/traced/service:traced',
]
target_vendor_available = [
'//:libperfetto_client_experimental',
'//src/shared_lib:libperfetto_c',
]
target_product_available = [
'//:libperfetto_client_experimental',
]
# Proto target groups which will be made public.
proto_groups = {
'trace': {
'types': ['lite'],
'targets': [
'//protos/perfetto/trace:non_minimal_source_set',
'//protos/perfetto/trace:minimal_source_set',
]
},
'winscope': {
'types': ['filegroup'],
'targets': [
'//protos/perfetto/trace:non_minimal_source_set',
'//protos/perfetto/trace/android:winscope_extensions_source_set',
]
},
'config': {
'types': ['lite', 'filegroup'],
'targets': [
'//protos/perfetto/config:source_set',
]
},
'metrics': {
'types': ['python'],
'targets': [
'//protos/perfetto/metrics:source_set',
]
},
}
needs_libfts = [
'//:perfetto_unittests',
'//src/trace_processor:trace_processor_shell',
'//src/traceconv:traceconv',
]
# All module names are prefixed with this string to avoid collisions.
module_prefix = 'perfetto_'
# Shared libraries which are directly translated to Android system equivalents.
shared_library_allowlist = [
'android',
'[email protected]',
'[email protected]',
'android.hardware.health-V2-ndk',
'[email protected]',
'android.hardware.power.stats-V1-cpp',
'android.system.suspend.control.internal-cpp',
'base',
'binder',
'binder_ndk',
'cutils',
'hidlbase',
'hidltransport',
'hwbinder',
'incident',
'log',
'services',
'statssocket',
'tracingproxy',
'utils',
'statspull',
]
# Static libraries which are directly translated to Android system equivalents.
static_library_allowlist = [
'statslog_perfetto',
]
# Name of the module which settings such as compiler flags for all other
# modules.
defaults_module = module_prefix + 'defaults'
# Location of the project in the Android source tree.
tree_path = 'external/perfetto'
# Path for the protobuf sources in the standalone build.
buildtools_protobuf_src = '//buildtools/protobuf/src'
# Location of the protobuf src dir in the Android source tree.
android_protobuf_src = 'external/protobuf/src'
# Compiler flags which are passed through to the blueprint.
cflag_allowlist = r'^-DPERFETTO.*$'
# Compiler defines which are passed through to the blueprint.
define_allowlist = r'^(GOOGLE_PROTO.*)|(ZLIB_.*)|(USE_MMAP)$'
# The directory where the generated perfetto_build_flags.h will be copied into.
buildflags_dir = 'include/perfetto/base/build_configs/android_tree'
def enumerate_data_deps():
with open(os.path.join(ROOT_DIR, 'tools', 'test_data.txt')) as f:
lines = f.readlines()
for line in (line.strip() for line in lines if not line.startswith('#')):
assert os.path.exists(line), 'file %s should exist' % line
if line.startswith('test/data/'):
# Skip test data files that require GCS. They are only for benchmarks.
# We don't run benchmarks in the android tree.
continue
if line.endswith('/.'):
yield line[:-1] + '**/*'
else:
yield line
# Additional arguments to apply to Android.bp rules.
additional_args = {
'heapprofd_client_api': [
('static_libs', {'libasync_safe'}),
# heapprofd_client_api MUST NOT have global constructors. Because it
# is loaded in an __attribute__((constructor)) of libc, we cannot
# guarantee that the global constructors get run before it is used.
('cflags', {'-Wglobal-constructors', '-Werror=global-constructors'}),
('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'),
('stubs', {
'versions': ['S'],
'symbol_file': 'src/profiling/memory/heapprofd_client_api.map.txt',
}),
('export_include_dirs', {'src/profiling/memory/include'}),
],
'heapprofd_api_noop': [
('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'),
('stubs', {
'versions': ['S'],
'symbol_file': 'src/profiling/memory/heapprofd_client_api.map.txt',
}),
('export_include_dirs', {'src/profiling/memory/include'}),
],
'heapprofd_client': [
('include_dirs', {'bionic/libc'}),
('static_libs', {'libasync_safe'}),
],
'heapprofd_standalone_client': [
('static_libs', {'libasync_safe'}),
('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'),
('export_include_dirs', {'src/profiling/memory/include'}),
('stl', 'libc++_static'),
],
'perfetto_unittests': [
('data', set(enumerate_data_deps())),
('include_dirs', {'bionic/libc/kernel'}),
],
'perfetto_integrationtests': [
('test_suites', {'general-tests'}),
('test_config', 'PerfettoIntegrationTests.xml'),
],
'libperfetto_android_internal': [('static_libs', {'libhealthhalutils'}),],
'trace_processor_shell': [
('strip', {
'all': True
}),
('host', {
'stl': 'libc++_static',
'dist': {
'targets': ['sdk_repo']
},
}),
],
'libperfetto_client_experimental': [
('apex_available', {
'//apex_available:platform', '//apex_available:anyapex'
}),
('min_sdk_version', '30'),
('shared_libs', {'liblog'}),
('export_include_dirs', {'include', buildflags_dir}),
],
'libperfetto_c': [
('min_sdk_version', '30'),
('export_include_dirs', {'include'}),
('cflags', {'-DPERFETTO_SHLIB_SDK_IMPLEMENTATION'}),
],
'perfetto_trace_protos': [
('apex_available', {
'//apex_available:platform', 'com.android.art',
'com.android.art.debug'
}),
('min_sdk_version', 'S'),
],
'libperfetto': [('export_include_dirs', {'include', buildflags_dir}),],
'perfetto': [('required', {'perfetto_persistent_cfg.pbtxt'}),],
'trace_redactor': [
('min_sdk_version', '35'),
('apex_available', {
'//apex_available:platform',
'com.android.profiling'
}),
],
}
def enable_base_platform(module):
module.srcs.add(':perfetto_base_default_platform')
def enable_gtest_and_gmock(module):
module.static_libs.add('libgmock')
module.static_libs.add('libgtest')
if module.name != 'perfetto_gtest_logcat_printer':
module.whole_static_libs.add('perfetto_gtest_logcat_printer')
def enable_protobuf_full(module):
if module.type == 'cc_binary_host':
module.static_libs.add('libprotobuf-cpp-full')
elif module.host_supported:
module.host.static_libs.add('libprotobuf-cpp-full')
module.android.shared_libs.add('libprotobuf-cpp-full')
else:
module.shared_libs.add('libprotobuf-cpp-full')
def enable_protobuf_lite(module):
module.shared_libs.add('libprotobuf-cpp-lite')
def enable_protoc_lib(module):
if module.type == 'cc_binary_host':
module.static_libs.add('libprotoc')
else:
module.shared_libs.add('libprotoc')
def enable_libunwindstack(module):
if module.name != 'heapprofd_standalone_client':
module.shared_libs.add('libunwindstack')
module.shared_libs.add('libprocinfo')
module.shared_libs.add('libbase')
else:
module.static_libs.add('libunwindstack')
module.static_libs.add('libprocinfo')
module.static_libs.add('libbase')
module.static_libs.add('liblzma')
module.static_libs.add('libdexfile_support')
module.runtime_libs.add('libdexfile') # libdexfile_support dependency
module.shared_libs.add('libz') # libunwindstack dependency
def enable_libunwind(module):
# libunwind is disabled on Darwin so we cannot depend on it.
pass
def enable_sqlite(module):
if module.type == 'cc_binary_host':
module.static_libs.add('libsqlite_static_noicu')
module.static_libs.add('sqlite_ext_percentile')
elif module.host_supported:
# Copy what the sqlite3 command line tool does.
module.android.shared_libs.add('libsqlite')
module.android.shared_libs.add('libicu')
module.android.shared_libs.add('liblog')
module.android.shared_libs.add('libutils')
module.android.static_libs.add('sqlite_ext_percentile')
module.host.static_libs.add('libsqlite_static_noicu')
module.host.static_libs.add('sqlite_ext_percentile')
else:
module.shared_libs.add('libsqlite')
module.shared_libs.add('libicu')
module.shared_libs.add('liblog')
module.shared_libs.add('libutils')
module.static_libs.add('sqlite_ext_percentile')
def enable_zlib(module):
if module.type == 'cc_binary_host':
module.static_libs.add('libz')
elif module.host_supported:
module.android.shared_libs.add('libz')
module.host.static_libs.add('libz')
else:
module.shared_libs.add('libz')
def enable_expat(module):
if module.type == 'cc_binary_host':
module.static_libs.add('libexpat')
elif module.host_supported:
module.android.shared_libs.add('libexpat')
module.host.static_libs.add('libexpat')
else:
module.shared_libs.add('libexpat')
def enable_uapi_headers(module):
module.include_dirs.add('bionic/libc/kernel')
def enable_bionic_libc_platform_headers_on_android(module):
module.header_libs.add('bionic_libc_platform_headers')
# Android equivalents for third-party libraries that the upstream project
# depends on.
builtin_deps = {
'//gn:default_deps':
lambda x: None,
'//gn:gtest_main':
lambda x: None,
'//gn:protoc':
lambda x: None,
'//gn:base_platform':
enable_base_platform,
'//gn:gtest_and_gmock':
enable_gtest_and_gmock,
'//gn:libunwind':
enable_libunwind,
'//gn:protobuf_full':
enable_protobuf_full,
'//gn:protobuf_lite':
enable_protobuf_lite,
'//gn:protoc_lib':
enable_protoc_lib,
'//gn:libunwindstack':
enable_libunwindstack,
'//gn:sqlite':
enable_sqlite,
'//gn:zlib':
enable_zlib,
'//gn:expat':
enable_expat,
'//gn:bionic_kernel_uapi_headers':
enable_uapi_headers,
'//src/profiling/memory:bionic_libc_platform_headers_on_android':
enable_bionic_libc_platform_headers_on_android,
}
# ----------------------------------------------------------------------------
# End of configuration.
# ----------------------------------------------------------------------------
class Error(Exception):
pass
class ThrowingArgumentParser(argparse.ArgumentParser):
def __init__(self, context):
super(ThrowingArgumentParser, self).__init__()
self.context = context
def error(self, message):
raise Error('%s: %s' % (self.context, message))
def write_blueprint_key_value(output, name, value, sort=True):
"""Writes a Blueprint key-value pair to the output"""
if isinstance(value, bool):
if value:
output.append(' %s: true,' % name)
else:
output.append(' %s: false,' % name)
return
if not value:
return
if isinstance(value, set):
value = sorted(value)
if isinstance(value, list):
output.append(' %s: [' % name)
for item in sorted(value) if sort else value:
output.append(' "%s",' % item)
output.append(' ],')
return
if isinstance(value, Target):
value.to_string(output)
return
if isinstance(value, dict):
kv_output = []
for k, v in value.items():
write_blueprint_key_value(kv_output, k, v)
output.append(' %s: {' % name)
for line in kv_output:
output.append(' %s' % line)
output.append(' },')
return
output.append(' %s: "%s",' % (name, value))
class Target(object):
"""A target-scoped part of a module"""
def __init__(self, name):
self.name = name
self.shared_libs = set()
self.static_libs = set()
self.whole_static_libs = set()
self.cflags = set()
self.dist = dict()
self.strip = dict()
self.stl = None
def to_string(self, output):
nested_out = []
self._output_field(nested_out, 'shared_libs')
self._output_field(nested_out, 'static_libs')
self._output_field(nested_out, 'whole_static_libs')
self._output_field(nested_out, 'cflags')
self._output_field(nested_out, 'stl')
self._output_field(nested_out, 'dist')
self._output_field(nested_out, 'strip')
if nested_out:
output.append(' %s: {' % self.name)
for line in nested_out:
output.append(' %s' % line)
output.append(' },')
def _output_field(self, output, name, sort=True):
value = getattr(self, name)
return write_blueprint_key_value(output, name, value, sort)
class Module(object):
"""A single module (e.g., cc_binary, cc_test) in a blueprint."""
def __init__(self, mod_type, name, gn_target):
assert (gn_target)
self.type = mod_type
self.gn_target = gn_target
self.name = name
self.srcs = set()
self.main: Optional[str] = None
self.comment = 'GN: ' + gn_utils.label_without_toolchain(gn_target)
self.shared_libs = set()
self.static_libs = set()
self.whole_static_libs = set()
self.runtime_libs = set()
self.tools = set()
self.cmd: Optional[str] = None
self.host_supported = False
self.vendor_available = False
self.product_available = False
self.init_rc = set()
self.out = set()
self.export_include_dirs = set()
self.generated_headers = set()
self.export_generated_headers = set()
self.defaults = set()
self.cflags = set()
self.include_dirs = set()
self.header_libs = set()
self.required = set()
self.user_debug_flag = False
self.tool_files: Optional[List[str]] = None
self.android = Target('android')
self.host = Target('host')
self.musl = Target('musl')
self.lto: Optional[bool] = None
self.stl = None
self.dist = dict()
self.strip = dict()
self.data = set()
self.apex_available = set()
self.sdk_version = None
self.min_sdk_version = None
self.proto = dict()
self.output_extension: Optional[str] = None
# The genrule_XXX below are properties that must to be propagated back
# on the module(s) that depend on the genrule.
self.genrule_headers = set()
self.genrule_srcs = set()
self.genrule_shared_libs = set()
self.version_script = None
self.test_suites = set()
self.test_config = None
self.stubs = {}
self.manifest: Optional[str] = None
self.resource_dirs: Optional[List[str]] = None
self.jni_libs: Optional[List[str]] = None
self.jni_uses_platform_apis: Optional[bool] = None
def to_string(self, output):
if self.comment:
output.append('// %s' % self.comment)
output.append('%s {' % self.type)
self._output_field(output, 'name')
self._output_field(output, 'srcs')
self._output_field(output, 'resource_dirs')
self._output_field(output, 'manifest')
self._output_field(output, 'shared_libs')
self._output_field(output, 'static_libs')
self._output_field(output, 'whole_static_libs')
self._output_field(output, 'runtime_libs')
self._output_field(output, 'tools')
self._output_field(output, 'cmd', sort=False)
if self.host_supported:
self._output_field(output, 'host_supported')
if self.vendor_available:
self._output_field(output, 'vendor_available')
if self.product_available:
self._output_field(output, 'product_available')
self._output_field(output, 'init_rc')
self._output_field(output, 'out')
self._output_field(output, 'export_include_dirs')
self._output_field(output, 'generated_headers')
self._output_field(output, 'export_generated_headers')
self._output_field(output, 'defaults')
self._output_field(output, 'cflags')
self._output_field(output, 'include_dirs')
self._output_field(output, 'header_libs')
self._output_field(output, 'required')
self._output_field(output, 'dist')
self._output_field(output, 'strip')
self._output_field(output, 'tool_files')
self._output_field(output, 'data')
self._output_field(output, 'stl')
self._output_field(output, 'apex_available')
self._output_field(output, 'sdk_version')
self._output_field(output, 'min_sdk_version')
self._output_field(output, 'version_script')
self._output_field(output, 'test_suites')
self._output_field(output, 'test_config')
self._output_field(output, 'stubs')
self._output_field(output, 'proto')
self._output_field(output, 'main')
self._output_field(output, 'output_extension')
self._output_field(output, 'jni_libs')
self._output_field(output, 'jni_uses_platform_apis')
target_out = []
self._output_field(target_out, 'android')
self._output_field(target_out, 'host')
self._output_field(target_out, 'musl')
if target_out:
output.append(' target: {')
for line in target_out:
output.append(' %s' % line)
output.append(' },')
if self.user_debug_flag:
output.append(' product_variables: {')
output.append(' debuggable: {')
output.append(
' cflags: ["-DPERFETTO_BUILD_WITH_ANDROID_USERDEBUG"],')
output.append(' },')
output.append(' },')
if self.lto is not None:
output.append(' target: {')
output.append(' android: {')
output.append(' lto: {')
output.append(' thin: %s,' %
'true' if self.lto else 'false')
output.append(' },')
output.append(' },')
output.append(' },')
output.append('}')
output.append('')
def add_android_static_lib(self, lib):
if self.type == 'cc_binary_host':
raise Exception('Adding Android static lib for host tool is unsupported')
elif self.host_supported:
self.android.static_libs.add(lib)
else:
self.static_libs.add(lib)
def add_android_shared_lib(self, lib):
if self.type == 'cc_binary_host':
raise Exception('Adding Android shared lib for host tool is unsupported')
elif self.host_supported:
self.android.shared_libs.add(lib)
else:
self.shared_libs.add(lib)
def _output_field(self, output, name, sort=True):
value = getattr(self, name)
return write_blueprint_key_value(output, name, value, sort)
class Blueprint(object):
"""In-memory representation of an Android.bp file."""
def __init__(self):
self.modules: Dict[str, Module] = {}
self.gn_target_to_module: Dict[str, Module] = {}
def add_module(self, module: Module):
"""Adds a new module to the blueprint, replacing any existing module
with the same name.
Args:
module: Module instance.
"""
self.modules[module.name] = module
self.gn_target_to_module[module.gn_target] = module
def to_string(self, output):
for m in sorted(itervalues(self.modules), key=lambda m: m.name):
m.to_string(output)
def label_to_module_name(label: str):
"""Turn a GN label (e.g., //:perfetto_tests) into a module name."""
# If the label is explicibly listed in the default target list, don't prefix
# its name and return just the target name. This is so tools like
# "traceconv" stay as such in the Android tree.
label_without_toolchain = gn_utils.label_without_toolchain(label)
if label in default_targets or label_without_toolchain in default_targets:
return label_without_toolchain.split(':')[-1]
module = re.sub(r'^//:?', '', label_without_toolchain)
module = re.sub(r'[^a-zA-Z0-9_]', '_', module)
if not module.startswith(module_prefix):
return module_prefix + module
return module
def is_supported_source_file(name: str):
"""Returns True if |name| can appear in a 'srcs' list."""
return os.path.splitext(name)[1] in ['.c', '.cc', '.java', '.proto']
def create_proto_modules(blueprint: Blueprint, gn: GnParser,
target: GnParser.Target):
"""Generate genrules for a proto GN target.
GN actions are used to dynamically generate files during the build. The
Soong equivalent is a genrule. This function turns a specific kind of
genrule which turns .proto files into source and header files into a pair
equivalent genrules.
Args:
blueprint: Blueprint instance which is being generated.
target: gn_utils.Target object.
Returns:
The source_genrule module.
"""
assert (target.type == 'proto_library')
# We don't generate any targets for source_set proto modules because
# they will be inlined into other modules if required.
if target.proto_plugin == 'source_set':
return None
tools = {'aprotoc'}
cpp_out_dir = '$(genDir)/%s/' % tree_path
target_module_name = label_to_module_name(target.name)
# In GN builds the proto path is always relative to the output directory
# (out/tmp.xxx).
cmd = ['mkdir -p %s &&' % cpp_out_dir, '$(location aprotoc)']
cmd += ['--proto_path=%s' % tree_path]
tool_files = set()
if buildtools_protobuf_src in target.proto_paths:
cmd += ['--proto_path=%s' % android_protobuf_src]
# Add `google/protobuf/descriptor.proto` to implicit deps
tool_files.add(':libprotobuf-internal-descriptor-proto')
# Descriptor targets only generate a single target.
if target.proto_plugin == 'descriptor':
out = '{}.bin'.format(target_module_name)
cmd += ['--descriptor_set_out=$(out)']
cmd += ['$(in)']
descriptor_module = Module('genrule', target_module_name, target.name)
descriptor_module.cmd = ' '.join(cmd)
descriptor_module.out.add(out)
descriptor_module.tools = tools
blueprint.add_module(descriptor_module)
# Recursively extract the .proto files of all the dependencies and
# add them to srcs.
descriptor_module.srcs.update(
gn_utils.label_to_path(src) for src in target.sources)
# Add the tool_files to srcs so that they get copied if this action is
# sandboxed in Soong.
# Add to `srcs` instead of `tool_files` (the latter will require a
# --proto_path that depends on Soong's sandbox implementation.)
descriptor_module.srcs.update(
src for src in tool_files)
for dep in target.transitive_proto_deps():
current_target = gn.get_target(dep.name)
descriptor_module.srcs.update(
gn_utils.label_to_path(src) for src in current_target.sources)
return descriptor_module
# We create two genrules for each proto target: one for the headers and
# another for the sources. This is because the module that depends on the
# generated files needs to declare two different types of dependencies --
# source files in 'srcs' and headers in 'generated_headers' -- and it's not
# valid to generate .h files from a source dependency and vice versa.
#
# We create an additional filegroup for .proto
# The .proto filegroup will be added to `tool_files` of rdeps so that the
# genrules can be sandboxed.
for proto_dep in target.proto_deps().union(target.transitive_proto_deps()):
tool_files.add(":" + label_to_module_name(proto_dep.name))
filegroup_module = Module('filegroup', target_module_name, target.name)
filegroup_module.srcs.update(
gn_utils.label_to_path(src) for src in target.sources)
blueprint.add_module(filegroup_module)
source_module_name = target_module_name + '_gen'
source_module = Module('genrule', source_module_name, target.name)
# Add the "root" .proto filegroup to srcs
source_module.srcs = set([':' + target_module_name])
# Add the tool_files to srcs so that they get copied if this action is
# sandboxed in Soong.
# Add to `srcs` instead of `tool_files` (the latter will require a
# --proto_path that depends on Soong's sandbox implementation.)
source_module.srcs.update(
src for src in tool_files)
blueprint.add_module(source_module)
header_module = Module('genrule', source_module_name + '_headers',
target.name)
blueprint.add_module(header_module)
header_module.srcs = set(source_module.srcs)
# TODO(primiano): at some point we should remove this. This was introduced
# by aosp/1108421 when adding "protos/" to .proto include paths, in order to
# avoid doing multi-repo changes and allow old clients in the android tree
# to still do the old #include "perfetto/..." rather than
# #include "protos/perfetto/...".
header_module.export_include_dirs = {'.', 'protos'}
source_module.genrule_srcs.add(':' + source_module.name)
source_module.genrule_headers.add(header_module.name)
if target.proto_plugin == 'proto':
suffixes = ['pb']
source_module.genrule_shared_libs.add('libprotobuf-cpp-lite')
cmd += ['--cpp_out=lite=true:' + cpp_out_dir]
elif target.proto_plugin == 'protozero':
suffixes = ['pbzero']
plugin = create_modules_from_target(blueprint, gn, protozero_plugin)
assert (plugin)
tools.add(plugin.name)
cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name]
cmd += ['--plugin_out=wrapper_namespace=pbzero:' + cpp_out_dir]
elif target.proto_plugin == 'cppgen':
suffixes = ['gen']
plugin = create_modules_from_target(blueprint, gn, cppgen_plugin)
assert (plugin)
tools.add(plugin.name)
cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name]
cmd += ['--plugin_out=wrapper_namespace=gen:' + cpp_out_dir]
elif target.proto_plugin == 'ipc':
suffixes = ['ipc']
plugin = create_modules_from_target(blueprint, gn, ipc_plugin)
assert (plugin)
tools.add(plugin.name)
cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name]
cmd += ['--plugin_out=wrapper_namespace=gen:' + cpp_out_dir]
else:
raise Error('Unsupported proto plugin: %s' % target.proto_plugin)
cmd += ['$(locations :%s)' % target_module_name]
source_module.cmd = ' '.join(cmd)
header_module.cmd = source_module.cmd
source_module.tools = tools
header_module.tools = tools
for sfx in suffixes:
source_module.out.update('%s/%s' %
(tree_path, src.replace('.proto', '.%s.cc' % sfx))
for src in filegroup_module.srcs)
header_module.out.update('%s/%s' %
(tree_path, src.replace('.proto', '.%s.h' % sfx))
for src in filegroup_module.srcs)
return source_module
def create_tp_tables_module(blueprint: Blueprint, gn: GnParser,
target: GnParser.Target):
bp_module_name = label_to_module_name(target.name)
bp_binary_module_name = f'{bp_module_name}_binary'
srcs = [gn_utils.label_to_path(src) for src in target.sources]
binary_module = Module('python_binary_host', bp_binary_module_name,
target.name)
for dep in target.non_proto_or_source_set_deps():
binary_module.srcs.update([
gn_utils.label_to_path(src) for src in gn.get_target(dep.name).sources
])
binary_module.srcs.update(srcs)
binary_module.srcs.add('tools/gen_tp_table_headers.py')
binary_module.main = 'tools/gen_tp_table_headers.py'
blueprint.add_module(binary_module)
module = Module('genrule', bp_module_name, target.name)
module.tools = set([
bp_binary_module_name,
])
module.cmd = ' '.join([
f'$(location {bp_binary_module_name})',
'--gen-dir=$(genDir)',
'--relative-input-dir=external/perfetto',
'--inputs',
'$(in)',
])
module.out.update(target.outputs)
module.genrule_headers.add(module.name)
module.srcs.update(srcs)
blueprint.add_module(module)
return module
def create_amalgamated_sql_module(blueprint: Blueprint, gn: GnParser,
target: GnParser.Target):
bp_module_name = label_to_module_name(target.name)
def find_arg(name):
for i, arg in enumerate(target.args):
if arg.startswith(f'--{name}'):
return target.args[i + 1]
namespace = find_arg('namespace')
module = Module('genrule', bp_module_name, target.name)
module.tool_files = [
'tools/gen_amalgamated_sql.py',
]
module.cmd = ' '.join([
'$(location tools/gen_amalgamated_sql.py)',
f'--namespace={namespace}',
'--cpp-out=$(out)',
'$(in)',
])
module.genrule_headers.add(module.name)
module.out.update(target.outputs)
for dep in target.transitive_deps:
# Use globs for the chrome stdlib so the file does not need to be
# regenerated in autoroll CLs.
if dep.name.startswith(
'//src/trace_processor/perfetto_sql/stdlib/chrome:chrome_sql'):
module.srcs.add('src/trace_processor/perfetto_sql/stdlib/chrome/**/*.sql')
continue
module.srcs.update(
[gn_utils.label_to_path(src) for src in gn.get_target(dep.name).inputs])
blueprint.add_module(module)
return module
def create_cc_proto_descriptor_module(blueprint: Blueprint,
target: GnParser.Target):
bp_module_name = label_to_module_name(target.name)
module = Module('genrule', bp_module_name, target.name)
module.tool_files = [
'tools/gen_cc_proto_descriptor.py',
]
module.cmd = ' '.join([
'$(location tools/gen_cc_proto_descriptor.py)', '--gen_dir=$(genDir)',
'--cpp_out=$(out)', '$(in)'
])
module.genrule_headers.add(module.name)
module.srcs.update(
':' + label_to_module_name(dep.name) for dep in target.proto_deps())
module.srcs.update(
gn_utils.label_to_path(src)
for src in target.inputs
if "tmp.gn_utils" not in src)
module.out.update(target.outputs)
blueprint.add_module(module)
return module
def create_gen_version_module(blueprint: Blueprint, target: GnParser.Target,
bp_module_name: str):
module = Module('genrule', bp_module_name, gn_utils.GEN_VERSION_TARGET)
script_path = gn_utils.label_to_path(target.script)
module.genrule_headers.add(bp_module_name)
module.tool_files = [script_path]
module.out.update(target.outputs)
module.srcs.update(gn_utils.label_to_path(src) for src in target.inputs)
module.cmd = ' '.join([
'python3 $(location %s)' % script_path, '--no_git',
'--changelog=$(location CHANGELOG)', '--cpp_out=$(out)'
])
blueprint.add_module(module)
return module
def create_proto_group_modules(blueprint, gn: GnParser, module_name: str,
group):
target_names = group['targets']
module_types = group['types']
module_sources = set()
for name in target_names:
target = gn.get_target(name)
module_sources.update(gn_utils.label_to_path(src) for src in target.sources)
for dep_label in target.transitive_proto_deps():
dep = gn.get_target(dep_label.name)
module_sources.update(gn_utils.label_to_path(src) for src in dep.sources)
for type in module_types:
if type == 'filegroup':
name = label_to_module_name(module_name) + '_filegroup_proto'
module = Module('filegroup', name, name)
module.comment = f'''GN: [{', '.join(target_names)}]'''
module.srcs = module_sources
blueprint.add_module(module)
elif type == 'lite':
name = label_to_module_name(module_name) + '_java_protos'
module = Module('java_library', name, name)
module.comment = f'''GN: [{', '.join(target_names)}]'''
module.proto = {'type': 'lite', 'canonical_path_from_root': False}
module.srcs = module_sources
blueprint.add_module(module)
elif type == 'python':
name = label_to_module_name(module_name) + '_python_protos'
module = Module('python_library_host', name, name)
module.comment = f'''GN: [{', '.join(target_names)}]'''
module.proto = {'canonical_path_from_root': False}
module.srcs = module_sources
blueprint.add_module(module)
else:
raise Error('Unhandled proto group type: {}'.format(group.type))
def is_target_android_jni_lib(target: GnParser.Target):
custom_target_type = target.custom_target_type()
return custom_target_type == 'perfetto_android_jni_library'
def android_jni_lib_custom_target_name(target: GnParser.Target):
"""
Android.bp 'cc_library_shared' rule generates binary with the name equal to
the rule name (plus the ".so" suffix).
So we change the target name to get the JNI library with the expected name.
"""
assert is_target_android_jni_lib(target)
return target.binary_name().rstrip('.so')
class AndroidJavaSDKModulesGenerator:
"""
This code is split into its own class to hide implementation details.
"""
def __init__(self, blueprint: Blueprint, gn: GnParser):
self.blueprint = blueprint
self.gn = gn
def create_android_app_module(self, target: GnParser.Target,
bp_module_name: str):
module = Module('android_app', bp_module_name, target.name)
module.srcs = [gn_utils.label_to_path(src) for src in target.sources]
non_jni_lib_deps = self.generate_deps_modules_and_return_non_jni_lib_deps(
target)
module.static_libs = [self._bp_module_name_from_gn_target(dep)
for dep in non_jni_lib_deps]
maybe_jni_lib_targets = set()
self._collect_all_transitive_deps(target, maybe_jni_lib_targets)
jni_lib_targets = [target for target in maybe_jni_lib_targets
if is_target_android_jni_lib(target)]
module.jni_libs = [android_jni_lib_custom_target_name(target)
for target in jni_lib_targets]
module.manifest = gn_utils.label_to_path(target.manifest)
module.resource_dirs = [
gn_utils.label_to_path(target.resource_files.rstrip('/**/*'))]
module.jni_uses_platform_apis = True
module.sdk_version = 30
self.blueprint.add_module(module)
return module
def create_android_library_module(self, target: GnParser.Target,
bp_module_name: str):
module = Module('android_library', bp_module_name, target.name)
module.srcs = [gn_utils.label_to_path(src) for src in target.sources]
# All JNI libs will be added to the 'jni_libs' argument of the 'android_app'
# rule that depends on this 'android_library' rule.
non_jni_lib_deps = self.generate_deps_modules_and_return_non_jni_lib_deps(
target)
module.static_libs = [self._bp_module_name_from_gn_target(dep)
for dep in non_jni_lib_deps]
module.manifest = gn_utils.label_to_path(target.manifest)
module.sdk_version = 30
self.blueprint.add_module(module)
return module
def generate_deps_modules_and_return_non_jni_lib_deps(self,
target: GnParser.Target):
non_jni_lib_deps = []
for dep in target.non_proto_or_source_set_deps():
create_modules_from_target(self.blueprint, self.gn, dep.name)
if not is_target_android_jni_lib(dep):
non_jni_lib_deps.append(dep)
return non_jni_lib_deps
def _bp_module_name_from_gn_target(self, target: GnParser.Target):
module = create_modules_from_target(self.blueprint, self.gn, target.name)
return module.name if module else None
def _collect_all_transitive_deps(self, target: GnParser.Target, visited):
if target in visited:
return
visited.add(target)
for d in target.non_proto_or_source_set_deps():
self._collect_all_transitive_deps(d, visited)
def _get_cflags(target: GnParser.Target):
cflags = {flag for flag in target.cflags if re.match(cflag_allowlist, flag)}
cflags |= set("-D%s" % define
for define in target.defines
if re.match(define_allowlist, define))
return cflags
def create_modules_from_target(blueprint: Blueprint, gn: GnParser,
gn_target_name: str) -> Optional[Module]:
"""Generate module(s) for a given GN target.
Given a GN target name, generate one or more corresponding modules into a
blueprint. The only case when this generates >1 module is proto libraries.
Args:
blueprint: Blueprint instance which is being generated.
gn: gn_utils.GnParser object.
gn_target_name: GN target for module generation.
"""
if gn_target_name in blueprint.gn_target_to_module:
return blueprint.gn_target_to_module[gn_target_name]
target = gn.get_target(gn_target_name)
bp_module_name = label_to_module_name(gn_target_name)
name_without_toolchain = gn_utils.label_without_toolchain(target.name)
if target.type == 'executable':
if target.toolchain == gn_utils.HOST_TOOLCHAIN:
module_type = 'cc_binary_host'
elif target.testonly:
module_type = 'cc_test'
else:
module_type = 'cc_binary'
module = Module(module_type, bp_module_name, gn_target_name)
elif target.type == 'static_library':
module = Module('cc_library_static', bp_module_name, gn_target_name)
elif target.type == 'shared_library':
if is_target_android_jni_lib(target):
bp_module_name = android_jni_lib_custom_target_name(target)
module = Module('cc_library_shared', bp_module_name, gn_target_name)
elif target.type == 'source_set':
module = Module('filegroup', bp_module_name, gn_target_name)
elif target.type == 'group':
# "group" targets are resolved recursively by gn_utils.get_target().
# There's nothing we need to do at this level for them.
return None
elif target.type == 'proto_library':
module = create_proto_modules(blueprint, gn, target)
if module is None:
return None
elif target.type == 'action':
if target.custom_action_type == 'sql_amalgamation':
return create_amalgamated_sql_module(blueprint, gn, target)
if target.custom_action_type == 'tp_tables':
return create_tp_tables_module(blueprint, gn, target)
if target.custom_action_type == 'perfetto_android_library':
return AndroidJavaSDKModulesGenerator(blueprint,
gn).create_android_library_module(
target, bp_module_name)
if target.custom_action_type == 'perfetto_android_app':
return AndroidJavaSDKModulesGenerator(blueprint,
gn).create_android_app_module(
target, bp_module_name)
if target.custom_action_type == 'cc_proto_descriptor':
module = create_cc_proto_descriptor_module(blueprint, target)
elif name_without_toolchain == gn_utils.GEN_VERSION_TARGET:
module = create_gen_version_module(blueprint, target, bp_module_name)
else:
raise Error('Unhandled action: {}'.format(target.name))
else:
raise Error('Unknown target %s (%s)' % (target.name, target.type))
blueprint.add_module(module)
module.host_supported = (name_without_toolchain in target_host_supported)
module.vendor_available = (name_without_toolchain in target_vendor_available)
module.product_available = (name_without_toolchain in target_product_available)
module.init_rc.update(target_initrc.get(target.name, []))
if target.type != 'proto_library':
# proto_library embeds a "root" filegroup in its srcs.
# Skip to prevent adding dups
module.srcs.update(
gn_utils.label_to_path(src)
for src in target.sources
if is_supported_source_file(src))
if name_without_toolchain in needs_libfts:
module.musl.static_libs.add('libfts')
if target.type in gn_utils.LINKER_UNIT_TYPES:
module.cflags.update(_get_cflags(target))
module_is_compiled = module.type not in ('genrule', 'filegroup')
if module_is_compiled:
# Don't try to inject library/source dependencies into genrules or
# filegroups because they are not compiled in the traditional sense.
module.defaults.update([defaults_module])
for lib in target.libs:
# Generally library names should be mangled as 'libXXX', unless they
# are HAL libraries (e.g., [email protected]) or AIDL c++ / NDK
# libraries (e.g. "android.hardware.power.stats-V1-cpp")
android_lib = lib if '@' in lib or "-cpp" in lib or "-ndk" in lib \
else 'lib' + lib
if lib in shared_library_allowlist:
module.add_android_shared_lib(android_lib)
if lib in static_library_allowlist:
module.add_android_static_lib(android_lib)
# If the module is a static library, export all the generated headers.
if module.type == 'cc_library_static':
module.export_generated_headers = module.generated_headers
if is_target_android_jni_lib(target):
module.header_libs.add('jni_headers')
# Merge in additional hardcoded arguments.
for key, add_val in additional_args.get(module.name, []):
curr = getattr(module, key)
if add_val and isinstance(add_val, set) and isinstance(curr, set):
curr.update(add_val)
elif isinstance(add_val, str) and (not curr or isinstance(curr, str)):
setattr(module, key, add_val)
elif isinstance(add_val, bool) and (not curr or isinstance(curr, bool)):
setattr(module, key, add_val)
elif isinstance(add_val, dict) and isinstance(curr, dict):
curr.update(add_val)
elif isinstance(add_val, dict) and isinstance(curr, Target):
curr.__dict__.update(add_val)
else:
raise Error('Unimplemented type %r of additional_args: %r' %
(type(add_val), key))
# dep_name is an unmangled GN target name (e.g. //foo:bar(toolchain)).
all_deps = target.non_proto_or_source_set_deps()
all_deps |= target.transitive_source_set_deps()
all_deps |= target.transitive_proto_deps()
for dep in all_deps:
# If the dependency refers to a library which we can replace with an
# Android equivalent, stop recursing and patch the dependency in.
# Don't recurse into //buildtools, builtin_deps are intercepted at
# the //gn:xxx level.
dep_name = dep.name
if dep_name.startswith('//buildtools'):
continue
# Ignore the dependency on the gen_buildflags genrule. That is run
# separately in this generator and the generated file is copied over
# into the repo (see usage of |buildflags_dir| in this script).
if dep_name.startswith(gn_utils.BUILDFLAGS_TARGET):
continue
dep_module = create_modules_from_target(blueprint, gn, dep_name)
# For filegroups and genrule, recurse but don't apply the deps.
if not module_is_compiled:
continue
# |builtin_deps| override GN deps with Android-specific ones. See the
# config in the top of this file.
if gn_utils.label_without_toolchain(dep_name) in builtin_deps:
builtin_deps[gn_utils.label_without_toolchain(dep_name)](module)
continue
# Don't recurse in any other //gn dep if not handled by builtin_deps.
if dep_name.startswith('//gn:'):
continue
if dep_module is None:
continue
if dep_module.type == 'cc_library_shared':
module.shared_libs.add(dep_module.name)
elif dep_module.type == 'cc_library_static':
module.static_libs.add(dep_module.name)
elif dep_module.type == 'filegroup':
module.srcs.add(':' + dep_module.name)
elif dep_module.type == 'genrule':
module.generated_headers.update(dep_module.genrule_headers)
module.srcs.update(dep_module.genrule_srcs)
module.shared_libs.update(dep_module.genrule_shared_libs)
elif dep_module.type == 'cc_binary':
continue # Ignore executables deps (used by cmdline integration tests).
else:
raise Error('Unknown dep %s (%s) for target %s' %
(dep_module.name, dep_module.type, module.name))
return module
def create_blueprint_for_targets(gn: GnParser, targets: List[str]):
"""Generate a blueprint for a list of GN targets."""
blueprint = Blueprint()
# Default settings used by all modules.
defaults = Module('cc_defaults', defaults_module, '//gn:default_deps')
# We have to use include_dirs passing the path relative to the android tree.
# This is because: (i) perfetto_cc_defaults is used also by
# test/**/Android.bp; (ii) if we use local_include_dirs instead, paths
# become relative to the Android.bp that *uses* cc_defaults (not the one
# that defines it).s
defaults.include_dirs = {
tree_path, tree_path + '/include', tree_path + '/' + buildflags_dir,
tree_path + '/src/profiling/memory/include'
}
defaults.cflags.update([
'-Wno-error=return-type',
'-Wno-sign-compare',
'-Wno-sign-promo',
'-Wno-unused-parameter',
'-fvisibility=hidden',
'-O2',
])
defaults.user_debug_flag = True
defaults.lto = True
blueprint.add_module(defaults)
for target in targets:
create_modules_from_target(blueprint, gn, target)
return blueprint
def main():
parser = argparse.ArgumentParser(
description='Generate Android.bp from a GN description.')
parser.add_argument(
'--check-only',
help='Don\'t keep the generated files',
action='store_true')
parser.add_argument(
'--desc',
help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
)
parser.add_argument(
'--extras',
help='Extra targets to include at the end of the Blueprint file',
default=os.path.join(gn_utils.repo_root(), 'Android.bp.extras'),
)
parser.add_argument(
'--output',
help='Blueprint file to create',
default=os.path.join(gn_utils.repo_root(), 'Android.bp'),
)
parser.add_argument(
'targets',
nargs=argparse.REMAINDER,
help='Targets to include in the blueprint (e.g., "//:perfetto_tests")')
args = parser.parse_args()
if args.desc:
with open(args.desc) as f:
desc = json.load(f)
else:
desc = gn_utils.create_build_description(gn_args)
gn = gn_utils.GnParser(desc)
blueprint = create_blueprint_for_targets(gn, args.targets or default_targets)
project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
tool_name = os.path.relpath(os.path.abspath(__file__), project_root)
# TODO(primiano): enable this on Android after the TODO in
# perfetto_component.gni is fixed.
# Check for ODR violations
# for target_name in default_targets:
# checker = gn_utils.ODRChecker(gn, target_name)
# Add any proto groups to the blueprint.
for name, group in proto_groups.items():
create_proto_group_modules(blueprint, gn, name, group)
output = [
"""// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file is automatically generated by %s. Do not edit.
""" % (tool_name)
]
blueprint.to_string(output)
with open(args.extras, 'r') as r:
for line in r:
output.append(line.rstrip("\n\r"))
out_files = []
# Generate the Android.bp file.
out_files.append(args.output + '.swp')
with open(out_files[-1], 'w') as f:
f.write('\n'.join(output))
# Text files should have a trailing EOL.
f.write('\n')
# Generate the perfetto_build_flags.h file.
out_files.append(os.path.join(buildflags_dir, 'perfetto_build_flags.h.swp'))
gn_utils.gen_buildflags(gn_args, out_files[-1])
# Either check the contents or move the files to their final destination.
return gn_utils.check_or_commit_generated_files(out_files, args.check_only)
if __name__ == '__main__':
sys.exit(main())