blob: a3e2620ad22a69e22b32dc4b4abb3b7941091147 [file] [log] [blame] [edit]
# Copyright 2023 The Bazel Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Bazel Dex Commands."""
load("//rules:attrs.bzl", _attrs = "attrs")
load("//rules:common.bzl", _common = "common")
load("//rules:java.bzl", _java = "java")
load("@bazel_skylib//lib:collections.bzl", "collections")
load(":providers.bzl", "StarlarkAndroidDexInfo")
load(":utils.bzl", "ANDROID_TOOLCHAIN_TYPE", "get_android_toolchain", "utils")
_DEX_MEMORY = 4096
_DEX_THREADS = 5
_tristate = _attrs.tristate
def _resource_set_for_monolithic_dexing():
return {"cpu": _DEX_THREADS, "memory": _DEX_MEMORY}
def _process_incremental_dexing(
ctx,
output,
deps = [],
runtime_jars = [],
dexopts = [],
main_dex_list = None,
min_sdk_version = 0,
proguarded_jar = None,
java_info = None,
desugar_dict = {},
shuffle_jars = None,
dexbuilder = None,
dexbuilder_after_proguard = None,
dexmerger = None,
dexsharder = None,
toolchain_type = None):
info = _merge_infos(utils.collect_providers(StarlarkAndroidDexInfo, deps))
incremental_dexopts = _filter_dexopts(dexopts, ctx.fragments.android.get_dexopts_supported_in_incremental_dexing)
inclusion_filter_jar = proguarded_jar
if not proguarded_jar:
dex_archives = []
for jar in runtime_jars:
dex_archive = _get_dx_artifact(ctx, jar.basename + ".dex.zip")
_dex(
ctx,
input = desugar_dict[jar] if jar in desugar_dict else jar,
output = dex_archive,
incremental_dexopts = incremental_dexopts,
min_sdk_version = min_sdk_version,
dex_exec = dexbuilder,
toolchain_type = toolchain_type,
)
dex_archives.append(dex_archive)
dex_archives += _to_dexed_classpath(
dex_archives_dict = {d.jar: d.dex for d in info.dex_archives_dict.get("".join(incremental_dexopts), depset()).to_list()},
classpath = _filter(java_info.transitive_runtime_jars.to_list(), excludes = _get_library_r_jars(deps)),
runtime_jars = runtime_jars,
)
else:
java_resource_jar = ctx.actions.declare_file(ctx.label.name + "_files/java_resources.jar")
if ctx.fragments.android.incremental_dexing_shards_after_proguard > 1:
dex_archives = _shard_proguarded_jar_and_dex(
ctx,
java_resource_jar = java_resource_jar,
num_shards = ctx.fragments.android.incremental_dexing_shards_after_proguard,
dexopts = incremental_dexopts,
proguarded_jar = proguarded_jar,
main_dex_list = main_dex_list,
min_sdk_version = min_sdk_version,
shuffle_jars = shuffle_jars,
dexbuilder_after_proguard = dexbuilder_after_proguard,
toolchain_type = toolchain_type,
)
inclusion_filter_jar = None
else:
# No need to shuffle if there is only one shard
dex_archive = _get_dx_artifact(ctx, "classes.jar")
_dex(
ctx,
input = proguarded_jar,
output = dex_archive,
incremental_dexopts = incremental_dexopts,
min_sdk_version = min_sdk_version,
dex_exec = dexbuilder_after_proguard,
toolchain_type = toolchain_type,
)
dex_archives = [dex_archive]
if len(dex_archives) == 1:
_dex_merge(
ctx,
output = output,
inputs = dex_archives,
multidex_strategy = "minimal",
main_dex_list = main_dex_list,
dexopts = _filter_dexopts(dexopts, ctx.fragments.android.get_dexopts_supported_in_dex_merger),
dexmerger = dexmerger,
toolchain_type = toolchain_type,
)
else:
shards = ctx.actions.declare_directory("dexsplits/" + ctx.label.name)
dexes = ctx.actions.declare_directory("dexfiles/" + ctx.label.name)
_shard_dexes(
ctx,
output = shards,
inputs = dex_archives,
dexopts = _filter_dexopts(dexopts, ctx.fragments.android.get_dexopts_supported_in_dex_sharder),
main_dex_list = main_dex_list,
inclusion_filter_jar = inclusion_filter_jar,
dexsharder = dexsharder,
toolchain_type = toolchain_type,
)
# TODO(b/130571505): Implement this after SpawnActionTemplate is supported in Starlark
android_common.create_dex_merger_actions(
ctx,
output = dexes,
input = shards,
dexopts = dexopts,
dexmerger = dexmerger,
)
_java.singlejar(
ctx,
output = output,
inputs = [dexes],
mnemonic = "MergeDexZips",
progress_message = "Merging dex shards for %s." % ctx.label,
java_toolchain = _common.get_java_toolchain(ctx),
)
def _process_optimized_dexing(
ctx,
output,
input = None,
proguard_output_map = None,
postprocessing_output_map = None,
dexopts = [],
native_multidex = True,
min_sdk_version = 0,
main_dex_list = None,
library_jar = None,
startup_profile = None,
optimizing_dexer = None,
toolchain_type = None):
inputs = [input]
outputs = [output]
args = ctx.actions.args()
args.add(input)
args.add("--release")
args.add("--no-desugaring")
args.add("--output", output)
args.add_all(dexopts)
if proguard_output_map:
args.add("--pg-map", proguard_output_map)
args.add("--pg-map-output", postprocessing_output_map)
inputs.append(proguard_output_map)
outputs.append(postprocessing_output_map)
if startup_profile and native_multidex:
args.add("--startup-profile", startup_profile)
inputs.append(startup_profile)
# TODO(b/261110876): Pass min SDK through here based on the value in the merged manifest. The
# current value is statically defined for the entire depot.
# We currently set the minimum SDK version to 21 if you are doing native multidex as that is
# required for native multidex to work in the first place and as a result is required for
# correct behavior from the dexer.
sdk = max(min_sdk_version, 21) if native_multidex else min_sdk_version
if sdk != 0:
args.add("--min-api", sdk)
if main_dex_list:
args.add("--main-dex-list", main_dex_list)
inputs.append(main_dex_list)
if library_jar:
args.add("--lib", library_jar)
inputs.append(library_jar)
ctx.actions.run(
outputs = outputs,
executable = optimizing_dexer,
inputs = inputs,
arguments = [args],
mnemonic = "OptimizingDex",
progress_message = "Optimized dexing for " + str(ctx.label),
use_default_shell_env = True,
toolchain = toolchain_type,
)
def _process_monolithic_dexing(
ctx,
output,
input,
dexopts = [],
min_sdk_version = 0,
main_dex_list = None,
dexbuilder = None,
toolchain_type = None):
# Create an artifact for the intermediate zip output generated by AndroidDexer that includes
# non-.dex files. A subsequent TrimDexZip action will filter out all non-.dex files.
classes_dex_intermediate = _get_dx_artifact(ctx, "intermediate_classes.dex.zip")
inputs = [input]
args = ctx.actions.args()
args.add("--dex")
args.add_all(dexopts)
if min_sdk_version > 0:
args.add("--min_sdk_version", min_sdk_version)
args.add("--multi-dex")
if main_dex_list:
args.add(main_dex_list, format = "--main-dex-list=%s")
inputs.append(main_dex_list)
args.add(classes_dex_intermediate, format = "--output=%s")
args.add(input)
ctx.actions.run(
executable = dexbuilder,
inputs = inputs,
outputs = [classes_dex_intermediate],
arguments = [args],
progress_message = "Converting %s to dex format" % input.short_path,
mnemonic = "AndroidDexer",
use_default_shell_env = True,
resource_set = _resource_set_for_monolithic_dexing,
toolchain = toolchain_type,
)
# Because the dexer also places resources into this zip, we also need to create a cleanup
# action that removes all non-.dex files before staging for apk building.
_java.singlejar(
ctx,
inputs = [classes_dex_intermediate],
output = output,
include_prefixes = ["classes"],
java_toolchain = _common.get_java_toolchain(ctx),
mnemonic = "TrimDexZip",
progress_message = "Trimming %s." % classes_dex_intermediate.short_path,
)
def _shard_proguarded_jar_and_dex(
ctx,
java_resource_jar,
num_shards = 50,
dexopts = [],
proguarded_jar = None,
main_dex_list = None,
min_sdk_version = 0,
shuffle_jars = None,
dexbuilder_after_proguard = None,
toolchain_type = None):
if num_shards <= 1:
fail("num_shards expects to be larger than 1.")
shards = _make_shard_artifacts(ctx, num_shards, ".jar.dex.zip")
shuffle_outputs = _make_shard_artifacts(ctx, num_shards, ".jar")
inputs = []
args = ctx.actions.args()
args.add_all(shuffle_outputs, before_each = "--output_jar")
args.add("--output_resources", java_resource_jar)
if main_dex_list:
args.add("--main_dex_filter", main_dex_list)
inputs.append(main_dex_list)
# If we need to run Proguard, all the class files will be in the Proguarded jar, which has to
# be converted to dex.
args.add("--input_jar", proguarded_jar)
inputs.append(proguarded_jar)
ctx.actions.run(
executable = shuffle_jars,
outputs = shuffle_outputs + [java_resource_jar],
inputs = inputs,
arguments = [args],
mnemonic = "ShardClassesToDex",
progress_message = "Sharding classes for dexing for " + str(ctx.label),
use_default_shell_env = True,
toolchain = toolchain_type,
)
for i in range(len(shards)):
_dex(
ctx,
input = shuffle_outputs[i],
output = shards[i],
incremental_dexopts = dexopts,
min_sdk_version = min_sdk_version,
dex_exec = dexbuilder_after_proguard,
toolchain_type = toolchain_type,
)
return shards
def _make_shard_artifacts(ctx, n, suffix):
return [_get_dx_artifact(ctx, "shard" + str(i) + suffix) for i in range(1, n + 1)]
def _shard_dexes(
ctx,
output,
inputs = [],
dexopts = [],
main_dex_list = None,
inclusion_filter_jar = None,
dexsharder = None,
toolchain_type = None):
args = ctx.actions.args().use_param_file(param_file_arg = "@%s")
args.add_all(inputs, before_each = "--input")
args.add("--output", output.path)
if main_dex_list:
inputs.append(main_dex_list)
args.add("--main-dex-list", main_dex_list)
if inclusion_filter_jar:
inputs.append(inclusion_filter_jar)
args.add("--inclusion_filter_jar", inclusion_filter_jar)
args.add_all(dexopts)
ctx.actions.run(
executable = dexsharder,
outputs = [output],
inputs = inputs,
arguments = [args],
mnemonic = "ShardForMultidex",
progress_message = "Assembling dex files for " + ctx.label.name,
use_default_shell_env = True,
toolchain = toolchain_type,
)
return output
def _append_java8_legacy_dex(
ctx,
output = None,
input = None,
java8_legacy_dex = None,
dex_zips_merger = None):
args = ctx.actions.args()
# Order matters here: we want java8_legacy_dex to be the highest-numbered classesN.dex
args.add("--input_zip", input)
args.add("--input_zip", java8_legacy_dex)
args.add("--output_zip", output)
ctx.actions.run(
executable = dex_zips_merger,
inputs = [input, java8_legacy_dex],
outputs = [output],
arguments = [args],
mnemonic = "AppendJava8LegacyDex",
use_default_shell_env = True,
progress_message = "Adding Java8 legacy library for %s" % ctx.label,
toolchain = ANDROID_TOOLCHAIN_TYPE,
)
def _to_dexed_classpath(dex_archives_dict = {}, classpath = [], runtime_jars = []):
dexed_classpath = []
for jar in classpath:
if jar not in dex_archives_dict:
if jar not in runtime_jars:
fail("Dependencies on .jar artifacts are not allowed in Android binaries, please use " +
"a java_import to depend on " + jar.short_path +
". If this is an implicit dependency then the rule that " +
"introduces it will need to be fixed to account for it correctly.")
else:
dexed_classpath.append(dex_archives_dict[jar])
return dexed_classpath
def _dex(
ctx,
input,
output = None,
incremental_dexopts = [],
min_sdk_version = 0,
dex_exec = None,
toolchain_type = None):
"""Dexes a JAR.
Args:
ctx: The context.
input: File. The jar to be dexed.
output: File. The archive file containing all of the dexes.
incremental_dexopts: List of strings. Additional command-line flags for the dexing tool when building dexes.
min_sdk_version: Integer. The minimum targeted sdk version.
dex_exec: File. The executable dex builder file.
"""
args = ctx.actions.args()
args.use_param_file("@%s", use_always = True) # Required for workers.
args.set_param_file_format("multiline")
args.add("--input_jar", input)
args.add("--output_zip", output)
args.add_all(incremental_dexopts)
if min_sdk_version > 0:
args.add("--min_sdk_version", min_sdk_version)
execution_requirements = {}
if ctx.fragments.android.persistent_android_dex_desugar:
execution_requirements["supports-workers"] = "1"
if ctx.fragments.android.persistent_multiplex_android_dex_desugar:
execution_requirements["supports-multiplex-workers"] = "1"
ctx.actions.run(
executable = dex_exec,
arguments = [args],
inputs = [input],
outputs = [output],
mnemonic = "DexBuilder",
progress_message = "Dexing " + input.path + " with applicable dexopts " + str(incremental_dexopts),
execution_requirements = execution_requirements,
toolchain = toolchain_type,
)
def _get_dx_artifact(ctx, basename):
return ctx.actions.declare_file("_dx_migrated/" + ctx.label.name + "/" + basename)
def _get_effective_incremental_dexing(
force_incremental_dexing = _tristate.auto,
has_forbidden_dexopts = False,
incremental_dexing_after_proguard_by_default = True,
incremental_dexing_shards_after_proguard = True,
is_binary_optimized = False,
use_incremental_dexing = True):
if (is_binary_optimized and
force_incremental_dexing == _tristate.yes and incremental_dexing_shards_after_proguard <= 0):
fail("Target cannot be incrementally dexed because it uses Proguard")
if force_incremental_dexing == _tristate.yes:
return True
if force_incremental_dexing == _tristate.no:
return False
# If there are incompatible dexopts and the incremental_dexing attr is not set, we silently don't run
# incremental dexing.
if has_forbidden_dexopts or (is_binary_optimized and not incremental_dexing_after_proguard_by_default):
return False
# use_incremental_dexing config flag will take effect if incremental_dexing attr is not set
return use_incremental_dexing
def _get_java8_legacy_dex_and_map(ctx, build_customized_files = False, binary_jar = None, android_jar = None):
if not build_customized_files:
return utils.only(get_android_toolchain(ctx).java8_legacy_dex.files.to_list()), None
else:
java8_legacy_dex_rules = _get_dx_artifact(ctx, "_java8_legacy.dex.pgcfg")
java8_legacy_dex_map = _get_dx_artifact(ctx, "_java8_legacy.dex.map")
java8_legacy_dex = _get_dx_artifact(ctx, "_java8_legacy.dex.zip")
args = ctx.actions.args()
args.add("--rules", java8_legacy_dex_rules)
args.add("--binary", binary_jar)
args.add("--android_jar", android_jar)
args.add("--output", java8_legacy_dex)
args.add("--output_map", java8_legacy_dex_map)
ctx.actions.run(
executable = get_android_toolchain(ctx).build_java8_legacy_dex.files_to_run,
inputs = [binary_jar, android_jar],
outputs = [java8_legacy_dex_rules, java8_legacy_dex_map, java8_legacy_dex],
arguments = [args],
mnemonic = "BuildLegacyDex",
progress_message = "Building Java8 legacy library for %s" % ctx.label,
toolchain = ANDROID_TOOLCHAIN_TYPE,
)
return java8_legacy_dex, java8_legacy_dex_map
def _get_library_r_jars(deps):
transitive_resource_jars = []
for dep in utils.collect_providers(AndroidLibraryResourceClassJarProvider, deps):
transitive_resource_jars += dep.jars.to_list()
return transitive_resource_jars
def _dex_merge(
ctx,
output = None,
inputs = [],
multidex_strategy = "minimal",
main_dex_list = None,
dexopts = [],
dexmerger = None,
toolchain_type = None):
args = ctx.actions.args()
args.add("--multidex", multidex_strategy)
args.add_all(inputs, before_each = "--input")
args.add("--output", output)
args.add_all(dexopts)
if main_dex_list:
inputs.append(main_dex_list)
args.add("--main-dex-list", main_dex_list)
ctx.actions.run(
executable = dexmerger,
arguments = [args],
inputs = inputs,
outputs = [output],
mnemonic = "DexMerger",
progress_message = "Assembling dex files into " + output.short_path,
toolchain = toolchain_type,
)
def _merge_infos(infos):
dex_archives_dict = {}
for info in infos:
for dexopts in info.dex_archives_dict:
if dexopts not in dex_archives_dict:
dex_archives_dict[dexopts] = [info.dex_archives_dict[dexopts]]
else:
dex_archives_dict[dexopts].append(info.dex_archives_dict[dexopts])
return StarlarkAndroidDexInfo(
dex_archives_dict =
{dexopts: depset(direct = [], transitive = dex_archives) for dexopts, dex_archives in dex_archives_dict.items()},
)
def _filter_dexopts(tokenized_dexopts, includes):
return _normalize_dexopts(_filter(tokenized_dexopts, includes = includes))
def _filter(candidates, includes = [], excludes = []):
if excludes and includes:
fail("Only one of excludes list and includes list can be set.")
if includes:
return [c for c in candidates if c in includes]
if excludes:
return [c for c in candidates if c not in excludes]
return candidates
def _normalize_dexopts(tokenized_dexopts):
def _dx_to_dexbuilder(opt):
return opt.replace("--no-", "--no")
return collections.uniq(sorted([_dx_to_dexbuilder(token) for token in tokenized_dexopts]))
def _generate_main_dex_list(
ctx,
jar,
android_jar = None,
desugar_java8_libs = True,
main_dex_classes = None,
main_dex_list_opts = [],
main_dex_proguard_spec = None,
proguard_specs = [],
legacy_apis = [],
shrinked_android_jar = None,
toolchain_type = None,
main_dex_list_creator = None,
legacy_main_dex_list_generator = None,
proguard_tool = None):
main_dex_list = _get_dx_artifact(ctx, "main_dex_list.txt")
if not proguard_specs:
proguard_specs.append(main_dex_classes)
if main_dex_proguard_spec:
proguard_specs.append(main_dex_proguard_spec)
# If legacy_main_dex_list_generator is not set by either the SDK or the flag, use ProGuard and
# the main dext list creator specified by the android_sdk rule. If
# legacy_main_dex_list_generator is provided, use that tool instead.
# TODO(b/147692286): Remove the old main-dex list generation that relied on ProGuard.
if not legacy_main_dex_list_generator:
if not shrinked_android_jar:
fail("In \"legacy\" multidex mode, either legacy_main_dex_list_generator or " +
"shrinked_android_jar must be set in the android_sdk.")
# Process the input jar through Proguard into an intermediate, streamlined jar.
stripped_jar = _get_dx_artifact(ctx, "main_dex_intermediate.jar")
args = ctx.actions.args()
args.add("-forceprocessing")
args.add("-injars", jar)
args.add("-libraryjars", shrinked_android_jar)
args.add("-outjars", stripped_jar)
args.add("-dontwarn")
args.add("-dontnote")
args.add("-dontoptimize")
args.add("-dontobfuscate")
ctx.actions.run(
outputs = [stripped_jar],
executable = proguard_tool,
args = [args],
inputs = [jar, shrinked_android_jar],
mnemonic = "MainDexClassesIntermediate",
progress_message = "Generating streamlined input jar for main dex classes list",
use_default_shell_dev = True,
toolchain = toolchain_type,
)
args = ctx.actions.args()
args.add_all([main_dex_list, stripped_jar, jar])
args.add_all(main_dex_list_opts)
ctx.actions.run(
outputs = [main_dex_list],
executable = main_dex_list_creator,
arguments = [args],
inputs = [jar, stripped_jar],
mnemonic = "MainDexClasses",
progress_message = "Generating main dex classes list",
toolchain = toolchain_type,
)
else:
inputs = [jar, android_jar] + proguard_specs
args = ctx.actions.args()
args.add("--main-dex-list-output", main_dex_list)
args.add("--lib", android_jar)
if desugar_java8_libs:
args.add_all(legacy_apis, before_each = "--lib")
inputs += legacy_apis
args.add_all(proguard_specs, before_each = "--main-dex-rules")
args.add(jar)
ctx.actions.run(
executable = legacy_main_dex_list_generator,
arguments = [args],
outputs = [main_dex_list],
inputs = inputs,
mnemonic = "MainDexClasses",
progress_message = "Generating main dex classes list",
toolchain = toolchain_type,
)
return main_dex_list
def _transform_dex_list_through_proguard_map(
ctx,
proguard_output_map = None,
main_dex_list = None,
toolchain_type = None,
dex_list_obfuscator = None):
if not proguard_output_map:
return main_dex_list
obfuscated_main_dex_list = _get_dx_artifact(ctx, "main_dex_list_obfuscated.txt")
args = ctx.actions.args()
args.add("--input", main_dex_list)
args.add("--output", obfuscated_main_dex_list)
args.add("--obfuscation_map", proguard_output_map)
ctx.actions.run(
executable = dex_list_obfuscator,
arguments = [args],
outputs = [obfuscated_main_dex_list],
inputs = [main_dex_list],
mnemonic = "MainDexProguardClasses",
progress_message = "Obfuscating main dex classes list",
toolchain = toolchain_type,
)
return obfuscated_main_dex_list
dex = struct(
append_java8_legacy_dex = _append_java8_legacy_dex,
dex = _dex,
dex_merge = _dex_merge,
generate_main_dex_list = _generate_main_dex_list,
get_dx_artifact = _get_dx_artifact,
get_effective_incremental_dexing = _get_effective_incremental_dexing,
get_java8_legacy_dex_and_map = _get_java8_legacy_dex_and_map,
filter_dexopts = _filter_dexopts,
merge_infos = _merge_infos,
normalize_dexopts = _normalize_dexopts,
process_monolithic_dexing = _process_monolithic_dexing,
process_incremental_dexing = _process_incremental_dexing,
process_optimized_dexing = _process_optimized_dexing,
transform_dex_list_through_proguard_map = _transform_dex_list_through_proguard_map,
)