rules_java_gapic/java_gapic_pkg.bzl

# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

load("@com_google_api_gax_java_properties//:dependencies.properties.bzl", "PROPERTIES")

def _wrapPropertyNamesInBraces(properties):
    wrappedProperties = {}
    for k, v in properties.items():
        wrappedProperties["{{%s}}" % k] = v
    return wrappedProperties

_PROPERTIES = _wrapPropertyNamesInBraces(PROPERTIES)

# ========================================================================
# General packaging helpers.
# ========================================================================

def _construct_package_dir_paths(attr_package_dir, out_pkg, label_name):
    if attr_package_dir:
        package_dir = attr_package_dir
        package_dir_expr = "../{}/".format(package_dir)
        tar_cd_suffix = ".."
        tar_prefix = attr_package_dir
    else:
        package_dir = label_name
        package_dir_expr = "./"
        tar_cd_suffix = "."
        tar_prefix = "."

    # We need to include label in the path to eliminate possible output files duplicates
    # (labels are guaranteed to be unique by bazel itself)
    package_dir_path = "%s/%s/%s" % (out_pkg.dirname, label_name, package_dir)
    return struct(
        package_dir = package_dir,
        package_dir_expr = package_dir_expr,
        package_dir_path = package_dir_path,
        package_dir_sibling_parent = out_pkg,
        package_dir_sibling_basename = label_name,
        tar_cd_suffix = tar_cd_suffix,
        tar_prefix = tar_prefix,
    )

def _put_dep_in_a_bucket(dep, dep_bucket, processed_deps):
    if processed_deps.get(dep):
        return
    dep_bucket.append(dep)
    processed_deps[dep] = True

def _gapic_pkg_tar_impl(ctx):
    deps = []
    for dep in ctx.attr.deps:
        for f in dep.files.to_list():
            deps.append(f)

    samples =[]
    for s in ctx.attr.samples:
        for f in s.files.to_list():
            samples.append(f)

    paths = _construct_package_dir_paths(
        ctx.attr.package_dir,
        ctx.outputs.pkg,
        ctx.label.name,
    )

    script = """
    for s in {samples}; do
        mkdir -p {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/
        unzip -q ./$s -d {package_dir_path}/{tar_cd_suffix}/{tar_prefix}/samples/snippets/generated/
    done

    mkdir -p {package_dir_path}
    for dep in {deps}; do
        tar -xzpf $dep -C {package_dir_path}
    done
    cd {package_dir_path}/{tar_cd_suffix}

    tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/*
    cd - > /dev/null
    mv {package_dir_path}/{package_dir}.tar.gz {pkg}
    rm -rf {package_dir_path}
    """.format(
        deps = " ".join(["'%s'" % d.path for d in deps]),
        package_dir_path = paths.package_dir_path,
        package_dir = paths.package_dir,
        samples = " ".join(["'%s'" % s.path for s in samples]),
        pkg = ctx.outputs.pkg.path,
        tar_cd_suffix = paths.tar_cd_suffix,
        tar_prefix = paths.tar_prefix,
    )

    ctx.actions.run_shell(
        inputs = deps + samples,
        command = script,
        outputs = [ctx.outputs.pkg],
    )

# The Bazel's native gapic_pkg_tar rule behaves weirdly when package_dir parameter
# is specified (at least on some Linux machines it does not put all the files
# under the package_dir). As a workaround for that bug we provide the custom
# implementation of the gapic_pkg_tar rule.
gapic_pkg_tar = rule(
    attrs = {
        "deps": attr.label_list(mandatory = True),
        "samples": attr.label_list(mandatory = False),
        "package_dir": attr.string(mandatory = False, default = ""),
        "extension": attr.string(mandatory = False, default = "tar.gz"),
    },
    outputs = {"pkg": "%{name}.%{extension}"},
    implementation = _gapic_pkg_tar_impl,
)

# ========================================================================
# Java Gapic package helpers.
# ========================================================================
def _construct_extra_deps(scope_to_deps, versions_map):
    label_name_to_maven_artifact = {
        "policy_proto": "maven.com_google_api_grpc_proto_google_iam_v1",
        "iam_policy_proto": "maven.com_google_api_grpc_proto_google_iam_v1",
        "iam_java_proto": "maven.com_google_api_grpc_proto_google_iam_v1",
        "iam_java_grpc": "maven.com_google_api_grpc_grpc_google_iam_v1",
        "iam_policy_java_grpc": "maven.com_google_api_grpc_grpc_google_iam_v1",
        "location_java_grpc": "maven.com_google_api_grpc_grpc_google_common_protos",
    }
    extra_deps = {}
    for scope, deps in scope_to_deps.items():
        for dep in deps:
            pkg_dependency = _get_gapic_pkg_dependency_name(dep)
            if pkg_dependency:
                key = "{{%s}}" % pkg_dependency
                if not extra_deps.get(key):
                    extra_deps[key] = "%s project(':%s')" % (scope, pkg_dependency)
            elif _is_java_dependency(dep):
                for f in dep[JavaInfo].transitive_deps.to_list():
                    maven_artifact = label_name_to_maven_artifact.get(f.owner.name)
                    if not maven_artifact:
                        continue
                    key = "{{%s}}" % maven_artifact
                    if not extra_deps.get(key):
                        extra_deps[key] = "%s '%s'" % (scope, versions_map[key])

    return "\n  ".join(extra_deps.values())

def _is_java_dependency(dep):
    return JavaInfo in dep

def _is_source_dependency(dep):
    return _is_java_dependency(dep) and hasattr(dep[JavaInfo], "source_jars") and dep.label.package != "jar"

def _is_proto_dependency(dep):
    return ProtoInfo in dep

def _get_gapic_pkg_dependency_name(dep):
    files_list = dep.files.to_list()
    if not files_list or len(files_list) != 1:
        return None
    for ext in (".tar.gz", ".gz", ".tgz"):
        if files_list[0].basename.endswith(ext):
            return files_list[0].basename[:-len(ext)]
    return None

# ========================================================================
# Java Gapic package rules.
# ========================================================================

def _java_gapic_build_configs_pkg_impl(ctx):
    expanded_templates = []
    paths = _construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name)

    substitutions = dict(ctx.attr.static_substitutions)
    substitutions["{{extra_deps}}"] = _construct_extra_deps({
        "api": ctx.attr.deps,
        "testImplementation": ctx.attr.test_deps,
    }, substitutions)

    for template in ctx.attr.templates.items():
        expanded_template = ctx.actions.declare_file(
            "%s/%s" % (paths.package_dir_sibling_basename, template[1]),
            sibling = paths.package_dir_sibling_parent,
        )
        expanded_templates.append(expanded_template)
        ctx.actions.expand_template(
            template = template[0].files.to_list()[0],
            substitutions = substitutions,
            output = expanded_template,
        )

    # Note the script is more complicated than it intuitively should be because of the limitations
    # inherent to bazel execution environment: no absolute paths allowed, the generated artifacts
    # must ensure uniqueness within a build. The template output directory manipulations are
    # to modify default 555 file permissions on any generated by bazel file (exectuable read-only,
    # which is not at all what we need for build files). There is no bazel built-in way to change
    # the generated files permissions, also the actual files accessible by the script are symlinks
    # and `chmod`, when applied to a directory, does not change the attributes of symlink targets
    # inside the directory. Chaning the symlink target's permissions is also not an option, because
    # they are on a read-only file system.
    script = """
    mkdir -p {package_dir_path}
    for templ in {templates}; do
        cp $templ {package_dir_path}/
    done
    chmod 644 {package_dir_path}/*
    cd {package_dir_path}/{tar_cd_suffix}
    tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/*
    cd - > /dev/null
    mv {package_dir_path}/{package_dir}.tar.gz {pkg}
    """.format(
        templates = " ".join(["'%s'" % f.path for f in expanded_templates]),
        package_dir_path = paths.package_dir_path,
        package_dir = paths.package_dir,
        pkg = ctx.outputs.pkg.path,
        tar_cd_suffix = paths.tar_cd_suffix,
        tar_prefix = paths.tar_prefix,
    )

    ctx.actions.run_shell(
        inputs = expanded_templates,
        command = script,
        outputs = [ctx.outputs.pkg],
    )

java_gapic_build_configs_pkg = rule(
    attrs = {
        "deps": attr.label_list(mandatory = True),
        "test_deps": attr.label_list(mandatory = False, allow_empty = True),
        "package_dir": attr.string(mandatory = False),
        "templates": attr.label_keyed_string_dict(mandatory = False, allow_files = True),
        "static_substitutions": attr.string_dict(mandatory = False, allow_empty = True, default = {}),
    },
    outputs = {"pkg": "%{name}.tar.gz"},
    implementation = _java_gapic_build_configs_pkg_impl,
)

def _java_gapic_srcs_pkg_impl(ctx):
    srcs = []
    proto_srcs = []
    for src_dep in ctx.attr.deps:
        if _is_source_dependency(src_dep):
            srcs.extend(src_dep[JavaInfo].source_jars)
        if _is_proto_dependency(src_dep):
            proto_srcs.extend(src_dep[ProtoInfo].check_deps_sources.to_list())

    test_srcs = []
    for test_src_dep in ctx.attr.test_deps:
        if _is_source_dependency(test_src_dep):
            test_srcs.extend(test_src_dep[JavaInfo].source_jars)

    paths = _construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name)

    # Note the script is more complicated than it intuitively should be because of limitations
    # inherent to bazel execution environment: no absolute paths allowed, the generated artifacts
    # must ensure uniqueness within a build.
    script = """
    for src in {srcs}; do
        mkdir -p {package_dir_path}/src/main/java
        unzip -q -o $src -d {package_dir_path}/src/main/java
        rm -r -f {package_dir_path}/src/main/java/META-INF

        # Remove empty files. If there are no resource names, one such file might have
        # been created. See java_gapic.bzl.
        find {package_dir_path}/src/main/java -type f -size 0 | while read f; do rm -f $f; done

        if [ -d {package_dir_path}/src/main/java/samples ]; then
            mv {package_dir_path}/src/main/java/samples {package_dir_path}
        fi
    done
    for proto_src in {proto_srcs}; do
        mkdir -p {package_dir_path}/src/main/proto
        cp -f --parents $proto_src {package_dir_path}/src/main/proto
    done
    for test_src in {test_srcs}; do
        mkdir -p {package_dir_path}/src/test/java
        unzip -q -o $test_src -d {package_dir_path}/src/test/java
        rm -r -f {package_dir_path}/src/test/java/META-INF
    done
    cd {package_dir_path}/{tar_cd_suffix}
    tar -zchpf {tar_prefix}/{package_dir}.tar.gz {tar_prefix}/*
    cd - > /dev/null
    mv {package_dir_path}/{package_dir}.tar.gz {pkg}
    """.format(
        srcs = " ".join(["'%s'" % f.path for f in srcs]),
        proto_srcs = " ".join(["'%s'" % f.path for f in proto_srcs]),
        test_srcs = " ".join(["'%s'" % f.path for f in test_srcs]),
        package_dir_path = paths.package_dir_path,
        package_dir = paths.package_dir,
        pkg = ctx.outputs.pkg.path,
        tar_cd_suffix = paths.tar_cd_suffix,
        tar_prefix = paths.tar_prefix,
    )

    ctx.actions.run_shell(
        inputs = srcs + proto_srcs + test_srcs,
        command = script,
        outputs = [ctx.outputs.pkg],
    )

java_gapic_srcs_pkg = rule(
    attrs = {
        "deps": attr.label_list(mandatory = True),
        "test_deps": attr.label_list(mandatory = False, allow_empty = True),
        "package_dir": attr.string(mandatory = True),
    },
    outputs = {"pkg": "%{name}.tar.gz"},
    implementation = _java_gapic_srcs_pkg_impl,
)

def java_gapic_assembly_gradle_pkg(
        name,
        deps,
        include_samples = False,
        assembly_name = None,
        transport = None,
        **kwargs):
    package_dir = name
    if assembly_name:
        package_dir = "google-cloud-%s-%s" % (assembly_name, name)

    # Rename to avoid target conflicts with the monolith.
    proto_target = "proto-%s" % package_dir
    proto_target_dep = []
    grpc_target = "grpc-%s" % package_dir
    grpc_target_dep = []
    client_target = "gapic-%s" % package_dir
    client_target_dep = []

    client_deps = []
    client_test_deps = []
    grpc_deps = []
    proto_deps = []
    samples = []

    processed_deps = {}  #there is no proper Set in Starlark
    for dep in deps:
        # Use contains instead of endswith since microgenerator testing may use differently-named targets.
        if "_java_gapic" in dep:
            if include_samples:
                samples.append(dep + "_samples")
            _put_dep_in_a_bucket(dep, client_deps, processed_deps)
            _put_dep_in_a_bucket("%s_test" % dep, client_test_deps, processed_deps)
            _put_dep_in_a_bucket("%s_resource_name" % dep, proto_deps, processed_deps)
        elif dep.endswith("_java_grpc"):
            _put_dep_in_a_bucket(dep, grpc_deps, processed_deps)
        else:
            _put_dep_in_a_bucket(dep, proto_deps, processed_deps)

    if proto_deps:
        _java_gapic_gradle_pkg(
            name = proto_target,
            template_label = Label("//rules_java_gapic:resources/gradle/proto.gradle.tmpl"),
            deps = proto_deps,
            **kwargs
        )
        proto_target_dep = [":%s" % proto_target]

    if grpc_deps:
        _java_gapic_gradle_pkg(
            name = grpc_target,
            template_label = Label("//rules_java_gapic:resources/gradle/grpc.gradle.tmpl"),
            deps = proto_target_dep + grpc_deps,
            **kwargs
        )
        grpc_target_dep = ["%s" % grpc_target]

    if client_deps:
        if not transport or transport == "grpc":
            template_label = Label("//rules_java_gapic:resources/gradle/client_grpc.gradle.tmpl")
        elif transport == "rest":
            template_label = Label("//rules_java_gapic:resources/gradle/client_rest.gradle.tmpl")
        elif transport == "grpc+rest":
            template_label = Label("//rules_java_gapic:resources/gradle/client_grpcrest.gradle.tmpl")

        _java_gapic_gradle_pkg(
            name = client_target,
            template_label = template_label,
            deps = proto_target_dep + client_deps,
            test_deps = grpc_target_dep + client_test_deps,
            **kwargs
        )
        client_target_dep = ["%s" % client_target]

    _java_gapic_assembly_gradle_pkg(
        name = name,
        assembly_name = package_dir,
        deps = proto_target_dep + grpc_target_dep + client_target_dep,
        samples = samples,
    )

def _java_gapic_gradle_pkg(
        name,
        template_label,
        deps,
        test_deps = None,
        project_deps = None,
        test_project_deps = None,
        **kwargs):
    resource_target_name = "%s-resources" % name

    static_substitutions = dict(_PROPERTIES)
    static_substitutions["{{name}}"] = name

    java_gapic_build_configs_pkg(
        name = resource_target_name,
        deps = deps,
        test_deps = test_deps,
        package_dir = name,
        templates = {
            template_label: "build.gradle",
        },
        static_substitutions = static_substitutions,
    )

    srcs_gapic_pkg_target_name = "%s-srcs_pkg" % name
    java_gapic_srcs_pkg(
        name = srcs_gapic_pkg_target_name,
        deps = deps,
        test_deps = test_deps,
        package_dir = name,
        **kwargs
    )

    gapic_pkg_tar(
        name = name,
        extension = "tar.gz",
        deps = [
            resource_target_name,
            srcs_gapic_pkg_target_name,
        ],
        **kwargs
    )

def _java_gapic_assembly_gradle_pkg(name, assembly_name, deps, samples = None, visibility = None):
    resource_target_name = "%s-resources" % assembly_name
    java_gapic_build_configs_pkg(
        name = resource_target_name,
        deps = deps,
        templates = {
            Label("//rules_java_gapic:resources/gradle/assembly.gradle.tmpl"): "build.gradle",
            Label("//rules_java_gapic:resources/gradle/settings.gradle.tmpl"): "settings.gradle",
        },
    )

    gapic_pkg_tar(
        name = name,
        extension = "tar.gz",
        deps = [
            Label("//rules_java_gapic:gradlew"),
            resource_target_name,
        ] + deps,
        samples = samples,
        package_dir = assembly_name,
        visibility = visibility,
    )