Add a test for bazel-built dist archives (#9929)
This also fixes some packaging rules that are needed for the result to be buildable.
diff --git a/BUILD.bazel b/BUILD.bazel
index ae50fa6..044e01e 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -1284,6 +1284,7 @@
"generate_descriptor_proto.sh",
"maven_install.json",
"update_file_lists.sh",
+ "//third_party:BUILD.bazel",
"//third_party:zlib.BUILD",
"//util/python:BUILD.bazel",
],
diff --git a/benchmarks/BUILD.bazel b/benchmarks/BUILD.bazel
index 83b2c7e..36f6283 100644
--- a/benchmarks/BUILD.bazel
+++ b/benchmarks/BUILD.bazel
@@ -75,7 +75,6 @@
],
exclude = [
"__init__.py", # not in autotools dist
- "BUILD.bazel",
"go/*",
],
),
diff --git a/benchmarks/cpp/BUILD.bazel b/benchmarks/cpp/BUILD.bazel
index ba93f64..7a3d3ba 100644
--- a/benchmarks/cpp/BUILD.bazel
+++ b/benchmarks/cpp/BUILD.bazel
@@ -17,7 +17,10 @@
pkg_files(
name = "dist_files",
- srcs = ["cpp_benchmark.cc"],
+ srcs = [
+ "BUILD.bazel",
+ "cpp_benchmark.cc",
+ ],
strip_prefix = strip_prefix.from_root(""),
visibility = ["//benchmarks:__pkg__"],
)
diff --git a/benchmarks/datasets/google_message1/proto2/BUILD.bazel b/benchmarks/datasets/google_message1/proto2/BUILD.bazel
index e315932..30caed5 100644
--- a/benchmarks/datasets/google_message1/proto2/BUILD.bazel
+++ b/benchmarks/datasets/google_message1/proto2/BUILD.bazel
@@ -46,7 +46,7 @@
pkg_files(
name = "dist_files",
- srcs = glob(["*.proto"]),
+ srcs = glob(["*"]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//benchmarks:__pkg__"],
)
diff --git a/benchmarks/datasets/google_message1/proto3/BUILD.bazel b/benchmarks/datasets/google_message1/proto3/BUILD.bazel
index 2628d02..0dc59d1 100644
--- a/benchmarks/datasets/google_message1/proto3/BUILD.bazel
+++ b/benchmarks/datasets/google_message1/proto3/BUILD.bazel
@@ -46,7 +46,7 @@
pkg_files(
name = "dist_files",
- srcs = glob(["*.proto"]),
+ srcs = glob(["*"]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//benchmarks:__pkg__"],
)
diff --git a/benchmarks/datasets/google_message2/BUILD.bazel b/benchmarks/datasets/google_message2/BUILD.bazel
index a3208d0..f3d66a2 100644
--- a/benchmarks/datasets/google_message2/BUILD.bazel
+++ b/benchmarks/datasets/google_message2/BUILD.bazel
@@ -46,7 +46,7 @@
pkg_files(
name = "dist_files",
- srcs = glob(["*.proto"]),
+ srcs = glob(["*"]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//benchmarks:__pkg__"],
)
diff --git a/benchmarks/datasets/google_message3/BUILD.bazel b/benchmarks/datasets/google_message3/BUILD.bazel
index 130c874..a729e50 100644
--- a/benchmarks/datasets/google_message3/BUILD.bazel
+++ b/benchmarks/datasets/google_message3/BUILD.bazel
@@ -52,7 +52,7 @@
pkg_files(
name = "dist_files",
- srcs = glob(["*.proto"]),
+ srcs = glob(["*"]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//benchmarks:__pkg__"],
)
diff --git a/benchmarks/datasets/google_message4/BUILD.bazel b/benchmarks/datasets/google_message4/BUILD.bazel
index 9c7190d..33de093 100644
--- a/benchmarks/datasets/google_message4/BUILD.bazel
+++ b/benchmarks/datasets/google_message4/BUILD.bazel
@@ -47,7 +47,7 @@
pkg_files(
name = "dist_files",
- srcs = glob(["*.proto"]),
+ srcs = glob(["*"]),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//benchmarks:__pkg__"],
)
diff --git a/conformance/BUILD.bazel b/conformance/BUILD.bazel
index 2b30604..7de2b8e 100644
--- a/conformance/BUILD.bazel
+++ b/conformance/BUILD.bazel
@@ -1,7 +1,13 @@
# Conformance testing for Protobuf.
-load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test", "objc_library", "cc_proto_library")
-load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library")
+load(
+ "@rules_pkg//:mappings.bzl",
+ "pkg_attributes",
+ "pkg_filegroup",
+ "pkg_files",
+ "strip_prefix",
+)
exports_files([
"conformance_test_runner.sh",
@@ -145,16 +151,33 @@
srcs = glob(
["**/*"],
exclude = [
+ # Handled by dist_scripts:
+ "conformance_test_runner.sh",
+
# The following are not in autotools dist:
- "conformance/autoload.php",
- "conformance/conformance_nodejs.js",
- "conformance/conformance_test_runner.sh",
- "conformance/failure_list_java_lite.txt",
- "conformance/failure_list_jruby.txt",
- "conformance/text_format_failure_list_*.txt",
- "conformance/update_failure_list.py",
+ "autoload.php",
+ "conformance_nodejs.js",
+ "failure_list_jruby.txt",
+ "update_failure_list.py",
],
),
strip_prefix = strip_prefix.from_root(""),
visibility = ["//pkg:__pkg__"],
)
+
+pkg_files(
+ name = "dist_scripts",
+ srcs = ["conformance_test_runner.sh"],
+ attributes = pkg_attributes(mode = "0555"),
+ strip_prefix = strip_prefix.from_root(""),
+ visibility = ["//pkg:__pkg__"],
+)
+
+pkg_filegroup(
+ name = "all_dist_files",
+ srcs = [
+ ":dist_files",
+ ":dist_scripts",
+ ],
+ visibility = ["//pkg:__pkg__"],
+)
diff --git a/kokoro/linux/bazel_distcheck/build.sh b/kokoro/linux/bazel_distcheck/build.sh
new file mode 100755
index 0000000..4eb6e11
--- /dev/null
+++ b/kokoro/linux/bazel_distcheck/build.sh
@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+# Build file to set up and run tests using bazel-build dist archive
+#
+# Note that the builds use WORKSPACE to fetch external sources, not
+# git submodules.
+
+set -eux
+
+BUILD_ONLY_TARGETS=(
+ //pkg:all
+ //:protoc
+ //:protobuf
+ //:protobuf_python
+)
+
+TEST_TARGETS=(
+ //build_defs:all
+ //conformance:all
+ //java:tests
+ //:protobuf_test
+)
+
+use_bazel.sh 5.0.0 || true
+bazel version
+
+# Change to repo root
+cd $(dirname $0)/../../..
+
+# Construct temp directory for running the dist build.
+# If you want to run locally and keep the build dir, create a directory
+# and pass it in the DIST_WORK_ROOT env var.
+if [[ -z ${DIST_WORK_ROOT:-} ]]; then
+ : ${DIST_WORK_ROOT:=$(mktemp -d)}
+ function dist_cleanup() {
+ rm -rf ${DIST_WORK_ROOT}
+ }
+ trap dist_cleanup EXIT
+fi
+
+# Let Bazel share the distdir.
+TMP_DISTDIR=${DIST_WORK_ROOT}/bazel-distdir
+mkdir -p ${TMP_DISTDIR}
+
+# Build distribution archive
+date
+bazel fetch --distdir=${TMP_DISTDIR} //pkg:dist_all_tar
+bazel build --distdir=${TMP_DISTDIR} //pkg:dist_all_tar
+DIST_ARCHIVE=$(readlink $(bazel info bazel-bin)/pkg/dist_all_tar.tar.gz)
+bazel shutdown
+
+# The `pkg_tar` rule emits a symlink based on the rule name. The actual
+# file is named with the current version.
+date
+echo "Resolved archive path: ${DIST_ARCHIVE}"
+
+# Extract the dist archive.
+date
+DIST_WORKSPACE=${DIST_WORK_ROOT}/protobuf
+mkdir -p ${DIST_WORKSPACE}
+tar -C ${DIST_WORKSPACE} --strip-components=1 -axf ${DIST_ARCHIVE}
+
+# Perform build steps in the extracted dist sources.
+
+cd ${DIST_WORKSPACE}
+FAILED=false
+
+date
+bazel fetch --distdir=${TMP_DISTDIR} "${BUILD_ONLY_TARGETS[@]}" "${TEST_TARGETS[@]}"
+
+date
+bazel build --distdir=${TMP_DISTDIR} -k \
+ "${BUILD_ONLY_TARGETS[@]}" || FAILED=true
+
+date
+bazel test --distdir=${TMP_DISTDIR} --test_output=errors -k \
+ "${TEST_TARGETS[@]}" || FAILED=true
+
+date
+cd examples
+bazel build --distdir=${TMP_DISTDIR} //... || FAILED=true
+
+if ${FAILED}; then
+ echo FAILED
+ exit 1
+fi
+echo PASS
diff --git a/kokoro/linux/bazel_distcheck/continuous.cfg b/kokoro/linux/bazel_distcheck/continuous.cfg
new file mode 100644
index 0000000..4ea8b21
--- /dev/null
+++ b/kokoro/linux/bazel_distcheck/continuous.cfg
@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh"
+timeout_mins: 15
diff --git a/kokoro/linux/bazel_distcheck/presubmit.cfg b/kokoro/linux/bazel_distcheck/presubmit.cfg
new file mode 100644
index 0000000..4ea8b21
--- /dev/null
+++ b/kokoro/linux/bazel_distcheck/presubmit.cfg
@@ -0,0 +1,5 @@
+# Config file for running tests in Kokoro
+
+# Location of the build script in repository
+build_file: "protobuf/kokoro/linux/bazel_distcheck/build.sh"
+timeout_mins: 15
diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel
index 271810f..0de27d3 100644
--- a/pkg/BUILD.bazel
+++ b/pkg/BUILD.bazel
@@ -4,6 +4,7 @@
"pkg_attributes",
"pkg_filegroup",
"pkg_files",
+ "strip_prefix",
)
load("//:protobuf_release.bzl", "package_naming")
load(":build_systems.bzl", "gen_automake_file_lists", "gen_file_lists")
@@ -101,11 +102,12 @@
pkg_filegroup(
name = "dist_common",
srcs = [
+ ":dist_files",
"//:common_dist_files",
- "//:conformance_dist_files",
"//:cpp_dist_files", # to build protoc
"//benchmarks:all_dist_files",
"//build_defs:dist_files",
+ "//conformance:all_dist_files",
"@com_google_protobuf_examples//:dist_files",
],
)
@@ -332,7 +334,7 @@
src_libs = {
# source rule: name in generated file
"//:common_dist_files": "dist_common",
- "//:conformance_dist_files": "dist_conformance",
+ "//conformance:dist_files": "dist_conformance",
"//benchmarks:all_dist_files": "dist_benchmark",
"@com_google_protobuf_examples//:dist_files": "dist_example",
"//:csharp_dist_files": "dist_csharp",
@@ -374,3 +376,13 @@
"//:protobuf_lite",
],
)
+
+################################################################################
+# Distribution sources
+################################################################################
+
+pkg_files(
+ name = "dist_files",
+ srcs = glob(["*"]),
+ strip_prefix = strip_prefix.from_root(""),
+)
diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel
index a8b35ef..d12b3e3 100644
--- a/third_party/BUILD.bazel
+++ b/third_party/BUILD.bazel
@@ -1 +1,4 @@
-exports_files(["zlib.BUILD"])
+exports_files([
+ "BUILD.bazel",
+ "zlib.BUILD",
+])
