[bazel] make it possible to build the whole world, update CI (#78870)
Fixes https://github.com/pytorch/pytorch/issues/77509
This PR supersedes https://github.com/pytorch/pytorch/pull/77510.
It allows both `bazel query //...` and `bazel build --config=gpu //...` to work.
Concretely the changes are:
1. Add "GenerateAten" mnemonic -- this is a convenience thing, so anybody who uses [Remote Execution](https://bazel.build/docs/remote-execution) can add a
```
build:rbe --strategy=GenerateAten=sandboxed,local
```
line to the `~/.bazelrc` and build this action locally (it doesn't have hermetic dependencies at the moment).
2. Replaced few `http_archive` repos by the proper existing submodules to avoid code drift.
3. Updated `pybind11_bazel` and added `python_version="3"` to `python_configure`. This prevents hard-to-debug error that are caused by an attempt to build with python2 on the systems where it's a default python (Ubuntu 18.04 for example).
4. Added `unused_` repos, they purpose is to hide the unwanted submodules of submodules that often have bazel targets in them.
5. Updated CI to build //... -- this is a great step forward to prevent regressions in targets not only in the top-level BUILD.bazel file, but in other folders too.
6. Switch default bazel build to use gpu support.
Pull Request resolved: https://github.com/pytorch/pytorch/pull/78870
Approved by: https://github.com/ezyang
diff --git a/WORKSPACE b/WORKSPACE
index fb15aad..96eea42 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -26,25 +26,10 @@
)
http_archive(
- name = "com_google_googletest",
- strip_prefix = "googletest-cd6b9ae3243985d4dc725abd513a874ab4161f3e",
- urls = [
- "https://github.com/google/googletest/archive/cd6b9ae3243985d4dc725abd513a874ab4161f3e.tar.gz",
- ],
-)
-
-http_archive(
- name = "google_benchmark",
- sha256 = "6132883bc8c9b0df5375b16ab520fac1a85dc9e4cf5be59480448ece74b278d4",
- strip_prefix = "benchmark-1.6.1/",
- urls = ["https://github.com/google/benchmark/archive/refs/tags/v1.6.1.tar.gz"],
-)
-
-http_archive(
name = "pybind11_bazel",
- strip_prefix = "pybind11_bazel-7f397b5d2cc2434bbd651e096548f7b40c128044",
- urls = ["https://github.com/pybind/pybind11_bazel/archive/7f397b5d2cc2434bbd651e096548f7b40c128044.zip"],
- sha256 = "e4a9536f49d4a88e3c5a09954de49c4a18d6b1632c457a62d6ec4878c27f1b5b",
+ strip_prefix = "pybind11_bazel-992381ced716ae12122360b0fbadbc3dda436dbf",
+ urls = ["https://github.com/pybind/pybind11_bazel/archive/992381ced716ae12122360b0fbadbc3dda436dbf.zip"],
+ sha256 = "3dc6435bd41c058453efe102995ef084d0a86b0176fd6a67a6b7100a2e9a940e",
)
new_local_repository(
@@ -183,7 +168,7 @@
)
load("@pybind11_bazel//:python_configure.bzl", "python_configure")
-python_configure(name = "local_config_python")
+python_configure(name = "local_config_python", python_version="3")
load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
@@ -209,3 +194,70 @@
name = "com_github_google_flatbuffers",
path = "third_party/flatbuffers",
)
+
+local_repository(
+ name = "google_benchmark",
+ path = "third_party/benchmark",
+)
+
+local_repository(
+ name = "com_google_googletest",
+ path = "third_party/googletest",
+)
+
+local_repository(
+ name = "pthreadpool",
+ path = "third_party/pthreadpool",
+ repo_mapping = {"@com_google_benchmark" : "@google_benchmark"}
+)
+
+local_repository(
+ name = "FXdiv",
+ path = "third_party/FXdiv",
+ repo_mapping = {"@com_google_benchmark" : "@google_benchmark"}
+)
+
+local_repository(
+ name = "XNNPACK",
+ path = "third_party/XNNPACK",
+ repo_mapping = {"@com_google_benchmark" : "@google_benchmark"}
+)
+
+local_repository(
+ name = "gemmlowp",
+ path = "third_party/gemmlowp/gemmlowp",
+)
+
+### Unused repos start
+
+# `unused` repos are defined to hide bazel files from submodules of submodules.
+# This allows us to run `bazel build //...` and not worry about the submodules madness.
+# Otherwise everything traverses recursively and a lot of submodules of submodules have
+# they own bazel build files.
+
+local_repository(
+ name = "unused_tensorpipe_googletest",
+ path = "third_party/tensorpipe/third_party/googletest",
+)
+
+local_repository(
+ name = "unused_fbgemm",
+ path = "third_party/fbgemm",
+)
+
+local_repository(
+ name = "unused_kineto_googletest",
+ path = "third_party/kineto/libkineto/third_party/googletest",
+)
+
+local_repository(
+ name = "unused_onnx_benchmark",
+ path = "third_party/onnx/third_party/benchmark",
+)
+
+local_repository(
+ name = "unused_onnx_tensorrt_benchmark",
+ path = "third_party/onnx-tensorrt/third_party/onnx/third_party/benchmark",
+)
+
+### Unused repos end
