| # Copyright 2016 gRPC authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import errno |
| import os |
| import os.path |
| import platform |
| import re |
| import shlex |
| import shutil |
| import subprocess |
| from subprocess import PIPE |
| import sys |
| import sysconfig |
| |
| import setuptools |
| from setuptools import Extension |
| from setuptools.command import build_ext |
| |
| # TODO(atash) add flag to disable Cython use |
| |
| _PACKAGE_PATH = os.path.realpath(os.path.dirname(__file__)) |
| _README_PATH = os.path.join(_PACKAGE_PATH, "README.rst") |
| |
| os.chdir(os.path.dirname(os.path.abspath(__file__))) |
| sys.path.insert(0, os.path.abspath(".")) |
| |
| import _parallel_compile_patch |
| import protoc_lib_deps |
| |
| import grpc_version |
| |
| _EXT_INIT_SYMBOL = None |
| if sys.version_info[0] == 2: |
| _EXT_INIT_SYMBOL = "init_protoc_compiler" |
| else: |
| _EXT_INIT_SYMBOL = "PyInit__protoc_compiler" |
| |
| _parallel_compile_patch.monkeypatch_compile_maybe() |
| |
| CLASSIFIERS = [ |
| "Development Status :: 5 - Production/Stable", |
| "Programming Language :: Python", |
| "Programming Language :: Python :: 3", |
| "License :: OSI Approved :: Apache Software License", |
| ] |
| |
| PY3 = sys.version_info.major == 3 |
| |
| |
| def _env_bool_value(env_name, default): |
| """Parses a bool option from an environment variable""" |
| return os.environ.get(env_name, default).upper() not in ["FALSE", "0", ""] |
| |
| |
| # Environment variable to determine whether or not the Cython extension should |
| # *use* Cython or use the generated C files. Note that this requires the C files |
| # to have been generated by building first *with* Cython support. |
| BUILD_WITH_CYTHON = _env_bool_value("GRPC_PYTHON_BUILD_WITH_CYTHON", "False") |
| |
| # Export this variable to force building the python extension with a statically linked libstdc++. |
| # At least on linux, this is normally not needed as we can build manylinux-compatible wheels on linux just fine |
| # without statically linking libstdc++ (which leads to a slight increase in the wheel size). |
| # This option is useful when crosscompiling wheels for aarch64 where |
| # it's difficult to ensure that the crosscompilation toolchain has a high-enough version |
| # of GCC (we require >=5.1) but still uses old-enough libstdc++ symbols. |
| # TODO(jtattermusch): remove this workaround once issues with crosscompiler version are resolved. |
| BUILD_WITH_STATIC_LIBSTDCXX = _env_bool_value( |
| "GRPC_PYTHON_BUILD_WITH_STATIC_LIBSTDCXX", "False" |
| ) |
| |
| |
| def check_linker_need_libatomic(): |
| """Test if linker on system needs libatomic.""" |
| code_test = ( |
| b"#include <atomic>\n" |
| + b"int main() { return std::atomic<int64_t>{}; }" |
| ) |
| cxx = os.environ.get("CXX", "c++") |
| cpp_test = subprocess.Popen( |
| [cxx, "-x", "c++", "-std=c++14", "-"], |
| stdin=PIPE, |
| stdout=PIPE, |
| stderr=PIPE, |
| ) |
| cpp_test.communicate(input=code_test) |
| if cpp_test.returncode == 0: |
| return False |
| # Double-check to see if -latomic actually can solve the problem. |
| # https://github.com/grpc/grpc/issues/22491 |
| cpp_test = subprocess.Popen( |
| [cxx, "-x", "c++", "-std=c++14", "-", "-latomic"], |
| stdin=PIPE, |
| stdout=PIPE, |
| stderr=PIPE, |
| ) |
| cpp_test.communicate(input=code_test) |
| return cpp_test.returncode == 0 |
| |
| |
| class BuildExt(build_ext.build_ext): |
| """Custom build_ext command.""" |
| |
| def get_ext_filename(self, ext_name): |
| # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value |
| # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets. |
| # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so" |
| # When crosscompiling python wheels, we need to be able to override this suffix |
| # so that the resulting file name matches the target architecture and we end up with a well-formed |
| # wheel. |
| filename = build_ext.build_ext.get_ext_filename(self, ext_name) |
| orig_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX") |
| new_ext_suffix = os.getenv("GRPC_PYTHON_OVERRIDE_EXT_SUFFIX") |
| if new_ext_suffix and filename.endswith(orig_ext_suffix): |
| filename = filename[: -len(orig_ext_suffix)] + new_ext_suffix |
| return filename |
| |
| def build_extensions(self): |
| # This special conditioning is here due to difference of compiler |
| # behavior in gcc and clang. The clang doesn't take --stdc++11 |
| # flags but gcc does. Since the setuptools of Python only support |
| # all C or all C++ compilation, the mix of C and C++ will crash. |
| # *By default*, macOS and FreBSD use clang and Linux use gcc |
| # |
| # If we are not using a permissive compiler that's OK with being |
| # passed wrong std flags, swap out compile function by adding a filter |
| # for it. |
| old_compile = self.compiler._compile |
| |
| def new_compile(obj, src, ext, cc_args, extra_postargs, pp_opts): |
| if src.endswith(".c"): |
| extra_postargs = [ |
| arg for arg in extra_postargs if "-std=c++" not in arg |
| ] |
| elif src.endswith(".cc") or src.endswith(".cpp"): |
| extra_postargs = [ |
| arg for arg in extra_postargs if "-std=c11" not in arg |
| ] |
| return old_compile(obj, src, ext, cc_args, extra_postargs, pp_opts) |
| |
| self.compiler._compile = new_compile |
| |
| build_ext.build_ext.build_extensions(self) |
| |
| |
| # There are some situations (like on Windows) where CC, CFLAGS, and LDFLAGS are |
| # entirely ignored/dropped/forgotten by distutils and its Cygwin/MinGW support. |
| # We use these environment variables to thus get around that without locking |
| # ourselves in w.r.t. the multitude of operating systems this ought to build on. |
| # We can also use these variables as a way to inject environment-specific |
| # compiler/linker flags. We assume GCC-like compilers and/or MinGW as a |
| # reasonable default. |
| EXTRA_ENV_COMPILE_ARGS = os.environ.get("GRPC_PYTHON_CFLAGS", None) |
| EXTRA_ENV_LINK_ARGS = os.environ.get("GRPC_PYTHON_LDFLAGS", None) |
| if EXTRA_ENV_COMPILE_ARGS is None: |
| EXTRA_ENV_COMPILE_ARGS = "" |
| if "win32" in sys.platform: |
| # MSVC by defaults uses C++14 so C11 needs to be specified. |
| EXTRA_ENV_COMPILE_ARGS += " /std:c11" |
| # We need to statically link the C++ Runtime, only the C runtime is |
| # available dynamically |
| EXTRA_ENV_COMPILE_ARGS += " /MT" |
| elif "linux" in sys.platform: |
| # GCC by defaults uses C17 so only C++14 needs to be specified. |
| EXTRA_ENV_COMPILE_ARGS += " -std=c++14" |
| EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti" |
| # Reduce the optimization level from O3 (in many cases) to O1 to |
| # workaround gcc misalignment bug with MOVAPS (internal b/329134877) |
| EXTRA_ENV_COMPILE_ARGS += " -O1" |
| elif "darwin" in sys.platform: |
| # AppleClang by defaults uses C17 so only C++14 needs to be specified. |
| EXTRA_ENV_COMPILE_ARGS += " -std=c++14" |
| EXTRA_ENV_COMPILE_ARGS += " -fno-wrapv -frtti" |
| EXTRA_ENV_COMPILE_ARGS += " -stdlib=libc++ -DHAVE_UNISTD_H" |
| if EXTRA_ENV_LINK_ARGS is None: |
| EXTRA_ENV_LINK_ARGS = "" |
| # This is needed for protobuf/main.cc |
| if "win32" in sys.platform: |
| EXTRA_ENV_LINK_ARGS += " Shell32.lib" |
| # NOTE(rbellevi): Clang on Mac OS will make all static symbols (both |
| # variables and objects) global weak symbols. When a process loads the |
| # protobuf wheel's shared object library before loading *this* C extension, |
| # the runtime linker will prefer the protobuf module's version of symbols. |
| # This results in the process using a mixture of symbols from the protobuf |
| # wheel and this wheel, which may be using different versions of |
| # libprotobuf. In the case that they *are* using different versions of |
| # libprotobuf *and* there has been a change in data layout (or in other |
| # invariants) segfaults, data corruption, or "bad things" may happen. |
| # |
| # This flag ensures that on Mac, the only global symbol is the one loaded by |
| # the Python interpreter. The problematic global weak symbols become local |
| # weak symbols. This is not required on Linux since the compiler does not |
| # produce global weak symbols. This is not required on Windows as our ".pyd" |
| # file does not contain any symbols. |
| # |
| # Finally, the leading underscore here is part of the Mach-O ABI. Unlike |
| # more modern ABIs (ELF et al.), Mach-O prepends an underscore to the names |
| # of C functions. |
| if "darwin" in sys.platform: |
| EXTRA_ENV_LINK_ARGS += " -Wl,-exported_symbol,_{}".format( |
| _EXT_INIT_SYMBOL |
| ) |
| if "linux" in sys.platform or "darwin" in sys.platform: |
| EXTRA_ENV_LINK_ARGS += " -lpthread" |
| if check_linker_need_libatomic(): |
| EXTRA_ENV_LINK_ARGS += " -latomic" |
| |
| # Explicitly link Core Foundation framework for MacOS to ensure no symbol is |
| # missing when compiled using package managers like Conda. |
| if "darwin" in sys.platform: |
| EXTRA_ENV_LINK_ARGS += " -framework CoreFoundation" |
| |
| EXTRA_COMPILE_ARGS = shlex.split(EXTRA_ENV_COMPILE_ARGS) |
| EXTRA_LINK_ARGS = shlex.split(EXTRA_ENV_LINK_ARGS) |
| |
| if BUILD_WITH_STATIC_LIBSTDCXX: |
| EXTRA_LINK_ARGS.append("-static-libstdc++") |
| |
| CC_FILES = [os.path.normpath(cc_file) for cc_file in protoc_lib_deps.CC_FILES] |
| PROTO_FILES = [ |
| os.path.normpath(proto_file) for proto_file in protoc_lib_deps.PROTO_FILES |
| ] |
| CC_INCLUDES = [ |
| os.path.normpath(include_dir) for include_dir in protoc_lib_deps.CC_INCLUDES |
| ] |
| PROTO_INCLUDE = os.path.normpath(protoc_lib_deps.PROTO_INCLUDE) |
| |
| GRPC_PYTHON_TOOLS_PACKAGE = "grpc_tools" |
| GRPC_PYTHON_PROTO_RESOURCES_NAME = "_proto" |
| |
| DEFINE_MACROS = () |
| if "win32" in sys.platform: |
| DEFINE_MACROS += ( |
| ("WIN32_LEAN_AND_MEAN", 1), |
| # avoid https://github.com/abseil/abseil-cpp/issues/1425 |
| ("NOMINMAX", 1), |
| ) |
| if "64bit" in platform.architecture()[0]: |
| DEFINE_MACROS += (("MS_WIN64", 1),) |
| elif "linux" in sys.platform or "darwin" in sys.platform: |
| DEFINE_MACROS += (("HAVE_PTHREAD", 1),) |
| |
| |
| def package_data(): |
| tools_path = GRPC_PYTHON_TOOLS_PACKAGE.replace(".", os.path.sep) |
| proto_resources_path = os.path.join( |
| tools_path, GRPC_PYTHON_PROTO_RESOURCES_NAME |
| ) |
| proto_files = [] |
| for proto_file in PROTO_FILES: |
| source = os.path.join(PROTO_INCLUDE, proto_file) |
| target = os.path.join(proto_resources_path, proto_file) |
| relative_target = os.path.join( |
| GRPC_PYTHON_PROTO_RESOURCES_NAME, proto_file |
| ) |
| try: |
| os.makedirs(os.path.dirname(target)) |
| except OSError as error: |
| if error.errno == errno.EEXIST: |
| pass |
| else: |
| raise |
| shutil.copy(source, target) |
| proto_files.append(relative_target) |
| return {GRPC_PYTHON_TOOLS_PACKAGE: proto_files} |
| |
| |
| def extension_modules(): |
| if BUILD_WITH_CYTHON: |
| plugin_sources = [os.path.join("grpc_tools", "_protoc_compiler.pyx")] |
| else: |
| plugin_sources = [os.path.join("grpc_tools", "_protoc_compiler.cpp")] |
| |
| plugin_sources += [ |
| os.path.join("grpc_tools", "main.cc"), |
| os.path.join("grpc_root", "src", "compiler", "python_generator.cc"), |
| os.path.join("grpc_root", "src", "compiler", "proto_parser_helper.cc"), |
| ] + CC_FILES |
| |
| plugin_ext = Extension( |
| name="grpc_tools._protoc_compiler", |
| sources=plugin_sources, |
| include_dirs=[ |
| ".", |
| "grpc_root", |
| os.path.join("grpc_root", "include"), |
| ] |
| + CC_INCLUDES, |
| define_macros=list(DEFINE_MACROS), |
| extra_compile_args=list(EXTRA_COMPILE_ARGS), |
| extra_link_args=list(EXTRA_LINK_ARGS), |
| ) |
| extensions = [plugin_ext] |
| if BUILD_WITH_CYTHON: |
| from Cython import Build |
| |
| return Build.cythonize(extensions) |
| else: |
| return extensions |
| |
| |
| setuptools.setup( |
| name="grpcio-tools", |
| version=grpc_version.VERSION, |
| description="Protobuf code generator for gRPC", |
| long_description_content_type="text/x-rst", |
| long_description=open(_README_PATH, "r").read(), |
| author="The gRPC Authors", |
| author_email="[email protected]", |
| url="https://grpc.io", |
| project_urls={ |
| "Source Code": "https://github.com/grpc/grpc/tree/master/tools/distrib/python/grpcio_tools", |
| "Bug Tracker": "https://github.com/grpc/grpc/issues", |
| }, |
| license="Apache License 2.0", |
| classifiers=CLASSIFIERS, |
| ext_modules=extension_modules(), |
| packages=setuptools.find_packages("."), |
| python_requires=">=3.8", |
| install_requires=[ |
| "protobuf>=5.26.1,<6.0dev", |
| "grpcio>={version}".format(version=grpc_version.VERSION), |
| "setuptools", |
| ], |
| package_data=package_data(), |
| cmdclass={ |
| "build_ext": BuildExt, |
| }, |
| ) |