| #!/usr/bin/env python3 |
| # Copyright 2015 gRPC authors. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| """Run interop (cross-language) tests in parallel.""" |
| |
| from __future__ import print_function |
| |
| import argparse |
| import atexit |
| import itertools |
| import json |
| import multiprocessing |
| import os |
| import re |
| import subprocess |
| import sys |
| import tempfile |
| import time |
| import traceback |
| import uuid |
| |
| import six |
| |
| import python_utils.dockerjob as dockerjob |
| import python_utils.jobset as jobset |
| import python_utils.report_utils as report_utils |
| |
| # It's ok to not import because this is only necessary to upload results to BQ. |
| try: |
| from python_utils.upload_test_results import upload_interop_results_to_bq |
| except ImportError as e: |
| print(e) |
| |
| # Docker doesn't clean up after itself, so we do it on exit. |
| atexit.register(lambda: subprocess.call(["stty", "echo"])) |
| |
| ROOT = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "../..")) |
| os.chdir(ROOT) |
| |
| _DEFAULT_SERVER_PORT = 8080 |
| |
| _SKIP_CLIENT_COMPRESSION = [ |
| "client_compressed_unary", |
| "client_compressed_streaming", |
| ] |
| |
| _SKIP_SERVER_COMPRESSION = [ |
| "server_compressed_unary", |
| "server_compressed_streaming", |
| ] |
| |
| _SKIP_COMPRESSION = _SKIP_CLIENT_COMPRESSION + _SKIP_SERVER_COMPRESSION |
| |
| _SKIP_ADVANCED = [ |
| "status_code_and_message", |
| "custom_metadata", |
| "unimplemented_method", |
| "unimplemented_service", |
| ] |
| |
| _SKIP_SPECIAL_STATUS_MESSAGE = ["special_status_message"] |
| |
| _ORCA_TEST_CASES = ["orca_per_rpc", "orca_oob"] |
| |
| _GOOGLE_DEFAULT_CREDS_TEST_CASE = "google_default_credentials" |
| |
| _SKIP_GOOGLE_DEFAULT_CREDS = [ |
| _GOOGLE_DEFAULT_CREDS_TEST_CASE, |
| ] |
| |
| _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE = "compute_engine_channel_credentials" |
| |
| _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS = [ |
| _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, |
| ] |
| |
| _TEST_TIMEOUT = 3 * 60 |
| |
| # disable this test on core-based languages, |
| # see https://github.com/grpc/grpc/issues/9779 |
| _SKIP_DATA_FRAME_PADDING = ["data_frame_padding"] |
| |
| # report suffix "sponge_log.xml" is important for reports to get picked up by internal CI |
| _DOCKER_BUILD_XML_REPORT = "interop_docker_build/sponge_log.xml" |
| _TESTS_XML_REPORT = "interop_test/sponge_log.xml" |
| |
| |
| class CXXLanguage: |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.http2_cwd = None |
| self.safename = "cxx" |
| |
| def client_cmd(self, args): |
| return ["cmake/build/interop_client"] + args |
| |
| def client_cmd_http2interop(self, args): |
| return ["cmake/build/http2_client"] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ["cmake/build/interop_server"] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_DATA_FRAME_PADDING |
| + _SKIP_SPECIAL_STATUS_MESSAGE |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return [] |
| |
| def __str__(self): |
| return "c++" |
| |
| |
| class AspNetCoreLanguage: |
| def __init__(self): |
| self.client_cwd = "../grpc-dotnet/output/InteropTestsClient" |
| self.server_cwd = "../grpc-dotnet/output/InteropTestsWebsite" |
| self.safename = str(self) |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def client_cmd(self, args): |
| return ["dotnet", "exec", "InteropTestsClient.dll"] + args |
| |
| def server_cmd(self, args): |
| return ["dotnet", "exec", "InteropTestsWebsite.dll"] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "aspnetcore" |
| |
| |
| class DartLanguage: |
| def __init__(self): |
| self.client_cwd = "../grpc-dart/interop" |
| self.server_cwd = "../grpc-dart/interop" |
| self.http2_cwd = "../grpc-dart/interop" |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ["dart", "bin/client.dart"] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ["dart", "bin/server.dart"] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_COMPRESSION |
| + _SKIP_SPECIAL_STATUS_MESSAGE |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return ( |
| _SKIP_COMPRESSION + _SKIP_SPECIAL_STATUS_MESSAGE + _ORCA_TEST_CASES |
| ) |
| |
| def __str__(self): |
| return "dart" |
| |
| |
| class JavaLanguage: |
| def __init__(self): |
| self.client_cwd = "../grpc-java" |
| self.server_cwd = "../grpc-java" |
| self.http2_cwd = "../grpc-java" |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ["./run-test-client.sh"] + args |
| |
| def client_cmd_http2interop(self, args): |
| return [ |
| "./interop-testing/build/install/grpc-interop-testing/bin/http2-client" |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ["./run-test-server.sh"] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return [] |
| |
| def unimplemented_test_cases_server(self): |
| # Does not support CompressedRequest feature. |
| # Only supports CompressedResponse feature for unary. |
| return _SKIP_CLIENT_COMPRESSION + ["server_compressed_streaming"] |
| |
| def __str__(self): |
| return "java" |
| |
| |
| class JavaOkHttpClient: |
| def __init__(self): |
| self.client_cwd = "../grpc-java" |
| self.safename = "java" |
| |
| def client_cmd(self, args): |
| return ["./run-test-client.sh", "--use_okhttp=true"] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_DATA_FRAME_PADDING |
| |
| def __str__(self): |
| return "javaokhttp" |
| |
| |
| class GoLanguage: |
| def __init__(self): |
| # TODO: this relies on running inside docker |
| self.client_cwd = "/go/src/google.golang.org/grpc/interop/client" |
| self.server_cwd = "/go/src/google.golang.org/grpc/interop/server" |
| self.http2_cwd = "/go/src/google.golang.org/grpc/interop/http2" |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ["go", "run", "client.go"] + args |
| |
| def client_cmd_http2interop(self, args): |
| return ["go", "run", "negative_http2_client.go"] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ["go", "run", "server.go"] + args |
| |
| def global_env(self): |
| return {"GO111MODULE": "on"} |
| |
| def unimplemented_test_cases(self): |
| return _SKIP_COMPRESSION |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION |
| |
| def __str__(self): |
| return "go" |
| |
| |
| class Http2Server: |
| """Represents the HTTP/2 Interop Test server |
| |
| This pretends to be a language in order to be built and run, but really it |
| isn't. |
| """ |
| |
| def __init__(self): |
| self.server_cwd = None |
| self.safename = str(self) |
| |
| def server_cmd(self, args): |
| return ["python test/http2_test/http2_test_server.py"] |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _TEST_CASES |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_SPECIAL_STATUS_MESSAGE |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _TEST_CASES |
| |
| def __str__(self): |
| return "http2" |
| |
| |
| class Http2Client: |
| """Represents the HTTP/2 Interop Test |
| |
| This pretends to be a language in order to be built and run, but really it |
| isn't. |
| """ |
| |
| def __init__(self): |
| self.client_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ["tools/http2_interop/http2_interop.test", "-test.v"] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _TEST_CASES |
| + _SKIP_SPECIAL_STATUS_MESSAGE |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _TEST_CASES |
| |
| def __str__(self): |
| return "http2" |
| |
| |
| class NodeLanguage: |
| def __init__(self): |
| self.client_cwd = "../../../../home/appuser/grpc-node" |
| self.server_cwd = "../../../../home/appuser/grpc-node" |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", |
| "node", |
| "--require", |
| "./test/fixtures/native_native", |
| "test/interop/interop_client.js", |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", |
| "node", |
| "--require", |
| "./test/fixtures/native_native", |
| "test/interop/interop_server.js", |
| ] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION + _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "node" |
| |
| |
| class NodePureJSLanguage: |
| def __init__(self): |
| self.client_cwd = "../../../../home/appuser/grpc-node" |
| self.server_cwd = "../../../../home/appuser/grpc-node" |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| "packages/grpc-native-core/deps/grpc/tools/run_tests/interop/with_nvm.sh", |
| "node", |
| "--require", |
| "./test/fixtures/js_js", |
| "test/interop/interop_client.js", |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "nodepurejs" |
| |
| |
| class PHP7Language: |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return ["src/php/bin/interop_client.sh"] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return ["src/php/bin/interop_server.sh"] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_SERVER_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION + _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "php7" |
| |
| |
| class ObjcLanguage: |
| def __init__(self): |
| self.client_cwd = "src/objective-c/tests" |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| # from args, extract the server port and craft xcodebuild command out of it |
| for arg in args: |
| port = re.search("--server_port=(\d+)", arg) |
| if port: |
| portnum = port.group(1) |
| cmdline = ( |
| "pod install && xcodebuild -workspace Tests.xcworkspace" |
| ' -scheme InteropTestsLocalSSL -destination name="iPhone 6"' |
| " HOST_PORT_LOCALSSL=localhost:%s test" % portnum |
| ) |
| return [cmdline] |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| # ObjC test runs all cases with the same command. It ignores the testcase |
| # cmdline argument. Here we return all but one test cases as unimplemented, |
| # and depend upon ObjC test's behavior that it runs all cases even when |
| # we tell it to run just one. |
| return ( |
| _TEST_CASES[1:] |
| + _SKIP_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_SPECIAL_STATUS_MESSAGE |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION + _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "objc" |
| |
| |
| class RubyLanguage: |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| "tools/run_tests/interop/with_rvm.sh", |
| "ruby", |
| "src/ruby/pb/test/client.rb", |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| "tools/run_tests/interop/with_rvm.sh", |
| "ruby", |
| "src/ruby/pb/test/server.rb", |
| ] + args |
| |
| def global_env(self): |
| return {} |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_SERVER_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION + _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "ruby" |
| |
| |
| _PYTHON_BINARY = "py39/bin/python" |
| |
| |
| class PythonLanguage: |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.http2_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| _PYTHON_BINARY, |
| "src/python/grpcio_tests/setup.py", |
| "run_interop", |
| "--client", |
| '--args="{}"'.format(" ".join(args)), |
| ] |
| |
| def client_cmd_http2interop(self, args): |
| return [ |
| _PYTHON_BINARY, |
| "src/python/grpcio_tests/tests/http2/negative_http2_client.py", |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| _PYTHON_BINARY, |
| "src/python/grpcio_tests/setup.py", |
| "run_interop", |
| "--server", |
| '--args="{}"'.format(" ".join(args)), |
| ] |
| |
| def global_env(self): |
| return { |
| "LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT), |
| "PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT), |
| } |
| |
| def unimplemented_test_cases(self): |
| return ( |
| _SKIP_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _SKIP_GOOGLE_DEFAULT_CREDS |
| + _SKIP_COMPUTE_ENGINE_CHANNEL_CREDS |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| return _SKIP_COMPRESSION + _ORCA_TEST_CASES |
| |
| def __str__(self): |
| return "python" |
| |
| |
| class PythonAsyncIOLanguage: |
| def __init__(self): |
| self.client_cwd = None |
| self.server_cwd = None |
| self.http2_cwd = None |
| self.safename = str(self) |
| |
| def client_cmd(self, args): |
| return [ |
| _PYTHON_BINARY, |
| "src/python/grpcio_tests/setup.py", |
| "run_interop", |
| "--use-asyncio", |
| "--client", |
| '--args="{}"'.format(" ".join(args)), |
| ] |
| |
| def client_cmd_http2interop(self, args): |
| return [ |
| _PYTHON_BINARY, |
| "src/python/grpcio_tests/tests/http2/negative_http2_client.py", |
| ] + args |
| |
| def cloud_to_prod_env(self): |
| return {} |
| |
| def server_cmd(self, args): |
| return [ |
| _PYTHON_BINARY, |
| "src/python/grpcio_tests/setup.py", |
| "run_interop", |
| "--use-asyncio", |
| "--server", |
| '--args="{}"'.format(" ".join(args)), |
| ] |
| |
| def global_env(self): |
| return { |
| "LD_LIBRARY_PATH": "{}/libs/opt".format(DOCKER_WORKDIR_ROOT), |
| "PYTHONPATH": "{}/src/python/gens".format(DOCKER_WORKDIR_ROOT), |
| } |
| |
| def unimplemented_test_cases(self): |
| # TODO(https://github.com/grpc/grpc/issues/21707) |
| return ( |
| _SKIP_COMPRESSION |
| + _SKIP_DATA_FRAME_PADDING |
| + _AUTH_TEST_CASES |
| + ["timeout_on_sleeping_server"] |
| + _ORCA_TEST_CASES |
| ) |
| |
| def unimplemented_test_cases_server(self): |
| # TODO(https://github.com/grpc/grpc/issues/21749) |
| return ( |
| _TEST_CASES |
| + _AUTH_TEST_CASES |
| + _HTTP2_TEST_CASES |
| + _HTTP2_SERVER_TEST_CASES |
| ) |
| |
| def __str__(self): |
| return "pythonasyncio" |
| |
| |
| _LANGUAGES = { |
| "c++": CXXLanguage(), |
| "aspnetcore": AspNetCoreLanguage(), |
| "dart": DartLanguage(), |
| "go": GoLanguage(), |
| "java": JavaLanguage(), |
| "javaokhttp": JavaOkHttpClient(), |
| "node": NodeLanguage(), |
| "nodepurejs": NodePureJSLanguage(), |
| "php7": PHP7Language(), |
| "objc": ObjcLanguage(), |
| "ruby": RubyLanguage(), |
| "python": PythonLanguage(), |
| "pythonasyncio": PythonAsyncIOLanguage(), |
| } |
| |
| # languages supported as cloud_to_cloud servers |
| _SERVERS = [ |
| "c++", |
| "node", |
| "aspnetcore", |
| "java", |
| "go", |
| "ruby", |
| "python", |
| "dart", |
| "pythonasyncio", |
| "php7", |
| ] |
| |
| _TEST_CASES = [ |
| "large_unary", |
| "empty_unary", |
| "ping_pong", |
| "empty_stream", |
| "client_streaming", |
| "server_streaming", |
| "cancel_after_begin", |
| "cancel_after_first_response", |
| "timeout_on_sleeping_server", |
| "custom_metadata", |
| "status_code_and_message", |
| "unimplemented_method", |
| "client_compressed_unary", |
| "server_compressed_unary", |
| "client_compressed_streaming", |
| "server_compressed_streaming", |
| "unimplemented_service", |
| "special_status_message", |
| "orca_per_rpc", |
| "orca_oob", |
| ] |
| |
| _AUTH_TEST_CASES = [ |
| "compute_engine_creds", |
| "jwt_token_creds", |
| "oauth2_auth_token", |
| "per_rpc_creds", |
| _GOOGLE_DEFAULT_CREDS_TEST_CASE, |
| _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE, |
| ] |
| |
| _HTTP2_TEST_CASES = ["tls", "framing"] |
| |
| _HTTP2_SERVER_TEST_CASES = [ |
| "rst_after_header", |
| "rst_after_data", |
| "rst_during_data", |
| "goaway", |
| "ping", |
| "max_streams", |
| "data_frame_padding", |
| "no_df_padding_sanity_test", |
| ] |
| |
| _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES = { |
| "data_frame_padding": "large_unary", |
| "no_df_padding_sanity_test": "large_unary", |
| } |
| |
| _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS = list( |
| _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES.keys() |
| ) |
| |
| _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ |
| "java", |
| "go", |
| "python", |
| "c++", |
| ] |
| |
| _LANGUAGES_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"] |
| |
| _SERVERS_FOR_ALTS_TEST_CASES = ["java", "go", "c++", "python"] |
| |
| _TRANSPORT_SECURITY_OPTIONS = ["tls", "alts", "insecure"] |
| |
| _CUSTOM_CREDENTIALS_TYPE_OPTIONS = [ |
| "tls", |
| "google_default_credentials", |
| "compute_engine_channel_creds", |
| ] |
| |
| DOCKER_WORKDIR_ROOT = "/var/local/git/grpc" |
| |
| |
| def docker_run_cmdline(cmdline, image, docker_args=[], cwd=None, environ=None): |
| """Wraps given cmdline array to create 'docker run' cmdline from it.""" |
| |
| # don't use '-t' even when TTY is available, since that would break |
| # the testcases generated by tools/interop_matrix/create_testcases.sh |
| docker_cmdline = ["docker", "run", "-i", "--rm=true"] |
| |
| # turn environ into -e docker args |
| if environ: |
| for k, v in list(environ.items()): |
| docker_cmdline += ["-e", "%s=%s" % (k, v)] |
| |
| # set working directory |
| workdir = DOCKER_WORKDIR_ROOT |
| if cwd: |
| workdir = os.path.join(workdir, cwd) |
| docker_cmdline += ["-w", workdir] |
| |
| docker_cmdline += docker_args + [image] + cmdline |
| return docker_cmdline |
| |
| |
| def manual_cmdline(docker_cmdline, docker_image): |
| """Returns docker cmdline adjusted for manual invocation.""" |
| print_cmdline = [] |
| for item in docker_cmdline: |
| if item.startswith("--name="): |
| continue |
| if item == docker_image: |
| item = "$docker_image" |
| item = item.replace('"', '\\"') |
| # add quotes when necessary |
| if any(character.isspace() for character in item): |
| item = '"%s"' % item |
| print_cmdline.append(item) |
| return " ".join(print_cmdline) |
| |
| |
| def write_cmdlog_maybe(cmdlog, filename): |
| """Returns docker cmdline adjusted for manual invocation.""" |
| if cmdlog: |
| with open(filename, "w") as logfile: |
| logfile.write("#!/bin/bash\n") |
| logfile.write("# DO NOT MODIFY\n") |
| logfile.write( |
| "# This file is generated by" |
| " run_interop_tests.py/create_testcases.sh\n" |
| ) |
| logfile.writelines("%s\n" % line for line in cmdlog) |
| print("Command log written to file %s" % filename) |
| |
| |
| def bash_cmdline(cmdline): |
| """Creates bash -c cmdline from args list.""" |
| # Use login shell: |
| # * makes error messages clearer if executables are missing |
| return ["bash", "-c", " ".join(cmdline)] |
| |
| |
| def compute_engine_creds_required(language, test_case): |
| """Returns True if given test requires access to compute engine creds.""" |
| language = str(language) |
| if test_case == "compute_engine_creds": |
| return True |
| if test_case == "oauth2_auth_token" and language == "c++": |
| # C++ oauth2 test uses GCE creds because C++ only supports JWT |
| return True |
| return False |
| |
| |
| def auth_options( |
| language, |
| test_case, |
| google_default_creds_use_key_file, |
| service_account_key_file, |
| default_service_account, |
| ): |
| """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" |
| |
| language = str(language) |
| cmdargs = [] |
| env = {} |
| |
| oauth_scope_arg = "--oauth_scope=https://www.googleapis.com/auth/xapi.zoo" |
| key_file_arg = "--service_account_key_file=%s" % service_account_key_file |
| default_account_arg = ( |
| "--default_service_account=%s" % default_service_account |
| ) |
| |
| if test_case in ["jwt_token_creds", "per_rpc_creds", "oauth2_auth_token"]: |
| if language in [ |
| "aspnetcore", |
| "node", |
| "php7", |
| "python", |
| "ruby", |
| "nodepurejs", |
| ]: |
| env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file |
| else: |
| cmdargs += [key_file_arg] |
| |
| if test_case in ["per_rpc_creds", "oauth2_auth_token"]: |
| cmdargs += [oauth_scope_arg] |
| |
| if test_case == "oauth2_auth_token" and language == "c++": |
| # C++ oauth2 test uses GCE creds and thus needs to know the default account |
| cmdargs += [default_account_arg] |
| |
| if test_case == "compute_engine_creds": |
| cmdargs += [oauth_scope_arg, default_account_arg] |
| |
| if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: |
| if google_default_creds_use_key_file: |
| env["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_file |
| cmdargs += [default_account_arg] |
| |
| if test_case == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE: |
| cmdargs += [default_account_arg] |
| |
| return (cmdargs, env) |
| |
| |
| def _job_kill_handler(job): |
| if job._spec.container_name: |
| dockerjob.docker_kill(job._spec.container_name) |
| # When the job times out and we decide to kill it, |
| # we need to wait a before restarting the job |
| # to prevent "container name already in use" error. |
| # TODO(jtattermusch): figure out a cleaner way to this. |
| time.sleep(2) |
| |
| |
| def cloud_to_prod_jobspec( |
| language, |
| test_case, |
| server_host_nickname, |
| server_host, |
| google_default_creds_use_key_file, |
| docker_image=None, |
| auth=False, |
| manual_cmd_log=None, |
| service_account_key_file=None, |
| default_service_account=None, |
| transport_security="tls", |
| ): |
| """Creates jobspec for cloud-to-prod interop test""" |
| container_name = None |
| cmdargs = [ |
| "--server_host=%s" % server_host, |
| "--server_port=443", |
| "--test_case=%s" % test_case, |
| ] |
| if transport_security == "tls": |
| transport_security_options = ["--use_tls=true"] |
| elif transport_security == "google_default_credentials" and str( |
| language |
| ) in ["c++", "go", "java", "javaokhttp"]: |
| transport_security_options = [ |
| "--custom_credentials_type=google_default_credentials" |
| ] |
| elif transport_security == "compute_engine_channel_creds" and str( |
| language |
| ) in ["go", "java", "javaokhttp"]: |
| transport_security_options = [ |
| "--custom_credentials_type=compute_engine_channel_creds" |
| ] |
| else: |
| print( |
| "Invalid transport security option %s in cloud_to_prod_jobspec." |
| " Lang: %s" % (str(language), transport_security) |
| ) |
| sys.exit(1) |
| cmdargs = cmdargs + transport_security_options |
| environ = dict(language.cloud_to_prod_env(), **language.global_env()) |
| if auth: |
| auth_cmdargs, auth_env = auth_options( |
| language, |
| test_case, |
| google_default_creds_use_key_file, |
| service_account_key_file, |
| default_service_account, |
| ) |
| cmdargs += auth_cmdargs |
| environ.update(auth_env) |
| cmdline = bash_cmdline(language.client_cmd(cmdargs)) |
| cwd = language.client_cwd |
| |
| if docker_image: |
| container_name = dockerjob.random_name( |
| "interop_client_%s" % language.safename |
| ) |
| cmdline = docker_run_cmdline( |
| cmdline, |
| image=docker_image, |
| cwd=cwd, |
| environ=environ, |
| docker_args=["--net=host", "--name=%s" % container_name], |
| ) |
| if manual_cmd_log is not None: |
| if manual_cmd_log == []: |
| manual_cmd_log.append( |
| 'echo "Testing ${docker_image:=%s}"' % docker_image |
| ) |
| manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
| cwd = None |
| environ = None |
| |
| suite_name = "cloud_to_prod_auth" if auth else "cloud_to_prod" |
| test_job = jobset.JobSpec( |
| cmdline=cmdline, |
| cwd=cwd, |
| environ=environ, |
| shortname="%s:%s:%s:%s:%s" |
| % ( |
| suite_name, |
| language, |
| server_host_nickname, |
| test_case, |
| transport_security, |
| ), |
| timeout_seconds=_TEST_TIMEOUT, |
| flake_retries=4 if args.allow_flakes else 0, |
| timeout_retries=2 if args.allow_flakes else 0, |
| kill_handler=_job_kill_handler, |
| ) |
| if docker_image: |
| test_job.container_name = container_name |
| return test_job |
| |
| |
| def cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| server_name, |
| server_host, |
| server_port, |
| docker_image=None, |
| transport_security="tls", |
| manual_cmd_log=None, |
| ): |
| """Creates jobspec for cloud-to-cloud interop test""" |
| interop_only_options = [ |
| "--server_host_override=foo.test.google.fr", |
| "--use_test_ca=true", |
| ] |
| if transport_security == "tls": |
| interop_only_options += ["--use_tls=true"] |
| elif transport_security == "alts": |
| interop_only_options += ["--use_tls=false", "--use_alts=true"] |
| elif transport_security == "insecure": |
| interop_only_options += ["--use_tls=false"] |
| else: |
| print( |
| "Invalid transport security option %s in cloud_to_cloud_jobspec." |
| % transport_security |
| ) |
| sys.exit(1) |
| |
| client_test_case = test_case |
| if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
| client_test_case = _GRPC_CLIENT_TEST_CASES_FOR_HTTP2_SERVER_TEST_CASES[ |
| test_case |
| ] |
| if client_test_case in language.unimplemented_test_cases(): |
| print( |
| "asking client %s to run unimplemented test case %s" |
| % (repr(language), client_test_case) |
| ) |
| sys.exit(1) |
| |
| if test_case in _ORCA_TEST_CASES: |
| interop_only_options += [ |
| '--service_config_json=\'{"loadBalancingConfig":[{"test_backend_metrics_load_balancer":{}}]}\'' |
| ] |
| |
| common_options = [ |
| "--test_case=%s" % client_test_case, |
| "--server_host=%s" % server_host, |
| "--server_port=%s" % server_port, |
| ] |
| |
| if test_case in _HTTP2_SERVER_TEST_CASES: |
| if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
| client_options = interop_only_options + common_options |
| cmdline = bash_cmdline(language.client_cmd(client_options)) |
| cwd = language.client_cwd |
| else: |
| cmdline = bash_cmdline( |
| language.client_cmd_http2interop(common_options) |
| ) |
| cwd = language.http2_cwd |
| else: |
| cmdline = bash_cmdline( |
| language.client_cmd(common_options + interop_only_options) |
| ) |
| cwd = language.client_cwd |
| |
| environ = language.global_env() |
| if docker_image and language.safename != "objc": |
| # we can't run client in docker for objc. |
| container_name = dockerjob.random_name( |
| "interop_client_%s" % language.safename |
| ) |
| cmdline = docker_run_cmdline( |
| cmdline, |
| image=docker_image, |
| environ=environ, |
| cwd=cwd, |
| docker_args=["--net=host", "--name=%s" % container_name], |
| ) |
| if manual_cmd_log is not None: |
| if manual_cmd_log == []: |
| manual_cmd_log.append( |
| 'echo "Testing ${docker_image:=%s}"' % docker_image |
| ) |
| manual_cmd_log.append(manual_cmdline(cmdline, docker_image)) |
| cwd = None |
| |
| test_job = jobset.JobSpec( |
| cmdline=cmdline, |
| cwd=cwd, |
| environ=environ, |
| shortname="cloud_to_cloud:%s:%s_server:%s:%s" |
| % (language, server_name, test_case, transport_security), |
| timeout_seconds=_TEST_TIMEOUT, |
| flake_retries=4 if args.allow_flakes else 0, |
| timeout_retries=2 if args.allow_flakes else 0, |
| kill_handler=_job_kill_handler, |
| ) |
| if docker_image: |
| test_job.container_name = container_name |
| return test_job |
| |
| |
| def server_jobspec( |
| language, docker_image, transport_security="tls", manual_cmd_log=None |
| ): |
| """Create jobspec for running a server""" |
| container_name = dockerjob.random_name( |
| "interop_server_%s" % language.safename |
| ) |
| server_cmd = ["--port=%s" % _DEFAULT_SERVER_PORT] |
| if transport_security == "tls": |
| server_cmd += ["--use_tls=true"] |
| elif transport_security == "alts": |
| server_cmd += ["--use_tls=false", "--use_alts=true"] |
| elif transport_security == "insecure": |
| server_cmd += ["--use_tls=false"] |
| else: |
| print( |
| "Invalid transport security option %s in server_jobspec." |
| % transport_security |
| ) |
| sys.exit(1) |
| cmdline = bash_cmdline(language.server_cmd(server_cmd)) |
| environ = language.global_env() |
| docker_args = ["--name=%s" % container_name] |
| if language.safename == "http2": |
| # we are running the http2 interop server. Open next N ports beginning |
| # with the server port. These ports are used for http2 interop test |
| # (one test case per port). |
| docker_args += list( |
| itertools.chain.from_iterable( |
| ("-p", str(_DEFAULT_SERVER_PORT + i)) |
| for i in range(len(_HTTP2_SERVER_TEST_CASES)) |
| ) |
| ) |
| # Enable docker's healthcheck mechanism. |
| # This runs a Python script inside the container every second. The script |
| # pings the http2 server to verify it is ready. The 'health-retries' flag |
| # specifies the number of consecutive failures before docker will report |
| # the container's status as 'unhealthy'. Prior to the first 'health_retries' |
| # failures or the first success, the status will be 'starting'. 'docker ps' |
| # or 'docker inspect' can be used to see the health of the container on the |
| # command line. |
| docker_args += [ |
| "--health-cmd=python test/http2_test/http2_server_health_check.py " |
| "--server_host=%s --server_port=%d" |
| % ("localhost", _DEFAULT_SERVER_PORT), |
| "--health-interval=1s", |
| "--health-retries=5", |
| "--health-timeout=10s", |
| ] |
| |
| else: |
| docker_args += ["-p", str(_DEFAULT_SERVER_PORT)] |
| |
| docker_cmdline = docker_run_cmdline( |
| cmdline, |
| image=docker_image, |
| cwd=language.server_cwd, |
| environ=environ, |
| docker_args=docker_args, |
| ) |
| if manual_cmd_log is not None: |
| if manual_cmd_log == []: |
| manual_cmd_log.append( |
| 'echo "Testing ${docker_image:=%s}"' % docker_image |
| ) |
| manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image)) |
| server_job = jobset.JobSpec( |
| cmdline=docker_cmdline, |
| environ=environ, |
| shortname="interop_server_%s" % language, |
| timeout_seconds=30 * 60, |
| ) |
| server_job.container_name = container_name |
| return server_job |
| |
| |
| def build_interop_image_jobspec(language, tag=None): |
| """Creates jobspec for building interop docker image for a language""" |
| if not tag: |
| tag = "grpc_interop_%s:%s" % (language.safename, uuid.uuid4()) |
| env = { |
| "INTEROP_IMAGE": tag, |
| "BASE_NAME": "grpc_interop_%s" % language.safename, |
| } |
| build_job = jobset.JobSpec( |
| cmdline=["tools/run_tests/dockerize/build_interop_image.sh"], |
| environ=env, |
| shortname="build_docker_%s" % (language), |
| timeout_seconds=30 * 60, |
| ) |
| build_job.tag = tag |
| return build_job |
| |
| |
| def aggregate_http2_results(stdout): |
| match = re.search(r'\{"cases[^\]]*\]\}', stdout) |
| if not match: |
| return None |
| |
| results = json.loads(match.group(0)) |
| skipped = 0 |
| passed = 0 |
| failed = 0 |
| failed_cases = [] |
| for case in results["cases"]: |
| if case.get("skipped", False): |
| skipped += 1 |
| else: |
| if case.get("passed", False): |
| passed += 1 |
| else: |
| failed += 1 |
| failed_cases.append(case.get("name", "NONAME")) |
| return { |
| "passed": passed, |
| "failed": failed, |
| "skipped": skipped, |
| "failed_cases": ", ".join(failed_cases), |
| "percent": 1.0 * passed / (passed + failed), |
| } |
| |
| |
| # A dictionary of prod servers to test against. |
| # See go/grpc-interop-tests (internal-only) for details. |
| prod_servers = { |
| "default": "grpc-test.sandbox.googleapis.com", |
| "gateway_v4": "grpc-test4.sandbox.googleapis.com", |
| } |
| |
| argp = argparse.ArgumentParser(description="Run interop tests.") |
| argp.add_argument( |
| "-l", |
| "--language", |
| choices=["all"] + sorted(_LANGUAGES), |
| nargs="+", |
| default=["all"], |
| help="Clients to run. Objc client can be only run on OSX.", |
| ) |
| argp.add_argument("-j", "--jobs", default=multiprocessing.cpu_count(), type=int) |
| argp.add_argument( |
| "--cloud_to_prod", |
| default=False, |
| action="store_const", |
| const=True, |
| help="Run cloud_to_prod tests.", |
| ) |
| argp.add_argument( |
| "--cloud_to_prod_auth", |
| default=False, |
| action="store_const", |
| const=True, |
| help="Run cloud_to_prod_auth tests.", |
| ) |
| argp.add_argument( |
| "--google_default_creds_use_key_file", |
| default=False, |
| action="store_const", |
| const=True, |
| help=( |
| "Whether or not we should use a key file for the " |
| "google_default_credentials test case, e.g. by " |
| "setting env var GOOGLE_APPLICATION_CREDENTIALS." |
| ), |
| ) |
| argp.add_argument( |
| "--prod_servers", |
| choices=list(prod_servers.keys()), |
| default=["default"], |
| nargs="+", |
| help=( |
| "The servers to run cloud_to_prod and cloud_to_prod_auth tests against." |
| ), |
| ) |
| argp.add_argument( |
| "-s", |
| "--server", |
| choices=["all"] + sorted(_SERVERS), |
| nargs="+", |
| help="Run cloud_to_cloud servers in a separate docker " |
| + "image. Servers can only be started automatically if " |
| + "--use_docker option is enabled.", |
| default=[], |
| ) |
| argp.add_argument( |
| "--override_server", |
| action="append", |
| type=lambda kv: kv.split("="), |
| help=( |
| "Use servername=HOST:PORT to explicitly specify a server. E.g." |
| " csharp=localhost:50000" |
| ), |
| default=[], |
| ) |
| # TODO(jtattermusch): the default service_account_key_file only works when --use_docker is used. |
| argp.add_argument( |
| "--service_account_key_file", |
| type=str, |
| help="The service account key file to use for some auth interop tests.", |
| default="/root/service_account/grpc-testing-ebe7c1ac7381.json", |
| ) |
| argp.add_argument( |
| "--default_service_account", |
| type=str, |
| help=( |
| "Default GCE service account email to use for some auth interop tests." |
| ), |
| default="[email protected]", |
| ) |
| argp.add_argument( |
| "-t", |
| "--travis", |
| default=False, |
| action="store_const", |
| const=True, |
| help=( |
| "When set, indicates that the script is running on CI (= not locally)." |
| ), |
| ) |
| argp.add_argument( |
| "-v", "--verbose", default=False, action="store_const", const=True |
| ) |
| argp.add_argument( |
| "--use_docker", |
| default=False, |
| action="store_const", |
| const=True, |
| help="Run all the interop tests under docker. That provides " |
| + "additional isolation and prevents the need to install " |
| + "language specific prerequisites. Only available on Linux.", |
| ) |
| argp.add_argument( |
| "--allow_flakes", |
| default=False, |
| action="store_const", |
| const=True, |
| help=( |
| "Allow flaky tests to show as passing (re-runs failed tests up to five" |
| " times)" |
| ), |
| ) |
| argp.add_argument( |
| "--manual_run", |
| default=False, |
| action="store_const", |
| const=True, |
| help="Prepare things for running interop tests manually. " |
| + "Preserve docker images after building them and skip " |
| "actually running the tests. Only print commands to run by " + "hand.", |
| ) |
| argp.add_argument( |
| "--http2_interop", |
| default=False, |
| action="store_const", |
| const=True, |
| help="Enable HTTP/2 client edge case testing. (Bad client, good server)", |
| ) |
| argp.add_argument( |
| "--http2_server_interop", |
| default=False, |
| action="store_const", |
| const=True, |
| help=( |
| "Enable HTTP/2 server edge case testing. (Includes positive and" |
| " negative tests" |
| ), |
| ) |
| argp.add_argument( |
| "--transport_security", |
| choices=_TRANSPORT_SECURITY_OPTIONS, |
| default="tls", |
| type=str, |
| nargs="?", |
| const=True, |
| help="Which transport security mechanism to use.", |
| ) |
| argp.add_argument( |
| "--custom_credentials_type", |
| choices=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, |
| default=_CUSTOM_CREDENTIALS_TYPE_OPTIONS, |
| nargs="+", |
| help=( |
| "Credential types to test in the cloud_to_prod setup. Default is to" |
| " test with all creds types possible." |
| ), |
| ) |
| argp.add_argument( |
| "--skip_compute_engine_creds", |
| default=False, |
| action="store_const", |
| const=True, |
| help="Skip auth tests requiring access to compute engine credentials.", |
| ) |
| argp.add_argument( |
| "--internal_ci", |
| default=False, |
| action="store_const", |
| const=True, |
| help=( |
| "(Deprecated, has no effect) Put reports into subdirectories to improve" |
| " presentation of results by Internal CI." |
| ), |
| ) |
| argp.add_argument( |
| "--bq_result_table", |
| default="", |
| type=str, |
| nargs="?", |
| help="Upload test results to a specified BQ table.", |
| ) |
| args = argp.parse_args() |
| |
| servers = set( |
| s |
| for s in itertools.chain.from_iterable( |
| _SERVERS if x == "all" else [x] for x in args.server |
| ) |
| ) |
| # ALTS servers are only available for certain languages. |
| if args.transport_security == "alts": |
| servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) |
| |
| if args.use_docker: |
| if not args.travis: |
| print("Seen --use_docker flag, will run interop tests under docker.") |
| print("") |
| print( |
| "IMPORTANT: The changes you are testing need to be locally" |
| " committed" |
| ) |
| print( |
| "because only the committed changes in the current branch will be" |
| ) |
| print("copied to the docker environment.") |
| time.sleep(5) |
| |
| if args.manual_run and not args.use_docker: |
| print("--manual_run is only supported with --use_docker option enabled.") |
| sys.exit(1) |
| |
| if not args.use_docker and servers: |
| print( |
| "Running interop servers is only supported with --use_docker option" |
| " enabled." |
| ) |
| sys.exit(1) |
| |
| # we want to include everything but objc in 'all' |
| # because objc won't run on non-mac platforms |
| all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(["objc"]) |
| languages = set( |
| _LANGUAGES[l] |
| for l in itertools.chain.from_iterable( |
| all_but_objc if x == "all" else [x] for x in args.language |
| ) |
| ) |
| # ALTS interop clients are only available for certain languages. |
| if args.transport_security == "alts": |
| alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) |
| languages = languages.intersection(alts_languages) |
| |
| languages_http2_clients_for_http2_server_interop = set() |
| if args.http2_server_interop: |
| languages_http2_clients_for_http2_server_interop = set( |
| _LANGUAGES[l] |
| for l in _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES |
| if "all" in args.language or l in args.language |
| ) |
| |
| http2Interop = Http2Client() if args.http2_interop else None |
| http2InteropServer = Http2Server() if args.http2_server_interop else None |
| |
| docker_images = {} |
| if args.use_docker: |
| # languages for which to build docker images |
| languages_to_build = set( |
| _LANGUAGES[k] |
| for k in set([str(l) for l in languages] + [s for s in servers]) |
| ) |
| languages_to_build = ( |
| languages_to_build | languages_http2_clients_for_http2_server_interop |
| ) |
| |
| if args.http2_interop: |
| languages_to_build.add(http2Interop) |
| |
| if args.http2_server_interop: |
| languages_to_build.add(http2InteropServer) |
| |
| build_jobs = [] |
| for l in languages_to_build: |
| if str(l) == "objc": |
| # we don't need to build a docker image for objc |
| continue |
| job = build_interop_image_jobspec(l) |
| docker_images[str(l)] = job.tag |
| build_jobs.append(job) |
| |
| if build_jobs: |
| jobset.message( |
| "START", "Building interop docker images.", do_newline=True |
| ) |
| if args.verbose: |
| print("Jobs to run: \n%s\n" % "\n".join(str(j) for j in build_jobs)) |
| |
| num_failures, build_resultset = jobset.run( |
| build_jobs, newline_on_success=True, maxjobs=args.jobs |
| ) |
| |
| report_utils.render_junit_xml_report( |
| build_resultset, _DOCKER_BUILD_XML_REPORT |
| ) |
| |
| if num_failures == 0: |
| jobset.message( |
| "SUCCESS", |
| "All docker images built successfully.", |
| do_newline=True, |
| ) |
| else: |
| jobset.message( |
| "FAILED", |
| "Failed to build interop docker images.", |
| do_newline=True, |
| ) |
| for image in six.itervalues(docker_images): |
| dockerjob.remove_image(image, skip_nonexistent=True) |
| sys.exit(1) |
| |
| server_manual_cmd_log = [] if args.manual_run else None |
| client_manual_cmd_log = [] if args.manual_run else None |
| |
| # Start interop servers. |
| server_jobs = {} |
| server_addresses = {} |
| try: |
| for s in servers: |
| lang = str(s) |
| spec = server_jobspec( |
| _LANGUAGES[lang], |
| docker_images.get(lang), |
| args.transport_security, |
| manual_cmd_log=server_manual_cmd_log, |
| ) |
| if not args.manual_run: |
| job = dockerjob.DockerJob(spec) |
| server_jobs[lang] = job |
| server_addresses[lang] = ( |
| "localhost", |
| job.mapped_port(_DEFAULT_SERVER_PORT), |
| ) |
| else: |
| # don't run the server, set server port to a placeholder value |
| server_addresses[lang] = ("localhost", "${SERVER_PORT}") |
| |
| http2_server_job = None |
| if args.http2_server_interop: |
| # launch a HTTP2 server emulator that creates edge cases |
| lang = str(http2InteropServer) |
| spec = server_jobspec( |
| http2InteropServer, |
| docker_images.get(lang), |
| manual_cmd_log=server_manual_cmd_log, |
| ) |
| if not args.manual_run: |
| http2_server_job = dockerjob.DockerJob(spec) |
| server_jobs[lang] = http2_server_job |
| else: |
| # don't run the server, set server port to a placeholder value |
| server_addresses[lang] = ("localhost", "${SERVER_PORT}") |
| |
| jobs = [] |
| if args.cloud_to_prod: |
| if args.transport_security not in ["tls"]: |
| print("TLS is always enabled for cloud_to_prod scenarios.") |
| for server_host_nickname in args.prod_servers: |
| for language in languages: |
| for test_case in _TEST_CASES: |
| if not test_case in language.unimplemented_test_cases(): |
| if ( |
| not test_case |
| in _SKIP_ADVANCED |
| + _SKIP_COMPRESSION |
| + _SKIP_SPECIAL_STATUS_MESSAGE |
| + _ORCA_TEST_CASES |
| ): |
| for ( |
| transport_security |
| ) in args.custom_credentials_type: |
| # google_default_credentials not yet supported by all languages |
| if ( |
| transport_security |
| == "google_default_credentials" |
| and str(language) |
| not in ["c++", "go", "java", "javaokhttp"] |
| ): |
| continue |
| # compute_engine_channel_creds not yet supported by all languages |
| if ( |
| transport_security |
| == "compute_engine_channel_creds" |
| and str(language) |
| not in ["go", "java", "javaokhttp"] |
| ): |
| continue |
| test_job = cloud_to_prod_jobspec( |
| language, |
| test_case, |
| server_host_nickname, |
| prod_servers[server_host_nickname], |
| google_default_creds_use_key_file=args.google_default_creds_use_key_file, |
| docker_image=docker_images.get( |
| str(language) |
| ), |
| manual_cmd_log=client_manual_cmd_log, |
| service_account_key_file=args.service_account_key_file, |
| default_service_account=args.default_service_account, |
| transport_security=transport_security, |
| ) |
| jobs.append(test_job) |
| if args.http2_interop: |
| for test_case in _HTTP2_TEST_CASES: |
| test_job = cloud_to_prod_jobspec( |
| http2Interop, |
| test_case, |
| server_host_nickname, |
| prod_servers[server_host_nickname], |
| google_default_creds_use_key_file=args.google_default_creds_use_key_file, |
| docker_image=docker_images.get(str(http2Interop)), |
| manual_cmd_log=client_manual_cmd_log, |
| service_account_key_file=args.service_account_key_file, |
| default_service_account=args.default_service_account, |
| transport_security=args.transport_security, |
| ) |
| jobs.append(test_job) |
| |
| if args.cloud_to_prod_auth: |
| if args.transport_security not in ["tls"]: |
| print("TLS is always enabled for cloud_to_prod scenarios.") |
| for server_host_nickname in args.prod_servers: |
| for language in languages: |
| for test_case in _AUTH_TEST_CASES: |
| if ( |
| not args.skip_compute_engine_creds |
| or not compute_engine_creds_required( |
| language, test_case |
| ) |
| ): |
| if not test_case in language.unimplemented_test_cases(): |
| if test_case == _GOOGLE_DEFAULT_CREDS_TEST_CASE: |
| transport_security = ( |
| "google_default_credentials" |
| ) |
| elif ( |
| test_case |
| == _COMPUTE_ENGINE_CHANNEL_CREDS_TEST_CASE |
| ): |
| transport_security = ( |
| "compute_engine_channel_creds" |
| ) |
| else: |
| transport_security = "tls" |
| if ( |
| transport_security |
| not in args.custom_credentials_type |
| ): |
| continue |
| test_job = cloud_to_prod_jobspec( |
| language, |
| test_case, |
| server_host_nickname, |
| prod_servers[server_host_nickname], |
| google_default_creds_use_key_file=args.google_default_creds_use_key_file, |
| docker_image=docker_images.get(str(language)), |
| auth=True, |
| manual_cmd_log=client_manual_cmd_log, |
| service_account_key_file=args.service_account_key_file, |
| default_service_account=args.default_service_account, |
| transport_security=transport_security, |
| ) |
| jobs.append(test_job) |
| for server in args.override_server: |
| server_name = server[0] |
| (server_host, server_port) = server[1].split(":") |
| server_addresses[server_name] = (server_host, server_port) |
| |
| for server_name, server_address in list(server_addresses.items()): |
| (server_host, server_port) = server_address |
| server_language = _LANGUAGES.get(server_name, None) |
| skip_server = [] # test cases unimplemented by server |
| if server_language: |
| skip_server = server_language.unimplemented_test_cases_server() |
| for language in languages: |
| for test_case in _TEST_CASES: |
| if not test_case in language.unimplemented_test_cases(): |
| if not test_case in skip_server: |
| test_job = cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| server_name, |
| server_host, |
| server_port, |
| docker_image=docker_images.get(str(language)), |
| transport_security=args.transport_security, |
| manual_cmd_log=client_manual_cmd_log, |
| ) |
| jobs.append(test_job) |
| |
| if args.http2_interop: |
| for test_case in _HTTP2_TEST_CASES: |
| if server_name == "go": |
| # TODO(carl-mastrangelo): Reenable after https://github.com/grpc/grpc-go/issues/434 |
| continue |
| test_job = cloud_to_cloud_jobspec( |
| http2Interop, |
| test_case, |
| server_name, |
| server_host, |
| server_port, |
| docker_image=docker_images.get(str(http2Interop)), |
| transport_security=args.transport_security, |
| manual_cmd_log=client_manual_cmd_log, |
| ) |
| jobs.append(test_job) |
| |
| if args.http2_server_interop: |
| if not args.manual_run: |
| http2_server_job.wait_for_healthy(timeout_seconds=600) |
| for language in languages_http2_clients_for_http2_server_interop: |
| for test_case in set(_HTTP2_SERVER_TEST_CASES) - set( |
| _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS |
| ): |
| offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
| server_port = _DEFAULT_SERVER_PORT + offset |
| if not args.manual_run: |
| server_port = http2_server_job.mapped_port(server_port) |
| test_job = cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| str(http2InteropServer), |
| "localhost", |
| server_port, |
| docker_image=docker_images.get(str(language)), |
| manual_cmd_log=client_manual_cmd_log, |
| ) |
| jobs.append(test_job) |
| for language in languages: |
| # HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS is a subset of |
| # HTTP_SERVER_TEST_CASES, in which clients use their gRPC interop clients rather |
| # than specialized http2 clients, reusing existing test implementations. |
| # For example, in the "data_frame_padding" test, use language's gRPC |
| # interop clients and make them think that they're running "large_unary" |
| # test case. This avoids implementing a new test case in each language. |
| for test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: |
| if test_case not in language.unimplemented_test_cases(): |
| offset = sorted(_HTTP2_SERVER_TEST_CASES).index(test_case) |
| server_port = _DEFAULT_SERVER_PORT + offset |
| if not args.manual_run: |
| server_port = http2_server_job.mapped_port(server_port) |
| if args.transport_security != "insecure": |
| print( |
| "Creating grpc client to http2 server test case " |
| "with insecure connection, even though " |
| "args.transport_security is not insecure. Http2 " |
| "test server only supports insecure connections." |
| ) |
| test_job = cloud_to_cloud_jobspec( |
| language, |
| test_case, |
| str(http2InteropServer), |
| "localhost", |
| server_port, |
| docker_image=docker_images.get(str(language)), |
| transport_security="insecure", |
| manual_cmd_log=client_manual_cmd_log, |
| ) |
| jobs.append(test_job) |
| |
| if not jobs: |
| print("No jobs to run.") |
| for image in six.itervalues(docker_images): |
| dockerjob.remove_image(image, skip_nonexistent=True) |
| sys.exit(1) |
| |
| if args.manual_run: |
| print("All tests will skipped --manual_run option is active.") |
| |
| if args.verbose: |
| print("Jobs to run: \n%s\n" % "\n".join(str(job) for job in jobs)) |
| |
| num_failures, resultset = jobset.run( |
| jobs, |
| newline_on_success=True, |
| maxjobs=args.jobs, |
| skip_jobs=args.manual_run, |
| ) |
| if args.bq_result_table and resultset: |
| upload_interop_results_to_bq(resultset, args.bq_result_table) |
| if num_failures: |
| jobset.message("FAILED", "Some tests failed", do_newline=True) |
| else: |
| jobset.message("SUCCESS", "All tests passed", do_newline=True) |
| |
| write_cmdlog_maybe(server_manual_cmd_log, "interop_server_cmds.sh") |
| write_cmdlog_maybe(client_manual_cmd_log, "interop_client_cmds.sh") |
| |
| report_utils.render_junit_xml_report(resultset, _TESTS_XML_REPORT) |
| |
| for name, job in list(resultset.items()): |
| if "http2" in name: |
| job[0].http2results = aggregate_http2_results(job[0].message) |
| |
| http2_server_test_cases = ( |
| _HTTP2_SERVER_TEST_CASES if args.http2_server_interop else [] |
| ) |
| |
| if num_failures: |
| sys.exit(1) |
| else: |
| sys.exit(0) |
| finally: |
| # Check if servers are still running. |
| for server, job in list(server_jobs.items()): |
| if not job.is_running(): |
| print('Server "%s" has exited prematurely.' % server) |
| |
| dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)]) |
| |
| for image in six.itervalues(docker_images): |
| if not args.manual_run: |
| print("Removing docker image %s" % image) |
| dockerjob.remove_image(image) |
| else: |
| print("Preserving docker image: %s" % image) |
