| # Copyright 2018 Google LLC |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import json |
| import os |
| import re |
| import shutil |
| import fnmatch |
| from copy import deepcopy |
| from pathlib import Path |
| from typing import Dict, List, Optional |
| import jinja2 |
| from datetime import date |
| |
| from synthtool import shell, _tracked_paths |
| from synthtool.gcp import partials |
| from synthtool.languages import node, node_mono_repo |
| from synthtool.log import logger |
| from synthtool.sources import git, templates |
| |
| PathOrStr = templates.PathOrStr |
| TEMPLATES_URL: str = git.make_repo_clone_url("googleapis/synthtool") |
| DEFAULT_TEMPLATES_PATH = "synthtool/gcp/templates" |
| LOCAL_TEMPLATES: Optional[str] = os.environ.get("SYNTHTOOL_TEMPLATES") |
| |
| |
| class CommonTemplates: |
| def __init__(self, template_path: Optional[Path] = None): |
| if template_path: |
| self._template_root = template_path |
| elif LOCAL_TEMPLATES: |
| logger.debug(f"Using local templates at {LOCAL_TEMPLATES}") |
| self._template_root = Path(LOCAL_TEMPLATES) |
| else: |
| templates_git = git.clone(TEMPLATES_URL) |
| self._template_root = templates_git / DEFAULT_TEMPLATES_PATH |
| |
| self._templates = templates.Templates(self._template_root) |
| self.excludes = [] # type: List[str] |
| |
| def _generic_library(self, directory: str, relative_dir=None, **kwargs) -> Path: |
| # load common repo meta information (metadata that's not language specific). |
| if "metadata" in kwargs: |
| self._load_generic_metadata(kwargs["metadata"], relative_dir=relative_dir) |
| # if no samples were found, don't attempt to render a |
| # samples/README.md. |
| if "samples" not in kwargs["metadata"] or not kwargs["metadata"]["samples"]: |
| self.excludes.append("samples/README.md") |
| |
| t = templates.TemplateGroup(self._template_root / directory, self.excludes) |
| |
| if "repository" in kwargs["metadata"] and "repo" in kwargs["metadata"]: |
| kwargs["metadata"]["repo"]["default_branch"] = _get_default_branch_name( |
| kwargs["metadata"]["repository"] |
| ) |
| |
| # TODO: migrate to python.py once old sample gen is deprecated |
| if directory == "python_samples": |
| t.env.globals["get_help"] = lambda filename: shell.run( |
| ["python", filename, "--help"] |
| ).stdout |
| |
| result = t.render(**kwargs) |
| _tracked_paths.add(result) |
| |
| return result |
| |
| def py_samples(self, **kwargs) -> List[Path]: |
| """ |
| Handles generation of README.md templates for Python samples |
| - Determines whether generation is being done in a client library or in a samples |
| folder automatically |
| - Otherwise accepts manually set sample_project_dir through kwargs metadata |
| - Delegates generation of additional sample documents alternate/overridden folders |
| through py_samples_override() |
| """ |
| # kwargs["metadata"] is required to load values from .repo-metadata.json |
| if "metadata" not in kwargs: |
| kwargs["metadata"] = {} |
| |
| # load common repo meta information (metadata that's not language specific). |
| self._load_generic_metadata(kwargs["metadata"]) |
| |
| # temporary exclusion prior to old templates being migrated out |
| self.excludes.extend( |
| [ |
| "README.rst", |
| "auth_api_key.tmpl.rst", |
| "auth.tmpl.rst", |
| "install_deps.tmpl.rst", |
| "install_portaudio.tmpl.rst", |
| "noxfile.py.j2", |
| ] |
| ) |
| |
| # ensure samples will generate |
| kwargs["metadata"]["samples"] = True |
| |
| # determine if in client lib and set custom root sample dir if specified, else None |
| in_client_library = Path("samples").exists() |
| sample_project_dir = kwargs["metadata"]["repo"].get("sample_project_dir") |
| |
| if sample_project_dir is None: # Not found in metadata |
| if in_client_library: |
| sample_project_dir = "samples" |
| else: |
| sample_project_dir = "." |
| elif not Path(sample_project_dir).exists(): |
| raise Exception(f"'{sample_project_dir}' does not exist") |
| |
| override_paths_to_samples: Dict[ |
| str, List[str] |
| ] = {} # Dict of format { override_path : sample(s) } |
| samples_dict = deepcopy(kwargs["metadata"]["repo"].get("samples")) |
| default_samples_dict = [] # Dict which will generate in sample_project_dir |
| |
| # Iterate through samples to store override_paths_to_samples for all existing |
| # override paths |
| for sample_idx, sample in enumerate(samples_dict): |
| override_path = samples_dict[sample_idx].get("override_path") |
| |
| if override_path is not None: |
| # add absolute path to metadata so `python foo.py --help` succeeds |
| if sample.get("file") is not None: |
| path = os.path.join( |
| sample_project_dir, override_path, sample.get("file") |
| ) |
| sample["abs_path"] = Path(path).resolve() |
| |
| cur_override_sample = override_paths_to_samples.get(override_path) |
| # Base case: No samples are yet planned to gen in this override dir |
| if cur_override_sample is None: |
| override_paths_to_samples[override_path] = [sample] |
| # Else: Sample docs will be generated in README merged with other |
| # sample doc(s) already planned to generate in this dir |
| else: |
| cur_override_sample.append(sample) |
| override_paths_to_samples[override_path] = cur_override_sample |
| # If override path none, will be generated in the default |
| # folder: sample_project_dir |
| else: |
| if sample.get("file") is not None: |
| path = os.path.join(sample_project_dir, sample.get("file")) |
| sample["abs_path"] = Path(path).resolve() |
| default_samples_dict.append(sample) |
| |
| # List of paths to tempdirs which will be copied into sample folders |
| result = [] |
| |
| # deep copy is req. here to avoid kwargs being affected |
| overridden_samples_kwargs = deepcopy(kwargs) |
| for override_path in override_paths_to_samples: |
| # Generate override sample docs |
| result.append( |
| self.py_samples_override( |
| root=sample_project_dir, |
| override_path=override_path, |
| override_samples=override_paths_to_samples[override_path], |
| **overridden_samples_kwargs, |
| ) |
| ) |
| kwargs["metadata"]["repo"]["samples"] = default_samples_dict |
| |
| logger.debug( |
| f"Generating templates for samples directory '{sample_project_dir}'" |
| ) |
| kwargs["subdir"] = sample_project_dir |
| # Generate default sample docs |
| result.append(self._generic_library("python_samples", **kwargs)) |
| |
| for path in result: |
| # .add() records the root of the paths and needs to be applied to each |
| _tracked_paths.add(path) |
| |
| return result |
| |
| def py_samples_override( |
| self, root, override_path, override_samples, **overridden_samples_kwargs |
| ) -> Path: |
| """ |
| Handles additional generation of READMEs where "override_path"s |
| are set in one or more samples' metadata |
| """ |
| overridden_samples_kwargs["metadata"]["repo"][ |
| "sample_project_dir" |
| ] = override_path |
| # Set samples metadata to ONLY samples intended to generate |
| # under this directory (override_path) |
| overridden_samples_kwargs["metadata"]["repo"]["samples"] = override_samples |
| if root != ".": |
| override_path = Path(root) / override_path |
| |
| logger.debug(f"Generating templates for override path '{override_path}'") |
| |
| overridden_samples_kwargs["subdir"] = override_path |
| return self._generic_library("python_samples", **overridden_samples_kwargs) |
| |
| def python_notebooks(self, **kwargs) -> Path: |
| # kwargs["metadata"] is required to load values from .repo-metadata.json |
| if "metadata" not in kwargs: |
| kwargs["metadata"] = {} |
| return self._generic_library("python_notebooks", **kwargs) |
| |
| def py_mono_repo_library(self, relative_dir, **kwargs) -> Path: |
| # kwargs["metadata"] is required to load values from .repo-metadata.json |
| if "metadata" not in kwargs: |
| kwargs["metadata"] = {} |
| |
| # load common repo meta information (metadata that's not language specific). |
| self._load_generic_metadata(kwargs["metadata"], relative_dir) |
| |
| # initialize default_version if it doesn't exist in kwargs["metadata"]['repo'] |
| if "default_version" not in kwargs["metadata"]["repo"]: |
| kwargs["metadata"]["repo"]["default_version"] = "" |
| |
| # Don't add `docs/index.rst` if `versions` is not provided or `default_version` is empty |
| if ( |
| "versions" not in kwargs |
| or not kwargs["metadata"]["repo"]["default_version"] |
| or kwargs["metadata"]["repo"]["default_version"] == "apiVersion" |
| ): |
| self.excludes += ["docs/index.rst"] |
| |
| # If the directory `google/cloud` exists, add kwargs to signal that the client library is for a Cloud API |
| if Path("google/cloud").exists(): |
| kwargs["is_google_cloud_api"] = True |
| |
| return self._generic_library("python_mono_repo_library", relative_dir, **kwargs) |
| |
| def py_library(self, **kwargs) -> Path: |
| # kwargs["metadata"] is required to load values from .repo-metadata.json |
| if "metadata" not in kwargs: |
| kwargs["metadata"] = {} |
| |
| # load common repo meta information (metadata that's not language specific). |
| self._load_generic_metadata(kwargs["metadata"]) |
| |
| # initialize default_version if it doesn't exist in kwargs["metadata"]['repo'] |
| if "default_version" not in kwargs["metadata"]["repo"]: |
| kwargs["metadata"]["repo"]["default_version"] = "" |
| |
| # rename variable to accommodate existing owlbot.py files |
| if "system_test_dependencies" in kwargs: |
| kwargs["system_test_local_dependencies"] = kwargs[ |
| "system_test_dependencies" |
| ] |
| logger.warning( |
| "Template argument 'system_test_dependencies' is deprecated." |
| "Use 'system_test_local_dependencies' or 'system_test_external_dependencies'" |
| "instead." |
| ) |
| |
| # Set default Python versions for noxfile.py |
| if "default_python_version" not in kwargs: |
| kwargs["default_python_version"] = "3.8" |
| if "unit_test_python_versions" not in kwargs: |
| kwargs["unit_test_python_versions"] = ["3.7", "3.8", "3.9", "3.10"] |
| |
| if "system_test_python_versions" not in kwargs: |
| kwargs["system_test_python_versions"] = ["3.8"] |
| |
| # If cov_level is not given, set it to None. |
| if "cov_level" not in kwargs: |
| kwargs["cov_level"] = None |
| |
| # Don't add samples templates if there are no samples |
| if "samples" not in kwargs: |
| self.excludes += ["samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md"] |
| |
| # Don't add `docs/index.rst` if `versions` is not provided or `default_version` is empty |
| if ( |
| "versions" not in kwargs |
| or not kwargs["metadata"]["repo"]["default_version"] |
| ): |
| self.excludes += ["docs/index.rst"] |
| |
| # Add kwargs to signal that UPGRADING.md should be included in docs/index.rst if it exists |
| if Path("docs/UPGRADING.md").exists() or Path("docs/UPGRADING.rst").exists(): |
| kwargs["include_uprading_doc"] = True |
| |
| # If the directory `google/cloud` exists, add kwargs to signal that the client library is for a Cloud API |
| if Path("google/cloud").exists(): |
| kwargs["is_google_cloud_api"] = True |
| |
| # If Dockerfile exists in .kokoro/docker/samples, add kwargs to |
| # signal that a custom docker image should be used when testing samples. |
| kwargs["custom_samples_dockerfile"] = Path( |
| ".kokoro/docker/samples/Dockerfile" |
| ).exists() |
| |
| ret = self._generic_library("python_library", **kwargs) |
| |
| # If split_system_tests is set to True, we disable the system |
| # test in the main presubmit build and create individual build |
| # configs for each python versions. |
| if kwargs.get("split_system_tests", False): |
| template_root = self._template_root / "py_library_split_systests" |
| # copy the main presubmit config |
| shutil.copy2( |
| template_root / ".kokoro/presubmit/presubmit.cfg", |
| ret / ".kokoro/presubmit/presubmit.cfg", |
| ) |
| env = jinja2.Environment(loader=jinja2.FileSystemLoader(str(template_root))) |
| tmpl = env.get_template(".kokoro/presubmit/system.cfg") |
| for v in kwargs["system_test_python_versions"]: |
| nox_session = f"system-{v}" |
| dest = ret / f".kokoro/presubmit/system-{v}.cfg" |
| content = tmpl.render(nox_session=nox_session) |
| with open(dest, "w") as f: |
| f.write(content) |
| return ret |
| |
| def java_library(self, **kwargs) -> Path: |
| # kwargs["metadata"] is required to load values from .repo-metadata.json |
| if "metadata" not in kwargs: |
| kwargs["metadata"] = {} |
| return self._generic_library("java_library", **kwargs) |
| |
| def node_library(self, **kwargs) -> Path: |
| # TODO: once we've migrated all Node.js repos to either having |
| # .repo-metadata.json, or excluding README.md, we can remove this. |
| if not os.path.exists("./.repo-metadata.json"): |
| self.excludes.append("README.md") |
| if "samples/README.md" not in self.excludes: |
| self.excludes.append("samples/README.md") |
| |
| kwargs["metadata"] = node.template_metadata() |
| kwargs["publish_token"] = node.get_publish_token(kwargs["metadata"]["name"]) |
| |
| ignore_src_index = [ |
| "yes" for f in self.excludes if fnmatch.fnmatch("src/index.ts", f) |
| ] |
| # generate root-level `src/index.ts` to export multiple versions and its default clients |
| if ( |
| "versions" in kwargs |
| and "default_version" in kwargs |
| and not ignore_src_index |
| ): |
| node.generate_index_ts( |
| versions=kwargs["versions"], default_version=kwargs["default_version"] |
| ) |
| |
| return self._generic_library("node_library", **kwargs) |
| |
| def node_mono_repo_library(self, relative_dir, **kwargs) -> Path: |
| # TODO: once we've migrated all Node.js repos to either having |
| # .repo-metadata.json, or excluding README.md, we can remove this. |
| if not os.path.exists(Path(relative_dir, ".repo-metadata.json").resolve()): |
| self.excludes.append("README.md") |
| if "samples/README.md" not in self.excludes: |
| self.excludes.append("samples/README.md") |
| |
| kwargs["metadata"] = node_mono_repo.template_metadata(relative_dir) |
| |
| ignore_src_index = [ |
| "yes" for f in self.excludes if fnmatch.fnmatch("src/index.ts", f) |
| ] |
| # generate root-level `src/index.ts` to export multiple versions and its default clients |
| if ( |
| "versions" in kwargs |
| and "default_version" in kwargs |
| and not ignore_src_index |
| ): |
| node_mono_repo.generate_index_ts( |
| versions=kwargs["versions"], |
| default_version=kwargs["default_version"], |
| relative_dir=relative_dir, |
| year=str(date.today().year), |
| ) |
| |
| return self._generic_library( |
| "node_mono_repo_library", relative_dir=relative_dir, **kwargs |
| ) |
| |
| def php_library(self, **kwargs) -> Path: |
| return self._generic_library("php_library", **kwargs) |
| |
| def ruby_library(self, **kwargs) -> Path: |
| # kwargs["metadata"] is required to load values from .repo-metadata.json |
| if "metadata" not in kwargs: |
| kwargs["metadata"] = {} |
| return self._generic_library("ruby_library", **kwargs) |
| |
| def render(self, template_name: str, **kwargs) -> Path: |
| template = self._templates.render(template_name, **kwargs) |
| _tracked_paths.add(template) |
| return template |
| |
| def _load_generic_metadata(self, metadata: Dict, relative_dir=None): |
| """ |
| loads additional meta information from .repo-metadata.json. |
| """ |
| metadata["partials"] = partials.load_partials() |
| |
| # Loads repo metadata information from the default location if it |
| # hasn't already been set. Some callers may have already loaded repo |
| # metadata, so we don't need to do it again or overwrite it. Also, only |
| # set the "repo" key. |
| if "repo" not in metadata: |
| metadata["repo"] = _load_repo_metadata(relative_dir=relative_dir) |
| |
| |
| def detect_versions( |
| path: str = "./src", |
| default_version: Optional[str] = None, |
| default_first: Optional[bool] = None, |
| ) -> List[str]: |
| """ |
| Detects the versions a library has, based on distinct folders |
| within path. This is based on the fact that our GAPIC libraries are |
| structured as follows: |
| |
| src/v1 |
| src/v1beta |
| src/v1alpha |
| |
| With folder names mapping directly to versions. |
| |
| Returns: a list of the sorted subdirectories; for the example above: |
| ['v1', 'v1alpha', 'v1beta'] |
| If the `default_version` argument is not provided, the `default_version` |
| will be read from `.repo-metadata.json`, if it exists. |
| If `default_version` is available, the `default_version` is moved to |
| at the front or the end of the sorted list depending on the value of `default_first`. |
| The `default_version` will be first in the list when `default_first` is `True`. |
| """ |
| |
| versions = [] |
| |
| if not default_version: |
| try: |
| # Get the `default_version` from ``.repo-metadata.json`. |
| default_version = json.load(open(".repo-metadata.json", "rt")).get( |
| "default_version" |
| ) |
| except FileNotFoundError: |
| pass |
| |
| # Detect versions up to a depth of 4 in directory hierarchy |
| for level in ("*v[1-9]*", "*/*v[1-9]*", "*/*/*v[1-9]*", "*/*/*/*v[1-9]*"): |
| # Sort the sub directories alphabetically. |
| sub_dirs = sorted([p.name for p in Path(path).glob(level) if p.is_dir()]) |
| # Don't proceed to the next level if we've detected versions in this depth level |
| if sub_dirs: |
| break |
| |
| if sub_dirs: |
| # if `default_version` is not specified, return the sorted directories. |
| if not default_version: |
| versions = sub_dirs |
| else: |
| # The subdirectory with the same suffix as the default_version |
| # will be the default client. |
| default_client = next( |
| iter([d for d in sub_dirs if d.endswith(default_version)]), None |
| ) |
| |
| # start with all the versions except for the default client |
| versions = [d for d in sub_dirs if not d.endswith(default_version)] |
| |
| if default_client: |
| # If `default_first` is true, the default_client will be first |
| # in the list. |
| if default_first: |
| versions = [default_client] + versions |
| else: |
| versions += [default_client] |
| return versions |
| |
| |
| def decamelize(value: str): |
| """Parser to convert fooBar.js to Foo Bar.""" |
| if not value: |
| return "" |
| str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. |
| str_decamelize = re.sub( |
| "([A-Z]+)([A-Z])([a-z0-9])", r"\1 \2\3", str_decamelize |
| ) # ACLBatman -> ACL Batman. |
| return re.sub("([a-z0-9])([A-Z])", r"\1 \2", str_decamelize) # FooBar -> Foo Bar. |
| |
| |
| def _load_repo_metadata( |
| relative_dir=None, metadata_file: str = "./.repo-metadata.json" |
| ) -> Dict: |
| """Parse a metadata JSON file into a Dict. |
| |
| Currently, the defined fields are: |
| * `name` - The service's API name |
| * `name_pretty` - The service's API title. This will be used for generating titles on READMEs |
| * `product_documentation` - The product documentation on cloud.google.com |
| * `client_documentation` - The client library reference documentation |
| * `issue_tracker` - The public issue tracker for the product |
| * `release_level` - The release level of the client library. One of: alpha, beta, |
| ga, deprecated, preview, stable |
| * `language` - The repo language. One of dotnet, go, java, nodejs, php, python, ruby |
| * `repo` - The GitHub repo in the format {owner}/{repo} |
| * `distribution_name` - The language-idiomatic package/distribution name |
| * `api_id` - The API ID associated with the service. Fully qualified identifier use to |
| enable a service in the cloud platform (e.g. monitoring.googleapis.com) |
| * `requires_billing` - Whether or not the API requires billing to be configured on the |
| customer's acocunt |
| |
| Args: |
| metadata_file (str, optional): Path to the metadata json file |
| |
| Returns: |
| A dictionary of metadata. This may not necessarily include all the defined fields above. |
| """ |
| if relative_dir is not None: |
| if os.path.exists(Path(relative_dir, metadata_file).resolve()): |
| with open(Path(relative_dir, metadata_file).resolve()) as f: |
| return json.load(f) |
| elif os.path.exists(metadata_file): |
| with open(metadata_file) as f: |
| return json.load(f) |
| return {} |
| |
| |
| def _get_default_branch_name(repository_name: str) -> str: |
| """Read the default branch name from the environment. |
| |
| First checks environment variable DEFAULT_BRANCH_PATH. If found, it |
| reads the contents of the file at DEFAULT_BRANCH_PATH and returns it. |
| |
| Then checks environment varabile DEFAULT_BRANCH, and returns it if found. |
| """ |
| default_branch_path = os.getenv("DEFAULT_BRANCH_PATH") |
| if default_branch_path: |
| return Path(default_branch_path).read_text().strip() |
| |
| # This default should be switched to "main" once we've migrated |
| # the majority of our repositories: |
| return os.getenv("DEFAULT_BRANCH", "master") |
