diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 9de0bc1c2c..9977160797 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -17,7 +17,7 @@ FROM gcr.io/cloud-devrel-public-resources/python SHELL [ "/bin/bash", "-c" ] -ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940 +ARG SYNTHTOOL_COMMITTISH=696c4bff721f5541cd75fdc97d413f8f39d2a2c1 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ARG PROTOC_VERSION=25.3 ENV HOME=/home diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 46e1491ccc..dc094c0b11 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -36,6 +36,7 @@ from library_generation.model.gapic_inputs import GapicInputs from library_generation.model.library_config import LibraryConfig from library_generation.model.gapic_inputs import parse as parse_build_file +from library_generation.model.repo_config import RepoConfig script_dir = os.path.dirname(os.path.realpath(__file__)) @@ -44,8 +45,7 @@ def generate_composed_library( config: GenerationConfig, library_path: str, library: LibraryConfig, - output_folder: str, - versions_file: str, + repo_config: RepoConfig, ) -> None: """ Generate libraries composed of more than one service or service version @@ -55,10 +55,10 @@ def generate_composed_library( :param library_path: the path to which the generated file goes :param library: a LibraryConfig object contained inside config, passed here for convenience and to prevent all libraries to be processed - :param output_folder: the folder to where tools go - :param versions_file: the file containing version of libraries + :param repo_config: :return None """ + output_folder = repo_config.output_folder util.pull_api_definition( config=config, library=library, output_folder=output_folder ) @@ -102,16 +102,20 @@ def generate_composed_library( cwd=output_folder, ) + library_version = repo_config.get_library_version( + artifact_id=library.get_artifact_id() + ) # call postprocess library util.run_process_and_print_output( [ f"{script_dir}/postprocess_library.sh", f"{library_path}", "", - versions_file, + repo_config.versions_file, owlbot_cli_source_folder, str(config.is_monorepo()).lower(), config.libraries_bom_version, + library_version, ], "Library postprocessing", ) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index b5cf9229fb..a4f6a5382e 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -44,14 +44,13 @@ def generate_from_yaml( gen_config=config, library_config=target_libraries, repo_path=repository_path ) - for library_path, library in repo_config.libraries.items(): + for library_path, library in repo_config.get_libraries().items(): print(f"generating library {library.get_library_name()}") generate_composed_library( config=config, library_path=library_path, library=library, - output_folder=repo_config.output_folder, - versions_file=repo_config.versions_file, + repo_config=repo_config, ) if not config.is_monorepo() or config.contains_common_protos(): diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py index 4d01698671..9855e4e834 100644 --- a/library_generation/model/library_config.py +++ b/library_generation/model/library_config.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. from hashlib import sha256 - from typing import Optional from library_generation.model.gapic_config import GapicConfig +MAVEN_COORDINATE_SEPARATOR = ":" + + class LibraryConfig: """ Class that represents a library in a generation_config.yaml file @@ -64,7 +66,6 @@ def __init__( self.excluded_dependencies = excluded_dependencies self.excluded_poms = excluded_poms self.client_documentation = client_documentation - self.distribution_name = distribution_name self.googleapis_commitish = googleapis_commitish self.group_id = group_id self.issue_tracker = issue_tracker @@ -76,6 +77,7 @@ def __init__( self.extra_versioned_modules = extra_versioned_modules self.recommended_package = recommended_package self.min_java_version = min_java_version + self.distribution_name = self.__get_distribution_name(distribution_name) def get_library_name(self) -> str: """ @@ -87,6 +89,34 @@ def get_library_name(self) -> str: def get_sorted_gapic_configs(self) -> list[GapicConfig]: return sorted(self.gapic_configs) + def get_maven_coordinate(self) -> str: + """ + Returns the Maven coordinate (group_id:artifact_id) of the library + """ + return self.distribution_name + + def get_artifact_id(self) -> str: + """ + Returns the artifact ID of the library + """ + return self.get_maven_coordinate().split(MAVEN_COORDINATE_SEPARATOR)[-1] + + def __get_distribution_name(self, distribution_name: Optional[str]) -> str: + LibraryConfig.__check_distribution_name(distribution_name) + if distribution_name: + return distribution_name + cloud_prefix = "cloud-" if self.cloud_api else "" + library_name = self.get_library_name() + return f"{self.group_id}:google-{cloud_prefix}{library_name}" + + @staticmethod + def __check_distribution_name(distribution_name: str) -> None: + if not distribution_name: + return + sections = distribution_name.split(MAVEN_COORDINATE_SEPARATOR) + if len(sections) != 2: + raise ValueError(f"{distribution_name} is not a valid distribution name.") + def __eq__(self, other): return ( self.api_shortname == other.api_shortname diff --git a/library_generation/model/repo_config.py b/library_generation/model/repo_config.py index 7f42720fe3..520c505823 100644 --- a/library_generation/model/repo_config.py +++ b/library_generation/model/repo_config.py @@ -12,10 +12,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict from library_generation.model.library_config import LibraryConfig +GRPC_PREFIX = "grpc-" +PROTO_PREFIX = "proto-" +NEW_CLIENT_VERSION = "0.0.0" + + class RepoConfig: """ Class that represents a generated repository @@ -24,11 +28,12 @@ class RepoConfig: def __init__( self, output_folder: str, - libraries: Dict[str, LibraryConfig], + libraries: dict[str, LibraryConfig], versions_file: str, ): """ Init a RepoConfig object + :param output_folder: the path to which the generated repo goes :param libraries: a mapping from library_path to LibraryConfig object :param versions_file: the path of versions.txt used in post-processing @@ -36,3 +41,34 @@ def __init__( self.output_folder = output_folder self.libraries = libraries self.versions_file = versions_file + self.library_versions = self.__parse_versions() + + def get_libraries(self) -> dict[str, LibraryConfig]: + return self.libraries + + def get_library_version(self, artifact_id: str) -> str: + """ + Returns the version of a given artifact ID. + If the artifact ID is not managed, i.e., a new client, returns `0.0.0`. + + :param artifact_id: the Maven artifact ID. + :return: the version of the artifact. + """ + return self.library_versions.get(artifact_id, NEW_CLIENT_VERSION) + + def __parse_versions(self) -> dict[str, str]: + library_versions = dict() + with open(self.versions_file) as f: + for line in f.readlines(): + sections = line.split(":") + # skip comments and empty lines. + if len(sections) != 3: + continue + artifact_id = sections[0] + released_version = sections[1] + if artifact_id.startswith(GRPC_PREFIX) or artifact_id.startswith( + PROTO_PREFIX + ): + continue + library_versions[artifact_id] = released_version + return library_versions diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index b5b84cf1ac..b64b12bdb6 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -22,6 +22,7 @@ # both to README and pom.xml files # 3: is_monorepo: whether we are postprocessing a monorepo # 4: libraries_bom_version: used to render the readme +# 5: library_version: used to render the readme # The scripts assumes the CWD is the folder where postprocessing is going to be # applied @@ -31,7 +32,7 @@ scripts_root=$1 versions_file=$2 is_monorepo=$3 libraries_bom_version=$4 - +library_version=$5 if [[ "${is_monorepo}" == "true" ]]; then mv owl-bot-staging/* temp @@ -48,10 +49,12 @@ then # synthtool library considering the way owlbot.py files are written export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates" export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}" + export SYNTHTOOL_LIBRARY_VERSION="${library_version}" # defaults to run owlbot.py python3 owlbot.py unset SYNTHTOOL_TEMPLATES unset SYNTHTOOL_LIBRARIES_BOM_VERSION + unset SYNTHTOOL_LIBRARY_VERSION fi echo "...done" diff --git a/library_generation/owlbot/templates/java_library/README.md b/library_generation/owlbot/templates/java_library/README.md index 04f1f4e834..21091251c0 100644 --- a/library_generation/owlbot/templates/java_library/README.md +++ b/library_generation/owlbot/templates/java_library/README.md @@ -1,5 +1,6 @@ {% set group_id = metadata['repo']['distribution_name'].split(':')|first -%} {% set artifact_id = metadata['repo']['distribution_name'].split(':')|last -%} +{% set version = metadata['library_version'] -%} {% set repo_short = metadata['repo']['repo'].split('/')|last -%} # Google {{ metadata['repo']['name_pretty'] }} Client for Java @@ -71,7 +72,7 @@ If you are using Maven, add this to your pom.xml file: {{ group_id }} {{ artifact_id }} - {{ metadata['latest_version'] }} + {{ version }} {% endif -%} ``` @@ -80,7 +81,7 @@ If you are using Maven, add this to your pom.xml file: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:{{metadata['latest_bom_version']}}') +implementation platform('com.google.cloud:libraries-bom:{{metadata['libraries_bom_version']}}') implementation '{{ group_id }}:{{ artifact_id }}' ``` @@ -89,13 +90,13 @@ implementation '{{ group_id }}:{{ artifact_id }}' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation '{{ group_id }}:{{ artifact_id }}:{{ metadata['latest_version'] }}' +implementation '{{ group_id }}:{{ artifact_id }}:{{ version }}' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ metadata['latest_version'] }}" +libraryDependencies += "{{ group_id }}" % "{{ artifact_id }}" % "{{ version }}" ``` @@ -264,7 +265,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/{{ repo_short }}/java11.html [stability-image]: https://img.shields.io/badge/stability-{% if metadata['repo']['release_level'] == 'stable' %}stable-green{% elif metadata['repo']['release_level'] == 'preview' %}preview-yellow{% else %}unknown-red{% endif %} [maven-version-image]: https://img.shields.io/maven-central/v/{{ group_id }}/{{ artifact_id }}.svg -[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ metadata['latest_version'] }} +[maven-version-link]: https://central.sonatype.com/artifact/{{ group_id }}/{{ artifact_id }}/{{ version }} [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 392301814f..eeec07156e 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -19,6 +19,8 @@ # different logic # 6 - libraries_bom_version: used by our implementation of owlbot to render the # readme +# 7 - library_version: used by our implementation of owlbot to render the +# readme set -exo pipefail scripts_root=$(dirname "$(readlink -f "$0")") @@ -28,6 +30,7 @@ versions_file=$3 owlbot_cli_source_folder=$4 is_monorepo=$5 libraries_bom_version=$6 +library_version=$7 owlbot_yaml_file_name=".OwlBot-hermetic.yaml" source "${scripts_root}"/utils/utilities.sh @@ -102,6 +105,7 @@ bash "${scripts_root}/owlbot/bin/entrypoint.sh" \ "${scripts_root}" \ "${versions_file}" \ "${is_monorepo}" \ - "${libraries_bom_version}" + "${libraries_bom_version}" \ + "${library_version}" popd # postprocessing_target diff --git a/library_generation/templates/owlbot.yaml.monorepo.j2 b/library_generation/templates/owlbot.yaml.monorepo.j2 index 5267a6f8a3..9ed63c4260 100644 --- a/library_generation/templates/owlbot.yaml.monorepo.j2 +++ b/library_generation/templates/owlbot.yaml.monorepo.j2 @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -{% if artifact_name %} +{% if artifact_id %} deep-remove-regex: - "/{{ module_name }}/grpc-google-.*/src" - "/{{ module_name }}/proto-google-.*/src" @@ -24,11 +24,11 @@ deep-preserve-regex: deep-copy-regex: - source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_name }}-$1/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_id }}-$1/src" - source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_name }}-$1/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_id }}-$1/src" - source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" - dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_name }}/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_id }}/src" - source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated" dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated" {%- endif %} diff --git a/library_generation/test/model/library_config_unit_tests.py b/library_generation/test/model/library_config_unit_tests.py index 35ba5be3e4..5d54737ced 100644 --- a/library_generation/test/model/library_config_unit_tests.py +++ b/library_generation/test/model/library_config_unit_tests.py @@ -64,3 +64,59 @@ def test_get_sorted_gapic_configs_returns_correct_order(self): ], library.get_sorted_gapic_configs(), ) + + def test_init_invalid_distribution_name_raise_value_error(self): + self.assertRaisesRegex( + ValueError, + "fake-distribution-name is not a valid distribution name.", + LibraryConfig, + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="fake-distribution-name", + ) + + def test_get_distribution_name_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + ) + self.assertEqual( + "com.google.cloud:google-cloud-baremetalsolution", + library.get_maven_coordinate(), + ) + self.assertEqual("google-cloud-baremetalsolution", library.get_artifact_id()) + + def test_get_distribution_name_non_cloud_api(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + cloud_api=False, + group_id="com.example", + ) + self.assertEqual( + "com.example:google-baremetalsolution", library.get_maven_coordinate() + ) + self.assertEqual("google-baremetalsolution", library.get_artifact_id()) + + def test_get_distribution_name_with_distribution_name(self): + library = LibraryConfig( + api_shortname="baremetalsolution", + name_pretty="Bare Metal Solution", + product_documentation="https://cloud.google.com/bare-metal/docs", + api_description="example api description", + gapic_configs=list(), + distribution_name="com.example:baremetalsolution", + ) + self.assertEqual( + "com.example:baremetalsolution", library.get_maven_coordinate() + ) + self.assertEqual("baremetalsolution", library.get_artifact_id()) diff --git a/library_generation/test/model/repo_config_unit_tests.py b/library_generation/test/model/repo_config_unit_tests.py new file mode 100644 index 0000000000..12d28fe254 --- /dev/null +++ b/library_generation/test/model/repo_config_unit_tests.py @@ -0,0 +1,56 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from library_generation.model.repo_config import RepoConfig + +script_dir = os.path.dirname(os.path.realpath(__file__)) +versions_file = os.path.join(script_dir, "..", "resources", "misc", "versions.txt") + + +class RepoConfigTest(unittest.TestCase): + def test_get_library_versions_with_existing_library(self): + repo_config = RepoConfig( + output_folder="test", libraries=dict(), versions_file=versions_file + ) + self.assertEqual( + "2.25.0", + repo_config.get_library_version("gapic-generator-java"), + ) + self.assertEqual( + "2.16.0", + repo_config.get_library_version("api-common"), + ) + self.assertEqual( + "2.33.0", + repo_config.get_library_version("gax"), + ) + self.assertEqual( + "2.34.0", + repo_config.get_library_version("gax-grpc"), + ) + self.assertEqual( + "0.118.0", + repo_config.get_library_version("gax-httpjson"), + ) + + def test_get_library_versions_with_new_library(self): + repo_config = RepoConfig( + output_folder="test", libraries=dict(), versions_file=versions_file + ) + self.assertEqual( + "0.0.0", + repo_config.get_library_version("example-artifact"), + ) diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index 0490ad9e2a..59f238aaea 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -16,7 +16,6 @@ import subprocess import os import shutil -import re from pathlib import Path from library_generation.model.generation_config import GenerationConfig from library_generation.model.library_config import LibraryConfig @@ -204,14 +203,8 @@ def generate_postprocessing_prerequisite_files( :param language: programming language of the library :return: None """ - cloud_prefix = "cloud-" if library.cloud_api else "" library_name = library.get_library_name() - distribution_name = ( - library.distribution_name - if library.distribution_name - else f"{library.group_id}:google-{cloud_prefix}{library_name}" - ) - distribution_name_short = re.split(r"[:/]", distribution_name)[-1] + artifact_id = library.get_artifact_id() if config.contains_common_protos(): repo = SDK_PLATFORM_JAVA elif config.is_monorepo(): @@ -224,7 +217,7 @@ def generate_postprocessing_prerequisite_files( client_documentation = ( library.client_documentation if library.client_documentation - else f"https://cloud.google.com/{language}/docs/reference/{distribution_name_short}/latest/overview" + else f"https://cloud.google.com/{language}/docs/reference/{artifact_id}/latest/overview" ) # The mapping is needed because transport in .repo-metadata.json @@ -247,7 +240,7 @@ def generate_postprocessing_prerequisite_files( "language": language, "repo": repo, "repo_short": f"{language}-{library_name}", - "distribution_name": distribution_name, + "distribution_name": library.get_maven_coordinate(), "api_id": api_id, "library_type": library.library_type, "requires_billing": library.requires_billing, @@ -298,7 +291,7 @@ def generate_postprocessing_prerequisite_files( render( template_name="owlbot.yaml.monorepo.j2", output_name=path_to_owlbot_yaml_file, - artifact_name=distribution_name_short, + artifact_id=artifact_id, proto_path=remove_version_from(proto_path), module_name=repo_metadata["repo_short"], api_shortname=library.api_shortname,