From 368d95ead34b899934db3923e79063e2ea83105f Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 27 May 2024 19:31:47 +0000 Subject: [PATCH 01/21] feat: enable hermetic library generation --- .../{.OwlBot.yaml => .OwlBot-hermetic.yaml} | 0 .../hermetic_library_generation.yaml | 39 ++++++ .../update_googleapis_committish.yaml | 42 ++++++ generation/hermetic_library_generation.sh | 125 ++++++++++++++++++ generation/update_googleapis_committish.sh | 92 +++++++++++++ generation_config.yaml | 42 ++++++ 6 files changed, 340 insertions(+) rename .github/{.OwlBot.yaml => .OwlBot-hermetic.yaml} (100%) create mode 100644 .github/workflows/hermetic_library_generation.yaml create mode 100644 .github/workflows/update_googleapis_committish.yaml create mode 100644 generation/hermetic_library_generation.sh create mode 100644 generation/update_googleapis_committish.sh create mode 100644 generation_config.yaml diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot-hermetic.yaml similarity index 100% rename from .github/.OwlBot.yaml rename to .github/.OwlBot-hermetic.yaml diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml new file mode 100644 index 000000000..a7e05557b --- /dev/null +++ b/.github/workflows/hermetic_library_generation.yaml @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Hermetic library generation upon generation config change through pull requests +on: + pull_request: + +jobs: + library_generation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Generate changed libraries + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash generation/hermetic_library_generation.sh \ + --target_branch ${{ github.base_ref }} \ + --current_branch ${{ github.head_ref }} \ + --image_tag $(cat generation_config.yaml | yq .gapic_generator_version) + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml new file mode 100644 index 000000000..aac08d039 --- /dev/null +++ b/.github/workflows/update_googleapis_committish.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update googleapis commit +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-googleapis-committish: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update googleapis committish to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash generation/update_googleapis_committish.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh new file mode 100644 index 000000000..f425d9c12 --- /dev/null +++ b/generation/hermetic_library_generation.sh @@ -0,0 +1,125 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. Find whether the last commit in this pull request contains changes to +# the generation configuration and exit early if it doesn't have such a change +# since the generation result would be the same. +# 2. Compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 3. Generate changed libraries using library_generation image; +# 4. Commit the changes to the pull request, if any. +# 5. Edit the PR body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --image_tag) + image_tag="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${image_tag}" ]; then + echo "missing required argument --image_tag" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Use default generation config: ${generation_config}" +fi + +workspace_name="/workspace" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# if the last commit doesn't contain changes to generation configuration, +# do not generate again as the result will be the same. +change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" +if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then + echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true + exit 0 +fi +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) + +# run hermetic code generation docker image. +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + -v "${HOME}/.m2:/home/.m2" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" + + +# commit the change to the pull request. +if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then + git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt +else + # The image leaves intermediate folders and files it works with. Here we remove them + rm -rdf output googleapis baseline_generation_config.yaml pr_description.txt + git add . +fi +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "There is no generated code change with the generation config change ${config_diff}." + echo "Skip committing to the pull request." + exit 0 +fi + +echo "Configuration diff:" +echo "${config_diff}" +git commit -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi diff --git a/generation/update_googleapis_committish.sh b/generation/update_googleapis_committish.sh new file mode 100644 index 000000000..5dfcaba56 --- /dev/null +++ b/generation/update_googleapis_committish.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis committish to latest in generation +# configuration at the time of running and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: update googleapis committish at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest googleapis commit is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push +fi diff --git a/generation_config.yaml b/generation_config.yaml new file mode 100644 index 000000000..09977eee8 --- /dev/null +++ b/generation_config.yaml @@ -0,0 +1,42 @@ +gapic_generator_version: 2.40.1 +protoc_version: '25.2' +googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 +libraries_bom_version: 26.38.0 +template_excludes: + - .kokoro/presubmit/integration.cfg + - .kokoro/presubmit/graalvm-native.cfg + - .kokoro/presubmit/graalvm-native-17.cfg + - .kokoro/presubmit/samples.cfg + - .kokoro/nightly/integration.cfg + - .kokoro/nightly/java11-integration.cfg + - .kokoro/nightly/samples.cfg + - .kokoro/build.sh + - samples/snapshot/pom.xml + - .kokoro/release/publish_javadoc.sh + - .kokoro/release/publish_javadoc11.sh + - .kokoro/release/stage.sh + - .kokoro/requirements.in + - .kokoro/requirements.txt + - renovate.json +libraries: + - api_shortname: "firestore" + name_pretty: "Cloud Firestore" + product_documentation: "https://cloud.google.com/firestore" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-firestore/latest/history" + issue_tracker: "https://issuetracker.google.com/savedsearches/5337669" + release_level: "stable" + language: "java" + repo: "googleapis/java-firestore" + repo_short: "java-firestore" + distribution_name: "com.google.cloud:google-cloud-firestore" + codeowner_team: "@googleapis/api-firestore @googleapis/firestore-dpe" + api_id: "firestore.googleapis.com" + library_type: "GAPIC_COMBO" + api_description: "is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions." + transport: "grpc" + excluded_poms: "google-cloud-firestore,google-cloud-firestore-bom" + recommended_package: "com.google.cloud.firestore" + GAPICs: + - proto_path: google/firestore/v1 + - proto_path: google/firestore/admin/v1 + - proto_path: google/firestore/bundle/v1 From c2cfd90dbefe3cb06c5f53bc54bd964aae230c2e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 27 May 2024 20:47:54 +0000 Subject: [PATCH 02/21] fix config yaml syntax --- generation_config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 09977eee8..4b85745c3 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -36,7 +36,7 @@ libraries: transport: "grpc" excluded_poms: "google-cloud-firestore,google-cloud-firestore-bom" recommended_package: "com.google.cloud.firestore" - GAPICs: - - proto_path: google/firestore/v1 - - proto_path: google/firestore/admin/v1 - - proto_path: google/firestore/bundle/v1 + GAPICs: + - proto_path: google/firestore/v1 + - proto_path: google/firestore/admin/v1 + - proto_path: google/firestore/bundle/v1 From 8c3274b35d4eb3d38829e1697112fcce2ca357e9 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 00:08:44 +0000 Subject: [PATCH 03/21] do not map runners home folder --- generation/hermetic_library_generation.sh | 1 - generation_config.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index f425d9c12..9464642d9 100644 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -93,7 +93,6 @@ docker run \ --rm \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ - -v "${HOME}/.m2:/home/.m2" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ --current-generation-config-path="${workspace_name}/${generation_config}" diff --git a/generation_config.yaml b/generation_config.yaml index 4b85745c3..8fc96095d 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -39,4 +39,4 @@ libraries: GAPICs: - proto_path: google/firestore/v1 - proto_path: google/firestore/admin/v1 - - proto_path: google/firestore/bundle/v1 + - proto_path: google/firestore/bundle From f828d39948573471bb2abe7edc92236d0c8cebbb Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 00:39:45 +0000 Subject: [PATCH 04/21] use copyright update comittish --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 8fc96095d..228e6ad5f 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,6 +1,6 @@ gapic_generator_version: 2.40.1 protoc_version: '25.2' -googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 +googleapis_committish: 3597f7db2191c00b100400991ef96e52d62f5841 libraries_bom_version: 26.38.0 template_excludes: - .kokoro/presubmit/integration.cfg From 1d3ebc77f18339b234bc3a51d6850ce991c8a397 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 00:41:54 +0000 Subject: [PATCH 05/21] correct proto_path --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 228e6ad5f..8fc96095d 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,6 +1,6 @@ gapic_generator_version: 2.40.1 protoc_version: '25.2' -googleapis_committish: 3597f7db2191c00b100400991ef96e52d62f5841 +googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 template_excludes: - .kokoro/presubmit/integration.cfg From 1a6e5ed9b9d9a13e66cb20bd7bcaf803b8a26c6d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 28 May 2024 14:29:03 +0000 Subject: [PATCH 06/21] update protoc --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 8fc96095d..7ea03523b 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,5 +1,5 @@ gapic_generator_version: 2.40.1 -protoc_version: '25.2' +protoc_version: '25.3' googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 template_excludes: From 5f952b2077c7e22daf281e129f96f62bb786769f Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 29 May 2024 21:32:30 +0000 Subject: [PATCH 07/21] preserve pr_description --- generation/hermetic_library_generation.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index 9464642d9..d7e55e3c6 100644 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -103,8 +103,8 @@ if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt else # The image leaves intermediate folders and files it works with. Here we remove them - rm -rdf output googleapis baseline_generation_config.yaml pr_description.txt - git add . + rm -rdf output googleapis "${baseline_generation_config}" + git add --all -- ':!pr_description.txt' fi changed_files=$(git diff --cached --name-only) if [[ "${changed_files}" == "" ]]; then From 82fc0e5174839c674df0680e1df7e95982713281 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 4 Jun 2024 20:50:21 +0000 Subject: [PATCH 08/21] update gapic_generator_version to 2.41.0 --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 7ea03523b..812212594 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,4 +1,4 @@ -gapic_generator_version: 2.40.1 +gapic_generator_version: 2.41.0 protoc_version: '25.3' googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 From 10406979406c28a405ea8a6f7c0d34712d2f7958 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Mon, 10 Jun 2024 21:10:47 +0000 Subject: [PATCH 09/21] generalize path to firestore/bundle in owlbot yaml --- .github/.OwlBot-hermetic.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/.OwlBot-hermetic.yaml b/.github/.OwlBot-hermetic.yaml index 5f2bf905c..0533c6856 100644 --- a/.github/.OwlBot-hermetic.yaml +++ b/.github/.OwlBot-hermetic.yaml @@ -29,5 +29,5 @@ deep-copy-regex: dest: "/owl-bot-staging/$1/grpc-google-cloud-firestore-admin-$1/src" - source: "/google/firestore/admin/(v\\d)/.*-java/gapic-google-.*/src" dest: "/owl-bot-staging/$1/google-cloud-firestore-admin/src" -- source: "/google/firestore/bundle/google-cloud-firestore-bundle-v1-java/proto-google-cloud-firestore-bundle-v1-java/src" +- source: "/google/firestore/bundle/.*-java/proto-google-.*/src" dest: "/owl-bot-staging/v1/proto-google-cloud-firestore-bundle-v1/src" From aa91b790fce99d9fd5b87f430be37f024d850e39 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 12 Jun 2024 17:11:43 +0000 Subject: [PATCH 10/21] infer image tag from config yaml --- .../workflows/hermetic_library_generation.yaml | 3 +-- generation/hermetic_library_generation.sh | 15 ++++----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index a7e05557b..23385a1be 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -33,7 +33,6 @@ jobs: [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash generation/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ - --current_branch ${{ github.head_ref }} \ - --image_tag $(cat generation_config.yaml | yq .gapic_generator_version) + --current_branch ${{ github.head_ref }} env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index d7e55e3c6..6c3f22d8f 100644 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -21,7 +21,6 @@ set -e # The parameters of this script is: # 1. target_branch, the branch into which the pull request is merged. # 2. current_branch, the branch with which the pull request is associated. -# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. # 3. [optional] generation_config, the path to the generation configuration, # the default value is generation_config.yaml in the repository root. while [[ $# -gt 0 ]]; do @@ -35,10 +34,6 @@ case "${key}" in current_branch="$2" shift ;; - --image_tag) - image_tag="$2" - shift - ;; --generation_config) generation_config="$2" shift @@ -61,14 +56,9 @@ if [ -z "${current_branch}" ]; then exit 1 fi -if [ -z "${image_tag}" ]; then - echo "missing required argument --image_tag" - exit 1 -fi - if [ -z "${generation_config}" ]; then generation_config=generation_config.yaml - echo "Use default generation config: ${generation_config}" + echo "Using default generation config: ${generation_config}" fi workspace_name="/workspace" @@ -88,6 +78,9 @@ fi git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) +# parse image tag from the generation configuration. +image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs) + # run hermetic code generation docker image. docker run \ --rm \ From fb861d3527eb1c4df8d144e757ac4daafad3a143 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 12 Jun 2024 17:18:26 +0000 Subject: [PATCH 11/21] correct workflow name --- .github/workflows/update_googleapis_committish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml index aac08d039..7cc9356f0 100644 --- a/.github/workflows/update_googleapis_committish.yaml +++ b/.github/workflows/update_googleapis_committish.yaml @@ -13,7 +13,7 @@ # limitations under the License. # GitHub action job to test core java library features on # downstream client libraries before they are released. -name: Update googleapis commit +name: Update googleapis committish on: schedule: - cron: '0 2 * * *' From b6ed4eda9b825b63323a8ec233debb9ab161f07f Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 12 Jun 2024 17:45:15 +0000 Subject: [PATCH 12/21] update config scripts and yamls --- .../scripts}/hermetic_library_generation.sh | 0 .../scripts/update_generation_config.sh | 39 ++++++++++++++--- .../workflows/update_generation_config.yaml | 42 +++++++++++++++++++ 3 files changed, 76 insertions(+), 5 deletions(-) rename {generation => .github/scripts}/hermetic_library_generation.sh (100%) rename generation/update_googleapis_committish.sh => .github/scripts/update_generation_config.sh (59%) create mode 100644 .github/workflows/update_generation_config.yaml diff --git a/generation/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh similarity index 100% rename from generation/hermetic_library_generation.sh rename to .github/scripts/hermetic_library_generation.sh diff --git a/generation/update_googleapis_committish.sh b/.github/scripts/update_generation_config.sh similarity index 59% rename from generation/update_googleapis_committish.sh rename to .github/scripts/update_generation_config.sh index 5dfcaba56..561a31304 100644 --- a/generation/update_googleapis_committish.sh +++ b/.github/scripts/update_generation_config.sh @@ -1,12 +1,32 @@ #!/bin/bash set -e # This script should be run at the root of the repository. -# This script is used to update googleapis committish to latest in generation -# configuration at the time of running and create a pull request. +# This script is used to update googleapis_commitish, gapic_generator_version, +# and libraries_bom_version in generation configuration at the time of running +# and create a pull request. # The following commands need to be installed before running the script: # 1. git # 2. gh +# 3. jq + +# Utility functions +# Get the latest released version of a Maven artifact. +function get_latest_released_version() { + local group_id=$1 + local artifact_id=$2 + latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1) + echo "${latest}" +} + +# Update a key to a new value in the generation config. +function update_config() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" +} # The parameters of this script is: # 1. base_branch, the base branch of the result pull request. @@ -52,7 +72,7 @@ if [ -z "${generation_config}" ]; then fi current_branch="generate-libraries-${base_branch}" -title="chore: update googleapis committish at $(date)" +title="chore: Update generation configuration at $(date)" # try to find a open pull request associated with the branch pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") @@ -72,12 +92,20 @@ git pull latest_commit=$(git rev-parse HEAD) popd rm -rf tmp-googleapis -sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" +update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" + +# update gapic-generator-java version to the latest +latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") +update_config "gapic_generator_version" "${latest_version}" "${generation_config}" + +# update libraries-bom version to the latest +latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") +update_config "libraries_bom_version" "${latest_version}" "${generation_config}" git add "${generation_config}" changed_files=$(git diff --cached --name-only) if [[ "${changed_files}" == "" ]]; then - echo "The latest googleapis commit is not changed." + echo "The latest generation config is not changed." echo "Skip committing to the pull request." exit 0 fi @@ -89,4 +117,5 @@ if [ -z "${pr_num}" ]; then gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" else git push + gh pr edit "${pr_num}" --title "${title}" --body "${title}" fi diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml new file mode 100644 index 000000000..70b513312 --- /dev/null +++ b/.github/workflows/update_generation_config.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update generation configuration +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-generation-config: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update params in generation config to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/update_generation_config.sh.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From 39fd9b3dd6e366f5867c8b647cbdb66808e39c0e Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 20 Jun 2024 20:41:48 +0000 Subject: [PATCH 13/21] remove old update_googleapis_committish workflow --- .../update_googleapis_committish.yaml | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 .github/workflows/update_googleapis_committish.yaml diff --git a/.github/workflows/update_googleapis_committish.yaml b/.github/workflows/update_googleapis_committish.yaml deleted file mode 100644 index 7cc9356f0..000000000 --- a/.github/workflows/update_googleapis_committish.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# GitHub action job to test core java library features on -# downstream client libraries before they are released. -name: Update googleapis committish -on: - schedule: - - cron: '0 2 * * *' - workflow_dispatch: - -jobs: - update-googleapis-committish: - runs-on: ubuntu-22.04 - env: - # the branch into which the pull request is merged - base_branch: main - steps: - - uses: actions/checkout@v4 - with: - token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - name: Update googleapis committish to latest - shell: bash - run: | - set -x - [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash generation/update_googleapis_committish.sh \ - --base_branch "${base_branch}"\ - --repo ${{ github.repository }} - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From 1cc293d471d93e1b064acc15a87804f2ae265a5d Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Thu, 20 Jun 2024 21:33:29 +0000 Subject: [PATCH 14/21] sync config structure with that of google-cloud-java --- generation_config.yaml | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 812212594..4cf9a05dc 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,25 +1,8 @@ gapic_generator_version: 2.41.0 -protoc_version: '25.3' googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 -template_excludes: - - .kokoro/presubmit/integration.cfg - - .kokoro/presubmit/graalvm-native.cfg - - .kokoro/presubmit/graalvm-native-17.cfg - - .kokoro/presubmit/samples.cfg - - .kokoro/nightly/integration.cfg - - .kokoro/nightly/java11-integration.cfg - - .kokoro/nightly/samples.cfg - - .kokoro/build.sh - - samples/snapshot/pom.xml - - .kokoro/release/publish_javadoc.sh - - .kokoro/release/publish_javadoc11.sh - - .kokoro/release/stage.sh - - .kokoro/requirements.in - - .kokoro/requirements.txt - - renovate.json libraries: - - api_shortname: "firestore" + - api_shortname: firestore name_pretty: "Cloud Firestore" product_documentation: "https://cloud.google.com/firestore" client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-firestore/latest/history" From ed8fade8c02cb3d7f396cecec796486bff429bdd Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 25 Jun 2024 18:54:22 +0000 Subject: [PATCH 15/21] remove quotes from config yamls --- generation_config.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 4cf9a05dc..3357e817a 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -3,22 +3,22 @@ googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 libraries: - api_shortname: firestore - name_pretty: "Cloud Firestore" - product_documentation: "https://cloud.google.com/firestore" - client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-firestore/latest/history" - issue_tracker: "https://issuetracker.google.com/savedsearches/5337669" - release_level: "stable" - language: "java" - repo: "googleapis/java-firestore" - repo_short: "java-firestore" - distribution_name: "com.google.cloud:google-cloud-firestore" - codeowner_team: "@googleapis/api-firestore @googleapis/firestore-dpe" - api_id: "firestore.googleapis.com" - library_type: "GAPIC_COMBO" - api_description: "is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions." - transport: "grpc" - excluded_poms: "google-cloud-firestore,google-cloud-firestore-bom" - recommended_package: "com.google.cloud.firestore" + name_pretty: Cloud Firestore + product_documentation: https://cloud.google.com/firestore + client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-firestore/latest/history + issue_tracker: https://issuetracker.google.com/savedsearches/5337669 + release_level: stable + language: java + repo: googleapis/java-firestore + repo_short: java-firestore + distribution_name: com.google.cloud:google-cloud-firestore + codeowner_team: @googleapis/api-firestore @googleapis/firestore-dpe + api_id: firestore.googleapis.com + library_type: GAPIC_COMBO + api_description: is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions. + transport: grpc + excluded_poms: google-cloud-firestore,google-cloud-firestore-bom + recommended_package: com.google.cloud.firestore GAPICs: - proto_path: google/firestore/v1 - proto_path: google/firestore/admin/v1 From d19c532be84516dbd7cb5beeaa6bf74a5bb40654 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 25 Jun 2024 20:20:08 +0000 Subject: [PATCH 16/21] fix typo in update_generation_config.yaml --- .github/workflows/update_generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml index 70b513312..3cf773992 100644 --- a/.github/workflows/update_generation_config.yaml +++ b/.github/workflows/update_generation_config.yaml @@ -35,7 +35,7 @@ jobs: set -x [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash .github/scripts/update_generation_config.sh.sh \ + bash .github/scripts/update_generation_config.sh \ --base_branch "${base_branch}"\ --repo ${{ github.repository }} env: From fbbaf9145c6b399a2f945c8eeec26a0ebae377e9 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Tue, 25 Jun 2024 20:26:04 +0000 Subject: [PATCH 17/21] correct --- .github/workflows/hermetic_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 23385a1be..0c7a82a3c 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -31,7 +31,7 @@ jobs: set -x [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" - bash generation/hermetic_library_generation.sh \ + bash .github/scriptes/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ --current_branch ${{ github.head_ref }} env: From 394583fd620da036c3ac59670ebba8029e1067b5 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 26 Jun 2024 20:47:13 +0000 Subject: [PATCH 18/21] quote codeowners_team in generation config --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index 3357e817a..beeb5936d 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -12,7 +12,7 @@ libraries: repo: googleapis/java-firestore repo_short: java-firestore distribution_name: com.google.cloud:google-cloud-firestore - codeowner_team: @googleapis/api-firestore @googleapis/firestore-dpe + codeowner_team: '@googleapis/api-firestore @googleapis/firestore-dpe' api_id: firestore.googleapis.com library_type: GAPIC_COMBO api_description: is a fully-managed NoSQL document database for mobile, web, and server development from Firebase and Google Cloud Platform. It's backed by a multi-region replicated database that ensures once data is committed, it's durable even in the face of unexpected disasters. Not only that, but despite being a distributed database, it's also strongly consistent and offers seamless integration with other Firebase and Google Cloud Platform products, including Google Cloud Functions. From 4cfe3c50098fe8a234738ca9dbea52386417ef26 Mon Sep 17 00:00:00 2001 From: diegomarquezp Date: Wed, 26 Jun 2024 20:49:56 +0000 Subject: [PATCH 19/21] update generator version --- generation_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation_config.yaml b/generation_config.yaml index beeb5936d..1bf35afd4 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,4 +1,4 @@ -gapic_generator_version: 2.41.0 +gapic_generator_version: 2.42.0 googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 libraries_bom_version: 26.38.0 libraries: From cbdfde9cfcbabac446dfb2c03c0c34a3915390f3 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 28 Jun 2024 18:54:34 +0000 Subject: [PATCH 20/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3ba476f83..585c3e2ba 100644 --- a/README.md +++ b/README.md @@ -50,20 +50,20 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.39.0') +implementation platform('com.google.cloud:libraries-bom:26.42.0') implementation 'com.google.cloud:google-cloud-firestore' ``` If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-firestore:3.21.3' +implementation 'com.google.cloud:google-cloud-firestore:3.22.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-firestore" % "3.21.3" +libraryDependencies += "com.google.cloud" % "google-cloud-firestore" % "3.22.0" ``` @@ -222,7 +222,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-firestore/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-firestore.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-firestore/3.21.3 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-firestore/3.22.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From d2930a1f9fb5b3dfe2939a35b3e6d8f931e852d2 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Sun, 30 Jun 2024 01:48:38 +0000 Subject: [PATCH 21/21] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20?= =?UTF-8?q?post-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b596b323e..319252fa5 100644 --- a/README.md +++ b/README.md @@ -223,7 +223,6 @@ Java is a registered trademark of Oracle and/or its affiliates. [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-firestore.svg [maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-firestore/3.22.0 -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-firestore/3.21.6 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles