From bd42477cf6e19761415cd70776daa13db4f43c1e Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 22 Nov 2022 10:35:33 -0500 Subject: [PATCH 1/7] chore: rename bootstrap to merge_repository --- .../{bootstrap.yaml => merge_repository.yaml} | 4 +- generation/README.md | 9 +- .../{bootstrap.sh => merge_repository.sh} | 0 generation/repos.txt | 131 +----------------- 4 files changed, 8 insertions(+), 136 deletions(-) rename .github/workflows/{bootstrap.yaml => merge_repository.yaml} (94%) rename generation/{bootstrap.sh => merge_repository.sh} (100%) diff --git a/.github/workflows/bootstrap.yaml b/.github/workflows/merge_repository.yaml similarity index 94% rename from .github/workflows/bootstrap.yaml rename to .github/workflows/merge_repository.yaml index fb30f0f681da..f68ed76774ca 100644 --- a/.github/workflows/bootstrap.yaml +++ b/.github/workflows/merge_repository.yaml @@ -1,4 +1,4 @@ -name: Bootstrap +name: Merge Repository on: schedule: - cron: '0 1 * * *' # nightly at 1 am UTC @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Generate - run: ./generation/bootstrap.sh + run: ./generation/merge_repository.sh env: USERNAME: ${{ github.actor }} - name: Compile diff --git a/generation/README.md b/generation/README.md index e75b73a3322e..974e2bf79047 100644 --- a/generation/README.md +++ b/generation/README.md @@ -3,13 +3,14 @@ ## Using scripts independently Most of the scripts in this directory can be used independently to do file updates across all modules. -They are also used and tested by `bootstrap.sh`. +They are also used and tested by `merge_repository.sh`. -## Generating the monorepo +## Merge repository into the monorepo -Built by running [bootstrap.sh](bootstrap.sh). +Built by running [erge_repository.sh](merge_repository.sh). -The script creates a new Git repository in `monorepo/google-cloud-java`. +The script creates a new Git repository in `monorepo/google-cloud-java` by merging +the repositories in the input. After running the script locally, you can `cd` into the generated repository and build the project. ```shell diff --git a/generation/bootstrap.sh b/generation/merge_repository.sh similarity index 100% rename from generation/bootstrap.sh rename to generation/merge_repository.sh diff --git a/generation/repos.txt b/generation/repos.txt index 99ca132909ae..a8d01cd26723 100644 --- a/generation/repos.txt +++ b/generation/repos.txt @@ -1,132 +1,3 @@ -java-accessapproval +google-cloud-java java-accesscontextmanager java-aiplatform -java-analytics-admin -java-analytics-data -java-api-gateway -java-apigee-connect -java-apigee-registry -java-apikeys -java-appengine-admin -java-area120-tables -java-artifact-registry -java-asset -java-assured-workloads -java-automl -java-bare-metal-solution -java-batch -java-beyondcorp-appconnections -java-beyondcorp-appconnectors -java-beyondcorp-appgateways -java-beyondcorp-clientconnectorservices -java-beyondcorp-clientgateways -java-bigquery-data-exchange -java-bigqueryconnection -java-bigquerydatatransfer -java-bigquerymigration -java-bigqueryreservation -java-billing -java-billingbudgets -java-binary-authorization -java-certificate-manager -java-channel -java-cloudbuild -java-cloudcommerceconsumerprocurement -java-compute -java-contact-center-insights -java-container -java-containeranalysis -java-data-fusion -java-datacatalog -java-dataflow -java-dataform -java-datalabeling -java-dataplex -java-dataproc -java-dataproc-metastore -java-datastream -java-debugger-client -java-deploy -java-dialogflow -java-dialogflow-cx -java-dlp -java-dms -java-dns -java-document-ai -java-domains -java-errorreporting -java-essential-contacts -java-eventarc -java-eventarc-publishing -java-filestore -java-functions -java-game-servers -java-gke-backup -java-gke-connect-gateway -java-gke-multi-cloud -java-gkehub -java-grafeas -java-gsuite-addons -java-iam-admin -java-iamcredentials -java-ids -java-iot -java-kms -java-language -java-life-sciences -java-managed-identities -java-mediatranslation -java-memcache -java-monitoring -java-monitoring-dashboards -java-network-management -java-network-security -java-networkconnectivity -java-notebooks -java-notification -java-optimization -java-orchestration-airflow -java-orgpolicy -java-os-config -java-os-login -java-phishingprotection -java-policy-troubleshooter -java-private-catalog -java-profiler -java-recaptchaenterprise -java-recommendations-ai -java-recommender -java-redis -java-resource-settings -java-resourcemanager -java-retail -java-run -java-scheduler -java-secretmanager -java-security-private-ca -java-securitycenter -java-securitycenter-settings -java-service-control -java-service-management -java-service-usage -java-servicedirectory -java-shell -java-speech -java-storage-transfer -java-talent -java-tasks -java-texttospeech -java-tpu -java-trace -java-translate -java-video-intelligence -java-video-live-stream -java-video-stitcher -java-video-transcoder -java-vision -java-vmmigration -java-vpcaccess -java-webrisk -java-websecurityscanner -java-workflow-executions -java-workflows From 9d32ea1fd3c6c234b0c9abfee025f7c695b3ce6a Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 22 Nov 2022 10:56:13 -0500 Subject: [PATCH 2/7] merge_repository.sh to copy itself --- generation/merge_repository.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation/merge_repository.sh b/generation/merge_repository.sh index 5fe140bb8a41..2260acda2ccd 100755 --- a/generation/merge_repository.sh +++ b/generation/merge_repository.sh @@ -12,7 +12,7 @@ cd "$(dirname "$0")" rm -rf monorepo mkdir monorepo -cp bootstrap.sh monorepo +cp merge_repository.sh monorepo cp repos.txt monorepo cd monorepo From 1383240dec1d78f823382db48ea9c975815e6441 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 22 Nov 2022 11:15:16 -0500 Subject: [PATCH 3/7] avoid conflicting repository name --- generation/merge_repository.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/generation/merge_repository.sh b/generation/merge_repository.sh index 2260acda2ccd..892c990bb9d3 100755 --- a/generation/merge_repository.sh +++ b/generation/merge_repository.sh @@ -20,9 +20,10 @@ cd monorepo git clone https://github.com/newren/git-filter-repo.git export PATH=$PATH:`pwd`/git-filter-repo -mkdir google-cloud-java +merged_repository=google-cloud-java-merged +mkdir "${merged_repository}" -cd google-cloud-java +cd "${merged_repository}" git init -b main cat ../repos.txt | while read service @@ -51,7 +52,7 @@ do git config --add secrets.allowed "dest.*src" git commit -am "chore: setup owlbot configuration" - cd ../google-cloud-java + cd "../${merged_repository}" git remote add ${service} ../${service} git config --add secrets.allowed "dest.*src" git fetch ${service} #--tags @@ -60,7 +61,7 @@ do rm -rf ../${service} done -# cwd: monorepo/google-cloud-java +# cwd: monorepo/google-cloud-java-merged echo "Working directory: $(pwd)" cp -R ../../../google-cloud-jar-parent google-cloud-jar-parent @@ -117,7 +118,7 @@ git commit -am 'chore: set owlbot copy config' --allow-empty # create a monorepo/diff repo cd .. -cp -R google-cloud-java split +cp -R "${merged_repository}" split rm -rf split/.git git clone -b main --single-branch https://github.com/googleapis/google-cloud-java.git shadow cp -R shadow/.git split/.git From c20d31533b071faf766d08625801c363e505e564 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 22 Nov 2022 14:58:33 -0500 Subject: [PATCH 4/7] Update generation/README.md Co-authored-by: Mike Eltsufin --- generation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation/README.md b/generation/README.md index 974e2bf79047..b3c2056fbfcb 100644 --- a/generation/README.md +++ b/generation/README.md @@ -7,7 +7,7 @@ They are also used and tested by `merge_repository.sh`. ## Merge repository into the monorepo -Built by running [erge_repository.sh](merge_repository.sh). +Built by running [merge_repository.sh](merge_repository.sh). The script creates a new Git repository in `monorepo/google-cloud-java` by merging the repositories in the input. From 070427c65edcff5250421c295adaf846e772fe61 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Tue, 22 Nov 2022 15:25:59 -0500 Subject: [PATCH 5/7] start with google-cloud-java monorepo --- generation/merge_repository.sh | 3 +-- generation/repos.txt | 4 +--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/generation/merge_repository.sh b/generation/merge_repository.sh index 892c990bb9d3..53aec64b67b1 100755 --- a/generation/merge_repository.sh +++ b/generation/merge_repository.sh @@ -21,10 +21,9 @@ git clone https://github.com/newren/git-filter-repo.git export PATH=$PATH:`pwd`/git-filter-repo merged_repository=google-cloud-java-merged -mkdir "${merged_repository}" +git clone https://github.com/googleapis/google-cloud-java "${merged_repository}" cd "${merged_repository}" -git init -b main cat ../repos.txt | while read service do diff --git a/generation/repos.txt b/generation/repos.txt index a8d01cd26723..58c8f1201c44 100644 --- a/generation/repos.txt +++ b/generation/repos.txt @@ -1,3 +1 @@ -google-cloud-java -java-accesscontextmanager -java-aiplatform +java-logging \ No newline at end of file From c1bbde6b8ada6538d854023fc87b694ee0a8fa83 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Wed, 23 Nov 2022 11:51:14 -0500 Subject: [PATCH 6/7] chore: allow-emtpy when committing --- generation/merge_repository.sh | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/generation/merge_repository.sh b/generation/merge_repository.sh index 53aec64b67b1..f1bb06a9f93d 100755 --- a/generation/merge_repository.sh +++ b/generation/merge_repository.sh @@ -72,18 +72,20 @@ git commit -m 'chore: add template files' ../../generate_root_pom.sh git add pom.xml -git commit -am 'chore: create aggregator pom' +git commit -am 'chore: create aggregator pom' --allow-empty # Point modules poms and BOMs to the aggregator pom as parent bash ../../set_parent_pom.sh git add --all -git commit -am 'chore: point modules to the aggregator pom as parent' +git commit -am 'chore: point modules to the aggregator pom as parent' \ + --allow-empty ../../consolidate_config.sh git add --all -git commit -am 'chore: consolidate config to parent' +git commit -am 'chore: consolidate config to parent' \ + --allow-empty ../../generate_gapic_bom.sh @@ -91,26 +93,30 @@ git commit -am 'chore: consolidate config to parent' ../../generate_root_pom.sh git add gapic-libraries-bom/pom.xml -git commit -am 'chore: create gapic-libraries-bom' +git commit -am 'chore: create gapic-libraries-bom' \ + --allow-empty cp ../../gapic_bom_versions.txt gapic-libraries-bom/versions.txt ../../delete_non_generated_samples.sh git add --all -git commit -am 'chore: delete non-auto-generated samples' +git commit -am 'chore: delete non-auto-generated samples' \ + --allow-empty ../../generate_root_versions_txt.sh ../../update_versions.sh -s ../../apply_current_versions.sh git add --all -git commit -am 'chore: update versions to latest in maven' +git commit -am 'chore: update versions to latest in maven' \ + --allow-empty ../../update_owlbot_postprocessor_config.sh git add --all -git commit -am 'chore: remove and disable owlbot postprocessor templates' +git commit -am 'chore: remove and disable owlbot postprocessor templates' \ + --allow-empty for F in `find . -maxdepth 2 -name '.OwlBot.yaml'`; do sh ../../set_owlbot_config.sh $F; done git commit -am 'chore: set owlbot copy config' --allow-empty From cfcdd038f9e5bc8e1a1dc44f96e3472533b7dc63 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Thu, 1 Dec 2022 09:57:09 -0500 Subject: [PATCH 7/7] setting working directory google-cloud-java-merged --- .github/workflows/merge_repository.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/merge_repository.yaml b/.github/workflows/merge_repository.yaml index f68ed76774ca..b3b49330a876 100644 --- a/.github/workflows/merge_repository.yaml +++ b/.github/workflows/merge_repository.yaml @@ -23,15 +23,15 @@ jobs: env: USERNAME: ${{ github.actor }} - name: Compile - working-directory: generation/monorepo/google-cloud-java + working-directory: generation/monorepo/google-cloud-java-merged run: mvn compile -T C1 -B - name: Unit Tests - working-directory: generation/monorepo/google-cloud-java + working-directory: generation/monorepo/google-cloud-java-merged run: mvn test -T C1 -B - name: Push monorepo to branch if: ${{ github.event_name != 'pull_request' }} run: | - cd generation/monorepo/google-cloud-java + cd generation/monorepo/google-cloud-java-merged git remote add monorepo https://${{ github.actor }}:${{ github.token }}@github.com/${{ github.repository }}.git git push -f monorepo main:bootstrap_output - name: Push diff to branch