Skip to content

Commit

Permalink
Merge branch 'GoogleCloudPlatform:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
hao-nan-li authored Oct 8, 2024
2 parents b6ae3c4 + b8be569 commit 6e8a9a0
Show file tree
Hide file tree
Showing 6,908 changed files with 94,152 additions and 487,369 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
2 changes: 1 addition & 1 deletion .ci/containers/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,6 @@ Before you begin, set up Docker (including configuring it to [authenticate with
```
4. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
5. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
- Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.erb` will trigger builds for TPG, TPGB, and TGC.
- Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.tmpl` will trigger builds for TPG, TPGB, and TGC.
6. Create a PR with these changes.
7. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)
2 changes: 1 addition & 1 deletion .ci/gcb-pr-downstream-generation-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ steps:
- $COMMIT_SHA
- $BUILD_ID
- $PROJECT_ID
- "22" # Build step
- "23" # Build step

- name: 'gcr.io/graphite-docker-images/go-plus'
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
Expand Down
13 changes: 10 additions & 3 deletions .ci/gcb-vcr-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@
steps:
- name: 'gcr.io/graphite-docker-images/go-plus'
id: gcb-vcr-nightly
entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GITHUB_TOKEN_CLASSIC"]
env:
- "GOOGLE_REGION=us-central1"
- "GOOGLE_ZONE=us-central1-a"
- "USER=magician"
args:
- $BUILD_ID
- 'vcr-cassette-update'
- $BUILD_ID

# Long timeout to enable waiting on VCR test
timeout: 20000s
Expand Down Expand Up @@ -39,3 +44,5 @@ availableSecrets:
env: SA_KEY
- versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
- versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
env: GITHUB_TOKEN_CLASSIC
35 changes: 33 additions & 2 deletions .ci/infra/terraform/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -376,8 +376,6 @@ resource "google_project_service_identity" "sqladmin_sa" {
service = "sqladmin.googleapis.com"
}

# TODO: Replace these permissions with bootstrapped permissions

# TestAccComposerEnvironment_fixPyPiPackages
# TestAccComposerEnvironmentComposer2_private
# TestAccComposerEnvironment_withEncryptionConfigComposer1
Expand Down Expand Up @@ -409,6 +407,24 @@ resource "google_project_iam_member" "compute_agent_secret_accessor" {
member = "serviceAccount:${google_project.proj.number}[email protected]"
}

# TestAccHealthcarePipelineJob_healthcarePipelineJobMappingReconDestExample
# TestAccHealthcarePipelineJob_healthcarePipelineJobReconciliationExample
# TestAccHealthcarePipelineJob_healthcarePipelineJobWhistleMappingExample
resource "google_project_iam_member" "healthcare_agent_storage_object_admin" {
project = google_project.proj.project_id
role = "roles/storage.objectAdmin"
member = "serviceAccount:service-${google_project.proj.number}@gcp-sa-healthcare.iam.gserviceaccount.com"
}

# TestAccHealthcarePipelineJob_healthcarePipelineJobMappingReconDestExample
# TestAccHealthcarePipelineJob_healthcarePipelineJobReconciliationExample
# TestAccHealthcarePipelineJob_healthcarePipelineJobWhistleMappingExample
resource "google_project_iam_member" "healthcare_agent_fhir_resource_editor" {
project = google_project.proj.project_id
role = "roles/healthcare.fhirResourceEditor"
member = "serviceAccount:service-${google_project.proj.number}@gcp-sa-healthcare.iam.gserviceaccount.com"
}

# TestAccVertexAIEndpoint_vertexAiEndpointNetwork
# TestAccVertexAIFeaturestoreEntitytype_vertexAiFeaturestoreEntitytypeExample
# TestAccVertexAIFeaturestoreEntitytype_vertexAiFeaturestoreEntitytypeWithBetaFieldsExample
Expand All @@ -423,6 +439,21 @@ resource "google_project_iam_member" "aiplatform_agent_encrypter_decrypter" {
member = "serviceAccount:service-${google_project.proj.number}@gcp-sa-aiplatform.iam.gserviceaccount.com"
}

# TestAccComputeInstance_confidentialHyperDiskBootDisk
resource "google_project_iam_member" "compute_default_sa_encrypter_decrypter" {
project = google_project.proj.project_id
role = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
member = "serviceAccount:${google_project.proj.number}[email protected]"
}

# TestAccComputeInstance_confidentialHyperDiskBootDisk
resource "google_project_iam_member" "compute_agent_encrypter_decrypter" {
project = google_project.proj.project_id
role = "roles/cloudkms.cryptoKeyEncrypterDecrypter"
member = "serviceAccount:service-${google_project.proj.number}@compute-system.iam.gserviceaccount.com"
}


data "google_organization" "org2" {
organization = var.org2_id
}
Expand Down
14 changes: 0 additions & 14 deletions .ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl

This file was deleted.

7 changes: 5 additions & 2 deletions .ci/magician/cmd/check_cassettes.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ var checkCassettesCmd = &cobra.Command{

ctlr := source.NewController(env["GOPATH"], "modular-magician", githubToken, rnr)

vt, err := vcr.NewTester(env, rnr)
vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "vcr-check-cassettes", rnr)
if err != nil {
return fmt.Errorf("error creating VCR tester: %w", err)
}
Expand Down Expand Up @@ -110,7 +110,10 @@ func execCheckCassettes(commit string, vt *vcr.Tester, ctlr *source.Controller)
if err != nil {
fmt.Println("Error running VCR: ", err)
}
if err := vt.UploadLogs("vcr-check-cassettes", "", "", false, false, vcr.Replaying, provider.Beta); err != nil {
if err := vt.UploadLogs(vcr.UploadLogsOptions{
Mode: vcr.Replaying,
Version: provider.Beta,
}); err != nil {
return fmt.Errorf("error uploading logs: %w", err)
}
fmt.Println(len(result.FailedTests), " failed tests: ", result.FailedTests)
Expand Down
6 changes: 3 additions & 3 deletions .ci/magician/cmd/generate_comment.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ import (
)

var (
//go:embed DIFF_COMMENT.md
//go:embed templates/DIFF_COMMENT.md.tmpl
diffComment string
)

Expand Down Expand Up @@ -504,9 +504,9 @@ func detectMissingTests(diffProcessorPath, tpgbLocalPath string, rnr ExecRunner)
}

func formatDiffComment(data diffCommentData) (string, error) {
tmpl, err := template.New("DIFF_COMMENT.md").Parse(diffComment)
tmpl, err := template.New("DIFF_COMMENT.md.tmpl").Parse(diffComment)
if err != nil {
panic(fmt.Sprintf("Unable to parse DIFF_COMMENT.md: %s", err))
return "", fmt.Errorf("unable to parse template DIFF_COMMENT.md.tmpl: %s", err)
}
sb := new(strings.Builder)
err = tmpl.Execute(sb, data)
Expand Down
1 change: 1 addition & 0 deletions .ci/magician/cmd/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ type ExecRunner interface {
PushDir(path string) error
PopDir() error
WriteFile(name, data string) error
AppendFile(name, data string) error // Not used (yet).
Run(name string, args []string, env map[string]string) (string, error)
MustRun(name string, args []string, env map[string]string) string
}
4 changes: 4 additions & 0 deletions .ci/magician/cmd/mock_runner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,10 @@ func (mr *mockRunner) WriteFile(name, data string) error {
return nil
}

func (mr *mockRunner) AppendFile(name, data string) error {
return nil
}

func (mr *mockRunner) Copy(src, dest string) error {
mr.calledMethods["Copy"] = append(mr.calledMethods["Copy"], ParameterList{src, dest})
return nil
Expand Down
6 changes: 3 additions & 3 deletions .ci/magician/cmd/scheduled_pr_reminders.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,13 @@ var (
// used for flags
dryRun bool

//go:embed SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl
//go:embed templates/SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl
waitingForContributorTemplate string

//go:embed SCHEDULED_PR_WAITING_FOR_MERGE.md.tmpl
//go:embed templates/SCHEDULED_PR_WAITING_FOR_MERGE.md.tmpl
waitingForMergeTemplate string

//go:embed SCHEDULED_PR_WAITING_FOR_REVIEW.md.tmpl
//go:embed templates/SCHEDULED_PR_WAITING_FOR_REVIEW.md.tmpl
waitingForReviewTemplate string
)

Expand Down
23 changes: 23 additions & 0 deletions .ci/magician/cmd/templates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package cmd

import (
"fmt"
)

func color(color, text string) string {
if color == "" || text == "" {
return text
}
var emoji string
switch color {
case "red":
emoji = "🔴"
case "yellow":
emoji = "🟡"
case "green":
emoji = "🟢"
default:
return text
}
return fmt.Sprintf("%s %s", emoji, text)
}
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{{ if lt .SinceDays 30 -}}
@{{.User}}, this PR is waiting for action from you. If no action is taken, this PR will be closed in
{{- if eq .SinceDays 10 }} 28 days.{{end}}
{{- if eq .SinceDays 20 }} 14 days.{{end}}
{{- if eq .SinceDays 28 }} 2 weekdays.{{end}}

Please address any comments or change requests, or [re-request review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review) from a core reviewer if no action is required.

![Image showing the re-request review button](https://docs.github.com/assets/cb-28785/mw-1440/images/help/pull_requests/request-re-review.webp)

This notification can be disabled with the `disable-automatic-closure` label.
{{ else -}}
@{{.User}}, this PR is being closed due to inactivity.
{{ end -}}
31 changes: 31 additions & 0 deletions .ci/magician/cmd/templates/vcr/record_replay.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
{{- if gt (len .RecordingResult.PassedTests) 0 -}}
{{color "green" "Tests passed during RECORDING mode:"}}
{{range .RecordingResult.PassedTests}}`{{.}}`[[Debug log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
{{end}}

{{- if gt (len .ReplayingAfterRecordingResult.FailedTests ) 0 -}}
{{color "red" "Tests failed when rerunning REPLAYING mode:"}}
{{range .ReplayingAfterRecordingResult.FailedTests}}`{{.}}`[[Error message](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/build-log/replaying_build_after_recording/{{.}}_replaying_test.log)] [[Debug log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/replaying_after_recording/{{.}}.log)]
{{end}}

Tests failed due to non-determinism or randomness when the VCR replayed the response after the HTTP request was made.

Please fix these to complete your PR. If you believe these test failures to be incorrect or unrelated to your change, or if you have any questions, please raise the concern with your reviewer.

{{else}}
{{color "green" "No issues found for passed tests after REPLAYING rerun."}}
{{end}}{{/* end of if gt (len .ReplayingAfterRecordingResult.FailedTests ) 0 */}}
---
{{end}}{{/* end of if gt (len .RecordingResult.PassedTests) 0 */}}

{{if gt (len .RecordingResult.FailedTests) 0 -}}
{{color "red" "Tests failed during RECORDING mode:"}}
{{range .RecordingResult.FailedTests}}`{{.}}`[[Error message](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/build-log/recording_build/{{.}}_recording_test.log)] [[Debug log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{$.PRNumber}}/artifacts/{{$.BuildID}}/recording/{{.}}.log)]
{{end}}
{{end}} {{- /* end of if gt (len .RecordingResult.FailedTests) 0 */ -}}

{{if .HasTerminatedTests}}{{color "red" "Several tests got terminated during RECORDING mode."}}{{end}}
{{if .RecordingErr}}{{color "red" "Errors occurred during RECORDING mode. Please fix them to complete your PR."}}{{end}}
{{if .AllRecordingPassed}}{{color "green" "All tests passed!"}}{{end}}

View the [build log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{.PRNumber}}/artifacts/{{.BuildID}}/build-log/recording_test.log) or the [debug log](https://console.cloud.google.com/storage/browser/ci-vcr-logs/beta/refs/heads/auto-pr-{{.PRNumber}}/artifacts/{{.BuildID}}/recording) for each test
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ The provider crashed while running the VCR tests in RECORDING mode
{{- else -}}
#################################
RECORDING Tests Report
#################################

{{if .RecordingResult.PassedTests}}
Tests passed during RECORDING mode:
Expand All @@ -18,7 +19,7 @@ Tests failed during RECORDING mode:

{{if .HasTerminatedTests}}Several tests got terminated during RECORDING mode{{end}}

{{if .RecordingErr}}Errors occurred during RECORDING mode.{{end}}
{{if .RecordingErr}}Errors occurred during RECORDING mode: {{.RecordingErr}}.{{end}}

{{if .AllRecordingPassed}}All tests passed!{{end}}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ The provider crashed while running the VCR tests in REPLAYING mode
{{- else}}
#################################
Tests Analytics
#################################

Total tests: {{add (add (len .ReplayingResult.PassedTests) (len .ReplayingResult.SkippedTests)) (len .ReplayingResult.FailedTests) }}
Passed tests: {{len .ReplayingResult.PassedTests}}
Skipped tests: {{len .ReplayingResult.SkippedTests}}
Expand All @@ -15,7 +17,7 @@ Affected tests list:
{{- end}}
{{if .ReplayingErr}}
#################################
Errors occurred during REPLAYING mode.
Errors occurred during REPLAYING mode: {{.ReplayingErr}}.
#################################
{{- end}}
{{if .AllReplayingPassed}}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{{- if .ReplayingErr -}}
{{color "red" "Errors occurred during REPLAYING mode. Please fix them to complete your PR."}}
{{- else -}}
{{color "green" "All tests passed!"}}
{{- end}}

View the [build log](https://storage.cloud.google.com/ci-vcr-logs/beta/refs/heads/auto-pr-{{.PRNumber}}/artifacts/{{.BuildID}}/build-log/replaying_test.log)
58 changes: 58 additions & 0 deletions .ci/magician/cmd/templates_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package cmd

import (
"testing"
)

func TestColor(t *testing.T) {
cases := []struct {
name string
color string
text string
want string
}{
{
name: "red",
color: "red",
text: "Test text",
want: "🔴 Test text",
},
{
name: "yellow",
color: "yellow",
text: "Test text",
want: "🟡 Test text",
},
{
name: "green",
color: "green",
text: "Test text",
want: "🟢 Test text",
},
{
name: "unsupported color",
color: "mauve",
text: "Test text",
want: "Test text",
},
{
name: "empty color",
text: "Test text",
want: "Test text",
},
{
name: "empty text",
color: "green",
want: "",
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := color(tc.color, tc.text)
if got != tc.want {
t.Errorf("color(%s, %s) got %s; want %s", tc.color, tc.text, got, tc.want)
}
})
}
}
Loading

0 comments on commit 6e8a9a0

Please sign in to comment.