From bfbc43cfc786a0d87c7254fec2f686980dc8e7fd Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Wed, 22 Oct 2025 15:18:30 +0000 Subject: [PATCH 1/3] feat(internal/librariangen): initial implementation of configure This is completely lacking in tests, which should be provided before merging. The implementation will currently require a playbook along the lines of: 1. Run `librarian generate -api=google/cloud/abc/v1 -library=abc` 1. Run `(cd abc && go mod tidy)` 1. Run `go work use ./abc` 1. Commit changes and push Steps 2 and 3 are only needed for whole new libraries (as opposed to new APIs within existing libraries) It would be nice to be able to get rid of steps 2 and 3 entirely, but they're difficult. We can't run `go mod tidy` without source code, which either means running the generator during configure (which is ugly and somewhat against the spirit of configure) or do so in `generate`, but *only* on the very first run. We can't run `go work use` without copying *all* of the `go.mod` files referenced in there into the output directory (and then removing them again). We could modify `go.work` directly, but that's ugly too. These can be addressed in the longer term as we attempt to get to fully automated onboarding, but having the steps in the playbook introduces *relatively* little toil, while saving a *lot* of work in the container. --- .../librariangen/configure/_README.md.txt | 55 ++++ .../librariangen/configure/_version.go.txt | 23 ++ internal/librariangen/configure/configure.go | 251 ++++++++++++++++-- .../librariangen/configure/configure_test.go | 20 +- 4 files changed, 327 insertions(+), 22 deletions(-) create mode 100644 internal/librariangen/configure/_README.md.txt create mode 100644 internal/librariangen/configure/_version.go.txt diff --git a/internal/librariangen/configure/_README.md.txt b/internal/librariangen/configure/_README.md.txt new file mode 100644 index 000000000000..1b57b969d946 --- /dev/null +++ b/internal/librariangen/configure/_README.md.txt @@ -0,0 +1,55 @@ +# {{.Name}} + +[![Go Reference](https://pkg.go.dev/badge/{{.ModulePath}}.svg)](https://pkg.go.dev/{{.ModulePath}}) + +Go Client Library for {{.Name}}. + +## Install + +```bash +go get {{.ModulePath}} +``` + +## Stability + +The stability of this module is indicated by SemVer. + +However, a `v1+` module may have breaking changes in two scenarios: + +* Packages with `alpha` or `beta` in the import path +* The GoDoc has an explicit stability disclaimer (for example, for an experimental feature). + +### Which package to use? + +Generated client library surfaces can be found in packages who's import path +ends in `.../apivXXX`. The `XXX` could be something like `1` or `2` in the case +of a stable service backend or may be like `1beta2` or `2beta` in the case of a +more experimental service backend. Because of this fact, a given module can have +multiple clients for different service backends. In these cases it is generally +recommend to use clients with stable service backends, with import suffixes like +`apiv1`, unless you need to use features that are only present in a beta backend +or there is not yet a stable backend available. + +## Google Cloud Samples + +To browse ready to use code samples check [Google Cloud Samples](https://cloud.google.com/docs/samples?l=go). + +## Go Version Support + +See the [Go Versions Supported](https://github.com/googleapis/google-cloud-go#go-versions-supported) +section in the root directory's README. + +## Authorization + +See the [Authorization](https://github.com/googleapis/google-cloud-go#authorization) +section in the root directory's README. + +## Contributing + +Contributions are welcome. Please, see the [CONTRIBUTING](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md) +document for details. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Contributor Code of Conduct](https://github.com/GoogleCloudPlatform/google-cloud-go/blob/main/CONTRIBUTING.md#contributor-code-of-conduct) +for more information. diff --git a/internal/librariangen/configure/_version.go.txt b/internal/librariangen/configure/_version.go.txt new file mode 100644 index 000000000000..e1f2157d5581 --- /dev/null +++ b/internal/librariangen/configure/_version.go.txt @@ -0,0 +1,23 @@ +// Copyright {{.Year}} Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by gapicgen. DO NOT EDIT. + +package {{.Package}} + +import "{{.ModuleRootInternal}}" + +func init() { + versionClient = internal.Version +} diff --git a/internal/librariangen/configure/configure.go b/internal/librariangen/configure/configure.go index 149daef26857..f0033cd1361b 100644 --- a/internal/librariangen/configure/configure.go +++ b/internal/librariangen/configure/configure.go @@ -16,14 +16,30 @@ package configure import ( "context" + _ "embed" "encoding/json" "errors" "fmt" + "html/template" "log/slog" "os" "path/filepath" + "strings" + "time" + "cloud.google.com/go/internal/postprocessor/librarian/librariangen/config" + "cloud.google.com/go/internal/postprocessor/librarian/librariangen/execv" + "cloud.google.com/go/internal/postprocessor/librarian/librariangen/module" "cloud.google.com/go/internal/postprocessor/librarian/librariangen/request" + "gopkg.in/yaml.v3" +) + +// External string template vars. +var ( + //go:embed _README.md.txt + readmeTmpl string + //go:embed _version.go.txt + versionTmpl string ) // NewAPIStatus is the API.Status value used to represent "this is a new API being configured". @@ -31,6 +47,7 @@ const NewAPIStatus = "new" // Test substitution vars. var ( + execvRun = execv.Run requestParse = Parse responseSave = saveResponse ) @@ -79,17 +96,17 @@ func Configure(ctx context.Context, cfg *Config) error { if err := cfg.Validate(); err != nil { return fmt.Errorf("librariangen: invalid configuration: %w", err) } - slog.Debug("librariangen: configure command started") + slog.Info("librariangen: configure command started") configureReq, err := readConfigureReq(cfg.LibrarianDir) if err != nil { return fmt.Errorf("librariangen: failed to read request: %w", err) } - library, err := findLibraryToConfigure(configureReq) + library, api, err := findLibraryAndAPIToConfigure(configureReq) if err != nil { return err } - response, err := configureLibrary(cfg, library) + response, err := configureLibrary(ctx, cfg, library, api) if err != nil { return err } @@ -105,7 +122,7 @@ func Configure(ctx context.Context, cfg *Config) error { // It is prepared by the Librarian tool and mounted at /librarian. func readConfigureReq(librarianDir string) (*Request, error) { reqPath := filepath.Join(librarianDir, "configure-request.json") - slog.Debug("librariangen: reading generate request", "path", reqPath) + slog.Debug("librariangen: reading configure request", "path", reqPath) configureReq, err := requestParse(reqPath) if err != nil { @@ -128,35 +145,239 @@ func saveConfigureResp(resp *request.Library, librarianDir string) error { return nil } -func findLibraryToConfigure(req *Request) (*request.Library, error) { +func findLibraryAndAPIToConfigure(req *Request) (*request.Library, *request.API, error) { var library *request.Library + var api *request.API for _, candidate := range req.Libraries { - var hasNewAPI bool + var newAPI *request.API for _, api := range candidate.APIs { if api.Status == NewAPIStatus { - if hasNewAPI { - return nil, fmt.Errorf("librariangen: library %s has multiple new APIs", candidate.ID) + if newAPI != nil { + return nil, nil, fmt.Errorf("librariangen: library %s has multiple new APIs", candidate.ID) } - hasNewAPI = true + newAPI = &api } } - if hasNewAPI { + + if newAPI != nil { if library != nil { - return nil, fmt.Errorf("librariangen: multiple libraries have new APIs (at least %s and %s)", library.ID, candidate.ID) + return nil, nil, fmt.Errorf("librariangen: multiple libraries have new APIs (at least %s and %s)", library.ID, candidate.ID) } library = candidate + api = newAPI } } if library == nil { - return nil, fmt.Errorf("librariangen: no libraries have new APIs") + return nil, nil, fmt.Errorf("librariangen: no libraries have new APIs") } - return library, nil + return library, api, nil } // configureLibrary performs the real work of configuring a new or updated module, // creating files and populating the state file entry. -func configureLibrary(cfg *Config, library *request.Library) (*request.Library, error) { - return nil, errors.New("configure unimplemented") +// In theory we could just have a return type of "error", but logically this is +// returning the configure-response... it just happens to be "the library being configured" +// at the moment. If the format of configure-response ever changes, we'll need fewer +// changes if we don't make too many assumptions now. +func configureLibrary(ctx context.Context, cfg *Config, library *request.Library, api *request.API) (*request.Library, error) { + // It's just *possible* the new path has a manually configured + // client directory - but even if not, RepoConfig has the logic + // for figuring out the client directory. Even if the new path + // doesn't have a custom configuration, we can use this to + // work out the module path, e.g. if there's a major version other + // than v1. + repoConfig, err := config.LoadRepoConfig(cfg.LibrarianDir) + if err != nil { + return nil, err + } + var moduleConfig = repoConfig.GetModuleConfig(library.ID) + + moduleRoot := filepath.Join(cfg.OutputDir, library.ID) + if err := os.Mkdir(moduleRoot, 0755); err != nil { + return nil, err + } + // Only a single API path can be added on each configure call, so we can tell + // if this is a new library if it's got exactly one API path. + // In that case, we need to add: + // - CHANGES.md (static text: "# Changes") + // - README.md + // - internal/version.go + // - go.mod + if len(library.APIs) == 1 { + library.SourcePaths = []string{library.ID, "internal/generated/", "internal/generated/snippets/" + library.ID} + library.RemoveRegex = []string{"^internal/generated/snippets/" + library.ID + "/"} + library.TagFormat = "{id}/v{version}" + library.Version = "0.0.0" + if err := generateReadme(cfg, library); err != nil { + return nil, err + } + if err := generateChanges(cfg, library); err != nil { + return nil, err + } + if err := module.GenerateInternalVersionFile(moduleRoot, library.Version); err != nil { + return nil, err + } + if err := goModInit(ctx, moduleConfig.GetModulePath(), moduleRoot); err != nil { + return nil, err + } + if err := goModEditReplaceInSnippets(ctx, cfg, moduleConfig.GetModulePath(), "../../../"+library.ID); err != nil { + return nil, err + } + } + + // Whether it's a new library or not, generate a version file for the new client directory. + if err := generateClientVersionFile(cfg, moduleConfig, api.Path); err != nil { + return nil, err + } + + // Make changes in the Library object, to communicate state file changes back to + // Librarian. + if err := updateLibraryState(moduleConfig, library, api); err != nil { + return nil, err + } + + return library, nil +} + +// generateReadme generates a README.md file in the module's root directory, +// using the service config for the first API in the library to obtain the +// service's title. +func generateReadme(cfg *Config, library *request.Library) error { + readmePath := filepath.Join(cfg.OutputDir, library.ID, "README.md") + serviceYAMLPath := filepath.Join(cfg.SourceDir, library.APIs[0].Path, library.APIs[0].ServiceConfig) + title, err := readTitleFromServiceYAML(serviceYAMLPath) + if err != nil { + return fmt.Errorf("librariangen: failed to read title from service yaml: %w", err) + } + + slog.Info("librariangen: creating file", "path", readmePath) + readmeFile, err := os.Create(readmePath) + if err != nil { + return err + } + defer readmeFile.Close() + t := template.Must(template.New("readme").Parse(readmeTmpl)) + readmeData := struct { + Name string + ModulePath string + }{ + Name: title, + ModulePath: "cloud.google.com/go/" + library.ID, + } + return t.Execute(readmeFile, readmeData) +} + +// generateChanges generates a CHANGES.md file at the root of the module. +func generateChanges(cfg *Config, library *request.Library) error { + changesPath := filepath.Join(cfg.OutputDir, library.ID, "CHANGES.md") + slog.Info("librariangen: creating file", "path", changesPath) + content := "# Changes\n" + return os.WriteFile(changesPath, []byte(content), 0644) +} + +// generateClientVersionFile creates a version.go file for a client. +func generateClientVersionFile(cfg *Config, moduleConfig *config.ModuleConfig, apiPath string) error { + var apiConfig = moduleConfig.GetAPIConfig(apiPath) + clientDir, err := apiConfig.GetClientDirectory() + if err != nil { + return err + } + + fullClientDir := filepath.Join(cfg.OutputDir, moduleConfig.Name, clientDir) + if err := os.MkdirAll(fullClientDir, 0755); err != nil { + return err + } + versionPath := filepath.Join(fullClientDir, "version.go") + slog.Info("librariangen: creating file", "path", versionPath) + t := template.Must(template.New("version").Parse(versionTmpl)) + versionData := struct { + Year int + Package string + ModuleRootInternal string + }{ + Year: time.Now().Year(), + Package: moduleConfig.Name, + ModuleRootInternal: moduleConfig.GetModulePath() + "/internal", + } + f, err := os.Create(versionPath) + if err != nil { + return err + } + defer f.Close() + return t.Execute(f, versionData) +} + +// goModInit initializes a go.mod file in the given directory. +func goModInit(ctx context.Context, modulePath, moduleDir string) error { + slog.Info("librariangen: running go mod init", "directory", moduleDir, "modulePath", modulePath) + args := []string{"go", "mod", "init", modulePath} + return execvRun(ctx, args, moduleDir) +} + +func goModEditReplaceInSnippets(ctx context.Context, cfg *Config, modulePath, relativeDir string) error { + outputSnippetsDir := filepath.Join(cfg.OutputDir, "internal", "generated", "snippets") + if err := os.MkdirAll(outputSnippetsDir, 0755); err != nil { + return err + } + copyRepoFileToOutput(cfg, "internal/generated/snippets/go.mod") + args := []string{"go", "mod", "edit", "-replace", fmt.Sprintf("%s=%s", modulePath, relativeDir)} + return execvRun(ctx, args, outputSnippetsDir) +} + +func copyRepoFileToOutput(cfg *Config, path string) error { + src := filepath.Join(cfg.RepoDir, path) + dst := filepath.Join(cfg.OutputDir, path) + data, err := os.ReadFile(src) + if err != nil { + return err + } + return os.WriteFile(dst, data, 0644) +} + +// updateLibraryState updates the library to add any required removal/preservation +// regexes for the specified API. +func updateLibraryState(moduleConfig *config.ModuleConfig, library *request.Library, api *request.API) error { + apiConfig := moduleConfig.GetAPIConfig(api.Path) + clientDirectory, err := apiConfig.GetClientDirectory() + if err != nil { + return err + } + apiParts := strings.Split(api.Path, "/") + protobufDir := apiParts[len(apiParts)-2] + "pb/.*" + generatedPaths := []string{ + "[^/]*_client\\.go", + "[^/]*_client_example_go123_test\\.go", + "[^/]*_client_example_test\\.go", + "auxiliary\\.go", + "auxiliary_go123\\.go", + "doc\\.go", + "gapic_metadata\\.json", + "helpers\\.go", + protobufDir, + } + for _, generatedPath := range generatedPaths { + library.RemoveRegex = append(library.RemoveRegex, "^"+clientDirectory+"/"+generatedPath+"$") + } + return nil +} + +// readTitleFromServiceYAML reads the service YAML file and returns the title. +func readTitleFromServiceYAML(path string) (string, error) { + slog.Info("librariangen: reading service yaml", "path", path) + data, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("librariangen: failed to read service yaml file: %w", err) + } + var serviceConfig struct { + Title string `yaml:"title"` + } + if err := yaml.Unmarshal(data, &serviceConfig); err != nil { + return "", fmt.Errorf("librariangen: failed to unmarshal service yaml: %w", err) + } + if serviceConfig.Title == "" { + return "", errors.New("librariangen: title not found in service yaml") + } + return serviceConfig.Title, nil } // Request corresponds to a librarian configure request. diff --git a/internal/librariangen/configure/configure_test.go b/internal/librariangen/configure/configure_test.go index 2577b4b9a7e1..d7aea4109622 100644 --- a/internal/librariangen/configure/configure_test.go +++ b/internal/librariangen/configure/configure_test.go @@ -99,10 +99,11 @@ func TestConfig_Validate(t *testing.T) { func TestFindLibraryToConfigure(t *testing.T) { tests := []struct { - name string - req *Request - wantID string - wantErr bool + name string + req *Request + wantID string + wantPath string + wantErr bool }{ { name: "valid new library", @@ -135,7 +136,8 @@ func TestFindLibraryToConfigure(t *testing.T) { }, }, }, - wantID: "new", + wantID: "new", + wantPath: "a/b/c", }, { name: "valid updated library", @@ -174,7 +176,8 @@ func TestFindLibraryToConfigure(t *testing.T) { }, }, }, - wantID: "updated", + wantID: "updated", + wantPath: "e/f/g", }, { name: "invalid no new APIs", @@ -250,7 +253,7 @@ func TestFindLibraryToConfigure(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - lib, err := findLibraryToConfigure(tt.req) + lib, api, err := findLibraryAndAPIToConfigure(tt.req) if (err != nil) != tt.wantErr { t.Fatalf("findLibraryToConfigure error = %v, wantErr %v", err, tt.wantErr) } @@ -259,6 +262,9 @@ func TestFindLibraryToConfigure(t *testing.T) { if tt.wantID != "" && lib.ID != tt.wantID { t.Errorf("mismatched ID, got=%s, want=%s", lib.ID, tt.wantID) } + if tt.wantPath != "" && api.Path != tt.wantPath { + t.Errorf("mismatched API path, got=%s, want=%s", api.Path, tt.wantPath) + } }) } } From d04f9023644d75cb334f753b587d287cb33df27b Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Thu, 23 Oct 2025 09:03:56 +0000 Subject: [PATCH 2/3] chore: changes spaces to a tab in version.go template --- internal/librariangen/configure/_version.go.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/librariangen/configure/_version.go.txt b/internal/librariangen/configure/_version.go.txt index e1f2157d5581..ee90549ed6af 100644 --- a/internal/librariangen/configure/_version.go.txt +++ b/internal/librariangen/configure/_version.go.txt @@ -19,5 +19,5 @@ package {{.Package}} import "{{.ModuleRootInternal}}" func init() { - versionClient = internal.Version + versionClient = internal.Version } From 5a960b3ecb435f83a8bb59113c43f899c1a0c79a Mon Sep 17 00:00:00 2001 From: Jon Skeet Date: Thu, 23 Oct 2025 09:14:14 +0000 Subject: [PATCH 3/3] chore: more comments and a test rename --- internal/librariangen/configure/configure.go | 9 +++++++++ internal/librariangen/configure/configure_test.go | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/librariangen/configure/configure.go b/internal/librariangen/configure/configure.go index f0033cd1361b..6213577bb383 100644 --- a/internal/librariangen/configure/configure.go +++ b/internal/librariangen/configure/configure.go @@ -145,6 +145,9 @@ func saveConfigureResp(resp *request.Library, librarianDir string) error { return nil } +// findLibraryAndAPIToConfigure examines a request, and finds a single library +// containing a single new API, returning both of them. An error is returned +// if there is not exactly one library containing exactly one new API. func findLibraryAndAPIToConfigure(req *Request) (*request.Library, *request.API, error) { var library *request.Library var api *request.API @@ -314,6 +317,10 @@ func goModInit(ctx context.Context, modulePath, moduleDir string) error { return execvRun(ctx, args, moduleDir) } +// goModEditReplaceInSnippets copies internal/generated/snippets/go.mod from +// cfg.RepoDir to cfg.OutputDir, then runs go mod edit to replace the specified +// modulePath with relativeDir which is expected to the location of the module +// relative to internal/generated/snippets func goModEditReplaceInSnippets(ctx context.Context, cfg *Config, modulePath, relativeDir string) error { outputSnippetsDir := filepath.Join(cfg.OutputDir, "internal", "generated", "snippets") if err := os.MkdirAll(outputSnippetsDir, 0755); err != nil { @@ -324,6 +331,8 @@ func goModEditReplaceInSnippets(ctx context.Context, cfg *Config, modulePath, re return execvRun(ctx, args, outputSnippetsDir) } +// copyRepoFileToOutput copies a single file (identified via path) +// from cfg.RepoDir to cfg.OutputDir. func copyRepoFileToOutput(cfg *Config, path string) error { src := filepath.Join(cfg.RepoDir, path) dst := filepath.Join(cfg.OutputDir, path) diff --git a/internal/librariangen/configure/configure_test.go b/internal/librariangen/configure/configure_test.go index d7aea4109622..c4baa309a877 100644 --- a/internal/librariangen/configure/configure_test.go +++ b/internal/librariangen/configure/configure_test.go @@ -97,7 +97,7 @@ func TestConfig_Validate(t *testing.T) { } } -func TestFindLibraryToConfigure(t *testing.T) { +func TestFindLibraryAndAPIToConfigure(t *testing.T) { tests := []struct { name string req *Request