Skip to content

feat(internal/librarian/java): integrate native Go postprocessor into Java generator #6768

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yangyzs merged 3 commits into from
Jul 12, 2026
Merged

feat(internal/librarian/java): integrate native Go postprocessor into Java generator #6768

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bafc968
feat(internal/librarian/java): integrate native Go postprocessor into…
yangyzs Jul 9, 2026
83d4061
fix(internal/librarian/java): force legacy staging path in generateAP…
yangyzs Jul 11, 2026
80a1c47
refactor(internal/librarian/java): unexport toKeepSet and restructure…
yangyzs Jul 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions internal/librarian/java/generate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,10 @@ func TestGenerateAPI(t *testing.T) {
testhelper.RequireCommand(t, "protoc-gen-java_gapic")
testhelper.RequireCommand(t, "protoc-gen-java_grpc")
outdir := t.TempDir()
// Force routing to the legacy owlbot.py postprocessor.
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
t.Fatal(err)
}
cfg := &config.Config{
Repo: "googleapis/google-cloud-java",
Default: &config.Default{
Expand Down Expand Up @@ -337,6 +341,10 @@ func TestGenerateAPI_ProtoOnly(t *testing.T) {
testhelper.RequireCommand(t, "protoc")
testhelper.RequireCommand(t, "protoc-gen-java_grpc")
outdir := t.TempDir()
// Force routing to the legacy owlbot.py postprocessor.
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
t.Fatal(err)
}
cfg := &config.Config{
Repo: "googleapis/google-cloud-java",
Default: &config.Default{
Expand Down Expand Up @@ -474,6 +482,10 @@ func TestGenerateAPI_WithAdditionalProtosToGenerateAndCopy(t *testing.T) {
testhelper.RequireCommand(t, "protoc-gen-java_gapic")
testhelper.RequireCommand(t, "protoc-gen-java_grpc")
outdir := t.TempDir()
// Force routing to the legacy owlbot.py postprocessor.
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
t.Fatal(err)
}
cfg := &config.Config{
Repo: "googleapis/google-cloud-java",
Default: &config.Default{
Expand Down Expand Up @@ -1044,6 +1056,10 @@ func TestGenerateAPI_Gating(t *testing.T) {
} {
t.Run(test.name, func(t *testing.T) {
outdir := t.TempDir()
// Force routing to the legacy owlbot.py postprocessor.
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("#!/usr/bin/env python3\npass"), 0755); err != nil {
t.Fatal(err)
}
api := &config.API{Path: "google/cloud/secretmanager/v1"}
cfg := &config.Config{
Repo: "googleapis/google-cloud-java",
Expand Down
114 changes: 84 additions & 30 deletions internal/librarian/java/postprocess.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/googleapis/librarian/internal/config"
"github.com/googleapis/librarian/internal/filesystem"
"github.com/googleapis/librarian/internal/license"
"github.com/googleapis/librarian/internal/postprocessing"
"github.com/googleapis/librarian/internal/serviceconfig"
)

Expand Down Expand Up @@ -70,18 +71,40 @@ type libraryPostProcessParams struct {
// TODO(https://github.com/googleapis/librarian/issues/6627): Remove legacy owlbot.py
// postprocessing execution once native Go postprocessing is enabled.
func postProcessLibrary(ctx context.Context, params libraryPostProcessParams) error {
if err := createOrVerifyOwlbotPy(params.outDir); err != nil {
return err
}
bomVersion, err := findBOMVersion(params.cfg)
if err != nil {
return err
}
if err := removeKeptFilesFromStaging(params.library, params.outDir); err != nil {
return fmt.Errorf("failed to remove kept files from staging: %w", err)
owlbotPath := filepath.Join(params.outDir, "owlbot.py")
_, err := os.Stat(owlbotPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("failed to check for owlbot.py: %w", err)
}
if err := runOwlBot(ctx, params.library, params.outDir, bomVersion); err != nil {
return fmt.Errorf("%w: %w", errRunOwlBot, err)
owlbotExists := err == nil
Comment thread
yangyzs marked this conversation as resolved.

if owlbotExists {
if err := createOrVerifyOwlbotPy(params.outDir); err != nil {
return err
}
bomVersion, err := findBOMVersion(params.cfg)
if err != nil {
return err
}
if err := removeKeptFilesFromStaging(params.library, params.outDir); err != nil {
return fmt.Errorf("failed to remove kept files from staging: %w", err)
}
if err := runOwlBot(ctx, params.library, params.outDir, bomVersion); err != nil {
return fmt.Errorf("%w: %w", errRunOwlBot, err)
}
} else {
if params.library != nil && params.library.Postprocess != nil {
if err := postprocessing.Apply(params.outDir, params.library.Postprocess); err != nil {
return err
}
}
var keepSet map[string]bool
if params.library != nil {
keepSet = toKeepSet(params.library.Keep)
}
if err := renderREADME(params, keepSet); err != nil {
return fmt.Errorf("failed to render README: %w", err)
}
}

monorepoVersion, err := findMonorepoVersion(params.cfg)
Expand Down Expand Up @@ -140,23 +163,54 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error {
if err := copyFiles(params); err != nil {
return fmt.Errorf("failed to copy files: %w", err)
}
if err := restructureToStaging(params); err != nil {
return fmt.Errorf("failed to restructure to staging: %w", err)
}

// Generate clirr-ignored-differences.xml for the proto module.
// We target the staging directory because runOwlBot hasn't moved the files
// to their final destination yet.
coords := params.coords()
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
if err != nil {
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
owlbotPath := filepath.Join(params.outDir, "owlbot.py")
_, err := os.Stat(owlbotPath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("failed to check for owlbot.py: %w", err)
}
if shouldGenerate {
protoModuleStagingRoot := filepath.Join(stagingDir(params.outDir), params.apiBase, coords.Proto.ArtifactID)
if err := generateClirrIgnore(protoModuleStagingRoot); err != nil {
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
owlbotExists := err == nil
Comment thread
yangyzs marked this conversation as resolved.

if !owlbotExists {
var keepSet map[string]bool
if params.library != nil {
keepSet = toKeepSet(params.library.Keep)
}
if err := restructureToLibrary(params, params.outDir, keepSet); err != nil {
return fmt.Errorf("failed to restructure to library root: %w", err)
}

coords := params.coords()
// Generate clirr-ignored-differences.xml for the proto module.
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
if err != nil {
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
}
if shouldGenerate {
if err := generateClirrIgnore(protoModuleRepoRoot); err != nil {
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
}
}
} else {
if err := restructureToStaging(params); err != nil {
return fmt.Errorf("failed to restructure to staging: %w", err)
}

// Generate clirr-ignored-differences.xml for the proto module.
// We target the staging directory because runOwlBot hasn't moved the files
// to their final destination yet.
coords := params.coords()
protoModuleRepoRoot := filepath.Join(params.outDir, coords.Proto.ArtifactID)
shouldGenerate, err := clirrIgnoreShouldGenerate(coords.Proto.ArtifactID, protoModuleRepoRoot, params.javaAPI.Monolithic)
if err != nil {
return fmt.Errorf("failed to check for clirr ignore file: %w", err)
}
if shouldGenerate {
protoModuleStagingRoot := filepath.Join(stagingDir(params.outDir), params.apiBase, coords.Proto.ArtifactID)
if err := generateClirrIgnore(protoModuleStagingRoot); err != nil {
return fmt.Errorf("failed to generate clirr ignore file: %w", err)
}
}
}

Expand Down Expand Up @@ -560,8 +614,8 @@ func ApplyMoveActionsToLibrary(actions []moveAction, destRoot string, keepSet ma
return nil
}

// ToKeepSet normalizes a list of keep paths into a lookup map.
func ToKeepSet(keep []string) map[string]bool {
// toKeepSet normalizes a list of keep paths into a lookup map.
func toKeepSet(keep []string) map[string]bool {
keepSet := make(map[string]bool, len(keep))
for _, k := range keep {
normalized := strings.TrimSuffix(filepath.ToSlash(k), "/")
Expand All @@ -570,9 +624,9 @@ func ToKeepSet(keep []string) map[string]bool {
return keepSet
}

// RestructureToLibrary moves all generated source code to the library root directories.
// restructureToLibrary moves all generated source code to the library root directories.
// It also removes conflicting files, and copies public proto files to the library.
func RestructureToLibrary(params postProcessParams, destRoot string, keepSet map[string]bool) error {
func restructureToLibrary(params postProcessParams, destRoot string, keepSet map[string]bool) error {
tempProtoSrcDir := params.protoDir()
isCommonProtos := params.library.Name == commonProtosLibrary
if !isCommonProtos {
Expand Down
97 changes: 92 additions & 5 deletions internal/librarian/java/postprocess_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ import (
func TestPostProcessAPI(t *testing.T) {
t.Parallel()
outdir := t.TempDir()
// Force routing to the legacy owlbot.py postprocessor.
if err := os.WriteFile(filepath.Join(outdir, "owlbot.py"), []byte("# dummy"), 0755); err != nil {
t.Fatal(err)
}
libraryName := "secretmanager"
apiBase := "v1"
gapicDir := filepath.Join(outdir, apiBase, "gapic")
Expand Down Expand Up @@ -1336,7 +1340,7 @@ func TestRestructureToLibrary(t *testing.T) {
includeSamples: tc.includeSamples,
apiBase: "v1",
}
if err := RestructureToLibrary(params, destDir, nil); err != nil {
if err := restructureToLibrary(params, destDir, nil); err != nil {
t.Fatal(err)
}
gotFiles := readDirFiles(t, destDir)
Expand All @@ -1347,7 +1351,7 @@ func TestRestructureToLibrary(t *testing.T) {
}
}

// TestRestructureToLibrary_OverwritesExistingFiles verifies that existing files in the destination are overwritten.
// TestRestructureToLibrary_OverwritesExistingFiles verifies that restructureToLibrary overwrites existing files in the destination.
func TestRestructureToLibrary_OverwritesExistingFiles(t *testing.T) {
t.Parallel()
srcDir := t.TempDir()
Expand All @@ -1372,7 +1376,7 @@ func TestRestructureToLibrary_OverwritesExistingFiles(t *testing.T) {
apiBase: "v1",
}
// Pass nil keepSet to expect default overwriting of conflicting files.
if err := RestructureToLibrary(params, destDir, nil); err != nil {
if err := restructureToLibrary(params, destDir, nil); err != nil {
t.Fatal(err)
}
gotFiles := readDirFiles(t, destDir)
Expand Down Expand Up @@ -1404,7 +1408,7 @@ func TestRestructureToLibrary_CommonProtos(t *testing.T) {
includeSamples: false,
apiBase: "v1",
}
if err := RestructureToLibrary(params, destDir, nil); err != nil {
if err := restructureToLibrary(params, destDir, nil); err != nil {
t.Fatal(err)
}
gotFiles := readDirFiles(t, destDir)
Expand All @@ -1416,6 +1420,89 @@ func TestRestructureToLibrary_CommonProtos(t *testing.T) {
}
}

// TestPostProcessAPI_Go verifies that Go-native postprocessor correctly restructures
// generated Java files to their target directories and cleans up intermediate files.
func TestPostProcessAPI_Go(t *testing.T) {
t.Parallel()
dir := t.TempDir()
writeFiles(t, dir, map[string]string{
"v1/gapic/src/main/java/Foo.java": "class Foo {}",
"v1/grpc/dummy": "",
"v1/proto/dummy": "",
})
library := &config.Library{
Name: "test-lib",
APIs: []*config.API{{Path: "google/cloud/test/v1", Java: &config.JavaAPI{}}},
Java: &config.JavaModule{GroupID: "com.google.cloud", ArtifactID: "google-cloud-test", ReleasedVersion: "1.2.3"},
}
postParams := postProcessParams{
cfg: &config.Config{},
library: library,
javaAPI: library.APIs[0].Java,
outDir: dir,
includeSamples: true,
apiBase: "v1",
}
if err := postProcessAPI(t.Context(), postParams); err != nil {
t.Fatal(err)
}
// Verify that files are relocated directly to target paths and staging is skipped.
want := map[string]string{
"google-cloud-test/src/main/java/Foo.java": "class Foo {}",
"proto-google-cloud-test-v1/src/main/java/dummy": "",
"grpc-google-cloud-test-v1/src/main/java/dummy": "",
}
got := readDirFiles(t, dir)
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("mismatch (-want +got):\n%s", diff)
}
}

// TestPostProcessLibrary_Go verifies that library-level postprocessing tasks
// (such as text replacements, POM updates, and README generation) execute
// correctly in the Go-native flow.
func TestPostProcessLibrary_Go(t *testing.T) {
t.Parallel()
dir := t.TempDir()
writeFiles(t, dir, map[string]string{"google-cloud-test/src/main/java/Foo.java": "class Foo {}"})
library := &config.Library{
Name: "test-lib",
Java: &config.JavaModule{
GroupID: "com.google.cloud",
ArtifactID: "google-cloud-test",
// Disable syncPOMs to simplify config requirements.
SkipPOMUpdates: true,
},
// Disable renderREADME to simplify config requirements.
Keep: []string{"README.md"},
Postprocess: &config.Postprocess{
Replace: []config.ReplaceConfig{
{Path: "google-cloud-test/src/main/java/Foo.java", Original: "class Foo", Replacement: "class RenamedFoo"},
},
},
}
params := libraryPostProcessParams{
cfg: &config.Config{
Libraries: []*config.Library{
{Name: "google-cloud-java", Version: "1.2.3"},
{Name: "google-cloud-pom-parent", Version: "1.2.3"},
},
},
library: library,
outDir: dir,
metadata: &repoMetadata{},
}
if err := postProcessLibrary(t.Context(), params); err != nil {
t.Fatal(err)
}
// Verify postprocessing rules were applied.
want := map[string]string{"google-cloud-test/src/main/java/Foo.java": "class RenamedFoo {}"}
got := readDirFiles(t, dir)
if diff := cmp.Diff(want, got); diff != "" {
t.Errorf("mismatch (-want +got):\n%s", diff)
}
}

func writeFiles(t *testing.T, dir string, files map[string]string) {
t.Helper()
if err := os.MkdirAll(dir, 0755); err != nil {
Expand Down Expand Up @@ -1462,7 +1549,7 @@ func readDirFiles(t *testing.T, dir string) map[string]string {
func TestToKeepSet(t *testing.T) {
t.Parallel()
input := []string{"foo/", "bar/baz", "qux/", ""}
got := ToKeepSet(input)
got := toKeepSet(input)
want := map[string]bool{
"foo": true,
"bar/baz": true,
Expand Down
Loading