feat: remove Error and Deploy from ModuleConfig (#3033)

Remove things from ModuleConfig in preparation for language plugins:
- `ModuleConfig.Errors`
- Removed logic to ever write errors to disk (except Java plugin, which
just hardcodes `errors.pb` until it moves to being an external plugin)
- `ModuleConfig.Deploy`
- This has been refactored to being part of a build result. This also
allows language plugins to dynamically include files to deploy.

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

frontend/cli/cmd_box.go

@@ -148,7 +148,7 @@ func (b *boxCmd) Run(ctx context.Context, client ftlv1connect.ControllerServiceC
 		destDir := filepath.Join(workDir, "modules", config.Module)
 
 		// Copy deployment artefacts.
-		files, err := buildengine.FindFilesToDeploy(config)
+		files, err := buildengine.FindFilesToDeploy(config, m.Deploy)
 		if err != nil {
 			return err
 		}

go-runtime/compile/build.go

@@ -20,12 +20,8 @@ import (
 	"golang.org/x/mod/modfile"
 	"golang.org/x/mod/semver"
 	"golang.org/x/sync/errgroup"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/TBD54566975/ftl"
-	languagepb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/language"
-	schemapb "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/schema"
 	extract "github.com/TBD54566975/ftl/go-runtime/schema"
 	"github.com/TBD54566975/ftl/internal"
 	"github.com/TBD54566975/ftl/internal/builderrors"
@@ -262,9 +258,9 @@ func buildDir(moduleDir string) string {
 }
 
 // Build the given module.
-func Build(ctx context.Context, projectRootDir, moduleDir string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, filesTransaction ModifyFilesTransaction, buildEnv []string, devMode bool) (err error) {
+func Build(ctx context.Context, projectRootDir, moduleDir string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, filesTransaction ModifyFilesTransaction, buildEnv []string, devMode bool) (moduleSch *schema.Module, buildErrors []builderrors.Error, err error) {
 	if err := filesTransaction.Begin(); err != nil {
-		return err
+		return nil, nil, fmt.Errorf("could not start a file transaction: %w", err)
 	}
 	defer func() {
 		if terr := filesTransaction.End(); terr != nil {
@@ -274,12 +270,12 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 
 	replacements, goModVersion, err := updateGoModule(filepath.Join(moduleDir, "go.mod"))
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	goVersion := runtime.Version()[2:]
 	if semver.Compare("v"+goVersion, "v"+goModVersion) < 0 {
-		return fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion)
+		return nil, nil, fmt.Errorf("go version %q is not recent enough for this module, needs minimum version %q", goVersion, goModVersion)
 	}
 
 	ftlVersion := ""
@@ -291,7 +287,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 	if pcpath, ok := projectconfig.DefaultConfigPath().Get(); ok {
 		pc, err := projectconfig.Load(ctx, pcpath)
 		if err != nil {
-			return fmt.Errorf("failed to load project config: %w", err)
+			return nil, nil, fmt.Errorf("failed to load project config: %w", err)
 		}
 		projectName = pc.Name
 	}
@@ -302,7 +298,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 	buildDir := buildDir(moduleDir)
 	err = os.MkdirAll(buildDir, 0750)
 	if err != nil {
-		return fmt.Errorf("failed to create build directory: %w", err)
+		return nil, nil, fmt.Errorf("failed to create build directory: %w", err)
 	}
 
 	var sharedModulesPaths []string
@@ -317,36 +313,30 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 		GoVersion:          goModVersion,
 		SharedModulesPaths: sharedModulesPaths,
 	}, scaffolder.Exclude("^go.mod$"), scaffolder.Functions(funcs)); err != nil {
-		return fmt.Errorf("failed to scaffold zip: %w", err)
+		return nil, nil, fmt.Errorf("failed to scaffold zip: %w", err)
 	}
 
 	logger.Debugf("Extracting schema")
 	result, err := extract.Extract(config.Dir)
 	if err != nil {
-		return err
+		return nil, nil, fmt.Errorf("could not extract schema: %w", err)
 	}
 
-	if err = writeSchemaErrors(config, result.Errors); err != nil {
-		return fmt.Errorf("failed to write schema errors: %w", err)
-	}
 	if builderrors.ContainsTerminalError(result.Errors) {
 		// Only bail if schema errors contain elements at level ERROR.
 		// If errors are only at levels below ERROR (e.g. INFO, WARN), the schema can still be used.
-		return nil
-	}
-	if err = writeSchema(config, result.Module); err != nil {
-		return fmt.Errorf("failed to write schema: %w", err)
+		return nil, result.Errors, nil
 	}
 
 	logger.Debugf("Generating main module")
 	mctx, err := buildMainModuleContext(sch, result, goModVersion, ftlVersion, projectName, sharedModulesPaths,
 		replacements)
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 	if err := internal.ScaffoldZip(buildTemplateFiles(), moduleDir, mctx, scaffolder.Exclude("^go.mod$"),
 		scaffolder.Functions(funcs)); err != nil {
-		return err
+		return nil, nil, fmt.Errorf("failed to scaffold build template: %w", err)
 	}
 
 	logger.Debugf("Tidying go.mod files")
@@ -374,7 +364,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 		return filesTransaction.ModifiedFiles(filepath.Join(mainDir, "go.mod"), filepath.Join(moduleDir, "go.sum"))
 	})
 	if err := wg.Wait(); err != nil {
-		return err
+		return nil, nil, err // nolint:wrapcheck
 	}
 
 	logger.Debugf("Compiling")
@@ -387,7 +377,7 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 	buildEnv = append(buildEnv, "GODEBUG=http2client=0")
 	err = exec.CommandWithEnv(ctx, log.Debug, mainDir, buildEnv, "go", args...).RunBuffered(ctx)
 	if err != nil {
-		return fmt.Errorf("failed to compile: %w", err)
+		return nil, nil, fmt.Errorf("failed to compile: %w", err)
 	}
 	err = os.WriteFile(filepath.Join(mainDir, "../../launch"), []byte(`#!/bin/bash
 	if [ -n "$FTL_DEBUG_PORT" ] && command -v dlv &> /dev/null ; then
@@ -397,9 +387,9 @@ func Build(ctx context.Context, projectRootDir, moduleDir string, config modulec
 	fi
 	`), 0770) // #nosec
 	if err != nil {
-		return fmt.Errorf("failed to write launch script: %w", err)
+		return nil, nil, fmt.Errorf("failed to write launch script: %w", err)
 	}
-	return nil
+	return result.Module, result.Errors, nil
 }
 
 // CleanStubs removes all generated stubs.
@@ -1154,47 +1144,6 @@ func shouldUpdateVersion(goModfile *modfile.File) bool {
 	return true
 }
 
-func writeSchema(config moduleconfig.AbsModuleConfig, module *schema.Module) error {
-	modulepb := module.ToProto().(*schemapb.Module) //nolint:forcetypeassert
-	// If user has overridden GOOS and GOARCH we want to use those values.
-	goos, ok := os.LookupEnv("GOOS")
-	if !ok {
-		goos = runtime.GOOS
-	}
-	goarch, ok := os.LookupEnv("GOARCH")
-	if !ok {
-		goarch = runtime.GOARCH
-	}
-
-	modulepb.Runtime = &schemapb.ModuleRuntime{
-		CreateTime: timestamppb.Now(),
-		Language:   "go",
-		Os:         &goos,
-		Arch:       &goarch,
-	}
-	schemaBytes, err := proto.Marshal(module.ToProto())
-	if err != nil {
-		return fmt.Errorf("failed to marshal schema: %w", err)
-	}
-	err = os.WriteFile(config.Schema(), schemaBytes, 0600)
-	if err != nil {
-		return fmt.Errorf("could not write schema: %w", err)
-	}
-	return nil
-}
-
-func writeSchemaErrors(config moduleconfig.AbsModuleConfig, errors []builderrors.Error) error {
-	elBytes, err := proto.Marshal(languagepb.ErrorsToProto(errors))
-	if err != nil {
-		return fmt.Errorf("failed to marshal errors: %w", err)
-	}
-	err = os.WriteFile(config.Errors, elBytes, 0600)
-	if err != nil {
-		return fmt.Errorf("could not write build errors: %w", err)
-	}
-	return nil
-}
-
 // returns the import path and directory name for a Go type
 // package and directory names are the same (dir=bar, pkg=bar): "github.com/foo/bar.A" => "github.com/foo/bar", none
 // package and directory names differ (dir=bar, pkg=baz): "github.com/foo/bar.baz.A" => "github.com/foo/bar", "baz"

internal/buildengine/build.go

@@ -20,7 +20,7 @@ import (
 // Build a module in the given directory given the schema and module config.
 //
 // A lock file is used to ensure that only one build is running at a time.
-func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, sch *schema.Schema, config moduleconfig.ModuleConfig, buildEnv []string, devMode bool) (*schema.Module, error) {
+func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRootDir string, sch *schema.Schema, config moduleconfig.ModuleConfig, buildEnv []string, devMode bool) (moduleSchema *schema.Module, deploy []string, err error) {
 	logger := log.FromContext(ctx).Module(config.Module).Scope("build")
 	ctx = log.ContextWithLogger(ctx, logger)
 
@@ -34,14 +34,14 @@ func build(ctx context.Context, plugin languageplugin.LanguagePlugin, projectRoo
 }
 
 // handleBuildResult processes the result of a build
-func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult either.Either[languageplugin.BuildResult, error]) (*schema.Module, error) {
+func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherResult either.Either[languageplugin.BuildResult, error]) (moduleSchema *schema.Module, deploy []string, err error) {
 	logger := log.FromContext(ctx)
 	config := c.Abs()
 
 	var result languageplugin.BuildResult
 	switch eitherResult := eitherResult.(type) {
 	case either.Right[languageplugin.BuildResult, error]:
-		return nil, fmt.Errorf("failed to build module: %w", eitherResult.Get())
+		return nil, nil, fmt.Errorf("failed to build module: %w", eitherResult.Get())
 	case either.Left[languageplugin.BuildResult, error]:
 		result = eitherResult.Get()
 	}
@@ -56,18 +56,18 @@ func handleBuildResult(ctx context.Context, c moduleconfig.ModuleConfig, eitherR
 	}
 
 	if len(errs) > 0 {
-		return nil, errors.Join(errs...)
+		return nil, nil, errors.Join(errs...)
 	}
 
 	logger.Infof("Module built (%.2fs)", time.Since(result.StartTime).Seconds())
 
 	// write schema proto to deploy directory
 	schemaBytes, err := proto.Marshal(result.Schema.ToProto())
 	if err != nil {
-		return nil, fmt.Errorf("failed to marshal schema: %w", err)
+		return nil, nil, fmt.Errorf("failed to marshal schema: %w", err)
 	}
 	if err := os.WriteFile(config.Schema(), schemaBytes, 0600); err != nil {
-		return nil, fmt.Errorf("failed to write schema: %w", err)
+		return nil, nil, fmt.Errorf("failed to write schema: %w", err)
 	}
-	return result.Schema, nil
+	return result.Schema, result.Deploy, nil
 }

internal/buildengine/deploy.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"connectrpc.com/connect"
@@ -38,13 +39,14 @@ type DeployClient interface {
 }
 
 // Deploy a module to the FTL controller with the given number of replicas. Optionally wait for the deployment to become ready.
-func Deploy(ctx context.Context, module Module, replicas int32, waitForDeployOnline bool, client DeployClient) error {
+func Deploy(ctx context.Context, module Module, deploy []string, replicas int32, waitForDeployOnline bool, client DeployClient) error {
+	fmt.Printf("Deplying with arg: %v", deploy)
 	logger := log.FromContext(ctx).Module(module.Config.Module).Scope("deploy")
 	ctx = log.ContextWithLogger(ctx, logger)
 	logger.Infof("Deploying module")
 
 	moduleConfig := module.Config.Abs()
-	files, err := FindFilesToDeploy(moduleConfig)
+	files, err := FindFilesToDeploy(moduleConfig, deploy)
 	if err != nil {
 		logger.Errorf(err, "failed to find files in %s", moduleConfig)
 		return err
@@ -159,9 +161,13 @@ func loadProtoSchema(config moduleconfig.AbsModuleConfig, replicas int32) (*sche
 }
 
 // FindFilesToDeploy returns a list of files to deploy for the given module.
-func FindFilesToDeploy(moduleConfig moduleconfig.AbsModuleConfig) ([]string, error) {
+func FindFilesToDeploy(config moduleconfig.AbsModuleConfig, deploy []string) ([]string, error) {
 	var out []string
-	for _, file := range moduleConfig.Deploy {
+	for _, f := range deploy {
+		file := filepath.Clean(filepath.Join(config.DeployDir, f))
+		if !strings.HasPrefix(file, config.DeployDir) {
+			return nil, fmt.Errorf("deploy path %q is not beneath deploy directory %q", file, config.DeployDir)
+		}
 		info, err := os.Stat(file)
 		if err != nil {
 			return nil, err

internal/buildengine/engine.go

@@ -338,11 +338,14 @@ func (e *Engine) Deploy(ctx context.Context, replicas int32, waitForDeployOnline
 				continue
 			}
 			deployGroup.Go(func() error {
-				module, ok := e.moduleMetas.Load(moduleName)
+				meta, ok := e.moduleMetas.Load(moduleName)
 				if !ok {
 					return fmt.Errorf("module %q not found", moduleName)
 				}
-				return Deploy(ctx, module.module, replicas, waitForDeployOnline, e.client)
+				if len(meta.module.Deploy) == 0 {
+					return fmt.Errorf("no files found to deploy for %q", moduleName)
+				}
+				return Deploy(ctx, meta.module, meta.module.Deploy, replicas, waitForDeployOnline, e.client)
 			})
 		}
 		if err := deployGroup.Wait(); err != nil {
@@ -599,7 +602,8 @@ func (e *Engine) BuildAndDeploy(ctx context.Context, replicas int32, waitForDepl
 			buildGroup.Go(func() error {
 				e.modulesToBuild.Store(module.Config.Module, false)
 				terminal.UpdateModuleState(ctx, module.Config.Module, terminal.BuildStateDeploying)
-				return Deploy(buildCtx, module, replicas, waitForDeployOnline, e.client)
+
+				return Deploy(buildCtx, module, module.Deploy, replicas, waitForDeployOnline, e.client)
 			})
 			return nil
 		}, moduleNames...)
@@ -758,7 +762,7 @@ func (e *Engine) tryBuild(ctx context.Context, mustBuild map[string]bool, module
 
 	meta, ok := e.moduleMetas.Load(moduleName)
 	if !ok {
-		return fmt.Errorf("Module %q not found", moduleName)
+		return fmt.Errorf("module %q not found", moduleName)
 	}
 
 	for _, dep := range meta.module.Dependencies {
@@ -770,6 +774,11 @@ func (e *Engine) tryBuild(ctx context.Context, mustBuild map[string]bool, module
 
 	err := e.build(ctx, moduleName, builtModules, schemas)
 	if err == nil && callback != nil {
+		// load latest meta as it may have been updated
+		meta, ok = e.moduleMetas.Load(moduleName)
+		if !ok {
+			return fmt.Errorf("module %q not found", moduleName)
+		}
 		return callback(ctx, meta.module)
 	}
 
@@ -802,11 +811,20 @@ func (e *Engine) build(ctx context.Context, moduleName string, builtModules map[
 		e.listener.OnBuildStarted(meta.module)
 	}
 
-	moduleSchema, err := build(ctx, meta.plugin, e.projectRoot, sch, meta.module.Config, e.buildEnv, e.devMode)
+	moduleSchema, deploy, err := build(ctx, meta.plugin, e.projectRoot, sch, meta.module.Config, e.buildEnv, e.devMode)
 	if err != nil {
 		terminal.UpdateModuleState(ctx, moduleName, terminal.BuildStateFailed)
 		return err
 	}
+	// update files to deploy
+	e.moduleMetas.Compute(moduleName, func(meta moduleMeta, exists bool) (out moduleMeta, shouldDelete bool) {
+		if !exists {
+			return moduleMeta{}, true
+		}
+		meta.module = meta.module.CopyWithDeploy(deploy)
+		return meta, false
+	})
+
 	terminal.UpdateModuleState(ctx, moduleName, terminal.BuildStateBuilt)
 	schemas <- moduleSchema
 	return nil
@@ -911,14 +929,16 @@ func (e *Engine) listenForBuildUpdates(originalCtx context.Context) {
 				}
 
 			case languageplugin.AutoRebuildEndedEvent:
-				if _, err := handleBuildResult(ctx, meta.module.Config, event.Result); err != nil {
+				_, deploy, err := handleBuildResult(ctx, meta.module.Config, event.Result)
+				if err != nil {
 					logger.Errorf(err, "build failed")
 					e.reportBuildFailed(err)
 					terminal.UpdateModuleState(ctx, event.Module, terminal.BuildStateFailed)
 					continue
 				}
+				// TODO: update deploy dirs
 				terminal.UpdateModuleState(ctx, event.Module, terminal.BuildStateDeploying)
-				if err := Deploy(ctx, meta.module, 1, true, e.client); err != nil {
+				if err := Deploy(ctx, meta.module, deploy, 1, true, e.client); err != nil {
 					logger.Errorf(err, "deploy failed")
 					e.reportBuildFailed(err)
 				} else {

internal/buildengine/languageplugin/go_plugin.go

@@ -51,7 +51,6 @@ func (p *goPlugin) ModuleConfigDefaults(ctx context.Context, dir string) (module
 	return moduleconfig.CustomDefaults{
 		Watch:     watch,
 		DeployDir: deployDir,
-		Deploy:    []string{"main", "launch"},
 	}, nil
 }
 
@@ -160,9 +159,14 @@ func (p *goPlugin) GetDependencies(ctx context.Context, config moduleconfig.Modu
 	})
 }
 
-func buildGo(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) error {
-	if err := compile.Build(ctx, projectRoot, config.Dir, config, sch, transaction, buildEnv, devMode); err != nil {
-		return CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", config.Module, err)}
+func buildGo(ctx context.Context, projectRoot string, config moduleconfig.AbsModuleConfig, sch *schema.Schema, buildEnv []string, devMode bool, transaction watch.ModifyFilesTransaction) (BuildResult, error) {
+	moduleSch, buildErrs, err := compile.Build(ctx, projectRoot, config.Dir, config, sch, transaction, buildEnv, devMode)
+	if err != nil {
+		return BuildResult{}, CompilerBuildError{err: fmt.Errorf("failed to build module %q: %w", config.Module, err)}
 	}
-	return nil
+	return BuildResult{
+		Errors: buildErrs,
+		Schema: moduleSch,
+		Deploy: []string{"main", "launch"},
+	}, nil
 }