Skip to content

Commit

Permalink
feat: add SQL verbs to build cycles (#4045)
Browse files Browse the repository at this point in the history
also generates SQL verb types in Go

fixes #3972
  • Loading branch information
worstell authored Jan 15, 2025
1 parent 48de2b8 commit f456ec1
Show file tree
Hide file tree
Showing 51 changed files with 880 additions and 373 deletions.
4 changes: 3 additions & 1 deletion Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@ SCHEMA_OUT := "common/protos/xyz/block/ftl/schema/v1/schema.proto"
ZIP_DIRS := "go-runtime/compile/build-template " + \
"go-runtime/compile/external-module-template " + \
"go-runtime/compile/main-work-template " + \
"go-runtime/compile/queries-template " + \
"internal/projectinit/scaffolding " + \
"go-runtime/scaffolding " + \
"jvm-runtime/java/scaffolding " + \
"jvm-runtime/kotlin/scaffolding " + \
"python-runtime/compile/build-template " + \
"python-runtime/compile/external-module-template " + \
"python-runtime/scaffolding"
"python-runtime/scaffolding " + \
"internal/sqlc/template"
CONSOLE_ROOT := "frontend/console"
FRONTEND_OUT := CONSOLE_ROOT + "/dist/index.html"
EXTENSION_OUT := "frontend/vscode/dist/extension.js"
Expand Down
37 changes: 37 additions & 0 deletions backend/controller/sql/database_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,13 @@
package sql_test

import (
"context"
"testing"

"github.com/alecthomas/assert/v2"

schemapb "github.com/block/ftl/common/protos/xyz/block/ftl/schema/v1"
"github.com/block/ftl/common/schema"
in "github.com/block/ftl/internal/integration"
)

Expand Down Expand Up @@ -39,5 +42,39 @@ func TestMySQL(t *testing.T) {
in.Call[in.Obj, in.Obj]("mysql", "query", map[string]any{}, func(t testing.TB, response in.Obj) {
assert.Equal(t, "hello", response["data"])
}),
in.IfLanguage("go", in.VerifySchemaVerb("mysql", "createRequest", func(ctx context.Context, t testing.TB, sch *schemapb.Schema, verb *schemapb.Verb) {
assert.True(t, verb.Response.GetUnit() != nil, "response was not a unit")
assert.True(t, verb.Request.GetRef() != nil, "request was not a ref")
fullSchema, err := schema.FromProto(sch)
assert.NoError(t, err, "failed to convert schema")
req := fullSchema.Resolve(schema.RefFromProto(verb.Request.GetRef()))
assert.True(t, req.Ok(), "request not found")

if data, ok := req.MustGet().(*schema.Data); ok {
assert.Equal(t, "CreateRequestQuery", data.Name)
assert.Equal(t, 1, len(data.Fields))
assert.Equal(t, "data", data.Fields[0].Name)
} else {
assert.False(t, true, "request not data")
}
})),
in.IfLanguage("go", in.VerifySchemaVerb("mysql", "getRequestData", func(ctx context.Context, t testing.TB, sch *schemapb.Schema, verb *schemapb.Verb) {
assert.True(t, verb.Response.GetArray() != nil, "response was not an array")
assert.True(t, verb.Response.GetArray().Element.GetRef() != nil, "array element was not a ref")
assert.True(t, verb.Request.GetUnit() != nil, "request was not a unit")
fullSchema, err := schema.FromProto(sch)
assert.NoError(t, err, "failed to convert schema")

resp := fullSchema.Resolve(schema.RefFromProto(verb.Response.GetArray().Element.GetRef()))
assert.True(t, resp.Ok(), "response not found")

if data, ok := resp.MustGet().(*schema.Data); ok {
assert.Equal(t, "GetRequestDataResult", data.Name)
assert.Equal(t, 1, len(data.Fields))
assert.Equal(t, "data", data.Fields[0].Name)
} else {
assert.False(t, true, "response not data")
}
})),
)
}
File renamed without changes.
29 changes: 29 additions & 0 deletions backend/controller/sql/testdata/go/mysql/queries.ftl.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

517 changes: 264 additions & 253 deletions backend/protos/xyz/block/ftl/language/v1/language.pb.go

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions backend/protos/xyz/block/ftl/language/v1/language.proto
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ message ModuleConfigDefaultsResponse {

// Default directory containing the SQL migration files
string sql_migration_dir = 8;

// Default directory containing the SQL query files
string sql_query_dir = 9;
}

message GetDependenciesRequest {
Expand Down
2 changes: 1 addition & 1 deletion backend/provisioner/sql_migration_provisioner.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func RunPostgresMigration(ctx context.Context, dsn string, moduleDir string, nam
}

func runDBMateMigration(ctx context.Context, dsn string, moduleDir string, name string) error {
migrationDir := filepath.Join(moduleDir, "db", name)
migrationDir := filepath.Join(moduleDir, "db", "schema", name)
_, err := os.Stat(migrationDir)
if err != nil {
return nil // No migration to run
Expand Down
15 changes: 15 additions & 0 deletions common/schema/verb.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,3 +178,18 @@ func (v *Verb) GetProvisioned() ResourceSet {
func (v *Verb) ResourceID() string {
return v.Name
}

// IsGenerated returns true if the Verb is in the schema but not in the source code.
func (v *Verb) IsGenerated() bool {
_, ok := v.GetRawSQLQuery()
return ok
}

// GetRawSQLQuery returns the raw SQL query for the Verb if it exists. If present, the Verb was generated from SQL.
func (v *Verb) GetRawSQLQuery() (string, bool) {
md, found := slices.FindVariant[*MetadataSQLQuery](v.Metadata)
if !found {
return "", false
}
return md.Query, true
}
4 changes: 2 additions & 2 deletions docs/content/docs/reference/databases.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ To use this in your FTL code you can then just use [Hibernate directly](https://

Note that this will likely change significantly in future once FTL has SQL Verbs.

An example showing DB usage with Pachance is shown below:
An example showing DB usage with Panache is shown below:

```java
package xyz.block.ftl.java.test.database;
Expand Down Expand Up @@ -159,4 +159,4 @@ The module name can be omitted if the current working directory only contains a

E.g. to create a new migration called `init` for the `testdb` datasource in the `mysql` module you would run `ftl new-sql-migration mysql.testdb init`.

When the modules are provisioned FTL will automatically run these migrations for you.
When the modules are provisioned FTL will automatically run these migrations for you.
6 changes: 6 additions & 0 deletions examples/go/mysql/db/queries/testdb/queries.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
-- name: GetRequestData :many
SELECT data FROM requests;

-- name: CreateRequest :exec
INSERT INTO requests (data) VALUES (?);

29 changes: 29 additions & 0 deletions examples/go/mysql/queries.ftl.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

71 changes: 66 additions & 5 deletions go-runtime/compile/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import (

var ErrInvalidateDependencies = errors.New("dependencies need to be updated")
var ftlTypesFilename = "types.ftl.go"
var ftlQueriesFilename = "queries.ftl.go"

type MainWorkContext struct {
GoVersion string
Expand All @@ -58,6 +59,7 @@ type mainDeploymentContext struct {
Replacements []*modfile.Replace
MainCtx mainFileContext
TypesCtx typesFileContext
QueriesCtx queriesFileContext
}

func (c *mainDeploymentContext) withImports(mainModuleImport string) {
Expand Down Expand Up @@ -187,6 +189,11 @@ type typesFileContext struct {
ExternalTypes []goExternalType
}

type queriesFileContext struct {
Module *schema.Module
Decls []schema.Decl
}

type goType interface {
getNativeType() nativeType
}
Expand Down Expand Up @@ -362,6 +369,7 @@ func (s *OngoingState) checkIfMainDeploymentContextChanged(moduleCtx mainDeploym
func (s *OngoingState) DetectedFileChanges(config moduleconfig.AbsModuleConfig, changes []watch.FileChange) {
paths := []string{
filepath.Join(config.Dir, ftlTypesFilename),
filepath.Join(config.Dir, ftlQueriesFilename),
filepath.Join(config.Dir, "go.mod"),
filepath.Join(config.Dir, "go.sum"),
}
Expand Down Expand Up @@ -453,7 +461,7 @@ func Build(ctx context.Context, projectRootDir, stubsRoot string, config modulec
extractResultChan := make(chan result.Result[extract.Result], 1)
go func() {
logger.Debugf("Extracting schema")
extractResultChan <- result.From(extract.Extract(config.Dir))
extractResultChan <- result.From(extract.Extract(config.Dir, sch))
}()
optimisticHashesChan := make(chan watch.FileHashes, 1)
optimisticCompileChan := make(chan error, 1)
Expand Down Expand Up @@ -569,6 +577,12 @@ func scaffoldBuildTemplateAndTidy(ctx context.Context, config moduleconfig.AbsMo
scaffolder.Functions(funcs)); err != nil {
return fmt.Errorf("failed to scaffold build template: %w", err)
}
if len(mctx.QueriesCtx.Decls) > 0 {
if err := internal.ScaffoldZip(queriesTemplateFiles(), config.Dir, mctx, scaffolder.Exclude("^go.mod$"),
scaffolder.Functions(funcs)); err != nil {
return fmt.Errorf("failed to scaffold queries template: %w", err)
}
}
if err := filesTransaction.ModifiedFiles(filepath.Join(config.Dir, ftlTypesFilename)); err != nil {
return fmt.Errorf("failed to mark %s as modified: %w", ftlTypesFilename, err)
}
Expand Down Expand Up @@ -661,6 +675,10 @@ func (b *mainDeploymentContextBuilder) build(goModVersion, ftlVersion, projectNa
SumTypes: []goSumType{},
ExternalTypes: []goExternalType{},
},
QueriesCtx: queriesFileContext{
Module: b.mainModule,
Decls: []schema.Decl{},
},
}

visited := sets.NewSet[string]()
Expand Down Expand Up @@ -707,8 +725,19 @@ func (b *mainDeploymentContextBuilder) visit(
visited sets.Set[string],
) error {
err := schema.Visit(node, func(node schema.Node, next func() error) error {
if ref, ok := node.(*schema.Ref); ok {
maybeResolved, maybeModule := b.sch.ResolveWithModule(ref)
switch n := node.(type) {
case *schema.Verb:
if _, isQuery := n.GetRawSQLQuery(); isQuery {
decls, err := b.getQueryDecls(n)
if err != nil {
return err
}
m := &schema.Module{Decls: decls}
schema.SortModuleDecls(m)
ctx.QueriesCtx.Decls = append(ctx.QueriesCtx.Decls, m.Decls...)
}
case *schema.Ref:
maybeResolved, maybeModule := b.sch.ResolveWithModule(n)
resolved, ok := maybeResolved.Get()
if !ok {
return next()
Expand All @@ -719,7 +748,7 @@ func (b *mainDeploymentContextBuilder) visit(
}
err := b.visit(ctx, m, resolved, visited)
if err != nil {
return fmt.Errorf("failed to visit children of %s: %w", ref, err)
return fmt.Errorf("failed to visit children of %s: %w", n, err)
}
return next()
}
Expand Down Expand Up @@ -759,11 +788,39 @@ func (b *mainDeploymentContextBuilder) visit(
return nil
}

func (b *mainDeploymentContextBuilder) getQueryDecls(node schema.Node) ([]schema.Decl, error) {
decls := []schema.Decl{}
err := schema.Visit(node, func(node schema.Node, next func() error) error {
switch n := node.(type) {
case *schema.Verb:
decls = append(decls, n)
case *schema.Data:
decls = append(decls, n)
case *schema.Ref:
maybeResolved, _ := b.sch.ResolveWithModule(n)
resolved, ok := maybeResolved.Get()
if !ok {
return next()
}
nested, err := b.getQueryDecls(resolved)
if err != nil {
return err
}
decls = append(decls, nested...)
}
return next()
})
if err != nil {
return nil, fmt.Errorf("failed to get query decls: %w", err)
}
return decls, nil
}

func (b *mainDeploymentContextBuilder) getGoType(module *schema.Module, node schema.Node) (gotype optional.Option[goType], isLocal bool, err error) {
isLocal = b.visitingMainModule(module.Name)
switch n := node.(type) {
case *schema.Verb:
if !isLocal {
if !isLocal || n.IsGenerated() {
return optional.None[goType](), false, nil
}
goverb, err := b.processVerb(n)
Expand Down Expand Up @@ -1256,6 +1313,10 @@ var scaffoldFuncs = scaffolder.FuncMap{
}
return nil
},
"getRawSQLQuery": func(verb *schema.Verb) string {
query, _ := verb.GetRawSQLQuery()
return query
},
}

func trimModuleQualifier(moduleName string, str string) string {
Expand Down
4 changes: 4 additions & 0 deletions go-runtime/compile/devel.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,7 @@ func externalModuleTemplateFiles() *zip.Reader {
func buildTemplateFiles() *zip.Reader {
return internal.ZipRelativeToCaller("build-template")
}

func queriesTemplateFiles() *zip.Reader {
return internal.ZipRelativeToCaller("queries-template")
}
Loading

0 comments on commit f456ec1

Please sign in to comment.