Port driver to transformation DSL (BIP-5)

.travis.yml

@@ -7,7 +7,7 @@ services:
   - docker
 
 before_script:
-  - go get -v gopkg.in/bblfsh/sdk.v1/...
+  - go get -v gopkg.in/bblfsh/sdk.v2/...
   - bblfsh-sdk prepare-build .
   - go get -v -t ./driver/...
 

Makefile

@@ -12,7 +12,6 @@ build-native-internal:
 generate:
 	cd driver; \
 	go run ./gen/gen.go && \
-	go fmt ./goast/goast.go && \
 	go fmt ./normalizer/ann_gen.go
 
 include .sdk/Makefile

README.md

@@ -7,7 +7,7 @@ Development Environment
 
 Requirements:
 - `docker`
-- [`bblfsh-sdk`](https://github.com/bblfsh/sdk) _(go get -u gopkg.in/bblfsh/sdk.v1/...)_
+- [`bblfsh-sdk`](https://github.com/bblfsh/sdk) _(go get -u gopkg.in/bblfsh/sdk.v2/...)_
 - UAST converter dependencies _(go get -t -v ./...)_
 
 To initialize the build system execute: `bblfsh-sdk prepare-build`, at the root of the project. This will install the SDK at `.sdk` for this driver.

driver/fixtures/fixtures_test.go

@@ -0,0 +1,35 @@
+package fixtures
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/bblfsh/go-driver/driver/golang"
+	"github.com/bblfsh/go-driver/driver/normalizer"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver/fixtures"
+)
+
+const projectRoot = "../../"
+
+var Suite = &fixtures.Suite{
+	Lang: "go",
+	Ext:  ".go",
+	Path: filepath.Join(projectRoot, fixtures.Dir),
+	NewDriver: func() driver.BaseDriver {
+		return golang.NewDriver()
+	},
+	Transforms: driver.Transforms{
+		Native: normalizer.Native,
+		Code:   normalizer.Code,
+	},
+	BenchName: "json",
+}
+
+func TestGoDriver(t *testing.T) {
+	Suite.RunTests(t)
+}
+
+func BenchmarkGoDriver(b *testing.B) {
+	Suite.RunBenchmarks(b)
+}

driver/gen/gen.go

@@ -61,7 +61,6 @@ func (a byName) Swap(i, j int) {
 }
 
 var (
-	fAnn     = flag.String("goast", "./goast/goast.go", "file for AST type annotations")
 	fAnnAuto = flag.String("ann", "./normalizer/ann_gen.go", "file for auto annotations")
 )
 
@@ -125,41 +124,24 @@ func main() {
 
 	const genHeader = "// Code generated by ./gen/gen.go DO NOT EDIT.\n\n"
 
-	genFile(*fAnn, func(w io.Writer) {
-		fmt.Fprintln(w, genHeader+`// Package goast defines constants from Go AST.
-package goast
-
-import "gopkg.in/bblfsh/sdk.v1/uast/ann"
-
-// all nodes that implements ast.Node
-var (`)
-		defer fmt.Fprintln(w, ")")
-
-		for _, n := range nodes {
-			name := n.Obj().Name()
-			fmt.Fprintf(w, "\t%s = ann.HasInternalType(%q)\n", name, name)
-		}
-	})
-
 	genFile(*fAnnAuto, func(w io.Writer) {
 		fmt.Fprintln(w, genHeader+`package normalizer
 
 import (
-	"github.com/bblfsh/go-driver/driver/goast"
-	"gopkg.in/bblfsh/sdk.v1/uast"
-	. "gopkg.in/bblfsh/sdk.v1/uast/ann"
+	"gopkg.in/bblfsh/sdk.v2/uast/role"
 )
-`)
+
+var typeRoles = map[string][]role.Role{`)
+		defer fmt.Fprintln(w, "}")
+
 		genRole := func(name, role string, list []*types.Named) {
-			fmt.Fprintf(w, `// all AST nodes that implements ast.%s
-var AnnotationRules%s = On(goast.File).Descendants(
-`, name, name)
-			defer fmt.Fprintln(w, ")")
+			fmt.Fprintf(w, "// all AST nodes that implements ast.%s\n", name)
 
 			for _, n := range list {
 				tp := n.Obj().Name()
-				fmt.Fprintf(w, "\tOn(goast.%s).Roles(uast.%s),\n", tp, role)
+				fmt.Fprintf(w, "\t%q: {role.%s},\n", tp, role)
 			}
+			fmt.Fprintln(w)
 		}
 
 		genRole("Expr", "Expression", expr)

driver/golang/golang.go

@@ -0,0 +1,137 @@
+package golang
+
+import (
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"reflect"
+
+	"gopkg.in/bblfsh/sdk.v2/protocol"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver"
+	"gopkg.in/bblfsh/sdk.v2/uast"
+)
+
+func ParseString(code string) (*ast.File, error) {
+	fs := token.NewFileSet()
+	tree, err := parser.ParseFile(fs, "input.go", code, parser.ParseComments)
+	if err != nil {
+		return nil, err
+	}
+	return tree, nil
+}
+
+func Parse(code string) (uast.Node, error) {
+	f, err := ParseString(code)
+	if err != nil {
+		return nil, err
+	}
+	return convValue(reflect.ValueOf(f))
+}
+
+var (
+	scopeType   = reflect.TypeOf((*ast.Scope)(nil))
+	objectType  = reflect.TypeOf((*ast.Object)(nil))
+	nodeType    = reflect.TypeOf((*ast.Node)(nil)).Elem()
+	posType     = reflect.TypeOf(token.Pos(0))
+	uastIntType = reflect.TypeOf(uast.Int(0))
+)
+
+func convPos(p token.Pos) uast.Node {
+	return uast.Position{Offset: uint32(p)}.ToObject()
+}
+
+// convValue takes an AST node/value and converts it to a tree of uast types
+// like Object and List. In this case we have a full control of json encoding
+// and can annotate the tree with native AST type names.
+func convValue(v reflect.Value) (uast.Node, error) {
+	if !v.IsValid() {
+		return nil, nil
+	}
+	t := v.Type()
+	switch t {
+	case posType:
+		return convPos(v.Interface().(token.Pos)), nil
+	}
+	switch t.Kind() {
+	case reflect.Slice:
+		if v.Len() == 0 {
+			return nil, nil
+		}
+		arr := make(uast.Array, 0, v.Len())
+		for i := 0; i < v.Len(); i++ {
+			el, err := convValue(v.Index(i))
+			if err != nil {
+				return nil, err
+			}
+			arr = append(arr, el)
+		}
+		return arr, nil
+	case reflect.Struct:
+		m := make(uast.Object, t.NumField())
+		m[uast.KeyType] = uast.String(t.Name()) // annotate nodes with type names
+		for i := 0; i < t.NumField(); i++ {
+			f := t.Field(i)
+			if f.Type == scopeType || f.Type == objectType {
+				// do not follow scope and object pointers - need a graph structure for it
+				continue
+			}
+			el, err := convValue(v.Field(i))
+			if err != nil {
+				return nil, err
+			}
+			m[f.Name] = el
+		}
+		return m, nil
+	case reflect.Ptr, reflect.Interface:
+		if v.IsNil() {
+			return nil, nil
+		}
+		o, err := convValue(v.Elem())
+		if err != nil {
+			return nil, err
+		}
+		if m, ok := o.(uast.Object); ok && v.Type().Implements(nodeType) {
+			n := v.Interface().(ast.Node)
+			m[uast.KeyStart] = convPos(n.Pos())
+			m[uast.KeyEnd] = convPos(n.End())
+		}
+		return o, nil
+	}
+	o := v.Interface()
+	if s, ok := o.(interface {
+		String() string
+	}); ok {
+		return uast.String(s.String()), nil
+	} else if t.ConvertibleTo(uastIntType) {
+		return v.Convert(uastIntType).Interface().(uast.Int), nil
+	}
+	return uast.ToNode(o)
+}
+
+func NewDriver() *Driver {
+	return &Driver{}
+}
+
+type Driver struct{}
+
+func (Driver) Start() error {
+	return nil
+}
+func (Driver) Close() error {
+	return nil
+}
+func (Driver) Parse(req *driver.InternalParseRequest) (*driver.InternalParseResponse, error) {
+	code, err := req.Encoding.Decode(req.Content)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode contents: %v", err)
+	}
+	n, err := Parse(code)
+	if err != nil {
+		return nil, err
+	}
+	return &driver.InternalParseResponse{
+		Status: driver.Status(protocol.Ok),
+		AST:    n,
+	}, nil
+}

driver/golang/golang_test.go

@@ -0,0 +1,47 @@
+package golang
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/bblfsh/sdk.v2/protocol"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver"
+	"gopkg.in/bblfsh/sdk.v2/uast"
+)
+
+func TestNative(t *testing.T) {
+	const code = `package main`
+
+	resp, err := NewDriver().Parse(&driver.InternalParseRequest{
+		Encoding: driver.Encoding(protocol.UTF8),
+		Content:  string(code),
+	})
+	require.NoError(t, err)
+	require.Equal(t, driver.Status(protocol.Ok), resp.Status)
+	require.Empty(t, resp.Errors)
+	pos := func(p int) uast.Node {
+		return uast.Position{Offset: uint32(p)}.ToObject()
+	}
+	str := func(s string) uast.Node {
+		return uast.String(s)
+	}
+	exp := uast.Object{
+		uast.KeyType:  str("File"),
+		uast.KeyStart: pos(1),
+		uast.KeyEnd:   pos(13),
+		"Package":     pos(1),
+		"Name": uast.Object{
+			uast.KeyType:  str("Ident"),
+			uast.KeyStart: pos(9),
+			uast.KeyEnd:   pos(13),
+			"Name":        str("main"),
+			"NamePos":     pos(9),
+		},
+		"Imports":    nil,
+		"Comments":   nil,
+		"Doc":        nil,
+		"Decls":      nil,
+		"Unresolved": nil,
+	}
+	require.Equal(t, exp, resp.AST)
+}

driver/impl/impl.go

@@ -0,0 +1,10 @@
+package impl
+
+import (
+	"github.com/bblfsh/go-driver/driver/golang"
+	"gopkg.in/bblfsh/sdk.v2/sdk/driver"
+)
+
+func init() {
+	driver.DefaultDriver = golang.NewDriver()
+}