Skip to content

Commit

Permalink
message/pipeline: converted to API
Browse files Browse the repository at this point in the history 
and changed cmd/gotext to use it.

Change-Id: I418957cfcbcad3acb2ebcd2f65c88a43e5e7f254
Reviewed-on: https://go-review.googlesource.com/82236
Run-TryBot: Marcel van Lohuizen <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Nigel Tao <[email protected]>
  • Loading branch information
@mpvl
mpvl committed Dec 9, 2017
1 parent bbb8cca commit 3b24cac
Show file tree
Hide file tree
Showing 12 changed files with 105 additions and 1,576 deletions.
4 changes: 1 addition & 3 deletions cmd/gotext/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,10 @@ var (
wrap = func(err error, msg string) error {
return fmt.Errorf("%s: %v", msg, err)
}
wrapf = func(err error, msg string, args ...interface{}) error {
return wrap(err, fmt.Sprintf(msg, args...))
}
errorf = fmt.Errorf
)

// TODO: still used. Remove when possible.
func loadPackages(conf *loader.Config, args []string) (*loader.Program, error) {
if len(args) == 0 {
args = []string{"."}
Expand Down
291 changes: 7 additions & 284 deletions cmd/gotext/extract.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,14 @@
package main

import (
"bytes"
"encoding/json"
"fmt"
"go/ast"
"go/constant"
"go/format"
"go/token"
"go/types"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"unicode"
"unicode/utf8"

"golang.org/x/text/internal"
fmtparser "golang.org/x/text/internal/format"
"golang.org/x/text/language"
"golang.org/x/tools/go/loader"
"golang.org/x/text/message/pipeline"
)

// TODO:
Expand All @@ -50,173 +38,16 @@ var cmdExtract = &Command{
}

func runExtract(cmd *Command, args []string) error {
conf := loader.Config{}
prog, err := loadPackages(&conf, args)
if err != nil {
return wrap(err, "")
}

// print returns Go syntax for the specified node.
print := func(n ast.Node) string {
var buf bytes.Buffer
format.Node(&buf, conf.Fset, n)
return buf.String()
}

var messages []Message

for _, info := range prog.AllPackages {
for _, f := range info.Files {
// Associate comments with nodes.
cmap := ast.NewCommentMap(prog.Fset, f, f.Comments)
getComment := func(n ast.Node) string {
cs := cmap.Filter(n).Comments()
if len(cs) > 0 {
return strings.TrimSpace(cs[0].Text())
}
return ""
}

// Find function calls.
ast.Inspect(f, func(n ast.Node) bool {
call, ok := n.(*ast.CallExpr)
if !ok {
return true
}

// Skip calls of functions other than
// (*message.Printer).{Sp,Fp,P}rintf.
sel, ok := call.Fun.(*ast.SelectorExpr)
if !ok {
return true
}
meth := info.Selections[sel]
if meth == nil || meth.Kind() != types.MethodVal {
return true
}
// TODO: remove cheap hack and check if the type either
// implements some interface or is specifically of type
// "golang.org/x/text/message".Printer.
m, ok := extractFuncs[path.Base(meth.Recv().String())]
if !ok {
return true
}

fmtType, ok := m[meth.Obj().Name()]
if !ok {
return true
}
// argn is the index of the format string.
argn := fmtType.arg
if argn >= len(call.Args) {
return true
}

args := call.Args[fmtType.arg:]

fmtMsg, ok := msgStr(info, args[0])
if !ok {
// TODO: identify the type of the format argument. If it
// is not a string, multiple keys may be defined.
return true
}
comment := ""
key := []string{}
if ident, ok := args[0].(*ast.Ident); ok {
key = append(key, ident.Name)
if v, ok := ident.Obj.Decl.(*ast.ValueSpec); ok && v.Comment != nil {
// TODO: get comment above ValueSpec as well
comment = v.Comment.Text()
}
}

arguments := []argument{}
args = args[1:]
simArgs := make([]interface{}, len(args))
for i, arg := range args {
expr := print(arg)
val := ""
if v := info.Types[arg].Value; v != nil {
val = v.ExactString()
simArgs[i] = val
switch arg.(type) {
case *ast.BinaryExpr, *ast.UnaryExpr:
expr = val
}
}
arguments = append(arguments, argument{
ArgNum: i + 1,
Type: info.Types[arg].Type.String(),
UnderlyingType: info.Types[arg].Type.Underlying().String(),
Expr: expr,
Value: val,
Comment: getComment(arg),
Position: posString(conf, info, arg.Pos()),
// TODO report whether it implements
// interfaces plural.Interface,
// gender.Interface.
})
}
msg := ""

ph := placeholders{index: map[string]string{}}

p := fmtparser.Parser{}
p.Reset(simArgs)
for p.SetFormat(fmtMsg); p.Scan(); {
switch p.Status {
case fmtparser.StatusText:
msg += p.Text()
case fmtparser.StatusSubstitution,
fmtparser.StatusBadWidthSubstitution,
fmtparser.StatusBadPrecSubstitution:
arguments[p.ArgNum-1].used = true
arg := arguments[p.ArgNum-1]
sub := p.Text()
if !p.HasIndex {
r, sz := utf8.DecodeLastRuneInString(sub)
sub = fmt.Sprintf("%s[%d]%c", sub[:len(sub)-sz], p.ArgNum, r)
}
msg += fmt.Sprintf("{%s}", ph.addArg(&arg, sub))
}
}
key = append(key, msg)

// Add additional Placeholders that can be used in translations
// that are not present in the string.
for _, arg := range arguments {
if arg.used {
continue
}
ph.addArg(&arg, fmt.Sprintf("%%[%d]v", arg.ArgNum))
}

if c := getComment(call.Args[0]); c != "" {
comment = c
}

messages = append(messages, Message{
ID: key,
Key: fmtMsg,
Message: Text{Msg: msg},
// TODO(fix): this doesn't get the before comment.
Comment: comment,
Placeholders: ph.slice,
Position: posString(conf, info, call.Lparen),
})
return true
})
}
}

tag, err := language.Parse(*srcLang)
if err != nil {
return wrap(err, "")
}
out := Locale{
Language: tag,
Messages: messages,
config := &pipeline.Config{
SourceLanguage: tag,
Packages: args,
}
out, err := pipeline.Extract(config)

data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return wrap(err, "")
Expand All @@ -226,7 +57,7 @@ func runExtract(cmd *Command, args []string) error {
// cycle with a init once and update cycle.
file := filepath.Join(*dir, extractFile)
if err := ioutil.WriteFile(file, data, 0644); err != nil {
return wrapf(err, "could not create file")
return wrap(err, "could not create file")
}

langs := append(getLangs(), tag)
Expand All @@ -248,111 +79,3 @@ func runExtract(cmd *Command, args []string) error {
}
return nil
}

func posString(conf loader.Config, info *loader.PackageInfo, pos token.Pos) string {
p := conf.Fset.Position(pos)
file := fmt.Sprintf("%s:%d:%d", filepath.Base(p.Filename), p.Line, p.Column)
return filepath.Join(info.Pkg.Path(), file)
}

// extractFuncs indicates the types and methods for which to extract strings,
// and which argument to extract.
// TODO: use the types in conf.Import("golang.org/x/text/message") to extract
// the correct instances.
var extractFuncs = map[string]map[string]extractType{
// TODO: Printer -> *golang.org/x/text/message.Printer
"message.Printer": {
"Printf": extractType{arg: 0, format: true},
"Sprintf": extractType{arg: 0, format: true},
"Fprintf": extractType{arg: 1, format: true},

"Lookup": extractType{arg: 0},
},
}

type extractType struct {
// format indicates if the next arg is a formatted string or whether to
// concatenate all arguments
format bool
// arg indicates the position of the argument to extract.
arg int
}

func getID(arg *argument) string {
s := getLastComponent(arg.Expr)
s = strip(s)
s = strings.Replace(s, " ", "", -1)
// For small variable names, use user-defined types for more info.
if len(s) <= 2 && arg.UnderlyingType != arg.Type {
s = getLastComponent(arg.Type)
}
return strings.Title(s)
}

// strip is a dirty hack to convert function calls to placeholder IDs.
func strip(s string) string {
s = strings.Map(func(r rune) rune {
if unicode.IsSpace(r) || r == '-' {
return '_'
}
if !unicode.In(r, unicode.Letter, unicode.Mark) {
return -1
}
return r
}, s)
// Strip "Get" from getter functions.
if strings.HasPrefix(s, "Get") || strings.HasPrefix(s, "get") {
if len(s) > len("get") {
r, _ := utf8.DecodeRuneInString(s)
if !unicode.In(r, unicode.Ll, unicode.M) { // not lower or mark
s = s[len("get"):]
}
}
}
return s
}

type placeholders struct {
index map[string]string
slice []Placeholder
}

func (p *placeholders) addArg(arg *argument, sub string) (id string) {
id = getID(arg)
id1 := id
alt, ok := p.index[id1]
for i := 1; ok && alt != sub; i++ {
id1 = fmt.Sprintf("%s_%d", id, i)
alt, ok = p.index[id1]
}
p.index[id1] = sub
p.slice = append(p.slice, Placeholder{
ID: id1,
String: sub,
Type: arg.Type,
UnderlyingType: arg.UnderlyingType,
ArgNum: arg.ArgNum,
Expr: arg.Expr,
Comment: arg.Comment,
})
return id1
}

func getLastComponent(s string) string {
return s[1+strings.LastIndexByte(s, '.'):]
}

func msgStr(info *loader.PackageInfo, e ast.Expr) (s string, ok bool) {
v := info.Types[e].Value
if v == nil || v.Kind() != constant.String {
return "", false
}
s = constant.StringVal(v)
// Only record strings with letters.
for _, r := range s {
if unicode.In(r, unicode.L) {
return s, true
}
}
return "", false
}
Loading

0 comments on commit 3b24cac

Please sign in to comment.