diff --git a/_tools/pygments2chroma.py b/_tools/pygments2chroma.py index 76efbaebc..103ac192d 100644 --- a/_tools/pygments2chroma.py +++ b/_tools/pygments2chroma.py @@ -16,10 +16,11 @@ import ( . "github.com/alecthomas/chroma" // nolint + "github.com/alecthomas/chroma/lexers/internal" ) // {{upper_name}} lexer. -var {{upper_name}} = Register(MustNewLexer( +var {{upper_name}} = internal.Register(MustNewLexer( &Config{ Name: "{{name}}", Aliases: []string{ {{#aliases}}"{{.}}", {{/aliases}} }, @@ -193,4 +194,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/cmd/chroma/main.go b/cmd/chroma/main.go index 4b5acd610..3e8448628 100644 --- a/cmd/chroma/main.go +++ b/cmd/chroma/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "fmt" + "github.com/alecthomas/kong" "io" "io/ioutil" "os" @@ -13,15 +14,13 @@ import ( "strconv" "strings" - "github.com/mattn/go-colorable" - "github.com/mattn/go-isatty" - "gopkg.in/alecthomas/kingpin.v3-unstable" - "github.com/alecthomas/chroma" "github.com/alecthomas/chroma/formatters" "github.com/alecthomas/chroma/formatters/html" "github.com/alecthomas/chroma/lexers" "github.com/alecthomas/chroma/styles" + "github.com/mattn/go-colorable" + "github.com/mattn/go-isatty" ) var ( @@ -30,34 +29,42 @@ var ( commit = "?" date = "?" - profileFlag = kingpin.Flag("profile", "Enable profiling to file.").Hidden().String() - listFlag = kingpin.Flag("list", "List lexers, styles and formatters.").Bool() - unbufferedFlag = kingpin.Flag("unbuffered", "Do not buffer output.").Bool() - traceFlag = kingpin.Flag("trace", "Trace lexer states as they are traversed.").Bool() - checkFlag = kingpin.Flag("check", "Do not format, check for tokenization errors instead.").Bool() - filenameFlag = kingpin.Flag("filename", "Filename to use for selecting a lexer when reading from stdin.").String() + description = ` +Chroma is a general purpose syntax highlighting library and corresponding +command, for Go. +` + + cli struct { + Version kong.VersionFlag `help:"Show version."` + Profile string `hidden:"" help:"Enable profiling to file."` + List bool `help:"List lexers, styles and formatters."` + Unbuffered bool `help:"Do not buffer output."` + Trace bool `help:"Trace lexer states as they are traversed."` + Check bool `help:"Do not format, check for tokenization errors instead."` + Filename string `help:"Filename to use for selecting a lexer when reading from stdin."` - lexerFlag = kingpin.Flag("lexer", "Lexer to use when formatting.").PlaceHolder("autodetect").Short('l').Enum(lexers.Names(true)...) - styleFlag = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...) - formatterFlag = kingpin.Flag("formatter", "Formatter to use.").Default("terminal").Short('f').Enum(formatters.Names()...) + Lexer string `help:"Lexer to use when formatting." placeholder:"autodetect" short:"l"` + Style string `help:"Style to use for formatting." default:"swapoff" short:"s"` + Formatter string `help:"Formatter to use." default:"terminal" short:"f"` - jsonFlag = kingpin.Flag("json", "Output JSON representation of tokens.").Bool() + JSON bool `help:"Output JSON representation of tokens."` - htmlFlag = kingpin.Flag("html", "Enable HTML mode (equivalent to '--formatter html').").Bool() - htmlPrefixFlag = kingpin.Flag("html-prefix", "HTML CSS class prefix.").PlaceHolder("PREFIX").String() - htmlStylesFlag = kingpin.Flag("html-styles", "Output HTML CSS styles.").Bool() - htmlOnlyFlag = kingpin.Flag("html-only", "Output HTML fragment.").Bool() - htmlInlineStyleFlag = kingpin.Flag("html-inline-styles", "Output HTML with inline styles (no classes).").Bool() - htmlTabWidthFlag = kingpin.Flag("html-tab-width", "Set the HTML tab width.").Default("8").Int() - htmlLinesFlag = kingpin.Flag("html-lines", "Include line numbers in output.").Bool() - htmlLinesTableFlag = kingpin.Flag("html-lines-table", "Split line numbers and code in a HTML table").Bool() - htmlLinesStyleFlag = kingpin.Flag("html-lines-style", "Style for line numbers.").String() - htmlHighlightFlag = kingpin.Flag("html-highlight", "Highlight these lines.").PlaceHolder("N[:M][,...]").String() - htmlHighlightStyleFlag = kingpin.Flag("html-highlight-style", "Style used for highlighting lines.").String() - htmlBaseLineFlag = kingpin.Flag("html-base-line", "Base line number.").Default("1").Int() - htmlPreventSurroundingPreFlag = kingpin.Flag("html-prevent-surrounding-pre", "Prevent the surrounding pre tag.").Bool() + HTML bool `help:"Enable HTML mode (equivalent to '--formatter html')."` + HTMLPrefix string `help:"HTML CSS class prefix." placeholder:"PREFIX"` + HTMLStyles bool `help:"Output HTML CSS styles."` + HTMLOnly bool `help:"Output HTML fragment."` + HTMLInlineStyles bool `help:"Output HTML with inline styles (no classes)."` + HTMLTabWidth int `help:"Set the HTML tab width." default:"8"` + HTMLLines bool `help:"Include line numbers in output."` + HTMLLinesTable bool `help:"Split line numbers and code in a HTML table"` + HTMLLinesStyle string `help:"Style for line numbers."` + HTMLHighlight string `help:"Highlight these lines." placeholder:"N[:M][,...]"` + HTMLHighlightStyle string `help:"Style used for highlighting lines."` + HTMLBaseLine int `help:"Base line number." default:"1"` + HTMLPreventSurroundingPre bool `help:"Prevent the surrounding pre tag."` - filesArgs = kingpin.Arg("files", "Files to highlight.").ExistingFiles() + Files []string `arg:"" help:"Files to highlight." type:"existingfile"` + } ) type flushableWriter interface { @@ -70,19 +77,16 @@ type nopFlushableWriter struct{ io.Writer } func (n *nopFlushableWriter) Flush() error { return nil } func main() { - kingpin.CommandLine.Version(fmt.Sprintf("%s-%s-%s", version, commit, date)) - kingpin.CommandLine.Help = ` -Chroma is a general purpose syntax highlighting library and corresponding -command, for Go. -` - kingpin.Parse() - if *listFlag { + ctx := kong.Parse(&cli, kong.Description(description), kong.Vars{ + "version": fmt.Sprintf("%s-%s-%s", version, commit, date), + }, ) + if cli.List { listAll() return } - if *profileFlag != "" { - f, err := os.Create(*profileFlag) - kingpin.FatalIfError(err, "") + if cli.Profile != "" { + f, err := os.Create(cli.Profile) + ctx.FatalIfErrorf(err) pprof.StartCPUProfile(f) signals := make(chan os.Signal, 1) signal.Notify(signals, os.Interrupt) @@ -99,75 +103,75 @@ command, for Go. out = colorable.NewColorableStdout() } var w flushableWriter - if *unbufferedFlag { + if cli.Unbuffered { w = &nopFlushableWriter{out} } else { w = bufio.NewWriterSize(out, 16384) } defer w.Flush() - if *jsonFlag { - *formatterFlag = "json" + if cli.JSON { + cli.Formatter = "json" } - if *htmlFlag { - *formatterFlag = "html" + if cli.HTML { + cli.Formatter = "html" } // Retrieve user-specified style, clone it, and add some overrides. - builder := styles.Get(*styleFlag).Builder() - if *htmlHighlightStyleFlag != "" { - builder.Add(chroma.LineHighlight, *htmlHighlightStyleFlag) + builder := styles.Get(cli.Style).Builder() + if cli.HTMLHighlightStyle != "" { + builder.Add(chroma.LineHighlight, cli.HTMLHighlightStyle) } - if *htmlLinesStyleFlag != "" { - builder.Add(chroma.LineNumbers, *htmlLinesStyleFlag) + if cli.HTMLLinesStyle != "" { + builder.Add(chroma.LineNumbers, cli.HTMLLinesStyle) } style, err := builder.Build() - kingpin.FatalIfError(err, "") + ctx.FatalIfErrorf(err) // Dump styles. - if *htmlStylesFlag { + if cli.HTMLStyles { formatter := html.New(html.WithClasses()) formatter.WriteCSS(w, style) return } - if *formatterFlag == "html" { + if cli.Formatter == "html" { options := []html.Option{ - html.TabWidth(*htmlTabWidthFlag), - html.BaseLineNumber(*htmlBaseLineFlag), + html.TabWidth(cli.HTMLTabWidth), + html.BaseLineNumber(cli.HTMLBaseLine), } - if *htmlPrefixFlag != "" { - options = append(options, html.ClassPrefix(*htmlPrefixFlag)) + if cli.HTMLPrefix != "" { + options = append(options, html.ClassPrefix(cli.HTMLPrefix)) } - if !*htmlInlineStyleFlag { + if !cli.HTMLInlineStyles { options = append(options, html.WithClasses()) } - if !*htmlOnlyFlag { + if !cli.HTMLOnly { options = append(options, html.Standalone()) } - if *htmlLinesFlag { + if cli.HTMLLines { options = append(options, html.WithLineNumbers()) } - if *htmlLinesTableFlag { + if cli.HTMLLinesTable { options = append(options, html.LineNumbersInTable()) } - if *htmlPreventSurroundingPreFlag { + if cli.HTMLPreventSurroundingPre { options = append(options, html.PreventSurroundingPre()) } - if len(*htmlHighlightFlag) > 0 { + if len(cli.HTMLHighlight) > 0 { ranges := [][2]int{} - for _, span := range strings.Split(*htmlHighlightFlag, ",") { + for _, span := range strings.Split(cli.HTMLHighlight, ",") { parts := strings.Split(span, ":") if len(parts) > 2 { - kingpin.Fatalf("range should be N[:M], not %q", span) + ctx.Fatalf("range should be N[:M], not %q", span) } start, err := strconv.ParseInt(parts[0], 10, 64) - kingpin.FatalIfError(err, "min value of range should be integer not %q", parts[0]) + ctx.FatalIfErrorf(err, "min value of range should be integer not %q", parts[0]) end := start if len(parts) == 2 { end, err = strconv.ParseInt(parts[1], 10, 64) - kingpin.FatalIfError(err, "max value of range should be integer not %q", parts[1]) + ctx.FatalIfErrorf(err, "max value of range should be integer not %q", parts[1]) } ranges = append(ranges, [2]int{int(start), int(end)}) } @@ -175,18 +179,18 @@ command, for Go. } formatters.Register("html", html.New(options...)) } - if len(*filesArgs) == 0 { + if len(cli.Files) == 0 { contents, err := ioutil.ReadAll(os.Stdin) - kingpin.FatalIfError(err, "") - format(w, style, lex(*filenameFlag, string(contents))) + ctx.FatalIfErrorf(err) + format(ctx, w, style, lex(ctx, cli.Filename, string(contents))) } else { - for _, filename := range *filesArgs { + for _, filename := range cli.Files { contents, err := ioutil.ReadFile(filename) - kingpin.FatalIfError(err, "") - if *checkFlag { - check(filename, lex(filename, string(contents))) + ctx.FatalIfErrorf(err) + if cli.Check { + check(filename, lex(ctx, filename, string(contents))) } else { - format(w, style, lex(filename, string(contents))) + format(ctx, w, style, lex(ctx, filename, string(contents))) } } } @@ -224,23 +228,23 @@ func listAll() { fmt.Println() } -func lex(path string, contents string) chroma.Iterator { +func lex(ctx *kong.Context, path string, contents string) chroma.Iterator { lexer := selexer(path, contents) if lexer == nil { lexer = lexers.Fallback } if rel, ok := lexer.(*chroma.RegexLexer); ok { - rel.Trace(*traceFlag) + rel.Trace(cli.Trace) } lexer = chroma.Coalesce(lexer) it, err := lexer.Tokenise(nil, string(contents)) - kingpin.FatalIfError(err, "") + ctx.FatalIfErrorf(err) return it } func selexer(path, contents string) (lexer chroma.Lexer) { - if *lexerFlag != "" { - return lexers.Get(*lexerFlag) + if cli.Lexer != "" { + return lexers.Get(cli.Lexer) } if path != "" { lexer := lexers.Match(path) @@ -251,10 +255,10 @@ func selexer(path, contents string) (lexer chroma.Lexer) { return lexers.Analyse(contents) } -func format(w io.Writer, style *chroma.Style, it chroma.Iterator) { - formatter := formatters.Get(*formatterFlag) +func format(ctx *kong.Context, w io.Writer, style *chroma.Style, it chroma.Iterator) { + formatter := formatters.Get(cli.Formatter) err := formatter.Format(w, style, it) - kingpin.FatalIfError(err, "") + ctx.FatalIfErrorf(err) } func check(filename string, it chroma.Iterator) { diff --git a/delegate_test.go b/delegate_test.go index b45fb58ae..670af5664 100644 --- a/delegate_test.go +++ b/delegate_test.go @@ -1,7 +1,6 @@ package chroma import ( - "fmt" "testing" "github.com/alecthomas/assert" @@ -105,7 +104,6 @@ func TestDelegate(t *testing.T) { it, err := delegate.Tokenise(nil, test.source) assert.NoError(t, err) actual := it.Tokens() - fmt.Println(actual) assert.Equal(t, test.expected, actual) }) } diff --git a/go.mod b/go.mod new file mode 100644 index 000000000..9d0aa86d1 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module github.com/alecthomas/chroma + +require ( + github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 + github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 // indirect + github.com/alecthomas/kong v0.1.15 + github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 // indirect + github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 + github.com/dlclark/regexp2 v1.1.6 + github.com/mattn/go-colorable v0.0.9 + github.com/mattn/go-isatty v0.0.4 + github.com/sergi/go-diff v1.0.0 // indirect + golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..27210bbb2 --- /dev/null +++ b/go.sum @@ -0,0 +1,26 @@ +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= +github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo= +github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= +github.com/alecthomas/kong v0.1.15 h1:IWBg+KrLvoHBicD50OzMI8fKjrtAa1okMR9g38HVM/s= +github.com/alecthomas/kong v0.1.15/go.mod h1:0m2VYms8rH0qbCqVB2gvGHk74bqLIq0HXjCs5bNbNQU= +github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY= +github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ= +github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg= +github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 h1:YAFjXN64LMvktoUZH9zgY4lGc/msGN7HQfoSuKCgaDU= +golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/lexers/testdata/vbnet.actual b/lexers/testdata/vbnet.actual new file mode 100644 index 000000000..99fd706da --- /dev/null +++ b/lexers/testdata/vbnet.actual @@ -0,0 +1,7 @@ +Imports System + +Public Module Hello + Public Sub Main( ) + Console.WriteLine("hello, world") + End Sub +End Module diff --git a/lexers/testdata/vbnet.expected b/lexers/testdata/vbnet.expected new file mode 100644 index 000000000..963c2ff13 --- /dev/null +++ b/lexers/testdata/vbnet.expected @@ -0,0 +1,36 @@ +[ + {"type":"Keyword","value":"Imports"}, + {"type":"Text","value":" "}, + {"type":"NameNamespace","value":"System"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"Public"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"Module"}, + {"type":"Text","value":" "}, + {"type":"NameNamespace","value":"Hello"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"Public"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"Sub"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"Main"}, + {"type":"Punctuation","value":"("}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"Console"}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"WriteLine"}, + {"type":"Punctuation","value":"("}, + {"type":"LiteralString","value":"\"hello, world\""}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"End"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"Sub"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"End"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"Module"}, + {"type":"Text","value":"\n"} +] diff --git a/lexers/v/vb.go b/lexers/v/vb.go new file mode 100644 index 000000000..38a9185c8 --- /dev/null +++ b/lexers/v/vb.go @@ -0,0 +1,75 @@ + +package v + +import ( + . "github.com/alecthomas/chroma" // nolint + "github.com/alecthomas/chroma/lexers/internal" +) + +const vbName = `[_\w][\w]*` + +// VB.Net lexer. +var VBNet = internal.Register(MustNewLexer( + &Config{ + Name: "VB.net", + Aliases: []string{ "vb.net", "vbnet", }, + Filenames: []string{ "*.vb", "*.bas", }, + MimeTypes: []string{ "text/x-vbnet", "text/x-vba", }, + CaseInsensitive: true, + }, + Rules{ + "root": { + { `^\s*<.*?>`, NameAttribute, nil }, + { `\s+`, Text, nil }, + { `\n`, Text, nil }, + { `rem\b.*?\n`, Comment, nil }, + { `'.*?\n`, Comment, nil }, + { `#If\s.*?\sThen|#ElseIf\s.*?\sThen|#Else|#End\s+If|#Const|#ExternalSource.*?\n|#End\s+ExternalSource|#Region.*?\n|#End\s+Region|#ExternalChecksum`, CommentPreproc, nil }, + { `[(){}!#,.:]`, Punctuation, nil }, + { `Option\s+(Strict|Explicit|Compare)\s+(On|Off|Binary|Text)`, KeywordDeclaration, nil }, + { Words(`(?>=|<<|>>|:=|<=|>=|<>|[-&*/\\^+=<>\[\]]`, Operator, nil }, + { `"`, LiteralString, Push("string") }, + { `_\n`, Text, nil }, + { vbName, Name, nil }, + { `#.*?#`, LiteralDate, nil }, + { `(\d+\.\d*|\d*\.\d+)(F[+-]?[0-9]+)?`, LiteralNumberFloat, nil }, + { `\d+([SILDFR]|US|UI|UL)?`, LiteralNumberInteger, nil }, + { `&H[0-9a-f]+([SILDFR]|US|UI|UL)?`, LiteralNumberInteger, nil }, + { `&O[0-7]+([SILDFR]|US|UI|UL)?`, LiteralNumberInteger, nil }, + }, + "string": { + { `""`, LiteralString, nil }, + { `"C?`, LiteralString, Pop(1) }, + { `[^"]+`, LiteralString, nil }, + }, + "dim": { + { vbName, NameVariable, Pop(1) }, + Default(Pop(1)), + }, + "funcname": { + { vbName, NameFunction, Pop(1) }, + }, + "classname": { + { vbName, NameClass, Pop(1) }, + }, + "namespace": { + { vbName, NameNamespace, nil }, + { `\.`, NameNamespace, nil }, + Default(Pop(1)), + }, + "end": { + { `\s+`, Text, nil }, + { `(Function|Sub|Property|Class|Structure|Enum|Module|Namespace)\b`, Keyword, Pop(1) }, + Default(Pop(1)), + }, + }, +)) +