Skip to content

Commit

Permalink
Support go modules + VB.Net lexer.
Browse files Browse the repository at this point in the history 
Fixes #201.
  • Loading branch information
@alecthomas
alecthomas committed Dec 4, 2018
1 parent 813e33c commit 3175fa5
Show file tree
Hide file tree
Showing 8 changed files with 246 additions and 85 deletions.
5 changes: 3 additions & 2 deletions _tools/pygments2chroma.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@
import (
. "github.com/alecthomas/chroma" // nolint
"github.com/alecthomas/chroma/lexers/internal"
)
// {{upper_name}} lexer.
var {{upper_name}} = Register(MustNewLexer(
var {{upper_name}} = internal.Register(MustNewLexer(
&Config{
Name: "{{name}}",
Aliases: []string{ {{#aliases}}"{{.}}", {{/aliases}} },
Expand Down Expand Up @@ -193,4 +194,4 @@ def main():


if __name__ == '__main__':
main()
main()
166 changes: 85 additions & 81 deletions cmd/chroma/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"bufio"
"fmt"
"github.com/alecthomas/kong"
"io"
"io/ioutil"
"os"
Expand All @@ -13,15 +14,13 @@ import (
"strconv"
"strings"

"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
"gopkg.in/alecthomas/kingpin.v3-unstable"

"github.com/alecthomas/chroma"
"github.com/alecthomas/chroma/formatters"
"github.com/alecthomas/chroma/formatters/html"
"github.com/alecthomas/chroma/lexers"
"github.com/alecthomas/chroma/styles"
"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
)

var (
Expand All @@ -30,34 +29,42 @@ var (
commit = "?"
date = "?"

profileFlag = kingpin.Flag("profile", "Enable profiling to file.").Hidden().String()
listFlag = kingpin.Flag("list", "List lexers, styles and formatters.").Bool()
unbufferedFlag = kingpin.Flag("unbuffered", "Do not buffer output.").Bool()
traceFlag = kingpin.Flag("trace", "Trace lexer states as they are traversed.").Bool()
checkFlag = kingpin.Flag("check", "Do not format, check for tokenization errors instead.").Bool()
filenameFlag = kingpin.Flag("filename", "Filename to use for selecting a lexer when reading from stdin.").String()
description = `
Chroma is a general purpose syntax highlighting library and corresponding
command, for Go.
`

cli struct {
Version kong.VersionFlag `help:"Show version."`
Profile string `hidden:"" help:"Enable profiling to file."`
List bool `help:"List lexers, styles and formatters."`
Unbuffered bool `help:"Do not buffer output."`
Trace bool `help:"Trace lexer states as they are traversed."`
Check bool `help:"Do not format, check for tokenization errors instead."`
Filename string `help:"Filename to use for selecting a lexer when reading from stdin."`

lexerFlag = kingpin.Flag("lexer", "Lexer to use when formatting.").PlaceHolder("autodetect").Short('l').Enum(lexers.Names(true)...)
styleFlag = kingpin.Flag("style", "Style to use for formatting.").Short('s').Default("swapoff").Enum(styles.Names()...)
formatterFlag = kingpin.Flag("formatter", "Formatter to use.").Default("terminal").Short('f').Enum(formatters.Names()...)
Lexer string `help:"Lexer to use when formatting." placeholder:"autodetect" short:"l"`
Style string `help:"Style to use for formatting." default:"swapoff" short:"s"`
Formatter string `help:"Formatter to use." default:"terminal" short:"f"`

jsonFlag = kingpin.Flag("json", "Output JSON representation of tokens.").Bool()
JSON bool `help:"Output JSON representation of tokens."`

htmlFlag = kingpin.Flag("html", "Enable HTML mode (equivalent to '--formatter html').").Bool()
htmlPrefixFlag = kingpin.Flag("html-prefix", "HTML CSS class prefix.").PlaceHolder("PREFIX").String()
htmlStylesFlag = kingpin.Flag("html-styles", "Output HTML CSS styles.").Bool()
htmlOnlyFlag = kingpin.Flag("html-only", "Output HTML fragment.").Bool()
htmlInlineStyleFlag = kingpin.Flag("html-inline-styles", "Output HTML with inline styles (no classes).").Bool()
htmlTabWidthFlag = kingpin.Flag("html-tab-width", "Set the HTML tab width.").Default("8").Int()
htmlLinesFlag = kingpin.Flag("html-lines", "Include line numbers in output.").Bool()
htmlLinesTableFlag = kingpin.Flag("html-lines-table", "Split line numbers and code in a HTML table").Bool()
htmlLinesStyleFlag = kingpin.Flag("html-lines-style", "Style for line numbers.").String()
htmlHighlightFlag = kingpin.Flag("html-highlight", "Highlight these lines.").PlaceHolder("N[:M][,...]").String()
htmlHighlightStyleFlag = kingpin.Flag("html-highlight-style", "Style used for highlighting lines.").String()
htmlBaseLineFlag = kingpin.Flag("html-base-line", "Base line number.").Default("1").Int()
htmlPreventSurroundingPreFlag = kingpin.Flag("html-prevent-surrounding-pre", "Prevent the surrounding pre tag.").Bool()
HTML bool `help:"Enable HTML mode (equivalent to '--formatter html')."`
HTMLPrefix string `help:"HTML CSS class prefix." placeholder:"PREFIX"`
HTMLStyles bool `help:"Output HTML CSS styles."`
HTMLOnly bool `help:"Output HTML fragment."`
HTMLInlineStyles bool `help:"Output HTML with inline styles (no classes)."`
HTMLTabWidth int `help:"Set the HTML tab width." default:"8"`
HTMLLines bool `help:"Include line numbers in output."`
HTMLLinesTable bool `help:"Split line numbers and code in a HTML table"`
HTMLLinesStyle string `help:"Style for line numbers."`
HTMLHighlight string `help:"Highlight these lines." placeholder:"N[:M][,...]"`
HTMLHighlightStyle string `help:"Style used for highlighting lines."`
HTMLBaseLine int `help:"Base line number." default:"1"`
HTMLPreventSurroundingPre bool `help:"Prevent the surrounding pre tag."`

filesArgs = kingpin.Arg("files", "Files to highlight.").ExistingFiles()
Files []string `arg:"" help:"Files to highlight." type:"existingfile"`
}
)

type flushableWriter interface {
Expand All @@ -70,19 +77,16 @@ type nopFlushableWriter struct{ io.Writer }
func (n *nopFlushableWriter) Flush() error { return nil }

func main() {
kingpin.CommandLine.Version(fmt.Sprintf("%s-%s-%s", version, commit, date))
kingpin.CommandLine.Help = `
Chroma is a general purpose syntax highlighting library and corresponding
command, for Go.
`
kingpin.Parse()
if *listFlag {
ctx := kong.Parse(&cli, kong.Description(description), kong.Vars{
"version": fmt.Sprintf("%s-%s-%s", version, commit, date),
}, )
if cli.List {
listAll()
return
}
if *profileFlag != "" {
f, err := os.Create(*profileFlag)
kingpin.FatalIfError(err, "")
if cli.Profile != "" {
f, err := os.Create(cli.Profile)
ctx.FatalIfErrorf(err)
pprof.StartCPUProfile(f)
signals := make(chan os.Signal, 1)
signal.Notify(signals, os.Interrupt)
Expand All @@ -99,94 +103,94 @@ command, for Go.
out = colorable.NewColorableStdout()
}
var w flushableWriter
if *unbufferedFlag {
if cli.Unbuffered {
w = &nopFlushableWriter{out}
} else {
w = bufio.NewWriterSize(out, 16384)
}
defer w.Flush()

if *jsonFlag {
*formatterFlag = "json"
if cli.JSON {
cli.Formatter = "json"
}

if *htmlFlag {
*formatterFlag = "html"
if cli.HTML {
cli.Formatter = "html"
}

// Retrieve user-specified style, clone it, and add some overrides.
builder := styles.Get(*styleFlag).Builder()
if *htmlHighlightStyleFlag != "" {
builder.Add(chroma.LineHighlight, *htmlHighlightStyleFlag)
builder := styles.Get(cli.Style).Builder()
if cli.HTMLHighlightStyle != "" {
builder.Add(chroma.LineHighlight, cli.HTMLHighlightStyle)
}
if *htmlLinesStyleFlag != "" {
builder.Add(chroma.LineNumbers, *htmlLinesStyleFlag)
if cli.HTMLLinesStyle != "" {
builder.Add(chroma.LineNumbers, cli.HTMLLinesStyle)
}
style, err := builder.Build()
kingpin.FatalIfError(err, "")
ctx.FatalIfErrorf(err)

// Dump styles.
if *htmlStylesFlag {
if cli.HTMLStyles {
formatter := html.New(html.WithClasses())
formatter.WriteCSS(w, style)
return
}

if *formatterFlag == "html" {
if cli.Formatter == "html" {
options := []html.Option{
html.TabWidth(*htmlTabWidthFlag),
html.BaseLineNumber(*htmlBaseLineFlag),
html.TabWidth(cli.HTMLTabWidth),
html.BaseLineNumber(cli.HTMLBaseLine),
}
if *htmlPrefixFlag != "" {
options = append(options, html.ClassPrefix(*htmlPrefixFlag))
if cli.HTMLPrefix != "" {
options = append(options, html.ClassPrefix(cli.HTMLPrefix))
}
if !*htmlInlineStyleFlag {
if !cli.HTMLInlineStyles {
options = append(options, html.WithClasses())
}
if !*htmlOnlyFlag {
if !cli.HTMLOnly {
options = append(options, html.Standalone())
}
if *htmlLinesFlag {
if cli.HTMLLines {
options = append(options, html.WithLineNumbers())
}
if *htmlLinesTableFlag {
if cli.HTMLLinesTable {
options = append(options, html.LineNumbersInTable())
}
if *htmlPreventSurroundingPreFlag {
if cli.HTMLPreventSurroundingPre {
options = append(options, html.PreventSurroundingPre())
}
if len(*htmlHighlightFlag) > 0 {
if len(cli.HTMLHighlight) > 0 {
ranges := [][2]int{}
for _, span := range strings.Split(*htmlHighlightFlag, ",") {
for _, span := range strings.Split(cli.HTMLHighlight, ",") {
parts := strings.Split(span, ":")
if len(parts) > 2 {
kingpin.Fatalf("range should be N[:M], not %q", span)
ctx.Fatalf("range should be N[:M], not %q", span)
}
start, err := strconv.ParseInt(parts[0], 10, 64)
kingpin.FatalIfError(err, "min value of range should be integer not %q", parts[0])
ctx.FatalIfErrorf(err, "min value of range should be integer not %q", parts[0])
end := start
if len(parts) == 2 {
end, err = strconv.ParseInt(parts[1], 10, 64)
kingpin.FatalIfError(err, "max value of range should be integer not %q", parts[1])
ctx.FatalIfErrorf(err, "max value of range should be integer not %q", parts[1])
}
ranges = append(ranges, [2]int{int(start), int(end)})
}
options = append(options, html.HighlightLines(ranges))
}
formatters.Register("html", html.New(options...))
}
if len(*filesArgs) == 0 {
if len(cli.Files) == 0 {
contents, err := ioutil.ReadAll(os.Stdin)
kingpin.FatalIfError(err, "")
format(w, style, lex(*filenameFlag, string(contents)))
ctx.FatalIfErrorf(err)
format(ctx, w, style, lex(ctx, cli.Filename, string(contents)))
} else {
for _, filename := range *filesArgs {
for _, filename := range cli.Files {
contents, err := ioutil.ReadFile(filename)
kingpin.FatalIfError(err, "")
if *checkFlag {
check(filename, lex(filename, string(contents)))
ctx.FatalIfErrorf(err)
if cli.Check {
check(filename, lex(ctx, filename, string(contents)))
} else {
format(w, style, lex(filename, string(contents)))
format(ctx, w, style, lex(ctx, filename, string(contents)))
}
}
}
Expand Down Expand Up @@ -224,23 +228,23 @@ func listAll() {
fmt.Println()
}

func lex(path string, contents string) chroma.Iterator {
func lex(ctx *kong.Context, path string, contents string) chroma.Iterator {
lexer := selexer(path, contents)
if lexer == nil {
lexer = lexers.Fallback
}
if rel, ok := lexer.(*chroma.RegexLexer); ok {
rel.Trace(*traceFlag)
rel.Trace(cli.Trace)
}
lexer = chroma.Coalesce(lexer)
it, err := lexer.Tokenise(nil, string(contents))
kingpin.FatalIfError(err, "")
ctx.FatalIfErrorf(err)
return it
}

func selexer(path, contents string) (lexer chroma.Lexer) {
if *lexerFlag != "" {
return lexers.Get(*lexerFlag)
if cli.Lexer != "" {
return lexers.Get(cli.Lexer)
}
if path != "" {
lexer := lexers.Match(path)
Expand All @@ -251,10 +255,10 @@ func selexer(path, contents string) (lexer chroma.Lexer) {
return lexers.Analyse(contents)
}

func format(w io.Writer, style *chroma.Style, it chroma.Iterator) {
formatter := formatters.Get(*formatterFlag)
func format(ctx *kong.Context, w io.Writer, style *chroma.Style, it chroma.Iterator) {
formatter := formatters.Get(cli.Formatter)
err := formatter.Format(w, style, it)
kingpin.FatalIfError(err, "")
ctx.FatalIfErrorf(err)
}

func check(filename string, it chroma.Iterator) {
Expand Down
2 changes: 0 additions & 2 deletions delegate_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package chroma

import (
"fmt"
"testing"

"github.com/alecthomas/assert"
Expand Down Expand Up @@ -105,7 +104,6 @@ func TestDelegate(t *testing.T) {
it, err := delegate.Tokenise(nil, test.source)
assert.NoError(t, err)
actual := it.Tokens()
fmt.Println(actual)
assert.Equal(t, test.expected, actual)
})
}
Expand Down
14 changes: 14 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module github.com/alecthomas/chroma

require (
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 // indirect
github.com/alecthomas/kong v0.1.15
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 // indirect
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964
github.com/dlclark/regexp2 v1.1.6
github.com/mattn/go-colorable v0.0.9
github.com/mattn/go-isatty v0.0.4
github.com/sergi/go-diff v1.0.0 // indirect
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
)
26 changes: 26 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
github.com/alecthomas/kong v0.1.15 h1:IWBg+KrLvoHBicD50OzMI8fKjrtAa1okMR9g38HVM/s=
github.com/alecthomas/kong v0.1.15/go.mod h1:0m2VYms8rH0qbCqVB2gvGHk74bqLIq0HXjCs5bNbNQU=
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY=
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964 h1:y5HC9v93H5EPKqaS1UYVg1uYah5Xf51mBfIoWehClUQ=
github.com/danwakefield/fnmatch v0.0.0-20160403171240-cbb64ac3d964/go.mod h1:Xd9hchkHSWYkEqJwUGisez3G1QY8Ryz0sdWrLPMGjLk=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.1.6 h1:CqB4MjHw0MFCDj+PHHjiESmHX+N7t0tJzKvC6M97BRg=
github.com/dlclark/regexp2 v1.1.6/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 h1:YAFjXN64LMvktoUZH9zgY4lGc/msGN7HQfoSuKCgaDU=
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
7 changes: 7 additions & 0 deletions lexers/testdata/vbnet.actual
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Imports System

Public Module Hello
Public Sub Main( )
Console.WriteLine("hello, world")
End Sub
End Module
Loading

0 comments on commit 3175fa5

Please sign in to comment.