Skip to content

Commit ae3dde1

Browse files
silverwindwxiaoguang
silverwind
and
wxiaoguang
authored
Rework file highlight rendering and fix yaml copy-paste (#19967)
* Rework file highlight rendering and fix yaml copy-paste * use Split+Trim to replace tag parser * remove unnecessary bytes.Count * remove newLineInHTML = "&#10;" Co-authored-by: wxiaoguang <[email protected]>
1 parent 0e61a74 commit ae3dde1

File tree

3 files changed

+177
-143
lines changed

3 files changed

+177
-143
lines changed

‎modules/highlight/highlight.go‎

Lines changed: 41 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"bytes"
1111
"fmt"
1212
gohtml "html"
13+
"io"
1314
"path/filepath"
1415
"strings"
1516
"sync"
@@ -26,7 +27,7 @@ import (
2627
)
2728

2829
// don't index files larger than this many bytes for performance purposes
29-
const sizeLimit = 1000000
30+
const sizeLimit = 1024 * 1024
3031

3132
var (
3233
// For custom user mapping
@@ -46,7 +47,6 @@ func NewContext() {
4647
highlightMapping[keys[i].Name()] = keys[i].Value()
4748
}
4849
}
49-
5050
// The size 512 is simply a conservative rule of thumb
5151
c, err := lru.New2Q(512)
5252
if err != nil {
@@ -60,7 +60,7 @@ func NewContext() {
6060
func Code(fileName, language, code string) string {
6161
NewContext()
6262

63-
// diff view newline will be passed as empty, change to literal \n so it can be copied
63+
// diff view newline will be passed as empty, change to literal '\n' so it can be copied
6464
// preserve literal newline in blame view
6565
if code == "" || code == "\n" {
6666
return "\n"
@@ -128,36 +128,32 @@ func CodeFromLexer(lexer chroma.Lexer, code string) string {
128128
return code
129129
}
130130

131-
htmlw.Flush()
131+
_ = htmlw.Flush()
132132
// Chroma will add newlines for certain lexers in order to highlight them properly
133-
// Once highlighted, strip them here so they don't cause copy/paste trouble in HTML output
133+
// Once highlighted, strip them here, so they don't cause copy/paste trouble in HTML output
134134
return strings.TrimSuffix(htmlbuf.String(), "\n")
135135
}
136136

137-
// File returns a slice of chroma syntax highlighted lines of code
138-
func File(numLines int, fileName, language string, code []byte) []string {
137+
// File returns a slice of chroma syntax highlighted HTML lines of code
138+
func File(fileName, language string, code []byte) ([]string, error) {
139139
NewContext()
140140

141141
if len(code) > sizeLimit {
142-
return plainText(string(code), numLines)
142+
return PlainText(code), nil
143143
}
144+
144145
formatter := html.New(html.WithClasses(true),
145146
html.WithLineNumbers(false),
146147
html.PreventSurroundingPre(true),
147148
)
148149

149-
if formatter == nil {
150-
log.Error("Couldn't create chroma formatter")
151-
return plainText(string(code), numLines)
152-
}
153-
154-
htmlbuf := bytes.Buffer{}
155-
htmlw := bufio.NewWriter(&htmlbuf)
150+
htmlBuf := bytes.Buffer{}
151+
htmlWriter := bufio.NewWriter(&htmlBuf)
156152

157153
var lexer chroma.Lexer
158154

159155
// provided language overrides everything
160-
if len(language) > 0 {
156+
if language != "" {
161157
lexer = lexers.Get(language)
162158
}
163159

@@ -168,9 +164,9 @@ func File(numLines int, fileName, language string, code []byte) []string {
168164
}
169165

170166
if lexer == nil {
171-
language := analyze.GetCodeLanguage(fileName, code)
167+
guessLanguage := analyze.GetCodeLanguage(fileName, code)
172168

173-
lexer = lexers.Get(language)
169+
lexer = lexers.Get(guessLanguage)
174170
if lexer == nil {
175171
lexer = lexers.Match(fileName)
176172
if lexer == nil {
@@ -181,54 +177,43 @@ func File(numLines int, fileName, language string, code []byte) []string {
181177

182178
iterator, err := lexer.Tokenise(nil, string(code))
183179
if err != nil {
184-
log.Error("Can't tokenize code: %v", err)
185-
return plainText(string(code), numLines)
180+
return nil, fmt.Errorf("can't tokenize code: %w", err)
186181
}
187182

188-
err = formatter.Format(htmlw, styles.GitHub, iterator)
183+
err = formatter.Format(htmlWriter, styles.GitHub, iterator)
189184
if err != nil {
190-
log.Error("Can't format code: %v", err)
191-
return plainText(string(code), numLines)
185+
return nil, fmt.Errorf("can't format code: %w", err)
192186
}
193187

194-
htmlw.Flush()
195-
finalNewLine := false
196-
if len(code) > 0 {
197-
finalNewLine = code[len(code)-1] == '\n'
198-
}
188+
_ = htmlWriter.Flush()
199189

200-
m := make([]string, 0, numLines)
201-
for _, v := range strings.SplitN(htmlbuf.String(), "\n", numLines) {
202-
content := v
203-
// need to keep lines that are only \n so copy/paste works properly in browser
204-
if content == "" {
205-
content = "\n"
206-
} else if content == `</span><span class="w">` {
207-
content += "\n</span>"
208-
} else if content == `</span></span><span class="line"><span class="cl">` {
209-
content += "\n"
210-
}
211-
content = strings.TrimSuffix(content, `<span class="w">`)
212-
content = strings.TrimPrefix(content, `</span>`)
213-
m = append(m, content)
190+
// at the moment, Chroma generates stable output `<span class="line"><span class="cl">...\n</span></span>` for each line
191+
htmlStr := htmlBuf.String()
192+
lines := strings.Split(htmlStr, `<span class="line"><span class="cl">`)
193+
m := make([]string, 0, len(lines))
194+
for i := 1; i < len(lines); i++ {
195+
line := lines[i]
196+
line = strings.TrimSuffix(line, "</span></span>")
197+
m = append(m, line)
214198
}
215-
if finalNewLine {
216-
m = append(m, "<span class=\"w\">\n</span>")
217-
}
218-
219-
return m
199+
return m, nil
220200
}
221201

222-
// return unhiglighted map
223-
func plainText(code string, numLines int) []string {
224-
m := make([]string, 0, numLines)
225-
for _, v := range strings.SplitN(code, "\n", numLines) {
226-
content := v
227-
// need to keep lines that are only \n so copy/paste works properly in browser
228-
if content == "" {
229-
content = "\n"
202+
// PlainText returns non-highlighted HTML for code
203+
func PlainText(code []byte) []string {
204+
r := bufio.NewReader(bytes.NewReader(code))
205+
m := make([]string, 0, bytes.Count(code, []byte{'\n'})+1)
206+
for {
207+
content, err := r.ReadString('\n')
208+
if err != nil && err != io.EOF {
209+
log.Error("failed to read string from buffer: %v", err)
210+
break
211+
}
212+
if content == "" && err == io.EOF {
213+
break
230214
}
231-
m = append(m, gohtml.EscapeString(content))
215+
s := gohtml.EscapeString(content)
216+
m = append(m, s)
232217
}
233218
return m
234219
}

‎modules/highlight/highlight_test.go‎

Lines changed: 123 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -8,97 +8,146 @@ import (
88
"strings"
99
"testing"
1010

11-
"code.gitea.io/gitea/modules/setting"
12-
"code.gitea.io/gitea/modules/util"
13-
1411
"github.com/stretchr/testify/assert"
15-
"gopkg.in/ini.v1"
1612
)
1713

14+
func lines(s string) []string {
15+
return strings.Split(strings.ReplaceAll(strings.TrimSpace(s), `\n`, "\n"), "\n")
16+
}
17+
1818
func TestFile(t *testing.T) {
19-
setting.Cfg = ini.Empty()
2019
tests := []struct {
21-
name string
22-
numLines int
23-
fileName string
24-
code string
25-
want string
20+
name string
21+
code string
22+
want []string
2623
}{
2724
{
28-
name: ".drone.yml",
29-
numLines: 12,
30-
fileName: ".drone.yml",
31-
code: util.Dedent(`
32-
kind: pipeline
33-
name: default
25+
name: "empty.py",
26+
code: "",
27+
want: lines(""),
28+
},
29+
{
30+
name: "tags.txt",
31+
code: "<>",
32+
want: lines("&lt;&gt;"),
33+
},
34+
{
35+
name: "tags.py",
36+
code: "<>",
37+
want: lines(`<span class="o">&lt;</span><span class="o">&gt;</span>`),
38+
},
39+
{
40+
name: "eol-no.py",
41+
code: "a=1",
42+
want: lines(`<span class="n">a</span><span class="o">=</span><span class="mi">1</span>`),
43+
},
44+
{
45+
name: "eol-newline1.py",
46+
code: "a=1\n",
47+
want: lines(`<span class="n">a</span><span class="o">=</span><span class="mi">1</span>\n`),
48+
},
49+
{
50+
name: "eol-newline2.py",
51+
code: "a=1\n\n",
52+
want: lines(`
53+
<span class="n">a</span><span class="o">=</span><span class="mi">1</span>\n
54+
\n
55+
`,
56+
),
57+
},
58+
{
59+
name: "empty-line-with-space.py",
60+
code: strings.ReplaceAll(strings.TrimSpace(`
61+
def:
62+
a=1
3463
35-
steps:
36-
- name: test
37-
image: golang:1.13
38-
environment:
39-
GOPROXY: https://goproxy.cn
40-
commands:
41-
- go get -u
42-
- go build -v
43-
- go test -v -race -coverprofile=coverage.txt -covermode=atomic
44-
`),
45-
want: util.Dedent(`
46-
<span class="line"><span class="cl"><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l">pipeline</span>
47-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">default</span>
48-
</span></span><span class="line"><span class="cl">
49-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">steps</span><span class="p">:</span>
50-
</span></span><span class="line"><span class="cl"><span class="w"></span>- <span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">test</span>
51-
</span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l">golang:1.13</span>
52-
</span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nt">environment</span><span class="p">:</span>
53-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w"> </span><span class="nt">GOPROXY</span><span class="p">:</span><span class="w"> </span><span class="l">https://goproxy.cn</span>
54-
</span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nt">commands</span><span class="p">:</span>
55-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w"> </span>- <span class="l">go get -u</span>
56-
</span></span><span class="line"><span class="cl"><span class="w"> </span>- <span class="l">go build -v</span>
57-
</span></span><span class="line"><span class="cl"><span class="w"> </span>- <span class="l">go test -v -race -coverprofile=coverage.txt -covermode=atomic</span></span></span>
64+
b=''
65+
{space}
66+
c=2
67+
`), "{space}", " "),
68+
want: lines(`
69+
<span class="n">def</span><span class="p">:</span>\n
70+
<span class="n">a</span><span class="o">=</span><span class="mi">1</span>\n
71+
\n
72+
<span class="n">b</span><span class="o">=</span><span class="sa"></span><span class="s1">&#39;</span><span class="s1">&#39;</span>\n
73+
\n
74+
<span class="n">c</span><span class="o">=</span><span class="mi">2</span>`,
75+
),
76+
},
77+
}
78+
79+
for _, tt := range tests {
80+
t.Run(tt.name, func(t *testing.T) {
81+
out, err := File(tt.name, "", []byte(tt.code))
82+
assert.NoError(t, err)
83+
expected := strings.Join(tt.want, "\n")
84+
actual := strings.Join(out, "\n")
85+
assert.Equal(t, strings.Count(actual, "<span"), strings.Count(actual, "</span>"))
86+
assert.EqualValues(t, expected, actual)
87+
})
88+
}
89+
}
90+
91+
func TestPlainText(t *testing.T) {
92+
tests := []struct {
93+
name string
94+
code string
95+
want []string
96+
}{
97+
{
98+
name: "empty.py",
99+
code: "",
100+
want: lines(""),
101+
},
102+
{
103+
name: "tags.py",
104+
code: "<>",
105+
want: lines("&lt;&gt;"),
106+
},
107+
{
108+
name: "eol-no.py",
109+
code: "a=1",
110+
want: lines(`a=1`),
111+
},
112+
{
113+
name: "eol-newline1.py",
114+
code: "a=1\n",
115+
want: lines(`a=1\n`),
116+
},
117+
{
118+
name: "eol-newline2.py",
119+
code: "a=1\n\n",
120+
want: lines(`
121+
a=1\n
122+
\n
58123
`),
59124
},
60125
{
61-
name: ".drone.yml - trailing space",
62-
numLines: 13,
63-
fileName: ".drone.yml",
64-
code: strings.Replace(util.Dedent(`
65-
kind: pipeline
66-
name: default
126+
name: "empty-line-with-space.py",
127+
code: strings.ReplaceAll(strings.TrimSpace(`
128+
def:
129+
a=1
67130
68-
steps:
69-
- name: test
70-
image: golang:1.13
71-
environment:
72-
GOPROXY: https://goproxy.cn
73-
commands:
74-
- go get -u
75-
- go build -v
76-
- go test -v -race -coverprofile=coverage.txt -covermode=atomic
77-
`)+"\n", "name: default", "name: default ", 1),
78-
want: util.Dedent(`
79-
<span class="line"><span class="cl"><span class="nt">kind</span><span class="p">:</span><span class="w"> </span><span class="l">pipeline</span>
80-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">default </span>
81-
</span></span><span class="line"><span class="cl">
82-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="nt">steps</span><span class="p">:</span>
83-
</span></span><span class="line"><span class="cl"><span class="w"></span>- <span class="nt">name</span><span class="p">:</span><span class="w"> </span><span class="l">test</span>
84-
</span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nt">image</span><span class="p">:</span><span class="w"> </span><span class="l">golang:1.13</span>
85-
</span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nt">environment</span><span class="p">:</span>
86-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w"> </span><span class="nt">GOPROXY</span><span class="p">:</span><span class="w"> </span><span class="l">https://goproxy.cn</span>
87-
</span></span><span class="line"><span class="cl"><span class="w"> </span><span class="nt">commands</span><span class="p">:</span>
88-
</span></span><span class="line"><span class="cl"><span class="w"></span><span class="w"> </span>- <span class="l">go get -u</span>
89-
</span></span><span class="line"><span class="cl"><span class="w"> </span>- <span class="l">go build -v</span>
90-
</span></span><span class="line"><span class="cl"><span class="w"> </span>- <span class="l">go test -v -race -coverprofile=coverage.txt -covermode=atomic</span>
91-
</span></span>
92-
<span class="w">
93-
</span>
94-
`),
131+
b=''
132+
{space}
133+
c=2
134+
`), "{space}", " "),
135+
want: lines(`
136+
def:\n
137+
a=1\n
138+
\n
139+
b=&#39;&#39;\n
140+
\n
141+
c=2`),
95142
},
96143
}
97144

98145
for _, tt := range tests {
99146
t.Run(tt.name, func(t *testing.T) {
100-
got := strings.Join(File(tt.numLines, tt.fileName, "", []byte(tt.code)), "\n")
101-
assert.Equal(t, tt.want, got)
147+
out := PlainText([]byte(tt.code))
148+
expected := strings.Join(tt.want, "\n")
149+
actual := strings.Join(out, "\n")
150+
assert.EqualValues(t, expected, actual)
102151
})
103152
}
104153
}

0 commit comments

Comments
 (0)