Skip to content

Commit 42b3935

Browse files
committed
Fix incorrect keyword lexing
For some weird reason this line is detected as a keyword line: ``` type: :module, ``` Though it clearly isn't. Ripper: ``` require 'ripper' pp Ripper.lex(<<~'EOM') { type: :module, } EOM ``` Produces: ``` [[[1, 0], :on_lbrace, "{", BEG|LABEL], [[1, 1], :on_ignored_nl, "\n", BEG|LABEL], [[2, 0], :on_sp, " ", BEG|LABEL], [[2, 2], :on_label, "type:", ARG|LABELED], [[2, 7], :on_sp, " ", ARG|LABELED], [[2, 8], :on_symbeg, ":", FNAME], [[2, 9], :on_kw, "module", ENDFN], [[2, 15], :on_comma, ",", BEG|LABEL], [[2, 16], :on_ignored_nl, "\n", BEG|LABEL], [[3, 0], :on_rbrace, "}", END], [[3, 1], :on_nl, "\n", BEG], [[4, 0], :on_const, "EOM", CMDARG]] ``` This is the problem line: ``` [[2, 9], :on_kw, "module", ENDFN], ``` Digging into the IRB source code they handled this case here ruby/ruby@776759e. Based on the description I believe this may be a bug in the lexer, but I'm not sure how to validate it.
1 parent fdf862b commit 42b3935

File tree

3 files changed

+25
-4
lines changed

3 files changed

+25
-4
lines changed

lib/dead_end/lex_all.rb

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@ def initialize(source:, source_lines: nil)
2525
lineno = @lex.last.pos.first + 1
2626
end
2727

28-
@lex.map! { |elem| LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state) }
28+
last_lex = nil
29+
@lex.map! { |elem|
30+
last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
31+
}
2932
end
3033

3134
def to_a

lib/dead_end/lex_value.rb

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,21 @@ module DeadEnd
1515
class LexValue
1616
attr_reader :line, :type, :token, :state
1717

18-
def initialize(line, type, token, state)
18+
def initialize(line, type, token, state, last_lex = nil)
1919
@line = line
2020
@type = type
2121
@token = token
2222
@state = state
2323

24-
set_kw_end
24+
set_kw_end(last_lex)
2525
end
2626

27-
private def set_kw_end
27+
private def set_kw_end(last_lex)
2828
@is_end = false
2929
@is_kw = false
3030
return if type != :on_kw
31+
#
32+
return if last_lex && last_lex.fname? # https://github.com/ruby/ruby/commit/776759e300e4659bb7468e2b97c8c2d4359a2953
3133

3234
case token
3335
when "if", "unless", "while", "until"
@@ -41,6 +43,10 @@ def initialize(line, type, token, state)
4143
end
4244
end
4345

46+
def fname?
47+
state.allbits?(Ripper::EXPR_FNAME)
48+
end
49+
4450
def ignore_newline?
4551
type == :on_ignored_nl
4652
end

spec/unit/code_line_spec.rb

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,18 @@
44

55
module DeadEnd
66
RSpec.describe CodeLine do
7+
it "bug in keyword detection" do
8+
lines = CodeLine.from_source(<<~'EOM')
9+
def to_json(*opts)
10+
{
11+
type: :module,
12+
}.to_json(*opts)
13+
end
14+
EOM
15+
expect(lines.count(&:is_kw?)).to eq(1)
16+
expect(lines.count(&:is_end?)).to eq(1)
17+
end
18+
719
it "supports endless method definitions" do
820
skip("Unsupported ruby version") unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3")
921

0 commit comments

Comments
 (0)