Issue with a nested structure

[gwd]

First, thank you so much for this library -- made my first grammar on Friday and I think the setup really helped make things straightforward.

I'm using participle v2.10.0, and trying to write a parser for strings like the following:

	tests := []string{
		"Xen 4.18.x",
		"Linux 4.7.x",
		"QEMU 4.7-RC series",
		"Linux 4.7.x - Linux 4.9.x",
		"xen-unstable",
		"Xen 4.18.x - xen-unstable",
		"QEMU 4.7-RC series - QEMU 4.10.x",
	}

i.e., a version that may be a single version, or a range (separated by -); that may be a single string, or <project> <version number>.

I wrote the following participle structures:

type ProjectVersion struct {
	Project string `parser:"@Project"`
	Version string `parser:"Space @(VersionString | RCVersionString)"`
}

type VersionSingle struct {
	TipVersion     *string         `parser:"@TipVersion"`
	ProjectVersion *ProjectVersion `parser:"| @@"`
}

type VersionRange struct {
	From VersionSingle `parser:"@@"`
	To   VersionSingle `parser:"VersionDash @@"`
}

type Version struct {
	Range  *VersionRange  `parser:"@@"`
	Single *VersionSingle `parser:"| @@"`
}

var rulesCommon []lexer.SimpleRule{	
	{"Project", `Xen|Linux|QEMU|xapi`},
	{"TipVersion", `xen-unstable`},
	{"RCVersionString", `\d+\.\d+-RC series`},
	{"VersionString", `\d+\.\d+\.x`},
	{"VersionDash", " - "},
	{"Space", ` `},
}

Unfortunately, I get errors like the following:

Parsing Xen 4.18.x: 1:11: unexpected token "<EOF>" (expected <versiondash> VersionSingle)
Parsing Linux 4.7.x: 1:12: unexpected token "<EOF>" (expected <versiondash> VersionSingle)
Parsing QEMU 4.7-RC series: 1:19: unexpected token "<EOF>" (expected <versiondash> VersionSingle)

In other words, it's somehow getting stuck on parsing something as a VersionRange, and not backing out and parsing it simply as a VersionSingle.

But this only happens if both Version and VersionSingle have at least two ways to be interpreted. If I replace the ProjectVersion with a single regexp that matches the same string, it works (here replacing the Version lexer token with a ProjectVersion token with the appropriate regexp).

// With the range, without the "project version"
type VersionSingleNoProjectVersion struct {
	TipVersion     *string `parser:"@TipVersion"`
	ProjectVersion *string `parser:"| @ProjectVersion"`
}

type VersionRangeNoProjectVersion struct {
	From VersionSingleNoProjectVersion `parser:"@@"`
	To   VersionSingleNoProjectVersion `parser:"VersionDash @@"`
}

type VersionNoProjectVersion struct {
	Range  *VersionRangeNoProjectVersion  `parser:"@@"`
	Single *VersionSingleNoProjectVersion `parser:"| @@"`
}

The problem for single items goes away if I get rid of the RangeVersion; but then of course you can't parse ranges:

// With "project version", without range
type VersionNoRange struct {
	Single *VersionSingle `parser:"@@"`
}

Any idea what's going on?

For completeness, here's a complete testing function you can use to trigger the issue:

package participletest_test

import (
	"testing"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

// What I'd like:
type ProjectVersion struct {
	Project string `parser:"@Project"`
	Version string `parser:"Space @(VersionString | RCVersionString)"`
}

type VersionSingle struct {
	TipVersion     *string         `parser:"@TipVersion"`
	ProjectVersion *ProjectVersion `parser:"| @@"`
}

type VersionRange struct {
	From VersionSingle `parser:"@@"`
	To   VersionSingle `parser:"VersionDash @@"`
}

type Version struct {
	Range  *VersionRange  `parser:"@@"`
	Single *VersionSingle `parser:"| @@"`
}

// With "project version", without range
type VersionNoRange struct {
	Single *VersionSingle `parser:"@@"`
}

// With the range, without the "project version"
type VersionSingleNoProjectVersion struct {
	TipVersion     *string `parser:"@TipVersion"`
	ProjectVersion *string `parser:"| @ProjectVersion"`
}

type VersionRangeNoProjectVersion struct {
	From VersionSingleNoProjectVersion `parser:"@@"`
	To   VersionSingleNoProjectVersion `parser:"VersionDash @@"`
}

type VersionNoProjectVersion struct {
	Range  *VersionRangeNoProjectVersion  `parser:"@@"`
	Single *VersionSingleNoProjectVersion `parser:"| @@"`
}

var rulesCommon = []lexer.SimpleRule{
	{"TipVersion", `xen-unstable`},
	{"RCVersionString", `\d+\.\d+-RC series`},
	{"VersionString", `\d+\.\d+\.x`},
	{"VersionDash", " - "},
	{"Space", ` `},
}

func TestVersion(t *testing.T) {
	simpletests := []string{
		"Xen 4.18.x",
		"Linux 4.7.x",
		"QEMU 4.7-RC series",
		"xen-unstable",
	}

	rangetests := []string{
		"Linux 4.7.x - Linux 4.9.x",
		"Xen 4.18.x - xen-unstable",
		"QEMU 4.7-RC series - QEMU 4.10.x",
	}

	lexProject := lexer.MustSimple(append(rulesCommon, lexer.SimpleRule{"Project", `Xen|Linux|QEMU|xapi`}))

	lexProjectVersion := lexer.MustSimple(append(rulesCommon,
		lexer.SimpleRule{"ProjectVersion", `(Xen|Linux|QEMU|xapi) (\d+\.\d+\.x|\d+\.\d+-RC series)`}))

	pVersion := participle.MustBuild[Version](participle.Lexer(lexProject))

	t.Log("Testing pVersion with simple and range")
	for _, in := range append(simpletests, rangetests...) {
		out, err := pVersion.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}

	pVersionNoRange := participle.MustBuild[VersionNoRange](participle.Lexer(lexProject))

	t.Log("Testing pVersionNoRange with simple only")
	for _, in := range simpletests {
		out, err := pVersionNoRange.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}

	pVersionNoProjcetVersion := participle.MustBuild[VersionNoProjectVersion](participle.Lexer(lexProjectVersion))

	t.Log("Testing pVersionNoProjectVersion with simple and range")
	for _, in := range append(simpletests, rangetests...) {
		out, err := pVersionNoProjcetVersion.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}
}

[gwd]

BTW, the following works for all tests:

type VersionSlice struct {
	Range []VersionSingle `parser:"@@ ( VersionDash @@ )*"`
}

...
	pVersionSlice := participle.MustBuild[VersionSlice](participle.Lexer(lexProject))

	t.Log("Testing pVersionSlice with simple and range")
	for _, in := range append(simpletests, rangetests...) {
		out, err := pVersionSlice.ParseString("", in)
		if err != nil {
			t.Errorf("ERROR: Parsing %v: %v", in, err)
		} else {
			t.Logf("Parsing %v resulted in %v", in, out)
		}
	}
...

So I think I have a work-around for now; but if the first version is supposed to work, it would be good to track down what's going on.