Feature/docclean Greatly improve the godoc comments in the runtime

README.md

@@ -3,20 +3,6 @@
 [![Java 11+](https://img.shields.io/badge/java-11+-4c7e9f.svg)](http://java.oracle.com)
 [![License](https://img.shields.io/badge/license-BSD-blue.svg)](https://raw.githubusercontent.com/antlr/antlr4/master/LICENSE.txt)
 
-
-## Versioning
-
-ANTLR 4 supports 10 target languages, and ensuring consistency across these targets is a unique and highly valuable feature.
-To ensure proper support of this feature, each release of ANTLR is a complete release of the tool and the 10 runtimes, all with the same version.
-As such, ANTLR versioning does not strictly follow semver semantics:
-
-* a component may be released with the latest version number even though nothing has changed within that component since the previous release
-* major version is bumped only when ANTLR is rewritten for a totally new "generation", such as ANTLR3 -> ANTLR4 (LL(\*) -> ALL(\*) parsing)
-* minor version updates may include minor breaking changes, the policy is to regenerate parsers with every release (4.11 -> 4.12)
-* backwards compatibility is only guaranteed for patch version bumps (4.11.1 -> 4.11.2)
-
-If you use a semver verifier in your CI, you probably want to apply special rules for ANTLR, such as treating minor change as a major change.
-
 **ANTLR** (ANother Tool for Language Recognition) is a powerful parser generator for reading, processing, executing, or translating structured text or binary files. It's widely used to build languages, tools, and frameworks. From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface (or visitor) that makes it easy to respond to the recognition of phrases of interest.
 
 **Dev branch build status**
@@ -32,6 +18,20 @@ If you use a semver verifier in your CI, you probably want to apply special rule
 [![Travis-CI Build Status (Swift-Linux)](https://img.shields.io/travis/antlr/antlr4.svg?label=Linux-Swift&branch=master)](https://travis-ci.com/github/antlr/antlr4)
 -->
 
+
+## Versioning
+
+ANTLR 4 supports 10 target languages, and ensuring consistency across these targets is a unique and highly valuable feature.
+To ensure proper support of this feature, each release of ANTLR is a complete release of the tool and the 10 runtimes, all with the same version.
+As such, ANTLR versioning does not strictly follow semver semantics:
+
+* a component may be released with the latest version number even though nothing has changed within that component since the previous release
+* major version is bumped only when ANTLR is rewritten for a totally new "generation", such as ANTLR3 -> ANTLR4 (LL(\*) -> ALL(\*) parsing)
+* minor version updates may include minor breaking changes, the policy is to regenerate parsers with every release (4.11 -> 4.12)
+* backwards compatibility is only guaranteed for patch version bumps (4.11.1 -> 4.11.2)
+
+If you use a semver verifier in your CI, you probably want to apply special rules for ANTLR, such as treating minor change as a major change.
+
 ## Repo branch structure
 
 The default branch for this repo is [`master`](https://github.com/antlr/antlr4/tree/master), which is the latest stable release and has tags for the various releases; e.g., see release tag [4.9.3](https://github.com/antlr/antlr4/tree/4.9.3).  Branch [`dev`](https://github.com/antlr/antlr4/tree/dev) is where development occurs between releases and all pull requests should be derived from that branch. The `dev` branch is merged back into `master` to cut a release and the release state is tagged (e.g., with `4.10-rc1` or `4.10`.) Visually our process looks roughly like this:

runtime-testsuite/test/org/antlr/v4/test/runtime/go/GoRunner.java

@@ -79,14 +79,14 @@ public String[] getExtraRunArgs() {
 	protected void initRuntime(RunOptions runOptions) throws Exception {
 		String cachePath = getCachePath();
 		mkdir(cachePath);
-		Path runtimeFilesPath = Paths.get(getRuntimePath("Go"), "antlr");
+		Path runtimeFilesPath = Paths.get(getRuntimePath("Go"), "antlr", "v4");
 		String runtimeToolPath = getRuntimeToolPath();
 		File goModFile = new File(cachePath, "go.mod");
 		if (goModFile.exists())
 			if (!goModFile.delete())
 				throw new IOException("Can't delete " + goModFile);
-		Processor.run(new String[] {runtimeToolPath, "mod", "init", "test"}, cachePath, environment);
-		Processor.run(new String[] {runtimeToolPath, "mod", "edit",
+		Processor.run(new String[]{runtimeToolPath, "mod", "init", "test"}, cachePath, environment);
+		Processor.run(new String[]{runtimeToolPath, "mod", "edit",
 				"-replace=" + GoRuntimeImportPath + "=" + runtimeFilesPath}, cachePath, environment);
 		cachedGoMod = readFile(cachePath + FileSeparator, "go.mod");
 	}
@@ -97,7 +97,18 @@ protected String grammarParseRuleToRecognizerName(String startRuleName) {
 			return null;
 		}
 
-		return startRuleName.substring(0, 1).toUpperCase() + startRuleName.substring(1);
+		// The rule name start is now translated to Start_ at runtime to avoid clashes with labels.
+		// Some tests use start as the first rule name, and we must cater for that
+		//
+		String rn = startRuleName.substring(0, 1).toUpperCase() + startRuleName.substring(1);
+		switch (rn) {
+			case "Start":
+			case "End":
+			case "Exception":
+				rn += "_";
+			default:
+		}
+		return rn;
 	}
 
 	@Override
@@ -126,7 +137,7 @@ protected CompiledState compile(RunOptions runOptions, GeneratedState generatedS
 		writeFile(tempDirPath, "go.mod", cachedGoMod);
 		Exception ex = null;
 		try {
-			Processor.run(new String[] {getRuntimeToolPath(), "mod", "tidy"}, tempDirPath, environment);
+			Processor.run(new String[]{getRuntimeToolPath(), "mod", "tidy"}, tempDirPath, environment);
 		} catch (InterruptedException | IOException e) {
 			ex = e;
 		}

runtime/Go/antlr/v4/antlrdoc.go

@@ -8,6 +8,19 @@ or translating structured text or binary files. It's widely used to build langua
 From a grammar, ANTLR generates a parser that can build parse trees and also generates a listener interface
 (or visitor) that makes it easy to respond to the recognition of phrases of interest.
 
+# Go Runtime
+
+At version 4.11.x and prior, the Go runtime was not properly versioned for go modules. After this point, the runtime
+source code is held in the `runtime/Go/antlr/v4` directory, and the go.mod file is updated to reflect the version of
+ANTLR4 that it is compatible with (I.E. uses the /v4 path). The runtime is now available as a go module, and can be
+imported as `github.com/antlr/antlr4/runtime/Go/antlr/v4` (the go get command should also be used with this path). See
+the main documentation for the ANTLR4 project for more information.
+
+This means that if you are using the source code without modules, you should also use the source code in /v4. Though
+we highly recommend that you use go modules, as they are now idiomatic Go.
+
+I am aware that this change will prove Hyrum's Law, but am prepared to live with it for teh common good. JI
+
 # Code Generation
 
 ANTLR supports the generation of code in a number of [target languages], and the generated code is supported by a
@@ -58,7 +71,7 @@ From the command line at the root of your package “myproject” you can then s
 
 # Copyright Notice
 
-Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
+Copyright (c) 2012-2023 The ANTLR Project. All rights reserved.
 
 Use of this file is governed by the BSD 3-clause license, which can be found in the [LICENSE.txt] file in the project root.
 

runtime/Go/antlr/v4/atn.go

@@ -20,10 +20,11 @@ var ATNInvalidAltNumber int
 // [ALL(*)]: https://www.antlr.org/papers/allstar-techreport.pdf
 // [Recursive Transition Network]: https://en.wikipedia.org/wiki/Recursive_transition_network
 type ATN struct {
-	// DecisionToState is the decision points for all rules, subrules, optional
-	// blocks, ()+, ()*, etc. Each subrule/rule is a decision point, and we must track them so we
+
+	// DecisionToState is the decision points for all rules, sub-rules, optional
+	// blocks, ()+, ()*, etc. Each sub-rule/rule is a decision point, and we must track them, so we
 	// can go back later and build DFA predictors for them.  This includes
-	// all the rules, subrules, optional blocks, ()+, ()* etc...
+	// all the rules, sub-rules, optional blocks, ()+, ()* etc...
 	DecisionToState []DecisionState
 
 	// grammarType is the ATN type and is used for deserializing ATNs from strings.
@@ -51,6 +52,8 @@ type ATN struct {
 	// specified, and otherwise is nil.
 	ruleToTokenType []int
 
+	// ATNStates is a list of all states in the ATN, ordered by state number.
+	//
 	states []ATNState
 
 	mu      sync.Mutex

runtime/Go/antlr/v4/atn_config.go

@@ -10,29 +10,44 @@ import (
 
 // ATNConfig is a tuple: (ATN state, predicted alt, syntactic, semantic
 // context). The syntactic context is a graph-structured stack node whose
-// path(s) to the root is the rule invocation(s) chain used to arrive at the
+// path(s) to the root is the rule invocation(s) chain used to arrive in the
 // state. The semantic context is the tree of semantic predicates encountered
 // before reaching an ATN state.
 type ATNConfig interface {
+
+	// Equals compares this ATNConfig to another for equality
 	Equals(o Collectable[ATNConfig]) bool
+
+	// Hash returns the hash code for this ATNConfig for use in maps and comparisons
 	Hash() int
 
+	// GetState returns the ATN state associated with this configuration
 	GetState() ATNState
+	// GetAlt returns the alternative associated with this configuration
 	GetAlt() int
+	// GetSemanticContext returns the semantic context associated with this configuration
 	GetSemanticContext() SemanticContext
 
+	// GetContext returns the rule invocation stack associated with this configuration
 	GetContext() PredictionContext
+	// SetContext sets the rule invocation stack associated with this configuration
 	SetContext(PredictionContext)
 
+	// GetReachesIntoOuterContext returns the count of references to an outer context from this configuration
 	GetReachesIntoOuterContext() int
+	// SetReachesIntoOuterContext sets the count of references to an outer context from this configuration
 	SetReachesIntoOuterContext(int)
 
+	// String returns a string representation of the configuration
 	String() string
 
 	getPrecedenceFilterSuppressed() bool
 	setPrecedenceFilterSuppressed(bool)
 }
 
+// BaseATNConfig is a base implementation of ATNConfig. Thi si s done to emulate Java's ability to have multiple
+// constructors for a single class. This is not idiomatic Go, but it works for now.
+// TODO: this isn't the way to do this I think, but it will take time to rework - JI Also, getters and setters are not Go. Might be better to just access the fields, though the compiler will probably eliminate the calls
 type BaseATNConfig struct {
 	precedenceFilterSuppressed bool
 	state                      ATNState
@@ -42,7 +57,8 @@ type BaseATNConfig struct {
 	reachesIntoOuterContext    int
 }
 
-func NewBaseATNConfig7(old *BaseATNConfig) ATNConfig { // TODO: Dup
+//goland:noinspection GoUnusedExportedFunction
+func NewBaseATNConfig7(old *BaseATNConfig) ATNConfig { // TODO: Dup - maybe delete this
 	return &BaseATNConfig{
 		state:                   old.state,
 		alt:                     old.alt,
@@ -52,10 +68,12 @@ func NewBaseATNConfig7(old *BaseATNConfig) ATNConfig { // TODO: Dup
 	}
 }
 
+// NewBaseATNConfig6 creates a new BaseATNConfig instance given a state, alt and context only
 func NewBaseATNConfig6(state ATNState, alt int, context PredictionContext) *BaseATNConfig {
 	return NewBaseATNConfig5(state, alt, context, SemanticContextNone)
 }
 
+// NewBaseATNConfig5 creates a new BaseATNConfig instance given a state, alt, context and semantic context
 func NewBaseATNConfig5(state ATNState, alt int, context PredictionContext, semanticContext SemanticContext) *BaseATNConfig {
 	if semanticContext == nil {
 		panic("semanticContext cannot be nil") // TODO: Necessary?
@@ -64,22 +82,28 @@ func NewBaseATNConfig5(state ATNState, alt int, context PredictionContext, seman
 	return &BaseATNConfig{state: state, alt: alt, context: context, semanticContext: semanticContext}
 }
 
+// NewBaseATNConfig4 creates a new BaseATNConfig instance given an existing config, and a state only
 func NewBaseATNConfig4(c ATNConfig, state ATNState) *BaseATNConfig {
 	return NewBaseATNConfig(c, state, c.GetContext(), c.GetSemanticContext())
 }
 
+// NewBaseATNConfig3 creates a new BaseATNConfig instance given an existing config, a state and a semantic context
 func NewBaseATNConfig3(c ATNConfig, state ATNState, semanticContext SemanticContext) *BaseATNConfig {
 	return NewBaseATNConfig(c, state, c.GetContext(), semanticContext)
 }
 
+// NewBaseATNConfig2 creates a new BaseATNConfig instance given an existing config, and a context only
 func NewBaseATNConfig2(c ATNConfig, semanticContext SemanticContext) *BaseATNConfig {
 	return NewBaseATNConfig(c, c.GetState(), c.GetContext(), semanticContext)
 }
 
+// NewBaseATNConfig1 creates a new BaseATNConfig instance given an existing config, a state, and a context only
 func NewBaseATNConfig1(c ATNConfig, state ATNState, context PredictionContext) *BaseATNConfig {
 	return NewBaseATNConfig(c, state, context, c.GetSemanticContext())
 }
 
+// NewBaseATNConfig creates a new BaseATNConfig instance given an existing config, a state, a context and a semantic context, other 'constructors'
+// are just wrappers around this one.
 func NewBaseATNConfig(c ATNConfig, state ATNState, context PredictionContext, semanticContext SemanticContext) *BaseATNConfig {
 	if semanticContext == nil {
 		panic("semanticContext cannot be nil")
@@ -103,29 +127,37 @@ func (b *BaseATNConfig) setPrecedenceFilterSuppressed(v bool) {
 	b.precedenceFilterSuppressed = v
 }
 
+// GetState returns the ATN state associated with this configuration
 func (b *BaseATNConfig) GetState() ATNState {
 	return b.state
 }
 
+// GetAlt returns the alternative associated with this configuration
 func (b *BaseATNConfig) GetAlt() int {
 	return b.alt
 }
 
+// SetContext sets the rule invocation stack associated with this configuration
 func (b *BaseATNConfig) SetContext(v PredictionContext) {
 	b.context = v
 }
+
+// GetContext returns the rule invocation stack associated with this configuration
 func (b *BaseATNConfig) GetContext() PredictionContext {
 	return b.context
 }
 
+// GetSemanticContext returns the semantic context associated with this configuration
 func (b *BaseATNConfig) GetSemanticContext() SemanticContext {
 	return b.semanticContext
 }
 
+// GetReachesIntoOuterContext returns the count of references to an outer context from this configuration
 func (b *BaseATNConfig) GetReachesIntoOuterContext() int {
 	return b.reachesIntoOuterContext
 }
 
+// SetReachesIntoOuterContext sets the count of references to an outer context from this configuration
 func (b *BaseATNConfig) SetReachesIntoOuterContext(v int) {
 	b.reachesIntoOuterContext = v
 }
@@ -182,6 +214,7 @@ func (b *BaseATNConfig) Hash() int {
 	return murmurFinish(h, 4)
 }
 
+// String returns a string representation of the BaseATNConfig, usually used for debugging purposes
 func (b *BaseATNConfig) String() string {
 	var s1, s2, s3 string
 
@@ -200,6 +233,9 @@ func (b *BaseATNConfig) String() string {
 	return fmt.Sprintf("(%v,%v%v%v%v)", b.state, b.alt, s1, s2, s3)
 }
 
+// LexerATNConfig represents a lexer ATN configuration which tracks the lexer action, and which "inherits" from the
+// BaseATNConfig struct.
+// TODO: Stop using a pointer and embed the struct instead as this saves allocations. Same for the LexerATNConfig "constructors"
 type LexerATNConfig struct {
 	*BaseATNConfig
 	lexerActionExecutor            *LexerActionExecutor
@@ -241,6 +277,7 @@ func NewLexerATNConfig2(c *LexerATNConfig, state ATNState, context PredictionCon
 	}
 }
 
+//goland:noinspection GoUnusedExportedFunction
 func NewLexerATNConfig1(state ATNState, alt int, context PredictionContext) *LexerATNConfig {
 	return &LexerATNConfig{BaseATNConfig: NewBaseATNConfig5(state, alt, context, SemanticContextNone)}
 }
@@ -271,29 +308,29 @@ func (l *LexerATNConfig) Equals(other Collectable[ATNConfig]) bool {
 	if l == other {
 		return true
 	}
-	var othert, ok = other.(*LexerATNConfig)
+	var otherT, ok = other.(*LexerATNConfig)
 
 	if l == other {
 		return true
 	} else if !ok {
 		return false
-	} else if l.passedThroughNonGreedyDecision != othert.passedThroughNonGreedyDecision {
+	} else if l.passedThroughNonGreedyDecision != otherT.passedThroughNonGreedyDecision {
 		return false
 	}
 
 	var b bool
 
 	if l.lexerActionExecutor != nil {
-		b = !l.lexerActionExecutor.Equals(othert.lexerActionExecutor)
+		b = !l.lexerActionExecutor.Equals(otherT.lexerActionExecutor)
 	} else {
-		b = othert.lexerActionExecutor != nil
+		b = otherT.lexerActionExecutor != nil
 	}
 
 	if b {
 		return false
 	}
 
-	return l.BaseATNConfig.Equals(othert.BaseATNConfig)
+	return l.BaseATNConfig.Equals(otherT.BaseATNConfig)
 }
 
 func checkNonGreedyDecision(source *LexerATNConfig, target ATNState) bool {