diff --git a/runtime/Go/antlr/v4/atn_config.go b/runtime/Go/antlr/v4/atn_config.go index 84b94e3989..3eebcf17af 100644 --- a/runtime/Go/antlr/v4/atn_config.go +++ b/runtime/Go/antlr/v4/atn_config.go @@ -18,7 +18,6 @@ const ( // path(s) to the root is the rule invocation(s) chain used to arrive in the // state. The semantic context is the tree of semantic predicates encountered // before reaching an ATN state. -// type ATNConfig struct { precedenceFilterSuppressed bool state ATNState @@ -31,18 +30,6 @@ type ATNConfig struct { passedThroughNonGreedyDecision bool } -//goland:noinspection GoUnusedExportedFunction -func NewATNConfig7(old *ATNConfig) *ATNConfig { // TODO: Dup - maybe delete this - return &ATNConfig{ - state: old.state, - alt: old.alt, - context: old.context, - semanticContext: old.semanticContext, - reachesIntoOuterContext: old.reachesIntoOuterContext, - cType: old.cType, - } -} - // NewATNConfig6 creates a new ATNConfig instance given a state, alt and context only func NewATNConfig6(state ATNState, alt int, context *PredictionContext) *ATNConfig { return NewATNConfig5(state, alt, context, SemanticContextNone) @@ -54,13 +41,13 @@ func NewATNConfig5(state ATNState, alt int, context *PredictionContext, semantic panic("semanticContext cannot be nil") // TODO: Necessary? } - return &ATNConfig{ - state: state, - alt: alt, - context: context, - semanticContext: semanticContext, - cType: parserConfig, - } + pac := &ATNConfig{} + pac.state = state + pac.alt = alt + pac.context = context + pac.semanticContext = semanticContext + pac.cType = parserConfig + return pac } // NewATNConfig4 creates a new ATNConfig instance given an existing config, and a state only @@ -89,11 +76,9 @@ func NewATNConfig(c *ATNConfig, state ATNState, context *PredictionContext, sema if semanticContext == nil { panic("semanticContext cannot be nil") // TODO: Remove this - probably put here for some bug that is now fixed } - b := &ATNConfig{ - cType: parserConfig, - } + b := &ATNConfig{} b.InitATNConfig(c, state, c.GetAlt(), context, semanticContext) - + b.cType = parserConfig return b } @@ -250,51 +235,37 @@ func (a *ATNConfig) String() string { } func NewLexerATNConfig6(state ATNState, alt int, context *PredictionContext) *ATNConfig { - return &ATNConfig{ - state: state, - alt: alt, - context: context, - semanticContext: SemanticContextNone, - cType: lexerConfig, - } -} - -func NewLexerATNConfig5(state ATNState, alt int, context *PredictionContext, lexerActionExecutor *LexerActionExecutor) *ATNConfig { - return &ATNConfig{ - state: state, - alt: alt, - context: context, - semanticContext: SemanticContextNone, - lexerActionExecutor: lexerActionExecutor, - cType: lexerConfig, - } + lac := &ATNConfig{} + lac.state = state + lac.alt = alt + lac.context = context + lac.semanticContext = SemanticContextNone + lac.cType = lexerConfig + return lac } func NewLexerATNConfig4(c *ATNConfig, state ATNState) *ATNConfig { - lac := &ATNConfig{ - lexerActionExecutor: c.lexerActionExecutor, - passedThroughNonGreedyDecision: checkNonGreedyDecision(c, state), - } + lac := &ATNConfig{} + lac.lexerActionExecutor = c.lexerActionExecutor + lac.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state) lac.InitATNConfig(c, state, c.GetAlt(), c.GetContext(), c.GetSemanticContext()) lac.cType = lexerConfig return lac } func NewLexerATNConfig3(c *ATNConfig, state ATNState, lexerActionExecutor *LexerActionExecutor) *ATNConfig { - lac := &ATNConfig{ - lexerActionExecutor: lexerActionExecutor, - passedThroughNonGreedyDecision: checkNonGreedyDecision(c, state), - } + lac := &ATNConfig{} + lac.lexerActionExecutor = lexerActionExecutor + lac.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state) lac.InitATNConfig(c, state, c.GetAlt(), c.GetContext(), c.GetSemanticContext()) lac.cType = lexerConfig return lac } func NewLexerATNConfig2(c *ATNConfig, state ATNState, context *PredictionContext) *ATNConfig { - lac := &ATNConfig{ - lexerActionExecutor: c.lexerActionExecutor, - passedThroughNonGreedyDecision: checkNonGreedyDecision(c, state), - } + lac := &ATNConfig{} + lac.lexerActionExecutor = c.lexerActionExecutor + lac.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state) lac.InitATNConfig(c, state, c.GetAlt(), context, c.GetSemanticContext()) lac.cType = lexerConfig return lac @@ -302,60 +273,59 @@ func NewLexerATNConfig2(c *ATNConfig, state ATNState, context *PredictionContext //goland:noinspection GoUnusedExportedFunction func NewLexerATNConfig1(state ATNState, alt int, context *PredictionContext) *ATNConfig { - lac := &ATNConfig{ - state: state, - alt: alt, - context: context, - semanticContext: SemanticContextNone, - cType: lexerConfig, - } + lac := &ATNConfig{} + lac.state = state + lac.alt = alt + lac.context = context + lac.semanticContext = SemanticContextNone + lac.cType = lexerConfig return lac } // LHash is the default hash function for Lexer ATNConfig objects, it can be used directly or via // the default comparator [ObjEqComparator]. -func (l *ATNConfig) LHash() int { +func (a *ATNConfig) LHash() int { var f int - if l.passedThroughNonGreedyDecision { + if a.passedThroughNonGreedyDecision { f = 1 } else { f = 0 } h := murmurInit(7) - h = murmurUpdate(h, l.state.GetStateNumber()) - h = murmurUpdate(h, l.alt) - h = murmurUpdate(h, l.context.Hash()) - h = murmurUpdate(h, l.semanticContext.Hash()) + h = murmurUpdate(h, a.state.GetStateNumber()) + h = murmurUpdate(h, a.alt) + h = murmurUpdate(h, a.context.Hash()) + h = murmurUpdate(h, a.semanticContext.Hash()) h = murmurUpdate(h, f) - h = murmurUpdate(h, l.lexerActionExecutor.Hash()) + h = murmurUpdate(h, a.lexerActionExecutor.Hash()) h = murmurFinish(h, 6) return h } // LEquals is the default comparison function for Lexer ATNConfig objects, it can be used directly or via // the default comparator [ObjEqComparator]. -func (l *ATNConfig) LEquals(other Collectable[*ATNConfig]) bool { +func (a *ATNConfig) LEquals(other Collectable[*ATNConfig]) bool { var otherT, ok = other.(*ATNConfig) if !ok { return false - } else if l == otherT { + } else if a == otherT { return true - } else if l.passedThroughNonGreedyDecision != otherT.passedThroughNonGreedyDecision { + } else if a.passedThroughNonGreedyDecision != otherT.passedThroughNonGreedyDecision { return false } switch { - case l.lexerActionExecutor == nil && otherT.lexerActionExecutor == nil: + case a.lexerActionExecutor == nil && otherT.lexerActionExecutor == nil: return true - case l.lexerActionExecutor != nil && otherT.lexerActionExecutor != nil: - if !l.lexerActionExecutor.Equals(otherT.lexerActionExecutor) { + case a.lexerActionExecutor != nil && otherT.lexerActionExecutor != nil: + if !a.lexerActionExecutor.Equals(otherT.lexerActionExecutor) { return false } default: return false // One but not both, are nil } - return l.PEquals(otherT) + return a.PEquals(otherT) } func checkNonGreedyDecision(source *ATNConfig, target ATNState) bool { diff --git a/runtime/Go/antlr/v4/atn_config_set.go b/runtime/Go/antlr/v4/atn_config_set.go index 38461fed6e..ab8d915860 100644 --- a/runtime/Go/antlr/v4/atn_config_set.go +++ b/runtime/Go/antlr/v4/atn_config_set.go @@ -21,7 +21,7 @@ type ATNConfigSet struct { // read-only because a set becomes a DFA state. configLookup *JStore[*ATNConfig, Comparator[*ATNConfig]] - // configs is the added elements. + // configs is the added elements that did not match an existing key in configLookup configs []*ATNConfig // TODO: These fields make me pretty uncomfortable, but it is nice to pack up @@ -68,7 +68,7 @@ func (b *ATNConfigSet) Alts() *BitSet { func NewATNConfigSet(fullCtx bool) *ATNConfigSet { return &ATNConfigSet{ cachedHash: -1, - configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst), + configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst, ATNConfigLookupCollection, "NewATNConfigSet()"), fullCtx: fullCtx, } } @@ -127,7 +127,7 @@ func (b *ATNConfigSet) GetStates() *JStore[ATNState, Comparator[ATNState]] { // states uses the standard comparator and Hash() provided by the ATNState instance // - states := NewJStore[ATNState, Comparator[ATNState]](aStateEqInst) + states := NewJStore[ATNState, Comparator[ATNState]](aStateEqInst, ATNStateCollection, "ATNConfigSet.GetStates()") for i := 0; i < len(b.configs); i++ { states.Put(b.configs[i].GetState()) @@ -155,13 +155,13 @@ func (b *ATNConfigSet) OptimizeConfigs(interpreter *BaseATNSimulator) { panic("set is read-only") } - if b.configLookup.Len() == 0 { + // Empty indicate no optimization is possible + if b.configLookup == nil || b.configLookup.Len() == 0 { return } for i := 0; i < len(b.configs); i++ { config := b.configs[i] - config.SetContext(interpreter.getCachedContext(config.GetContext())) } } @@ -174,12 +174,8 @@ func (b *ATNConfigSet) AddAll(coll []*ATNConfig) bool { return false } -// Compare is a hack function - O(n squared) at worst - just to verify that adding [DFA] states to the known -// set works, so long as comparison of [ATNConfigSet] works. For that to work, we -// need to make sure that the set of ATNConfigs in two sets are equivalent. The configs are -// only equal if they are in the same order too as Java uses ArrayList.equals(), which requires -// the same order. -// +// Compare The configs are only equal if they are in the same order and their Equals function returns true. +// Java uses ArrayList.equals(), which requires the same order. func (b *ATNConfigSet) Compare(bs *ATNConfigSet) bool { if len(b.configs) != len(bs.configs) { return false @@ -238,19 +234,17 @@ func (b *ATNConfigSet) hashCodeConfigs() int { } func (b *ATNConfigSet) Contains(item *ATNConfig) bool { - if b.configLookup == nil { + if b.readOnly { panic("not implemented for read-only sets") } - + if b.configLookup == nil { + return false + } return b.configLookup.Contains(item) } func (b *ATNConfigSet) ContainsFast(item *ATNConfig) bool { - if b.configLookup == nil { - panic("not implemented for read-only sets") - } - - return b.configLookup.Contains(item) // TODO: containsFast is not implemented for Set + return b.Contains(item) } func (b *ATNConfigSet) Clear() { @@ -259,11 +253,11 @@ func (b *ATNConfigSet) Clear() { } b.configs = make([]*ATNConfig, 0) b.cachedHash = -1 - b.configLookup = NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst) + b.configLookup = NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst, ATNConfigLookupCollection, "NewATNConfigSet()") } func (b *ATNConfigSet) String() string { - + s := "[" for i, c := range b.configs { @@ -301,7 +295,7 @@ func NewOrderedATNConfigSet() *ATNConfigSet { return &ATNConfigSet{ cachedHash: -1, // This set uses the standard Hash() and Equals() from ATNConfig - configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst), + configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ATNConfigCollection, "ATNConfigSet.NewOrderedATNConfigSet()"), fullCtx: false, } } diff --git a/runtime/Go/antlr/v4/atn_deserializer.go b/runtime/Go/antlr/v4/atn_deserializer.go index 1e5f6249dd..2dcb9ae11b 100644 --- a/runtime/Go/antlr/v4/atn_deserializer.go +++ b/runtime/Go/antlr/v4/atn_deserializer.go @@ -31,7 +31,7 @@ func NewATNDeserializer(options *ATNDeserializationOptions) *ATNDeserializer { if options == nil { options = &defaultATNDeserializationOptions } - + return &ATNDeserializer{options: options} } @@ -42,7 +42,7 @@ func stringInSlice(a string, list []string) int { return i } } - + return -1 } @@ -50,34 +50,34 @@ func (a *ATNDeserializer) Deserialize(data []int32) *ATN { a.data = data a.pos = 0 a.checkVersion() - + atn := a.readATN() - + a.readStates(atn) a.readRules(atn) a.readModes(atn) - + sets := a.readSets(atn, nil) - + a.readEdges(atn, sets) a.readDecisions(atn) a.readLexerActions(atn) a.markPrecedenceDecisions(atn) a.verifyATN(atn) - + if a.options.GenerateRuleBypassTransitions() && atn.grammarType == ATNTypeParser { a.generateRuleBypassTransitions(atn) // Re-verify after modification a.verifyATN(atn) } - + return atn - + } func (a *ATNDeserializer) checkVersion() { version := a.readInt() - + if version != serializedVersion { panic("Could not deserialize ATN with version " + strconv.Itoa(version) + " (expected " + strconv.Itoa(serializedVersion) + ").") } @@ -86,95 +86,95 @@ func (a *ATNDeserializer) checkVersion() { func (a *ATNDeserializer) readATN() *ATN { grammarType := a.readInt() maxTokenType := a.readInt() - + return NewATN(grammarType, maxTokenType) } func (a *ATNDeserializer) readStates(atn *ATN) { nstates := a.readInt() - + // Allocate worst case size. loopBackStateNumbers := make([]loopEndStateIntPair, 0, nstates) endStateNumbers := make([]blockStartStateIntPair, 0, nstates) - + // Preallocate states slice. atn.states = make([]ATNState, 0, nstates) - + for i := 0; i < nstates; i++ { stype := a.readInt() - + // Ignore bad types of states if stype == ATNStateInvalidType { atn.addState(nil) continue } - + ruleIndex := a.readInt() - + s := a.stateFactory(stype, ruleIndex) - + if stype == ATNStateLoopEnd { loopBackStateNumber := a.readInt() - + loopBackStateNumbers = append(loopBackStateNumbers, loopEndStateIntPair{s.(*LoopEndState), loopBackStateNumber}) } else if s2, ok := s.(BlockStartState); ok { endStateNumber := a.readInt() - + endStateNumbers = append(endStateNumbers, blockStartStateIntPair{s2, endStateNumber}) } - + atn.addState(s) } - + // Delay the assignment of loop back and end states until we know all the state // instances have been initialized for _, pair := range loopBackStateNumbers { pair.item0.loopBackState = atn.states[pair.item1] } - + for _, pair := range endStateNumbers { pair.item0.setEndState(atn.states[pair.item1].(*BlockEndState)) } - + numNonGreedyStates := a.readInt() for j := 0; j < numNonGreedyStates; j++ { stateNumber := a.readInt() - + atn.states[stateNumber].(DecisionState).setNonGreedy(true) } - + numPrecedenceStates := a.readInt() for j := 0; j < numPrecedenceStates; j++ { stateNumber := a.readInt() - + atn.states[stateNumber].(*RuleStartState).isPrecedenceRule = true } } func (a *ATNDeserializer) readRules(atn *ATN) { nrules := a.readInt() - + if atn.grammarType == ATNTypeLexer { atn.ruleToTokenType = make([]int, nrules) } - + atn.ruleToStartState = make([]*RuleStartState, nrules) - + for i := range atn.ruleToStartState { s := a.readInt() startState := atn.states[s].(*RuleStartState) - + atn.ruleToStartState[i] = startState - + if atn.grammarType == ATNTypeLexer { tokenType := a.readInt() - + atn.ruleToTokenType[i] = tokenType } } - + atn.ruleToStopState = make([]*RuleStopState, nrules) - + for _, state := range atn.states { if s2, ok := state.(*RuleStopState); ok { atn.ruleToStopState[s2.ruleIndex] = s2 @@ -186,50 +186,50 @@ func (a *ATNDeserializer) readRules(atn *ATN) { func (a *ATNDeserializer) readModes(atn *ATN) { nmodes := a.readInt() atn.modeToStartState = make([]*TokensStartState, nmodes) - + for i := range atn.modeToStartState { s := a.readInt() - + atn.modeToStartState[i] = atn.states[s].(*TokensStartState) } } func (a *ATNDeserializer) readSets(_ *ATN, sets []*IntervalSet) []*IntervalSet { m := a.readInt() - + // Preallocate the needed capacity. if cap(sets)-len(sets) < m { isets := make([]*IntervalSet, len(sets), len(sets)+m) copy(isets, sets) sets = isets } - + for i := 0; i < m; i++ { iset := NewIntervalSet() - + sets = append(sets, iset) - + n := a.readInt() containsEOF := a.readInt() - + if containsEOF != 0 { iset.addOne(-1) } - + for j := 0; j < n; j++ { i1 := a.readInt() i2 := a.readInt() - + iset.addRange(i1, i2) } } - + return sets } func (a *ATNDeserializer) readEdges(atn *ATN, sets []*IntervalSet) { nedges := a.readInt() - + for i := 0; i < nedges; i++ { var ( src = a.readInt() @@ -241,48 +241,48 @@ func (a *ATNDeserializer) readEdges(atn *ATN, sets []*IntervalSet) { trans = a.edgeFactory(atn, ttype, src, trg, arg1, arg2, arg3, sets) srcState = atn.states[src] ) - + srcState.AddTransition(trans, -1) } - + // Edges for rule stop states can be derived, so they are not serialized for _, state := range atn.states { for _, t := range state.GetTransitions() { var rt, ok = t.(*RuleTransition) - + if !ok { continue } - + outermostPrecedenceReturn := -1 - + if atn.ruleToStartState[rt.getTarget().GetRuleIndex()].isPrecedenceRule { if rt.precedence == 0 { outermostPrecedenceReturn = rt.getTarget().GetRuleIndex() } } - + trans := NewEpsilonTransition(rt.followState, outermostPrecedenceReturn) - + atn.ruleToStopState[rt.getTarget().GetRuleIndex()].AddTransition(trans, -1) } } - + for _, state := range atn.states { if s2, ok := state.(BlockStartState); ok { // We need to know the end state to set its start state if s2.getEndState() == nil { panic("IllegalState") } - + // Block end states can only be associated to a single block start state if s2.getEndState().startState != nil { panic("IllegalState") } - + s2.getEndState().startState = state } - + if s2, ok := state.(*PlusLoopbackState); ok { for _, t := range s2.GetTransitions() { if t2, ok := t.getTarget().(*PlusBlockStartState); ok { @@ -301,11 +301,11 @@ func (a *ATNDeserializer) readEdges(atn *ATN, sets []*IntervalSet) { func (a *ATNDeserializer) readDecisions(atn *ATN) { ndecisions := a.readInt() - + for i := 0; i < ndecisions; i++ { s := a.readInt() decState := atn.states[s].(DecisionState) - + atn.DecisionToState = append(atn.DecisionToState, decState) decState.setDecision(i) } @@ -314,9 +314,9 @@ func (a *ATNDeserializer) readDecisions(atn *ATN) { func (a *ATNDeserializer) readLexerActions(atn *ATN) { if atn.grammarType == ATNTypeLexer { count := a.readInt() - + atn.lexerActions = make([]LexerAction, count) - + for i := range atn.lexerActions { actionType := a.readInt() data1 := a.readInt() @@ -328,11 +328,11 @@ func (a *ATNDeserializer) readLexerActions(atn *ATN) { func (a *ATNDeserializer) generateRuleBypassTransitions(atn *ATN) { count := len(atn.ruleToStartState) - + for i := 0; i < count; i++ { atn.ruleToTokenType[i] = atn.maxTokenType + i + 1 } - + for i := 0; i < count; i++ { a.generateRuleBypassTransition(atn, i) } @@ -340,79 +340,79 @@ func (a *ATNDeserializer) generateRuleBypassTransitions(atn *ATN) { func (a *ATNDeserializer) generateRuleBypassTransition(atn *ATN, idx int) { bypassStart := NewBasicBlockStartState() - + bypassStart.ruleIndex = idx atn.addState(bypassStart) - + bypassStop := NewBlockEndState() - + bypassStop.ruleIndex = idx atn.addState(bypassStop) - + bypassStart.endState = bypassStop - + atn.defineDecisionState(&bypassStart.BaseDecisionState) - + bypassStop.startState = bypassStart - + var excludeTransition Transition var endState ATNState - + if atn.ruleToStartState[idx].isPrecedenceRule { // Wrap from the beginning of the rule to the StarLoopEntryState endState = nil - + for i := 0; i < len(atn.states); i++ { state := atn.states[i] - + if a.stateIsEndStateFor(state, idx) != nil { endState = state excludeTransition = state.(*StarLoopEntryState).loopBackState.GetTransitions()[0] - + break } } - + if excludeTransition == nil { panic("Couldn't identify final state of the precedence rule prefix section.") } } else { endState = atn.ruleToStopState[idx] } - + // All non-excluded transitions that currently target end state need to target // blockEnd instead for i := 0; i < len(atn.states); i++ { state := atn.states[i] - + for j := 0; j < len(state.GetTransitions()); j++ { transition := state.GetTransitions()[j] - + if transition == excludeTransition { continue } - + if transition.getTarget() == endState { transition.setTarget(bypassStop) } } } - + // All transitions leaving the rule start state need to leave blockStart instead ruleToStartState := atn.ruleToStartState[idx] count := len(ruleToStartState.GetTransitions()) - + for count > 0 { bypassStart.AddTransition(ruleToStartState.GetTransitions()[count-1], -1) ruleToStartState.SetTransitions([]Transition{ruleToStartState.GetTransitions()[len(ruleToStartState.GetTransitions())-1]}) } - + // Link the new states atn.ruleToStartState[idx].AddTransition(NewEpsilonTransition(bypassStart, -1), -1) bypassStop.AddTransition(NewEpsilonTransition(endState, -1), -1) - + MatchState := NewBasicState() - + atn.addState(MatchState) MatchState.AddTransition(NewAtomTransition(bypassStop, atn.ruleToTokenType[idx]), -1) bypassStart.AddTransition(NewEpsilonTransition(MatchState, -1), -1) @@ -422,23 +422,23 @@ func (a *ATNDeserializer) stateIsEndStateFor(state ATNState, idx int) ATNState { if state.GetRuleIndex() != idx { return nil } - + if _, ok := state.(*StarLoopEntryState); !ok { return nil } - + maybeLoopEndState := state.GetTransitions()[len(state.GetTransitions())-1].getTarget() - + if _, ok := maybeLoopEndState.(*LoopEndState); !ok { return nil } - + var _, ok = maybeLoopEndState.GetTransitions()[0].getTarget().(*RuleStopState) - + if maybeLoopEndState.(*LoopEndState).epsilonOnlyTransitions && ok { return state } - + return nil } @@ -456,10 +456,10 @@ func (a *ATNDeserializer) markPrecedenceDecisions(atn *ATN) { // precedence rule should continue or complete. if atn.ruleToStartState[state.GetRuleIndex()].isPrecedenceRule { maybeLoopEndState := state.GetTransitions()[len(state.GetTransitions())-1].getTarget() - + if s3, ok := maybeLoopEndState.(*LoopEndState); ok { var _, ok2 = maybeLoopEndState.GetTransitions()[0].getTarget().(*RuleStopState) - + if s3.epsilonOnlyTransitions && ok2 { state.(*StarLoopEntryState).precedenceRuleDecision = true } @@ -472,65 +472,65 @@ func (a *ATNDeserializer) verifyATN(atn *ATN) { if !a.options.VerifyATN() { return } - + // Verify assumptions for _, state := range atn.states { if state == nil { continue } - + a.checkCondition(state.GetEpsilonOnlyTransitions() || len(state.GetTransitions()) <= 1, "") - + switch s2 := state.(type) { case *PlusBlockStartState: a.checkCondition(s2.loopBackState != nil, "") - + case *StarLoopEntryState: a.checkCondition(s2.loopBackState != nil, "") a.checkCondition(len(s2.GetTransitions()) == 2, "") - + switch s2.transitions[0].getTarget().(type) { case *StarBlockStartState: _, ok := s2.transitions[1].getTarget().(*LoopEndState) - + a.checkCondition(ok, "") a.checkCondition(!s2.nonGreedy, "") - + case *LoopEndState: var _, ok = s2.transitions[1].getTarget().(*StarBlockStartState) - + a.checkCondition(ok, "") a.checkCondition(s2.nonGreedy, "") - + default: panic("IllegalState") } - + case *StarLoopbackState: a.checkCondition(len(state.GetTransitions()) == 1, "") - + var _, ok = state.GetTransitions()[0].getTarget().(*StarLoopEntryState) - + a.checkCondition(ok, "") - + case *LoopEndState: a.checkCondition(s2.loopBackState != nil, "") - + case *RuleStartState: a.checkCondition(s2.stopState != nil, "") - + case BlockStartState: a.checkCondition(s2.getEndState() != nil, "") - + case *BlockEndState: a.checkCondition(s2.startState != nil, "") - + case DecisionState: a.checkCondition(len(s2.GetTransitions()) <= 1 || s2.getDecision() >= 0, "") - + default: var _, ok = s2.(*RuleStopState) - + a.checkCondition(len(s2.GetTransitions()) <= 1 || ok, "") } } @@ -541,114 +541,114 @@ func (a *ATNDeserializer) checkCondition(condition bool, message string) { if message == "" { message = "IllegalState" } - + panic(message) } } func (a *ATNDeserializer) readInt() int { v := a.data[a.pos] - + a.pos++ - + return int(v) // data is 32 bits but int is at least that big } func (a *ATNDeserializer) edgeFactory(atn *ATN, typeIndex, _, trg, arg1, arg2, arg3 int, sets []*IntervalSet) Transition { target := atn.states[trg] - + switch typeIndex { case TransitionEPSILON: return NewEpsilonTransition(target, -1) - + case TransitionRANGE: if arg3 != 0 { return NewRangeTransition(target, TokenEOF, arg2) } - + return NewRangeTransition(target, arg1, arg2) - + case TransitionRULE: return NewRuleTransition(atn.states[arg1], arg2, arg3, target) - + case TransitionPREDICATE: return NewPredicateTransition(target, arg1, arg2, arg3 != 0) - + case TransitionPRECEDENCE: return NewPrecedencePredicateTransition(target, arg1) - + case TransitionATOM: if arg3 != 0 { return NewAtomTransition(target, TokenEOF) } - + return NewAtomTransition(target, arg1) - + case TransitionACTION: return NewActionTransition(target, arg1, arg2, arg3 != 0) - + case TransitionSET: return NewSetTransition(target, sets[arg1]) - + case TransitionNOTSET: return NewNotSetTransition(target, sets[arg1]) - + case TransitionWILDCARD: return NewWildcardTransition(target) } - + panic("The specified transition type is not valid.") } func (a *ATNDeserializer) stateFactory(typeIndex, ruleIndex int) ATNState { var s ATNState - + switch typeIndex { case ATNStateInvalidType: return nil - + case ATNStateBasic: s = NewBasicState() - + case ATNStateRuleStart: s = NewRuleStartState() - + case ATNStateBlockStart: s = NewBasicBlockStartState() - + case ATNStatePlusBlockStart: s = NewPlusBlockStartState() - + case ATNStateStarBlockStart: s = NewStarBlockStartState() - + case ATNStateTokenStart: s = NewTokensStartState() - + case ATNStateRuleStop: s = NewRuleStopState() - + case ATNStateBlockEnd: s = NewBlockEndState() - + case ATNStateStarLoopBack: s = NewStarLoopbackState() - + case ATNStateStarLoopEntry: s = NewStarLoopEntryState() - + case ATNStatePlusLoopBack: s = NewPlusLoopbackState() - + case ATNStateLoopEnd: s = NewLoopEndState() - + default: panic(fmt.Sprintf("state type %d is invalid", typeIndex)) } - + s.SetRuleIndex(ruleIndex) - + return s } @@ -656,28 +656,28 @@ func (a *ATNDeserializer) lexerActionFactory(typeIndex, data1, data2 int) LexerA switch typeIndex { case LexerActionTypeChannel: return NewLexerChannelAction(data1) - + case LexerActionTypeCustom: return NewLexerCustomAction(data1, data2) - + case LexerActionTypeMode: return NewLexerModeAction(data1) - + case LexerActionTypeMore: return LexerMoreActionINSTANCE - + case LexerActionTypePopMode: return LexerPopModeActionINSTANCE - + case LexerActionTypePushMode: return NewLexerPushModeAction(data1) - + case LexerActionTypeSkip: return LexerSkipActionINSTANCE - + case LexerActionTypeType: return NewLexerTypeAction(data1) - + default: panic(fmt.Sprintf("lexer action %d is invalid", typeIndex)) } diff --git a/runtime/Go/antlr/v4/atn_simulator.go b/runtime/Go/antlr/v4/atn_simulator.go index 902f26c9e8..6bf6516be4 100644 --- a/runtime/Go/antlr/v4/atn_simulator.go +++ b/runtime/Go/antlr/v4/atn_simulator.go @@ -23,9 +23,8 @@ func (b *BaseATNSimulator) getCachedContext(context *PredictionContext) *Predict return context } - // TODO: Should this be guarded by a mutex? - visited := NewJMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]](pContextEqInst) - + //visited := NewJMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]](pContextEqInst, PredictionVisitedCollection, "Visit map in getCachedContext()") + visited := NewVisitRecord() return getCachedBasePredictionContext(context, b.sharedContextCache, visited) } diff --git a/runtime/Go/antlr/v4/atn_state.go b/runtime/Go/antlr/v4/atn_state.go index 96d60db3b5..2ae5807cdb 100644 --- a/runtime/Go/antlr/v4/atn_state.go +++ b/runtime/Go/antlr/v4/atn_state.go @@ -25,7 +25,7 @@ const ( ATNStateStarLoopEntry = 10 ATNStatePlusLoopBack = 11 ATNStateLoopEnd = 12 - + ATNStateInvalidStateNumber = -1 ) @@ -34,25 +34,25 @@ var ATNStateInitialNumTransitions = 4 type ATNState interface { GetEpsilonOnlyTransitions() bool - + GetRuleIndex() int SetRuleIndex(int) - + GetNextTokenWithinRule() *IntervalSet SetNextTokenWithinRule(*IntervalSet) - + GetATN() *ATN SetATN(*ATN) - + GetStateType() int - + GetStateNumber() int SetStateNumber(int) - + GetTransitions() []Transition SetTransitions([]Transition) AddTransition(Transition, int) - + String() string Hash() int Equals(Collectable[ATNState]) bool @@ -61,19 +61,19 @@ type ATNState interface { type BaseATNState struct { // NextTokenWithinRule caches lookahead during parsing. Not used during construction. NextTokenWithinRule *IntervalSet - + // atn is the current ATN. atn *ATN - + epsilonOnlyTransitions bool - + // ruleIndex tracks the Rule index because there are no Rule objects at runtime. ruleIndex int - + stateNumber int - + stateType int - + // Track the transitions emanating from this ATN state. transitions []Transition } @@ -141,7 +141,7 @@ func (as *BaseATNState) Equals(other Collectable[ATNState]) bool { if ot, ok := other.(ATNState); ok { return as.stateNumber == ot.GetStateNumber() } - + return false } @@ -156,7 +156,7 @@ func (as *BaseATNState) AddTransition(trans Transition, index int) { _, _ = fmt.Fprintf(os.Stdin, "ATN state %d has both epsilon and non-epsilon transitions.\n", as.stateNumber) as.epsilonOnlyTransitions = false } - + // TODO: Check code for already present compared to the Java equivalent //alreadyPresent := false //for _, t := range as.transitions { @@ -197,10 +197,10 @@ func NewBasicState() *BasicState { type DecisionState interface { ATNState - + getDecision() int setDecision(int) - + getNonGreedy() bool setNonGreedy(bool) } @@ -239,7 +239,7 @@ func (s *BaseDecisionState) setNonGreedy(b bool) { type BlockStartState interface { DecisionState - + getEndState() *BlockEndState setEndState(*BlockEndState) } diff --git a/runtime/Go/antlr/v4/atnconfigset_test.go b/runtime/Go/antlr/v4/atnconfigset_test.go deleted file mode 100644 index 5d7c042258..0000000000 --- a/runtime/Go/antlr/v4/atnconfigset_test.go +++ /dev/null @@ -1,77 +0,0 @@ -package antlr - -import ( - "testing" -) - -// Test for Issue #3319 -// To run: -// -// cd antlr4/runtime/Go/antlr/v4 -// go test -// -// In the old runtime code, the test would crash because it would try -// to compare a *LexerActionExecutor with nil, causing a nil pointer dereference. -// It only happens if there were different states that had equal stateNumber mod 16, -// and you created that ATNConfig with a nil LexerActionExecutor. That's why this -// test code has a hardwired constant of 16. - -func TestCompare(t *testing.T) { - var set = NewOrderedATNConfigSet() - var s0 = NewATNState() - var s1 = NewATNState() - var s2 = NewATNState() - var s3 = NewATNState() - var s16 = NewATNState() - s16.SetStateNumber(16) - var s17 = NewATNState() - s17.SetStateNumber(17) - var s18 = NewATNState() - s18.SetStateNumber(18) - var s19 = NewATNState() - s19.SetStateNumber(19) - var la0 = NewBaseLexerAction(1) - var la1 = NewBaseLexerAction(2) - var laa = make([]LexerAction, 2) - laa[0] = la0 - laa[1] = la1 - var ae = NewLexerActionExecutor(laa) - set.Add(NewATNConfig5(s0, 0, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s0, 1, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s0, 2, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s1, 0, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s1, 1, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s1, 2, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s2, 0, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s2, 1, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s2, 2, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s3, 0, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s3, 1, BasePredictionContextEMPTY, ae), nil) - set.Add(NewATNConfig5(s3, 2, BasePredictionContextEMPTY, ae), nil) - - set.Add(NewATNConfig5(s0, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s0, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s0, 2, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s1, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s1, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s1, 2, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s2, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s2, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s2, 2, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s3, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s3, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s3, 2, BasePredictionContextEMPTY, nil), nil) - - set.Add(NewATNConfig5(s16, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s16, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s16, 2, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s17, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s17, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s17, 2, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s18, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s18, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s18, 2, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s19, 0, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s19, 1, BasePredictionContextEMPTY, nil), nil) - set.Add(NewATNConfig5(s19, 2, BasePredictionContextEMPTY, nil), nil) -} diff --git a/runtime/Go/antlr/v4/common_token_stream.go b/runtime/Go/antlr/v4/common_token_stream.go index 96f53e6aca..60a1188953 100644 --- a/runtime/Go/antlr/v4/common_token_stream.go +++ b/runtime/Go/antlr/v4/common_token_stream.go @@ -16,7 +16,7 @@ import ( // Token.HIDDEN_CHANNEL, use a filtering token stream such a CommonTokenStream. type CommonTokenStream struct { channel int - + // fetchedEOF indicates whether the Token.EOF token has been fetched from // tokenSource and added to tokens. This field improves performance for the // following cases: @@ -27,18 +27,18 @@ type CommonTokenStream struct { // fetch: The check to prevent adding multiple EOF symbols into tokens is // trivial with bt field. fetchedEOF bool - + // index into [tokens] of the current token (next token to consume). // tokens[p] should be LT(1). It is set to -1 when the stream is first // constructed or when SetTokenSource is called, indicating that the first token // has not yet been fetched from the token source. For additional information, // see the documentation of [IntStream] for a description of initializing methods. index int - + // tokenSource is the [TokenSource] from which tokens for the bt stream are // fetched. tokenSource TokenSource - + // tokens contains all tokens fetched from the token source. The list is considered a // complete view of the input once fetchedEOF is set to true. tokens []Token @@ -77,13 +77,13 @@ func (c *CommonTokenStream) Seek(index int) { func (c *CommonTokenStream) Get(index int) Token { c.lazyInit() - + return c.tokens[index] } func (c *CommonTokenStream) Consume() { SkipEOFCheck := false - + if c.index >= 0 { if c.fetchedEOF { // The last token in tokens is EOF. Skip the check if p indexes any fetched. @@ -97,11 +97,11 @@ func (c *CommonTokenStream) Consume() { // Not yet initialized SkipEOFCheck = false } - + if !SkipEOFCheck && c.LA(1) == TokenEOF { panic("cannot consume EOF") } - + if c.Sync(c.index + 1) { c.index = c.adjustSeekIndex(c.index + 1) } @@ -111,12 +111,12 @@ func (c *CommonTokenStream) Consume() { // located at index i and otherwise false. func (c *CommonTokenStream) Sync(i int) bool { n := i - len(c.tokens) + 1 // How many more elements do we need? - + if n > 0 { fetched := c.fetch(n) return fetched >= n } - + return true } @@ -126,20 +126,20 @@ func (c *CommonTokenStream) fetch(n int) int { if c.fetchedEOF { return 0 } - + for i := 0; i < n; i++ { t := c.tokenSource.NextToken() - + t.SetTokenIndex(len(c.tokens)) c.tokens = append(c.tokens, t) - + if t.GetTokenType() == TokenEOF { c.fetchedEOF = true - + return i + 1 } } - + return n } @@ -148,27 +148,27 @@ func (c *CommonTokenStream) GetTokens(start int, stop int, types *IntervalSet) [ if start < 0 || stop < 0 { return nil } - + c.lazyInit() - + subset := make([]Token, 0) - + if stop >= len(c.tokens) { stop = len(c.tokens) - 1 } - + for i := start; i < stop; i++ { t := c.tokens[i] - + if t.GetTokenType() == TokenEOF { break } - + if types == nil || types.contains(t.GetTokenType()) { subset = append(subset, t) } } - + return subset } @@ -203,23 +203,23 @@ func (c *CommonTokenStream) SetTokenSource(tokenSource TokenSource) { // no tokens on channel between 'i' and [TokenEOF]. func (c *CommonTokenStream) NextTokenOnChannel(i, _ int) int { c.Sync(i) - + if i >= len(c.tokens) { return -1 } - + token := c.tokens[i] - + for token.GetChannel() != c.channel { if token.GetTokenType() == TokenEOF { return -1 } - + i++ c.Sync(i) token = c.tokens[i] } - + return i } @@ -230,7 +230,7 @@ func (c *CommonTokenStream) previousTokenOnChannel(i, channel int) int { for i >= 0 && c.tokens[i].GetChannel() != channel { i-- } - + return i } @@ -239,23 +239,23 @@ func (c *CommonTokenStream) previousTokenOnChannel(i, channel int) int { // or EOF. If channel is -1, it finds any non-default channel token. func (c *CommonTokenStream) GetHiddenTokensToRight(tokenIndex, channel int) []Token { c.lazyInit() - + if tokenIndex < 0 || tokenIndex >= len(c.tokens) { panic(strconv.Itoa(tokenIndex) + " not in 0.." + strconv.Itoa(len(c.tokens)-1)) } - + nextOnChannel := c.NextTokenOnChannel(tokenIndex+1, LexerDefaultTokenChannel) from := tokenIndex + 1 - + // If no onChannel to the right, then nextOnChannel == -1, so set 'to' to the last token var to int - + if nextOnChannel == -1 { to = len(c.tokens) - 1 } else { to = nextOnChannel } - + return c.filterForChannel(from, to, channel) } @@ -264,30 +264,30 @@ func (c *CommonTokenStream) GetHiddenTokensToRight(tokenIndex, channel int) []To // -1, it finds any non default channel token. func (c *CommonTokenStream) GetHiddenTokensToLeft(tokenIndex, channel int) []Token { c.lazyInit() - + if tokenIndex < 0 || tokenIndex >= len(c.tokens) { panic(strconv.Itoa(tokenIndex) + " not in 0.." + strconv.Itoa(len(c.tokens)-1)) } - + prevOnChannel := c.previousTokenOnChannel(tokenIndex-1, LexerDefaultTokenChannel) - + if prevOnChannel == tokenIndex-1 { return nil } - + // If there are none on channel to the left and prevOnChannel == -1 then from = 0 from := prevOnChannel + 1 to := tokenIndex - 1 - + return c.filterForChannel(from, to, channel) } func (c *CommonTokenStream) filterForChannel(left, right, channel int) []Token { hidden := make([]Token, 0) - + for i := left; i < right+1; i++ { t := c.tokens[i] - + if channel == -1 { if t.GetChannel() != LexerDefaultTokenChannel { hidden = append(hidden, t) @@ -296,11 +296,11 @@ func (c *CommonTokenStream) filterForChannel(left, right, channel int) []Token { hidden = append(hidden, t) } } - + if len(hidden) == 0 { return nil } - + return hidden } @@ -325,7 +325,7 @@ func (c *CommonTokenStream) GetTextFromTokens(start, end Token) string { if start == nil || end == nil { return "" } - + return c.GetTextFromInterval(NewInterval(start.GetTokenIndex(), end.GetTokenIndex())) } @@ -336,37 +336,37 @@ func (c *CommonTokenStream) GetTextFromRuleContext(interval RuleContext) string func (c *CommonTokenStream) GetTextFromInterval(interval Interval) string { c.lazyInit() c.Sync(interval.Stop) - + start := interval.Start stop := interval.Stop - + if start < 0 || stop < 0 { return "" } - + if stop >= len(c.tokens) { stop = len(c.tokens) - 1 } - + s := "" - + for i := start; i < stop+1; i++ { t := c.tokens[i] - + if t.GetTokenType() == TokenEOF { break } - + s += t.GetText() } - + return s } // Fill gets all tokens from the lexer until EOF. func (c *CommonTokenStream) Fill() { c.lazyInit() - + for c.fetch(1000) == 1000 { continue } @@ -380,68 +380,68 @@ func (c *CommonTokenStream) LB(k int) Token { if k == 0 || c.index-k < 0 { return nil } - + i := c.index n := 1 - + // Find k good tokens looking backward for n <= k { // Skip off-channel tokens i = c.previousTokenOnChannel(i-1, c.channel) n++ } - + if i < 0 { return nil } - + return c.tokens[i] } func (c *CommonTokenStream) LT(k int) Token { c.lazyInit() - + if k == 0 { return nil } - + if k < 0 { return c.LB(-k) } - + i := c.index n := 1 // We know tokens[n] is valid - + // Find k good tokens for n < k { // Skip off-channel tokens, but make sure to not look past EOF if c.Sync(i + 1) { i = c.NextTokenOnChannel(i+1, c.channel) } - + n++ } - + return c.tokens[i] } // getNumberOfOnChannelTokens counts EOF once. func (c *CommonTokenStream) getNumberOfOnChannelTokens() int { var n int - + c.Fill() - + for i := 0; i < len(c.tokens); i++ { t := c.tokens[i] - + if t.GetChannel() == c.channel { n++ } - + if t.GetTokenType() == TokenEOF { break } } - + return n } diff --git a/runtime/Go/antlr/v4/common_token_stream_test.go b/runtime/Go/antlr/v4/common_token_stream_test.go deleted file mode 100644 index d8503e10a1..0000000000 --- a/runtime/Go/antlr/v4/common_token_stream_test.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. -// Use of this file is governed by the BSD 3-clause license that -// can be found in the LICENSE.txt file in the project root. - -package antlr - -import ( - "testing" -) - -type commonTokenStreamTestLexer struct { - *BaseLexer - - tokens []Token - i int -} - -func (l *commonTokenStreamTestLexer) NextToken() Token { - tmp := l.tokens[l.i] - l.i++ - return tmp -} - -func TestCommonTokenStreamOffChannel(t *testing.T) { - assert := assertNew(t) - lexEngine := &commonTokenStreamTestLexer{ - tokens: []Token{ - newTestCommonToken(1, " ", LexerHidden), // 0 - newTestCommonToken(1, "x", LexerDefaultTokenChannel), // 1 - newTestCommonToken(1, " ", LexerHidden), // 2 - newTestCommonToken(1, "=", LexerDefaultTokenChannel), // 3 - newTestCommonToken(1, "34", LexerDefaultTokenChannel), // 4 - newTestCommonToken(1, " ", LexerHidden), // 5 - newTestCommonToken(1, " ", LexerHidden), // 6 - newTestCommonToken(1, ";", LexerDefaultTokenChannel), // 7 - newTestCommonToken(1, "\n", LexerHidden), // 9 - newTestCommonToken(TokenEOF, "", LexerDefaultTokenChannel), // 10 - }, - } - tokens := NewCommonTokenStream(lexEngine, TokenDefaultChannel) - - assert.Equal("x", tokens.LT(1).GetText()) // must skip first off channel token - tokens.Consume() - assert.Equal("=", tokens.LT(1).GetText()) - assert.Equal("x", tokens.LT(-1).GetText()) - - tokens.Consume() - assert.Equal("34", tokens.LT(1).GetText()) - assert.Equal("=", tokens.LT(-1).GetText()) - - tokens.Consume() - assert.Equal(";", tokens.LT(1).GetText()) - assert.Equal("34", tokens.LT(-1).GetText()) - - tokens.Consume() - assert.Equal(TokenEOF, tokens.LT(1).GetTokenType()) - assert.Equal(";", tokens.LT(-1).GetText()) - - assert.Equal("34", tokens.LT(-2).GetText()) - assert.Equal("=", tokens.LT(-3).GetText()) - assert.Equal("x", tokens.LT(-4).GetText()) -} - -func TestCommonTokenStreamFetchOffChannel(t *testing.T) { - assert := assertNew(t) - lexEngine := &commonTokenStreamTestLexer{ - tokens: []Token{ - newTestCommonToken(1, " ", LexerHidden), // 0 - newTestCommonToken(1, "x", LexerDefaultTokenChannel), // 1 - newTestCommonToken(1, " ", LexerHidden), // 2 - newTestCommonToken(1, "=", LexerDefaultTokenChannel), // 3 - newTestCommonToken(1, "34", LexerDefaultTokenChannel), // 4 - newTestCommonToken(1, " ", LexerHidden), // 5 - newTestCommonToken(1, " ", LexerHidden), // 6 - newTestCommonToken(1, ";", LexerDefaultTokenChannel), // 7 - newTestCommonToken(1, " ", LexerHidden), // 8 - newTestCommonToken(1, "\n", LexerHidden), // 9 - newTestCommonToken(TokenEOF, "", LexerDefaultTokenChannel), // 10 - }, - } - tokens := NewCommonTokenStream(lexEngine, TokenDefaultChannel) - tokens.Fill() - - assert.Nil(tokens.GetHiddenTokensToLeft(0, -1)) - assert.Nil(tokens.GetHiddenTokensToRight(0, -1)) - - assert.Equal("[[@0,0:0=' ',<1>,channel=1,0:-1]]", tokensToString(tokens.GetHiddenTokensToLeft(1, -1))) - assert.Equal("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokensToString(tokens.GetHiddenTokensToRight(1, -1))) - - assert.Nil(tokens.GetHiddenTokensToLeft(2, -1)) - assert.Nil(tokens.GetHiddenTokensToRight(2, -1)) - - assert.Equal("[[@2,0:0=' ',<1>,channel=1,0:-1]]", tokensToString(tokens.GetHiddenTokensToLeft(3, -1))) - assert.Nil(tokens.GetHiddenTokensToRight(3, -1)) - - assert.Nil(tokens.GetHiddenTokensToLeft(4, -1)) - assert.Equal("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToRight(4, -1))) - - assert.Nil(tokens.GetHiddenTokensToLeft(5, -1)) - assert.Equal("[[@6,0:0=' ',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToRight(5, -1))) - - assert.Equal("[[@5,0:0=' ',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToLeft(6, -1))) - assert.Nil(tokens.GetHiddenTokensToRight(6, -1)) - - assert.Equal("[[@5,0:0=' ',<1>,channel=1,0:-1], [@6,0:0=' ',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToLeft(7, -1))) - assert.Equal("[[@8,0:0=' ',<1>,channel=1,0:-1], [@9,0:0='\\n',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToRight(7, -1))) - - assert.Nil(tokens.GetHiddenTokensToLeft(8, -1)) - assert.Equal("[[@9,0:0='\\n',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToRight(8, -1))) - - assert.Equal("[[@8,0:0=' ',<1>,channel=1,0:-1]]", - tokensToString(tokens.GetHiddenTokensToLeft(9, -1))) - assert.Nil(tokens.GetHiddenTokensToRight(9, -1)) - -} - -type commonTokenStreamTestLexerSingleEOF struct { - *BaseLexer - - tokens []Token - i int -} - -func (l *commonTokenStreamTestLexerSingleEOF) NextToken() Token { - return newTestCommonToken(TokenEOF, "", LexerDefaultTokenChannel) -} - -func TestCommonTokenStreamSingleEOF(t *testing.T) { - assert := assertNew(t) - lexEngine := &commonTokenStreamTestLexerSingleEOF{} - tokens := NewCommonTokenStream(lexEngine, TokenDefaultChannel) - tokens.Fill() - - assert.Equal(TokenEOF, tokens.LA(1)) - assert.Equal(0, tokens.index) - assert.Equal(1, tokens.Size()) -} - -func TestCommonTokenStreamCannotConsumeEOF(t *testing.T) { - assert := assertNew(t) - lexEngine := &commonTokenStreamTestLexerSingleEOF{} - tokens := NewCommonTokenStream(lexEngine, TokenDefaultChannel) - tokens.Fill() - assert.Equal(TokenEOF, tokens.LA(1)) - assert.Equal(0, tokens.index) - assert.Equal(1, tokens.Size()) - assert.Panics(tokens.Consume) -} - -func TestCommonTokenStreamGetTextFromInterval(t *testing.T) { - assert := assertNew(t) - lexEngine := &commonTokenStreamTestLexer{ - tokens: []Token{ - newTestCommonToken(1, " ", LexerHidden), // 0 - newTestCommonToken(1, "x", LexerDefaultTokenChannel), // 1 - newTestCommonToken(1, " ", LexerHidden), // 2 - newTestCommonToken(1, "=", LexerDefaultTokenChannel), // 3 - newTestCommonToken(1, "34", LexerDefaultTokenChannel), // 4 - newTestCommonToken(1, " ", LexerHidden), // 5 - newTestCommonToken(1, " ", LexerHidden), // 6 - newTestCommonToken(1, ";", LexerDefaultTokenChannel), // 7 - newTestCommonToken(1, " ", LexerHidden), // 8 - newTestCommonToken(1, "\n", LexerHidden), // 9 - newTestCommonToken(TokenEOF, "", LexerDefaultTokenChannel), // 10 - }, - } - tokens := NewCommonTokenStream(lexEngine, TokenDefaultChannel) - assert.Equal("x", tokens.GetTextFromInterval(Interval{Start: 1, Stop: 1})) - assert.Equal(len(tokens.tokens), 2) - assert.Equal(len(tokens.tokens), 11) -} diff --git a/runtime/Go/antlr/v4/comparators.go b/runtime/Go/antlr/v4/comparators.go index 0d27713885..d85c06fb60 100644 --- a/runtime/Go/antlr/v4/comparators.go +++ b/runtime/Go/antlr/v4/comparators.go @@ -24,7 +24,7 @@ type ObjEqComparator[T Collectable[T]] struct{} var ( aStateEqInst = &ObjEqComparator[ATNState]{} aConfEqInst = &ObjEqComparator[*ATNConfig]{} - + // aConfCompInst is the comparator used for the ATNConfigSet for the configLookup cache aConfCompInst = &ATNConfigComparator[*ATNConfig]{} atnConfCompInst = &BaseATNConfigComparator[*ATNConfig]{} @@ -41,7 +41,7 @@ func (c *ObjEqComparator[T]) Equals2(o1, o2 T) bool { // Hash1 delegates to the Hash() method of type T func (c *ObjEqComparator[T]) Hash1(o T) int { - + return o.Hash() } @@ -55,20 +55,20 @@ type ATNConfigComparator[T Collectable[T]] struct { // Equals2 is a custom comparator for ATNConfigs specifically for configLookup func (c *ATNConfigComparator[T]) Equals2(o1, o2 *ATNConfig) bool { - + // Same pointer, must be equal, even if both nil // if o1 == o2 { return true - + } - + // If either are nil, but not both, then the result is false // if o1 == nil || o2 == nil { return false } - + return o1.GetState().GetStateNumber() == o2.GetState().GetStateNumber() && o1.GetAlt() == o2.GetAlt() && o1.GetSemanticContext().Equals(o2.GetSemanticContext()) @@ -76,7 +76,7 @@ func (c *ATNConfigComparator[T]) Equals2(o1, o2 *ATNConfig) bool { // Hash1 is custom hash implementation for ATNConfigs specifically for configLookup func (c *ATNConfigComparator[T]) Hash1(o *ATNConfig) int { - + hash := 7 hash = 31*hash + o.GetState().GetStateNumber() hash = 31*hash + o.GetAlt() @@ -90,20 +90,20 @@ type ATNAltConfigComparator[T Collectable[T]] struct { // Equals2 is a custom comparator for ATNConfigs specifically for configLookup func (c *ATNAltConfigComparator[T]) Equals2(o1, o2 *ATNConfig) bool { - + // Same pointer, must be equal, even if both nil // if o1 == o2 { return true - + } - + // If either are nil, but not both, then the result is false // if o1 == nil || o2 == nil { return false } - + return o1.GetState().GetStateNumber() == o2.GetState().GetStateNumber() && o1.GetContext().Equals(o2.GetContext()) } @@ -124,20 +124,20 @@ type BaseATNConfigComparator[T Collectable[T]] struct { // Equals2 is a custom comparator for ATNConfigs specifically for baseATNConfigSet func (c *BaseATNConfigComparator[T]) Equals2(o1, o2 *ATNConfig) bool { - + // Same pointer, must be equal, even if both nil // if o1 == o2 { return true - + } - + // If either are nil, but not both, then the result is false // if o1 == nil || o2 == nil { return false } - + return o1.GetState().GetStateNumber() == o2.GetState().GetStateNumber() && o1.GetAlt() == o2.GetAlt() && o1.GetSemanticContext().Equals(o2.GetSemanticContext()) diff --git a/runtime/Go/antlr/v4/configuration.go b/runtime/Go/antlr/v4/configuration.go new file mode 100644 index 0000000000..c2b724514d --- /dev/null +++ b/runtime/Go/antlr/v4/configuration.go @@ -0,0 +1,214 @@ +package antlr + +type runtimeConfiguration struct { + statsTraceStacks bool + lexerATNSimulatorDebug bool + lexerATNSimulatorDFADebug bool + parserATNSimulatorDebug bool + parserATNSimulatorTraceATNSim bool + parserATNSimulatorDFADebug bool + parserATNSimulatorRetryDebug bool + lRLoopEntryBranchOpt bool + memoryManager bool +} + +// Global runtime configuration +var runtimeConfig = runtimeConfiguration{ + lRLoopEntryBranchOpt: true, +} + +type runtimeOption func(*runtimeConfiguration) error + +// ConfigureRuntime allows the runtime to be configured globally setting things like trace and statistics options. +// It uses the functional options pattern for go. This is a package global function as it operates on the runtime +// configuration regardless of the instantiation of anything higher up such as a parser or lexer. Generally this is +// used for debugging/tracing/statistics options, which are usually used by the runtime maintainers (or rather the +// only maintainer). However, it is possible that you might want to use this to set a global option concerning the +// memory allocation type used by the runtime such as sync.Pool or not. +// +// The options are applied in the order they are passed in, so the last option will override any previous options. +// +// For example, if you want to turn on the collection create point stack flag to true, you can do: +// +// antlr.ConfigureRuntime(antlr.WithStatsTraceStacks(true)) +// +// If you want to turn it off, you can do: +// +// antlr.ConfigureRuntime(antlr.WithStatsTraceStacks(false)) +func ConfigureRuntime(options ...runtimeOption) error { + for _, option := range options { + err := option(&runtimeConfig) + if err != nil { + return err + } + } + return nil +} + +// WithStatsTraceStacks sets the global flag indicating whether to collect stack traces at the create-point of +// certain structs, such as collections, or the use point of certain methods such as Put(). +// Because this can be expensive, it is turned off by default. However, it +// can be useful to track down exactly where memory is being created and used. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithStatsTraceStacks(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithStatsTraceStacks(false)) +func WithStatsTraceStacks(trace bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.statsTraceStacks = trace + return nil + } +} + +// WithLexerATNSimulatorDebug sets the global flag indicating whether to log debug information from the lexer [ATN] +// simulator. This is useful for debugging lexer issues by comparing the output with the Java runtime. Only useful +// to the runtime maintainers. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithLexerATNSimulatorDebug(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithLexerATNSimulatorDebug(false)) +func WithLexerATNSimulatorDebug(debug bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.lexerATNSimulatorDebug = debug + return nil + } +} + +// WithLexerATNSimulatorDFADebug sets the global flag indicating whether to log debug information from the lexer [ATN] [DFA] +// simulator. This is useful for debugging lexer issues by comparing the output with the Java runtime. Only useful +// to the runtime maintainers. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithLexerATNSimulatorDFADebug(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithLexerATNSimulatorDFADebug(false)) +func WithLexerATNSimulatorDFADebug(debug bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.lexerATNSimulatorDFADebug = debug + return nil + } +} + +// WithParserATNSimulatorDebug sets the global flag indicating whether to log debug information from the parser [ATN] +// simulator. This is useful for debugging parser issues by comparing the output with the Java runtime. Only useful +// to the runtime maintainers. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorDebug(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorDebug(false)) +func WithParserATNSimulatorDebug(debug bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.parserATNSimulatorDebug = debug + return nil + } +} + +// WithParserATNSimulatorTraceATNSim sets the global flag indicating whether to log trace information from the parser [ATN] simulator +// [DFA]. This is useful for debugging parser issues by comparing the output with the Java runtime. Only useful +// to the runtime maintainers. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorTraceATNSim(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorTraceATNSim(false)) +func WithParserATNSimulatorTraceATNSim(trace bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.parserATNSimulatorTraceATNSim = trace + return nil + } +} + +// WithParserATNSimulatorDFADebug sets the global flag indicating whether to log debug information from the parser [ATN] [DFA] +// simulator. This is useful for debugging parser issues by comparing the output with the Java runtime. Only useful +// to the runtime maintainers. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorDFADebug(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorDFADebug(false)) +func WithParserATNSimulatorDFADebug(debug bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.parserATNSimulatorDFADebug = debug + return nil + } +} + +// WithParserATNSimulatorRetryDebug sets the global flag indicating whether to log debug information from the parser [ATN] [DFA] +// simulator when retrying a decision. This is useful for debugging parser issues by comparing the output with the Java runtime. +// Only useful to the runtime maintainers. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorRetryDebug(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithParserATNSimulatorRetryDebug(false)) +func WithParserATNSimulatorRetryDebug(debug bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.parserATNSimulatorRetryDebug = debug + return nil + } +} + +// WithLRLoopEntryBranchOpt sets the global flag indicating whether let recursive loop operations should be +// optimized or not. This is useful for debugging parser issues by comparing the output with the Java runtime. +// It turns off the functionality of [canDropLoopEntryEdgeInLeftRecursiveRule] in [ParserATNSimulator]. +// +// Note that default is to use this optimization. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithLRLoopEntryBranchOpt(true)) +// +// You can turn it off at any time using: +// +// antlr.ConfigureRuntime(antlr.WithLRLoopEntryBranchOpt(false)) +func WithLRLoopEntryBranchOpt(off bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.lRLoopEntryBranchOpt = off + return nil + } +} + +// WithMemoryManager sets the global flag indicating whether to use the memory manager or not. This is useful +// for poorly constructed grammars that create a lot of garbage. It turns on the functionality of [memoryManager], which +// will intercept garbage collection and cause available memory to be reused. At the end of the day, this is no substitute +// for fixing your grammar by ridding yourself of extreme ambiguity. BUt if you are just trying to reuse an opensource +// grammar, this may help make it more practical. +// +// Note that default is to use normal Go memory allocation and not pool memory. +// +// Use: +// +// antlr.ConfigureRuntime(antlr.WithMemoryManager(true)) +// +// Note that if you turn this on, you should probably leave it on. You should use only one memory strategy or the other +// and should remember to nil out any references to the parser or lexer when you are done with them. +func WithMemoryManager(use bool) runtimeOption { + return func(config *runtimeConfiguration) error { + config.memoryManager = use + return nil + } +} diff --git a/runtime/Go/antlr/v4/dfa.go b/runtime/Go/antlr/v4/dfa.go index c7ce29354a..6b63eb1589 100644 --- a/runtime/Go/antlr/v4/dfa.go +++ b/runtime/Go/antlr/v4/dfa.go @@ -9,9 +9,9 @@ package antlr type DFA struct { // atnStartState is the ATN state in which this was created atnStartState DecisionState - + decision int - + // states is all the DFA states. Use Map to get the old state back; Set can only // indicate whether it is there. Go maps implement key hash collisions and so on and are very // good, but the DFAState is an object and can't be used directly as the key as it can in say Java @@ -19,11 +19,11 @@ type DFA struct { // to see if they really are the same object. Hence, we have our own map storage. // states *JStore[*DFAState, *ObjEqComparator[*DFAState]] - + numstates int - + s0 *DFAState - + // precedenceDfa is the backing field for isPrecedenceDfa and setPrecedenceDfa. // True if the DFA is for a precedence decision and false otherwise. precedenceDfa bool @@ -33,7 +33,7 @@ func NewDFA(atnStartState DecisionState, decision int) *DFA { dfa := &DFA{ atnStartState: atnStartState, decision: decision, - states: NewJStore[*DFAState, *ObjEqComparator[*DFAState]](dfaStateEqInst), + states: nil, // Lazy initialize } if s, ok := atnStartState.(*StarLoopEntryState); ok && s.precedenceRuleDecision { dfa.precedenceDfa = true @@ -52,12 +52,12 @@ func (d *DFA) getPrecedenceStartState(precedence int) *DFAState { if !d.getPrecedenceDfa() { panic("only precedence DFAs may contain a precedence start state") } - + // s0.edges is never nil for a precedence DFA if precedence < 0 || precedence >= len(d.getS0().getEdges()) { return nil } - + return d.getS0().getIthEdge(precedence) } @@ -67,11 +67,11 @@ func (d *DFA) setPrecedenceStartState(precedence int, startState *DFAState) { if !d.getPrecedenceDfa() { panic("only precedence DFAs may contain a precedence start state") } - + if precedence < 0 { return } - + // Synchronization on s0 here is ok. When the DFA is turned into a // precedence DFA, s0 will be initialized once and not updated again. s0.edges // is never nil for a precedence DFA. @@ -81,7 +81,7 @@ func (d *DFA) setPrecedenceStartState(precedence int, startState *DFAState) { s0.setEdges(edges) d.setS0(s0) } - + s0.setIthEdge(precedence, startState) } @@ -96,11 +96,11 @@ func (d *DFA) getPrecedenceDfa() bool { // true or nil otherwise, and d.precedenceDfa is updated. func (d *DFA) setPrecedenceDfa(precedenceDfa bool) { if d.getPrecedenceDfa() != precedenceDfa { - d.states = NewJStore[*DFAState, *ObjEqComparator[*DFAState]](dfaStateEqInst) + d.states = nil // Lazy initialize d.numstates = 0 - + if precedenceDfa { - precedenceState := NewDFAState(-1, NewATNConfigSet(false)) + precedenceState := NewDFAState(-1, NewATNConfigSet(false)) precedenceState.setEdges(make([]*DFAState, 0)) precedenceState.isAcceptState = false precedenceState.requiresFullContext = false @@ -108,11 +108,36 @@ func (d *DFA) setPrecedenceDfa(precedenceDfa bool) { } else { d.setS0(nil) } - + d.precedenceDfa = precedenceDfa } } +// Len returns the number of states in d. We use this instead of accessing states directly so that we can implement lazy +// instantiation of the states JMap. +func (d *DFA) Len() int { + if d.states == nil { + return 0 + } + return d.states.Len() +} + +// Get returns a state that matches s if it is present in the DFA state set. We defer to this +// function instead of accessing states directly so that we can implement lazy instantiation of the states JMap. +func (d *DFA) Get(s *DFAState) (*DFAState, bool) { + if d.states == nil { + return nil, false + } + return d.states.Get(s) +} + +func (d *DFA) Put(s *DFAState) (*DFAState, bool) { + if d.states == nil { + d.states = NewJStore[*DFAState, *ObjEqComparator[*DFAState]](dfaStateEqInst, DFAStateCollection, "DFA via DFA.Put") + } + return d.states.Put(s) +} + func (d *DFA) getS0() *DFAState { return d.s0 } @@ -121,13 +146,15 @@ func (d *DFA) setS0(s *DFAState) { d.s0 = s } -// sortedStates returns the states in d sorted by their state number. +// sortedStates returns the states in d sorted by their state number, or an empty set if d.states is nil. func (d *DFA) sortedStates() []*DFAState { - + if d.states == nil { + return []*DFAState{} + } vs := d.states.SortedSlice(func(i, j *DFAState) bool { return i.stateNumber < j.stateNumber }) - + return vs } @@ -135,7 +162,7 @@ func (d *DFA) String(literalNames []string, symbolicNames []string) string { if d.getS0() == nil { return "" } - + return NewDFASerializer(d, literalNames, symbolicNames).String() } @@ -143,6 +170,6 @@ func (d *DFA) ToLexerString() string { if d.getS0() == nil { return "" } - + return NewLexerDFASerializer(d).String() } diff --git a/runtime/Go/antlr/v4/dfa_state.go b/runtime/Go/antlr/v4/dfa_state.go index b22cf9efec..6541430745 100644 --- a/runtime/Go/antlr/v4/dfa_state.go +++ b/runtime/Go/antlr/v4/dfa_state.go @@ -47,26 +47,26 @@ func (p *PredPrediction) String() string { type DFAState struct { stateNumber int configs *ATNConfigSet - + // edges elements point to the target of the symbol. Shift up by 1 so (-1) // Token.EOF maps to the first element. edges []*DFAState - + isAcceptState bool // prediction is the 'ttype' we match or alt we predict if the state is 'accept'. // Set to ATN.INVALID_ALT_NUMBER when predicates != nil or // requiresFullContext. prediction int - + lexerActionExecutor *LexerActionExecutor - + // requiresFullContext indicates it was created during an SLL prediction that // discovered a conflict between the configurations in the state. Future // ParserATNSimulator.execATN invocations immediately jump doing // full context prediction if true. requiresFullContext bool - + // predicates is the predicates associated with the ATN configurations of the // DFA state during SLL parsing. When we have predicates, requiresFullContext // is false, since full context prediction evaluates predicates on-the-fly. If @@ -86,24 +86,24 @@ func NewDFAState(stateNumber int, configs *ATNConfigSet) *DFAState { if configs == nil { configs = NewATNConfigSet(false) } - + return &DFAState{configs: configs, stateNumber: stateNumber} } // GetAltSet gets the set of all alts mentioned by all ATN configurations in d. func (d *DFAState) GetAltSet() []int { var alts []int - + if d.configs != nil { for _, c := range d.configs.configs { alts = append(alts, c.GetAlt()) } } - + if len(alts) == 0 { return nil } - + return alts } @@ -140,7 +140,7 @@ func (d *DFAState) String() string { s = "=>" + fmt.Sprint(d.prediction) } } - + return fmt.Sprintf("%d:%s%s", d.stateNumber, fmt.Sprint(d.configs), s) } @@ -165,6 +165,6 @@ func (d *DFAState) Equals(o Collectable[*DFAState]) bool { if d == o { return true } - + return d.configs.Equals(o.(*DFAState).configs) } diff --git a/runtime/Go/antlr/v4/error_strategy.go b/runtime/Go/antlr/v4/error_strategy.go index e84adb4595..9db2be1c74 100644 --- a/runtime/Go/antlr/v4/error_strategy.go +++ b/runtime/Go/antlr/v4/error_strategy.go @@ -32,9 +32,9 @@ type DefaultErrorStrategy struct { var _ ErrorStrategy = &DefaultErrorStrategy{} func NewDefaultErrorStrategy() *DefaultErrorStrategy { - + d := new(DefaultErrorStrategy) - + // Indicates whether the error strategy is currently "recovering from an // error". This is used to suppress Reporting multiple error messages while // attempting to recover from a detected syntax error. @@ -42,7 +42,7 @@ func NewDefaultErrorStrategy() *DefaultErrorStrategy { // @see //InErrorRecoveryMode // d.errorRecoveryMode = false - + // The index into the input stream where the last error occurred. // This is used to prevent infinite loops where an error is found // but no token is consumed during recovery...another error is found, @@ -100,7 +100,7 @@ func (d *DefaultErrorStrategy) ReportError(recognizer Parser, e RecognitionExcep return // don't Report spurious errors } d.beginErrorCondition(recognizer) - + switch t := e.(type) { default: fmt.Println("unknown recognition error type: " + reflect.TypeOf(e).Name()) @@ -119,7 +119,7 @@ func (d *DefaultErrorStrategy) ReportError(recognizer Parser, e RecognitionExcep // It reSynchronizes the parser by consuming tokens until we find one in the reSynchronization set - // loosely the set of tokens that can follow the current rule. func (d *DefaultErrorStrategy) Recover(recognizer Parser, _ RecognitionException) { - + if d.lastErrorIndex == recognizer.GetInputStream().Index() && d.lastErrorStates != nil && d.lastErrorStates.contains(recognizer.GetState()) { // uh oh, another error at same token index and previously-Visited @@ -190,16 +190,16 @@ func (d *DefaultErrorStrategy) Sync(recognizer Parser) { if d.InErrorRecoveryMode(recognizer) { return } - + s := recognizer.GetInterpreter().atn.states[recognizer.GetState()] la := recognizer.GetTokenStream().LA(1) - + // try cheaper subset first might get lucky. seems to shave a wee bit off nextTokens := recognizer.GetATN().NextTokens(s, nil) if nextTokens.contains(TokenEpsilon) || nextTokens.contains(la) { return } - + switch s.GetStateType() { case ATNStateBlockStart, ATNStateStarBlockStart, ATNStatePlusBlockStart, ATNStateStarLoopEntry: // Report error and recover if possible @@ -393,7 +393,7 @@ func (d *DefaultErrorStrategy) SingleTokenInsertion(recognizer Parser) bool { d.ReportMissingToken(recognizer) return true } - + return false } @@ -427,7 +427,7 @@ func (d *DefaultErrorStrategy) SingleTokenDeletion(recognizer Parser) Token { d.ReportMatch(recognizer) // we know current token is correct return MatchedSymbol } - + return nil } @@ -458,7 +458,7 @@ func (d *DefaultErrorStrategy) GetMissingSymbol(recognizer Parser) Token { expecting := d.GetExpectedTokens(recognizer) expectedTokenType := expecting.first() var tokenText string - + if expectedTokenType == TokenEOF { tokenText = "" } else { @@ -474,9 +474,9 @@ func (d *DefaultErrorStrategy) GetMissingSymbol(recognizer Parser) Token { if current.GetTokenType() == TokenEOF && lookback != nil { current = lookback } - + tf := recognizer.GetTokenFactory() - + return tf.Create(current.GetSource(), expectedTokenType, tokenText, TokenDefaultChannel, -1, -1, current.GetLine(), current.GetColumn()) } @@ -664,11 +664,11 @@ var _ ErrorStrategy = &BailErrorStrategy{} //goland:noinspection GoUnusedExportedFunction func NewBailErrorStrategy() *BailErrorStrategy { - + b := new(BailErrorStrategy) - + b.DefaultErrorStrategy = NewDefaultErrorStrategy() - + return b } @@ -693,7 +693,7 @@ func (b *BailErrorStrategy) Recover(recognizer Parser, e RecognitionException) { // successfully recovers, it won't panic an exception. func (b *BailErrorStrategy) RecoverInline(recognizer Parser) Token { b.Recover(recognizer, NewInputMisMatchException(recognizer)) - + return nil } diff --git a/runtime/Go/antlr/v4/errors.go b/runtime/Go/antlr/v4/errors.go index 8239e13130..8f0f2f601f 100644 --- a/runtime/Go/antlr/v4/errors.go +++ b/runtime/Go/antlr/v4/errors.go @@ -26,7 +26,7 @@ type BaseRecognitionException struct { } func NewBaseRecognitionException(message string, recognizer Recognizer, input IntStream, ctx RuleContext) *BaseRecognitionException { - + // todo // Error.call(this) // @@ -36,20 +36,20 @@ func NewBaseRecognitionException(message string, recognizer Recognizer, input In // stack := NewError().stack // } // TODO: may be able to use - "runtime" func Stack(buf []byte, all bool) int - + t := new(BaseRecognitionException) - + t.message = message t.recognizer = recognizer t.input = input t.ctx = ctx - + // The current Token when an error occurred. Since not all streams // support accessing symbols by index, we have to track the {@link Token} // instance itself. // t.offendingToken = nil - + // Get the ATN state number the parser was in at the time the error // occurred. For NoViableAltException and LexerNoViableAltException exceptions, this is the // DecisionState number. For others, it is the state whose outgoing edge we couldn't Match. @@ -58,7 +58,7 @@ func NewBaseRecognitionException(message string, recognizer Recognizer, input In if t.recognizer != nil { t.offendingState = t.recognizer.GetState() } - + return t } @@ -84,11 +84,12 @@ func (b *BaseRecognitionException) GetInputStream() IntStream { // // The func returns the set of token types that could potentially follow the current // state in the {ATN}, or nil if the information is not available. + func (b *BaseRecognitionException) getExpectedTokens() *IntervalSet { if b.recognizer != nil { return b.recognizer.GetATN().getExpectedTokens(b.offendingState, b.ctx) } - + return nil } @@ -98,20 +99,20 @@ func (b *BaseRecognitionException) String() string { type LexerNoViableAltException struct { *BaseRecognitionException - + startIndex int deadEndConfigs *ATNConfigSet } func NewLexerNoViableAltException(lexer Lexer, input CharStream, startIndex int, deadEndConfigs *ATNConfigSet) *LexerNoViableAltException { - + l := new(LexerNoViableAltException) - + l.BaseRecognitionException = NewBaseRecognitionException("", lexer, input, nil) - + l.startIndex = startIndex l.deadEndConfigs = deadEndConfigs - + return l } @@ -125,7 +126,7 @@ func (l *LexerNoViableAltException) String() string { type NoViableAltException struct { *BaseRecognitionException - + startToken Token offendingToken Token ctx ParserRuleContext @@ -139,30 +140,30 @@ type NoViableAltException struct { // // Reported by [ReportNoViableAlternative] func NewNoViableAltException(recognizer Parser, input TokenStream, startToken Token, offendingToken Token, deadEndConfigs *ATNConfigSet, ctx ParserRuleContext) *NoViableAltException { - + if ctx == nil { ctx = recognizer.GetParserRuleContext() } - + if offendingToken == nil { offendingToken = recognizer.GetCurrentToken() } - + if startToken == nil { startToken = recognizer.GetCurrentToken() } - + if input == nil { input = recognizer.GetInputStream().(TokenStream) } - + n := new(NoViableAltException) n.BaseRecognitionException = NewBaseRecognitionException("", recognizer, input, ctx) - + // Which configurations did we try at input.Index() that couldn't Match // input.LT(1) n.deadEndConfigs = deadEndConfigs - + // The token object at the start index the input stream might // not be buffering tokens so get a reference to it. // @@ -170,7 +171,7 @@ func NewNoViableAltException(recognizer Parser, input TokenStream, startToken To // buffer of all the tokens, but later we might not have access to those. n.startToken = startToken n.offendingToken = offendingToken - + return n } @@ -181,14 +182,14 @@ type InputMisMatchException struct { // NewInputMisMatchException creates an exception that signifies any kind of mismatched input exceptions such as // when the current input does not Match the expected token. func NewInputMisMatchException(recognizer Parser) *InputMisMatchException { - + i := new(InputMisMatchException) i.BaseRecognitionException = NewBaseRecognitionException("", recognizer, recognizer.GetInputStream(), recognizer.GetParserRuleContext()) - + i.offendingToken = recognizer.GetCurrentToken() - + return i - + } // FailedPredicateException indicates that a semantic predicate failed during validation. Validation of predicates @@ -197,7 +198,7 @@ func NewInputMisMatchException(recognizer Parser) *InputMisMatchException { // prediction. type FailedPredicateException struct { *BaseRecognitionException - + ruleIndex int predicateIndex int predicate string @@ -205,11 +206,11 @@ type FailedPredicateException struct { //goland:noinspection GoUnusedExportedFunction func NewFailedPredicateException(recognizer Parser, predicate string, message string) *FailedPredicateException { - + f := new(FailedPredicateException) - + f.BaseRecognitionException = NewBaseRecognitionException(f.formatMessage(predicate, message), recognizer, recognizer.GetInputStream(), recognizer.GetParserRuleContext()) - + s := recognizer.GetInterpreter().atn.states[recognizer.GetState()] trans := s.GetTransitions()[0] if trans2, ok := trans.(*PredicateTransition); ok { @@ -221,7 +222,7 @@ func NewFailedPredicateException(recognizer Parser, predicate string, message st } f.predicate = predicate f.offendingToken = recognizer.GetCurrentToken() - + return f } @@ -229,7 +230,7 @@ func (f *FailedPredicateException) formatMessage(predicate, message string) stri if message != "" { return message } - + return "failed predicate: {" + predicate + "}?" } diff --git a/runtime/Go/antlr/v4/file_stream.go b/runtime/Go/antlr/v4/file_stream.go index e8d0efce47..7f072e25ba 100644 --- a/runtime/Go/antlr/v4/file_stream.go +++ b/runtime/Go/antlr/v4/file_stream.go @@ -41,8 +41,7 @@ func NewFileStream(fileName string) (*FileStream, error) { fs := new(FileStream) fs.filename = fileName - s := string(buf.Bytes()) - + s := buf.String() fs.InputStream = NewInputStream(s) return fs, nil diff --git a/runtime/Go/antlr/v4/go.mod b/runtime/Go/antlr/v4/go.mod index 0d8feb6214..ec024f41bb 100644 --- a/runtime/Go/antlr/v4/go.mod +++ b/runtime/Go/antlr/v4/go.mod @@ -1,5 +1,5 @@ module github.com/antlr/antlr4/runtime/Go/antlr/v4 -go 1.18 +go 1.20 require golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e diff --git a/runtime/Go/antlr/v4/interval_set.go b/runtime/Go/antlr/v4/interval_set.go index 649338ba33..cc5066067a 100644 --- a/runtime/Go/antlr/v4/interval_set.go +++ b/runtime/Go/antlr/v4/interval_set.go @@ -32,7 +32,7 @@ func (i Interval) String() string { if i.Start == i.Stop-1 { return strconv.Itoa(i.Start) } - + return strconv.Itoa(i.Start) + ".." + strconv.Itoa(i.Stop-1) } @@ -49,12 +49,12 @@ type IntervalSet struct { // NewIntervalSet creates a new empty, writable, interval set. func NewIntervalSet() *IntervalSet { - + i := new(IntervalSet) - + i.intervals = nil i.readOnly = false - + return i } @@ -62,13 +62,13 @@ func (i *IntervalSet) Equals(other *IntervalSet) bool { if len(i.intervals) != len(other.intervals) { return false } - + for k, v := range i.intervals { if v.Start != other.intervals[k].Start || v.Stop != other.intervals[k].Stop { return false } } - + return true } @@ -76,7 +76,7 @@ func (i *IntervalSet) first() int { if len(i.intervals) == 0 { return TokenInvalidType } - + return i.intervals[0].Start } @@ -104,7 +104,7 @@ func (i *IntervalSet) addInterval(v Interval) { return } else if v.Start <= interval.Stop { i.intervals[k] = NewInterval(intMin(interval.Start, v.Start), intMax(interval.Stop, v.Stop)) - + // if not applying to end, merge potential overlaps if k < len(i.intervals)-1 { l := i.intervals[k] @@ -158,11 +158,11 @@ func (i *IntervalSet) contains(item int) bool { func (i *IntervalSet) length() int { iLen := 0 - + for _, v := range i.intervals { iLen += v.Length() } - + return iLen } @@ -229,7 +229,7 @@ func (i *IntervalSet) String() string { } func (i *IntervalSet) StringVerbose(literalNames []string, symbolicNames []string, elemsAreChar bool) string { - + if i.intervals == nil { return "{}" } else if literalNames != nil || symbolicNames != nil { @@ -237,7 +237,7 @@ func (i *IntervalSet) StringVerbose(literalNames []string, symbolicNames []strin } else if elemsAreChar { return i.toCharString() } - + return i.toIndexString() } @@ -247,9 +247,9 @@ func (i *IntervalSet) GetIntervals() []Interval { func (i *IntervalSet) toCharString() string { names := make([]string, len(i.intervals)) - + var sb strings.Builder - + for j := 0; j < len(i.intervals); j++ { v := i.intervals[j] if v.Stop == v.Start+1 { @@ -275,12 +275,12 @@ func (i *IntervalSet) toCharString() string { if len(names) > 1 { return "{" + strings.Join(names, ", ") + "}" } - + return names[0] } func (i *IntervalSet) toIndexString() string { - + names := make([]string, 0) for j := 0; j < len(i.intervals); j++ { v := i.intervals[j] @@ -297,7 +297,7 @@ func (i *IntervalSet) toIndexString() string { if len(names) > 1 { return "{" + strings.Join(names, ", ") + "}" } - + return names[0] } @@ -311,7 +311,7 @@ func (i *IntervalSet) toTokenString(literalNames []string, symbolicNames []strin if len(names) > 1 { return "{" + strings.Join(names, ", ") + "}" } - + return names[0] } @@ -324,7 +324,7 @@ func (i *IntervalSet) elementName(literalNames []string, symbolicNames []string, if a < len(literalNames) && literalNames[a] != "" { return literalNames[a] } - + return symbolicNames[a] } } diff --git a/runtime/Go/antlr/v4/jcollect.go b/runtime/Go/antlr/v4/jcollect.go index a336e8f3cb..f1f9555565 100644 --- a/runtime/Go/antlr/v4/jcollect.go +++ b/runtime/Go/antlr/v4/jcollect.go @@ -5,7 +5,10 @@ package antlr // can be found in the LICENSE.txt file in the project root. import ( + "container/list" + "runtime/debug" "sort" + "sync" ) // Collectable is an interface that a struct should implement if it is to be @@ -20,6 +23,86 @@ type Comparator[T any] interface { Equals2(T, T) bool } +type CollectionSource int +type CollectionDescriptor struct { + SybolicName string + Description string +} + +const ( + UnknownCollection CollectionSource = iota + ATNConfigLookupCollection + ATNStateCollection + DFAStateCollection + ATNConfigCollection + PredictionContextCollection + SemanticContextCollection + ClosureBusyCollection + PredictionVisitedCollection + MergeCacheCollection + PredictionContextCacheCollection + AltSetCollection + ReachSetCollection +) + +var CollectionDescriptors = map[CollectionSource]CollectionDescriptor{ + UnknownCollection: { + SybolicName: "UnknownCollection", + Description: "Unknown collection type. Only used if the target author thought it was an unimportant collection.", + }, + ATNConfigCollection: { + SybolicName: "ATNConfigCollection", + Description: "ATNConfig collection. Used to store the ATNConfigs for a particular state in the ATN." + + "For instance, it is used to store the results of the closure() operation in the ATN.", + }, + ATNConfigLookupCollection: { + SybolicName: "ATNConfigLookupCollection", + Description: "ATNConfigLookup collection. Used to store the ATNConfigs for a particular state in the ATN." + + "This is used to prevent duplicating equivalent states in an ATNConfigurationSet.", + }, + ATNStateCollection: { + SybolicName: "ATNStateCollection", + Description: "ATNState collection. This is used to store the states of the ATN.", + }, + DFAStateCollection: { + SybolicName: "DFAStateCollection", + Description: "DFAState collection. This is used to store the states of the DFA.", + }, + PredictionContextCollection: { + SybolicName: "PredictionContextCollection", + Description: "PredictionContext collection. This is used to store the prediction contexts of the ATN and cache computes.", + }, + SemanticContextCollection: { + SybolicName: "SemanticContextCollection", + Description: "SemanticContext collection. This is used to store the semantic contexts of the ATN.", + }, + ClosureBusyCollection: { + SybolicName: "ClosureBusyCollection", + Description: "ClosureBusy collection. This is used to check and prevent infinite recursion right recursive rules." + + "It stores ATNConfigs that are currently being processed in the closure() operation.", + }, + PredictionVisitedCollection: { + SybolicName: "PredictionVisitedCollection", + Description: "A map that records whether we have visited a particular context when searching through cached entries.", + }, + MergeCacheCollection: { + SybolicName: "MergeCacheCollection", + Description: "A map that records whether we have already merged two particular contexts and can save effort by not repeating it.", + }, + PredictionContextCacheCollection: { + SybolicName: "PredictionContextCacheCollection", + Description: "A map that records whether we have already created a particular context and can save effort by not computing it again.", + }, + AltSetCollection: { + SybolicName: "AltSetCollection", + Description: "Used to eliminate duplicate alternatives in an ATN config set.", + }, + ReachSetCollection: { + SybolicName: "ReachSetCollection", + Description: "Used as merge cache to prevent us needing to compute the merge of two states if we have already done it.", + }, +} + // JStore implements a container that allows the use of a struct to calculate the key // for a collection of values akin to map. This is not meant to be a full-blown HashMap but just // serve the needs of the ANTLR Go runtime. @@ -35,9 +118,10 @@ type JStore[T any, C Comparator[T]] struct { store map[int][]T len int comparator Comparator[T] + stats *JStatRec } -func NewJStore[T any, C Comparator[T]](comparator Comparator[T]) *JStore[T, C] { +func NewJStore[T any, C Comparator[T]](comparator Comparator[T], cType CollectionSource, desc string) *JStore[T, C] { if comparator == nil { panic("comparator cannot be nil") @@ -47,6 +131,18 @@ func NewJStore[T any, C Comparator[T]](comparator Comparator[T]) *JStore[T, C] { store: make(map[int][]T, 1), comparator: comparator, } + if collectStats { + s.stats = &JStatRec{ + Source: cType, + Description: desc, + } + + // Track where we created it from if we are being asked to do so + if runtimeConfig.statsTraceStacks { + s.stats.CreateStack = debug.Stack() + } + Statistics.AddJStatRec(s.stats) + } return s } @@ -63,15 +159,42 @@ func NewJStore[T any, C Comparator[T]](comparator Comparator[T]) *JStore[T, C] { // If the given value is not present in the store, then the value is added to the store and returned as v and exists is set to false. func (s *JStore[T, C]) Put(value T) (v T, exists bool) { + if collectStats { + s.stats.Puts++ + } kh := s.comparator.Hash1(value) + var hClash bool for _, v1 := range s.store[kh] { + hClash = true if s.comparator.Equals2(value, v1) { + if collectStats { + s.stats.PutHits++ + s.stats.PutHashConflicts++ + } return v1, true } + if collectStats { + s.stats.PutMisses++ + } + } + if collectStats && hClash { + s.stats.PutHashConflicts++ } s.store[kh] = append(s.store[kh], value) + + if collectStats { + if len(s.store[kh]) > s.stats.MaxSlotSize { + s.stats.MaxSlotSize = len(s.store[kh]) + } + } s.len++ + if collectStats { + s.stats.CurSize = s.len + if s.len > s.stats.MaxSize { + s.stats.MaxSize = s.len + } + } return value, false } @@ -79,20 +202,35 @@ func (s *JStore[T, C]) Put(value T) (v T, exists bool) { // which would not generally be useful, but this is a specific thing for ANTLR where the key is // generated using the object we are going to store. func (s *JStore[T, C]) Get(key T) (T, bool) { - + if collectStats { + s.stats.Gets++ + } kh := s.comparator.Hash1(key) - + var hClash bool for _, v := range s.store[kh] { + hClash = true if s.comparator.Equals2(key, v) { + if collectStats { + s.stats.GetHits++ + s.stats.GetHashConflicts++ + } return v, true } + if collectStats { + s.stats.GetMisses++ + } + } + if collectStats { + if hClash { + s.stats.GetHashConflicts++ + } + s.stats.GetNoEnt++ } return key, false } // Contains returns true if the given key is present in the store func (s *JStore[T, C]) Contains(key T) bool { - _, present := s.Get(key) return present } @@ -124,9 +262,7 @@ func (s *JStore[T, C]) Len() int { func (s *JStore[T, C]) Values() []T { vs := make([]T, 0, len(s.store)) for _, e := range s.store { - for _, v := range e { - vs = append(vs, v) - } + vs = append(vs, e...) } return vs } @@ -140,20 +276,67 @@ type JMap[K, V any, C Comparator[K]] struct { store map[int][]*entry[K, V] len int comparator Comparator[K] + stats *JStatRec } -func NewJMap[K, V any, C Comparator[K]](comparator Comparator[K]) *JMap[K, V, C] { - return &JMap[K, V, C]{ +func NewJMap[K, V any, C Comparator[K]](comparator Comparator[K], cType CollectionSource, desc string) *JMap[K, V, C] { + m := &JMap[K, V, C]{ store: make(map[int][]*entry[K, V], 1), comparator: comparator, } + if collectStats { + m.stats = &JStatRec{ + Source: cType, + Description: desc, + } + // Track where we created it from if we are being asked to do so + if runtimeConfig.statsTraceStacks { + m.stats.CreateStack = debug.Stack() + } + Statistics.AddJStatRec(m.stats) + } + return m } -func (m *JMap[K, V, C]) Put(key K, val V) { +func (m *JMap[K, V, C]) Put(key K, val V) (V, bool) { + if collectStats { + m.stats.Puts++ + } kh := m.comparator.Hash1(key) + var hClash bool + for _, e := range m.store[kh] { + hClash = true + if m.comparator.Equals2(e.key, key) { + if collectStats { + m.stats.PutHits++ + m.stats.PutHashConflicts++ + } + return e.val, true + } + if collectStats { + m.stats.PutMisses++ + } + } + if collectStats { + if hClash { + m.stats.PutHashConflicts++ + } + } m.store[kh] = append(m.store[kh], &entry[K, V]{key, val}) + if collectStats { + if len(m.store[kh]) > m.stats.MaxSlotSize { + m.stats.MaxSlotSize = len(m.store[kh]) + } + } m.len++ + if collectStats { + m.stats.CurSize = m.len + if m.len > m.stats.MaxSize { + m.stats.MaxSize = m.len + } + } + return val, false } func (m *JMap[K, V, C]) Values() []V { @@ -167,19 +350,36 @@ func (m *JMap[K, V, C]) Values() []V { } func (m *JMap[K, V, C]) Get(key K) (V, bool) { - + if collectStats { + m.stats.Gets++ + } var none V kh := m.comparator.Hash1(key) + var hClash bool for _, e := range m.store[kh] { + hClash = true if m.comparator.Equals2(e.key, key) { + if collectStats { + m.stats.GetHits++ + m.stats.GetHashConflicts++ + } return e.val, true } + if collectStats { + m.stats.GetMisses++ + } + } + if collectStats { + if hClash { + m.stats.GetHashConflicts++ + } + m.stats.GetNoEnt++ } return none, false } func (m *JMap[K, V, C]) Len() int { - return len(m.store) + return m.len } func (m *JMap[K, V, C]) Delete(key K) { @@ -199,38 +399,287 @@ func (m *JMap[K, V, C]) Clear() { type JPCMap struct { store *JMap[*PredictionContext, *JMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]], *ObjEqComparator[*PredictionContext]] + size int + stats *JStatRec } -func NewJPCMap() *JPCMap { - return &JPCMap{ - store: NewJMap[*PredictionContext, *JMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]], *ObjEqComparator[*PredictionContext]](pContextEqInst), +func NewJPCMap(cType CollectionSource, desc string) *JPCMap { + m := &JPCMap{ + store: NewJMap[*PredictionContext, *JMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]], *ObjEqComparator[*PredictionContext]](pContextEqInst, cType, desc), + } + if collectStats { + m.stats = &JStatRec{ + Source: cType, + Description: desc, + } + // Track where we created it from if we are being asked to do so + if runtimeConfig.statsTraceStacks { + m.stats.CreateStack = debug.Stack() + } + Statistics.AddJStatRec(m.stats) } + return m } func (pcm *JPCMap) Get(k1, k2 *PredictionContext) (*PredictionContext, bool) { - + if collectStats { + pcm.stats.Gets++ + } // Do we have a map stored by k1? // m2, present := pcm.store.Get(k1) if present { + if collectStats { + pcm.stats.GetHits++ + } // We found a map of values corresponding to k1, so now we need to look up k2 in that map // return m2.Get(k2) } + if collectStats { + pcm.stats.GetMisses++ + } return nil, false } func (pcm *JPCMap) Put(k1, k2, v *PredictionContext) { + if collectStats { + pcm.stats.Puts++ + } // First does a map already exist for k1? // if m2, present := pcm.store.Get(k1); present { - m2.Put(k2, v) + if collectStats { + pcm.stats.PutHits++ + } + _, present = m2.Put(k2, v) + if !present { + pcm.size++ + if collectStats { + pcm.stats.CurSize = pcm.size + if pcm.size > pcm.stats.MaxSize { + pcm.stats.MaxSize = pcm.size + } + } + } } else { // No map found for k1, so we create it, add in our value, then store is // - m2 = NewJMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]](pContextEqInst) + if collectStats { + pcm.stats.PutMisses++ + m2 = NewJMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]](pContextEqInst, pcm.stats.Source, pcm.stats.Description+" map entry") + } else { + m2 = NewJMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]](pContextEqInst, PredictionContextCacheCollection, "map entry") + } + m2.Put(k2, v) pcm.store.Put(k1, m2) + pcm.size++ + } +} + +type JPCMap2 struct { + store map[int][]JPCEntry + size int + stats *JStatRec +} + +type JPCEntry struct { + k1, k2, v *PredictionContext +} + +func NewJPCMap2(cType CollectionSource, desc string) *JPCMap2 { + m := &JPCMap2{ + store: make(map[int][]JPCEntry, 1000), + } + if collectStats { + m.stats = &JStatRec{ + Source: cType, + Description: desc, + } + // Track where we created it from if we are being asked to do so + if runtimeConfig.statsTraceStacks { + m.stats.CreateStack = debug.Stack() + } + Statistics.AddJStatRec(m.stats) + } + return m +} + +func dHash(k1, k2 *PredictionContext) int { + return k1.cachedHash*31 + k2.cachedHash +} + +func (pcm *JPCMap2) Get(k1, k2 *PredictionContext) (*PredictionContext, bool) { + if collectStats { + pcm.stats.Gets++ + } + + h := dHash(k1, k2) + var hClash bool + for _, e := range pcm.store[h] { + hClash = true + if e.k1.Equals(k1) && e.k2.Equals(k2) { + if collectStats { + pcm.stats.GetHits++ + pcm.stats.GetHashConflicts++ + } + return e.v, true + } + if collectStats { + pcm.stats.GetMisses++ + } + } + if collectStats { + if hClash { + pcm.stats.GetHashConflicts++ + } + pcm.stats.GetNoEnt++ + } + return nil, false +} + +func (pcm *JPCMap2) Put(k1, k2, v *PredictionContext) (*PredictionContext, bool) { + if collectStats { + pcm.stats.Puts++ + } + h := dHash(k1, k2) + var hClash bool + for _, e := range pcm.store[h] { + hClash = true + if e.k1.Equals(k1) && e.k2.Equals(k2) { + if collectStats { + pcm.stats.PutHits++ + pcm.stats.PutHashConflicts++ + } + return e.v, true + } + if collectStats { + pcm.stats.PutMisses++ + } + } + if collectStats { + if hClash { + pcm.stats.PutHashConflicts++ + } + } + pcm.store[h] = append(pcm.store[h], JPCEntry{k1, k2, v}) + pcm.size++ + if collectStats { + pcm.stats.CurSize = pcm.size + if pcm.size > pcm.stats.MaxSize { + pcm.stats.MaxSize = pcm.size + } + } + return nil, false +} + +type VisitEntry struct { + k *PredictionContext + v *PredictionContext +} +type VisitRecord struct { + store map[*PredictionContext]*PredictionContext + len int + stats *JStatRec +} + +type VisitList struct { + cache *list.List + lock sync.RWMutex +} + +var visitListPool = VisitList{ + cache: list.New(), + lock: sync.RWMutex{}, +} + +// NewVisitRecord returns a new VisitRecord instance from the pool if available. +// Note that this "map" uses a pointer as a key because we are emulating the behavior of +// IdentityHashMap in Java, which uses the `==` operator to compare whether the keys are equal, +// which means is the key the same reference to an object rather than is it .equals() to another +// object. +func NewVisitRecord() *VisitRecord { + visitListPool.lock.Lock() + el := visitListPool.cache.Front() + defer visitListPool.lock.Unlock() + var vr *VisitRecord + if el == nil { + vr = &VisitRecord{ + store: make(map[*PredictionContext]*PredictionContext), + } + if collectStats { + vr.stats = &JStatRec{ + Source: PredictionContextCacheCollection, + Description: "VisitRecord", + } + // Track where we created it from if we are being asked to do so + if runtimeConfig.statsTraceStacks { + vr.stats.CreateStack = debug.Stack() + } + } + } else { + vr = el.Value.(*VisitRecord) + visitListPool.cache.Remove(el) + vr.store = make(map[*PredictionContext]*PredictionContext) + } + if collectStats { + Statistics.AddJStatRec(vr.stats) + } + return vr +} + +func (vr *VisitRecord) Release() { + vr.len = 0 + vr.store = nil + if collectStats { + vr.stats.MaxSize = 0 + vr.stats.CurSize = 0 + vr.stats.Gets = 0 + vr.stats.GetHits = 0 + vr.stats.GetMisses = 0 + vr.stats.GetHashConflicts = 0 + vr.stats.GetNoEnt = 0 + vr.stats.Puts = 0 + vr.stats.PutHits = 0 + vr.stats.PutMisses = 0 + vr.stats.PutHashConflicts = 0 + vr.stats.MaxSlotSize = 0 + } + visitListPool.lock.Lock() + visitListPool.cache.PushBack(vr) + visitListPool.lock.Unlock() +} + +func (vr *VisitRecord) Get(k *PredictionContext) (*PredictionContext, bool) { + if collectStats { + vr.stats.Gets++ + } + v := vr.store[k] + if v != nil { + if collectStats { + vr.stats.GetHits++ + } + return v, true + } + if collectStats { + vr.stats.GetNoEnt++ + } + return nil, false +} + +func (vr *VisitRecord) Put(k, v *PredictionContext) (*PredictionContext, bool) { + if collectStats { + vr.stats.Puts++ + } + vr.store[k] = v + vr.len++ + if collectStats { + vr.stats.CurSize = vr.len + if vr.len > vr.stats.MaxSize { + vr.stats.MaxSize = vr.len + } } + return v, false } diff --git a/runtime/Go/antlr/v4/jcollect_test.go b/runtime/Go/antlr/v4/jcollect_test.go deleted file mode 100644 index 816307a02c..0000000000 --- a/runtime/Go/antlr/v4/jcollect_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package antlr - -import "testing" - -func Test_try(t *testing.T) { - tests := []struct { - name string - }{ - {"Test_try"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - }) - } -} diff --git a/runtime/Go/antlr/v4/lexer.go b/runtime/Go/antlr/v4/lexer.go index 057e37f9e6..30d50af43a 100644 --- a/runtime/Go/antlr/v4/lexer.go +++ b/runtime/Go/antlr/v4/lexer.go @@ -257,8 +257,7 @@ func (b *BaseLexer) SetMode(m int) { // PushMode saves the current lexer mode so that it can be restored later. See [PopMode], then sets the // current lexer mode to the supplied mode m. func (b *BaseLexer) PushMode(m int) { - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("pushMode " + strconv.Itoa(m)) } b.modeStack.Push(b.mode) @@ -271,8 +270,7 @@ func (b *BaseLexer) PopMode() int { if len(b.modeStack) == 0 { panic("Empty Stack") } - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("popMode back to " + fmt.Sprint(b.modeStack[0:len(b.modeStack)-1])) } i, _ := b.modeStack.Pop() diff --git a/runtime/Go/antlr/v4/lexer_action.go b/runtime/Go/antlr/v4/lexer_action.go index 249afdb37e..eaa7393e06 100644 --- a/runtime/Go/antlr/v4/lexer_action.go +++ b/runtime/Go/antlr/v4/lexer_action.go @@ -9,25 +9,25 @@ import "strconv" const ( // LexerActionTypeChannel represents a [LexerChannelAction] action. LexerActionTypeChannel = 0 - + // LexerActionTypeCustom represents a [LexerCustomAction] action. LexerActionTypeCustom = 1 - + // LexerActionTypeMode represents a [LexerModeAction] action. LexerActionTypeMode = 2 - + // LexerActionTypeMore represents a [LexerMoreAction] action. LexerActionTypeMore = 3 - + // LexerActionTypePopMode represents a [LexerPopModeAction] action. LexerActionTypePopMode = 4 - + // LexerActionTypePushMode represents a [LexerPushModeAction] action. LexerActionTypePushMode = 5 - + // LexerActionTypeSkip represents a [LexerSkipAction] action. LexerActionTypeSkip = 6 - + // LexerActionTypeType represents a [LexerTypeAction] action. LexerActionTypeType = 7 ) @@ -47,10 +47,10 @@ type BaseLexerAction struct { func NewBaseLexerAction(action int) *BaseLexerAction { la := new(BaseLexerAction) - + la.actionType = action la.isPositionDependent = false - + return la } @@ -111,7 +111,7 @@ func (b *LexerSkipAction) Equals(other LexerAction) bool { // with the assigned type. type LexerTypeAction struct { *BaseLexerAction - + thetype int } @@ -155,10 +155,10 @@ type LexerPushModeAction struct { } func NewLexerPushModeAction(mode int) *LexerPushModeAction { - + l := new(LexerPushModeAction) l.BaseLexerAction = NewBaseLexerAction(LexerActionTypePushMode) - + l.mode = mode return l } @@ -199,11 +199,11 @@ type LexerPopModeAction struct { } func NewLexerPopModeAction() *LexerPopModeAction { - + l := new(LexerPopModeAction) - + l.BaseLexerAction = NewBaseLexerAction(LexerActionTypePopMode) - + return l } @@ -230,7 +230,7 @@ type LexerMoreAction struct { func NewLexerMoreAction() *LexerMoreAction { l := new(LexerMoreAction) l.BaseLexerAction = NewBaseLexerAction(LexerActionTypeMore) - + return l } @@ -415,14 +415,14 @@ type LexerIndexedCustomAction struct { // the token start index, at which the specified lexerAction should be // executed. func NewLexerIndexedCustomAction(offset int, lexerAction LexerAction) *LexerIndexedCustomAction { - + l := new(LexerIndexedCustomAction) l.BaseLexerAction = NewBaseLexerAction(lexerAction.getActionType()) - + l.offset = offset l.lexerAction = lexerAction l.isPositionDependent = true - + return l } diff --git a/runtime/Go/antlr/v4/lexer_action_executor.go b/runtime/Go/antlr/v4/lexer_action_executor.go index ab38e99799..dfc28c32b3 100644 --- a/runtime/Go/antlr/v4/lexer_action_executor.go +++ b/runtime/Go/antlr/v4/lexer_action_executor.go @@ -19,15 +19,15 @@ type LexerActionExecutor struct { } func NewLexerActionExecutor(lexerActions []LexerAction) *LexerActionExecutor { - + if lexerActions == nil { lexerActions = make([]LexerAction, 0) } - + l := new(LexerActionExecutor) - + l.lexerActions = lexerActions - + // Caches the result of {@link //hashCode} since the hash code is an element // of the performance-critical {@link ATNConfig//hashCode} operation. l.cachedHash = murmurInit(0) @@ -35,7 +35,7 @@ func NewLexerActionExecutor(lexerActions []LexerAction) *LexerActionExecutor { l.cachedHash = murmurUpdate(l.cachedHash, a.Hash()) } l.cachedHash = murmurFinish(l.cachedHash, len(lexerActions)) - + return l } @@ -47,7 +47,7 @@ func LexerActionExecutorappend(lexerActionExecutor *LexerActionExecutor, lexerAc if lexerActionExecutor == nil { return NewLexerActionExecutor([]LexerAction{lexerAction}) } - + return NewLexerActionExecutor(append(lexerActionExecutor.lexerActions, lexerAction)) } @@ -78,27 +78,22 @@ func LexerActionExecutorappend(lexerActionExecutor *LexerActionExecutor, lexerAc // // The func returns a [LexerActionExecutor] that stores input stream offsets // for all position-dependent lexer actions. -// func (l *LexerActionExecutor) fixOffsetBeforeMatch(offset int) *LexerActionExecutor { var updatedLexerActions []LexerAction for i := 0; i < len(l.lexerActions); i++ { _, ok := l.lexerActions[i].(*LexerIndexedCustomAction) if l.lexerActions[i].getIsPositionDependent() && !ok { if updatedLexerActions == nil { - updatedLexerActions = make([]LexerAction, 0) - - for _, a := range l.lexerActions { - updatedLexerActions = append(updatedLexerActions, a) - } + updatedLexerActions = make([]LexerAction, 0, len(l.lexerActions)) + updatedLexerActions = append(updatedLexerActions, l.lexerActions...) } - updatedLexerActions[i] = NewLexerIndexedCustomAction(offset, l.lexerActions[i]) } } if updatedLexerActions == nil { return l } - + return NewLexerActionExecutor(updatedLexerActions) } @@ -123,13 +118,13 @@ func (l *LexerActionExecutor) fixOffsetBeforeMatch(offset int) *LexerActionExecu func (l *LexerActionExecutor) execute(lexer Lexer, input CharStream, startIndex int) { requiresSeek := false stopIndex := input.Index() - + defer func() { if requiresSeek { input.Seek(stopIndex) } }() - + for i := 0; i < len(l.lexerActions); i++ { lexerAction := l.lexerActions[i] if la, ok := lexerAction.(*LexerIndexedCustomAction); ok { @@ -150,7 +145,7 @@ func (l *LexerActionExecutor) Hash() int { // TODO: Why is this here? l should not be nil return 61 } - + // TODO: This is created from the action itself when the struct is created - will this be an issue at some point? Java uses the runtime assign hashcode return l.cachedHash } diff --git a/runtime/Go/antlr/v4/lexer_atn_simulator.go b/runtime/Go/antlr/v4/lexer_atn_simulator.go index 8dff06324c..cb56aa0f3d 100644 --- a/runtime/Go/antlr/v4/lexer_atn_simulator.go +++ b/runtime/Go/antlr/v4/lexer_atn_simulator.go @@ -12,9 +12,6 @@ import ( //goland:noinspection GoUnusedGlobalVariable var ( - LexerATNSimulatorDebug = false - LexerATNSimulatorDFADebug = false - LexerATNSimulatorMinDFAEdge = 0 LexerATNSimulatorMaxDFAEdge = 127 // forces unicode to stay in ATN @@ -37,7 +34,7 @@ type LexerATNSimulator struct { recog Lexer predictionMode int - mergeCache *JPCMap + mergeCache *JPCMap2 startIndex int Line int CharPositionInLine int @@ -123,8 +120,7 @@ func (l *LexerATNSimulator) reset() { func (l *LexerATNSimulator) MatchATN(input CharStream) int { startState := l.atn.modeToStartState[l.mode] - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("MatchATN mode " + strconv.Itoa(l.mode) + " start: " + startState.String()) } oldMode := l.mode @@ -136,8 +132,7 @@ func (l *LexerATNSimulator) MatchATN(input CharStream) int { predict := l.execATN(input, next) - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("DFA after MatchATN: " + l.decisionToDFA[oldMode].ToLexerString()) } return predict @@ -145,8 +140,7 @@ func (l *LexerATNSimulator) MatchATN(input CharStream) int { func (l *LexerATNSimulator) execATN(input CharStream, ds0 *DFAState) int { - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("start state closure=" + ds0.configs.String()) } if ds0.isAcceptState { @@ -157,8 +151,7 @@ func (l *LexerATNSimulator) execATN(input CharStream, ds0 *DFAState) int { s := ds0 // s is current/from DFA state for { // while more work - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("execATN loop starting closure: " + s.configs.String()) } @@ -227,8 +220,7 @@ func (l *LexerATNSimulator) getExistingTargetState(s *DFAState, t int) *DFAState return nil } target := s.getIthEdge(t - LexerATNSimulatorMinDFAEdge) - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug && target != nil { + if runtimeConfig.lexerATNSimulatorDebug && target != nil { fmt.Println("reuse state " + strconv.Itoa(s.stateNumber) + " edge to " + strconv.Itoa(target.stateNumber)) } return target @@ -281,17 +273,16 @@ func (l *LexerATNSimulator) failOrAccept(prevAccept *SimState, input CharStream, // Parameter reach is a return parameter. func (l *LexerATNSimulator) getReachableConfigSet(input CharStream, closure *ATNConfigSet, reach *ATNConfigSet, t int) { // l is used to Skip processing for configs which have a lower priority - // than a config that already reached an accept state for the same rule + // than a runtimeConfig that already reached an accept state for the same rule SkipAlt := ATNInvalidAltNumber - + for _, cfg := range closure.configs { currentAltReachedAcceptState := cfg.GetAlt() == SkipAlt if currentAltReachedAcceptState && cfg.passedThroughNonGreedyDecision { continue } - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Printf("testing %s at %s\n", l.GetTokenName(t), cfg.String()) } @@ -317,8 +308,7 @@ func (l *LexerATNSimulator) getReachableConfigSet(input CharStream, closure *ATN } func (l *LexerATNSimulator) accept(input CharStream, lexerActionExecutor *LexerActionExecutor, startIndex, index, line, charPos int) { - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Printf("ACTION %v\n", lexerActionExecutor) } // seek to after last char in token @@ -352,23 +342,21 @@ func (l *LexerATNSimulator) computeStartState(input CharStream, p ATNState) *ATN // closure since the alternatives within any lexer decision are ordered by // preference, this method stops pursuing the closure as soon as an accept // state is reached. After the first accept state is reached by depth-first -// search from config, all other (potentially reachable) states for +// search from runtimeConfig, all other (potentially reachable) states for // this rule would have a lower priority. // // The func returns true if an accept state is reached. func (l *LexerATNSimulator) closure(input CharStream, config *ATNConfig, configs *ATNConfigSet, currentAltReachedAcceptState, speculative, treatEOFAsEpsilon bool) bool { - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("closure(" + config.String() + ")") } _, ok := config.state.(*RuleStopState) if ok { - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { if l.recog != nil { fmt.Printf("closure at %s rule stop %s\n", l.recog.GetRuleNames()[config.state.GetRuleIndex()], config) } else { @@ -449,8 +437,7 @@ func (l *LexerATNSimulator) getEpsilonTarget(input CharStream, config *ATNConfig pt := trans.(*PredicateTransition) - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("EVAL rule " + strconv.Itoa(trans.(*PredicateTransition).ruleIndex) + ":" + strconv.Itoa(pt.predIndex)) } configs.hasSemanticContext = true @@ -560,8 +547,7 @@ func (l *LexerATNSimulator) addDFAEdge(from *DFAState, tk int, to *DFAState, cfg // Only track edges within the DFA bounds return to } - if //goland:noinspection GoBoolExpressions - LexerATNSimulatorDebug { + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("EDGE " + from.String() + " -> " + to.String() + " upon " + strconv.Itoa(tk)) } l.atn.edgeMu.Lock() @@ -601,7 +587,7 @@ func (l *LexerATNSimulator) addDFAState(configs *ATNConfigSet, suppressEdge bool l.atn.stateMu.Lock() defer l.atn.stateMu.Unlock() - existing, present := dfa.states.Get(proposed) + existing, present := dfa.Get(proposed) if present { // This state was already present, so just return it. @@ -611,11 +597,11 @@ func (l *LexerATNSimulator) addDFAState(configs *ATNConfigSet, suppressEdge bool // We need to add the new state // - proposed.stateNumber = dfa.states.Len() + proposed.stateNumber = dfa.Len() configs.readOnly = true configs.configLookup = nil // Not needed now proposed.configs = configs - dfa.states.Put(proposed) + dfa.Put(proposed) } if !suppressEdge { dfa.setS0(proposed) diff --git a/runtime/Go/antlr/v4/ll1_analyzer.go b/runtime/Go/antlr/v4/ll1_analyzer.go index 3319f8642b..4955ac876f 100644 --- a/runtime/Go/antlr/v4/ll1_analyzer.go +++ b/runtime/Go/antlr/v4/ll1_analyzer.go @@ -38,11 +38,11 @@ func (la *LL1Analyzer) getDecisionLookahead(s ATNState) []*IntervalSet { count := len(s.GetTransitions()) look := make([]*IntervalSet, count) for alt := 0; alt < count; alt++ { - + look[alt] = NewIntervalSet() - lookBusy := NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst) + lookBusy := NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ClosureBusyCollection, "LL1Analyzer.getDecisionLookahead for lookBusy") la.look1(s.GetTransitions()[alt].getTarget(), nil, BasePredictionContextEMPTY, look[alt], lookBusy, NewBitSet(), false, false) - + // Wipe out lookahead for la alternative if we found nothing, // or we had a predicate when we !seeThruPreds if look[alt].length() == 0 || look[alt].contains(LL1AnalyzerHitPred) { @@ -75,7 +75,8 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet if ctx != nil { lookContext = predictionContextFromRuleContext(s.GetATN(), ctx) } - la.look1(s, stopState, lookContext, r, NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst), NewBitSet(), true, true) + la.look1(s, stopState, lookContext, r, NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ClosureBusyCollection, "LL1Analyzer.Look for la.look1()"), + NewBitSet(), true, true) return r } @@ -109,25 +110,26 @@ func (la *LL1Analyzer) Look(s, stopState ATNState, ctx RuleContext) *IntervalSet // outermost context is reached. This parameter has no effect if {@code ctx} // is {@code nil}. -func (la *LL1Analyzer) look2(_, stopState ATNState, ctx *PredictionContext, look *IntervalSet, lookBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) { - +func (la *LL1Analyzer) look2(_, stopState ATNState, ctx *PredictionContext, look *IntervalSet, lookBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], + calledRuleStack *BitSet, seeThruPreds, addEOF bool, i int) { + returnState := la.atn.states[ctx.getReturnState(i)] la.look1(returnState, stopState, ctx.GetParent(i), look, lookBusy, calledRuleStack, seeThruPreds, addEOF) - + } func (la *LL1Analyzer) look1(s, stopState ATNState, ctx *PredictionContext, look *IntervalSet, lookBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], calledRuleStack *BitSet, seeThruPreds, addEOF bool) { - + c := NewATNConfig6(s, 0, ctx) - + if lookBusy.Contains(c) { return } - + _, present := lookBusy.Put(c) if present { return - + } if s == stopState { if ctx == nil { @@ -138,9 +140,9 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx *PredictionContext, look return } } - + _, ok := s.(*RuleStopState) - + if ok { if ctx == nil { look.addOne(TokenEpsilon) @@ -149,7 +151,7 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx *PredictionContext, look look.addOne(TokenEOF) return } - + if ctx.pcType != PredictionContextEmpty { removed := calledRuleStack.contains(s.GetRuleIndex()) defer func() { @@ -166,17 +168,17 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx *PredictionContext, look return } } - + n := len(s.GetTransitions()) - + for i := 0; i < n; i++ { t := s.GetTransitions()[i] - + if t1, ok := t.(*RuleTransition); ok { if calledRuleStack.contains(t1.getTarget().GetRuleIndex()) { continue } - + newContext := SingletonBasePredictionContextCreate(ctx, t1.followState.GetStateNumber()) la.look3(stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF, t1) } else if t2, ok := t.(AbstractPredicateTransition); ok { @@ -201,15 +203,16 @@ func (la *LL1Analyzer) look1(s, stopState ATNState, ctx *PredictionContext, look } } -func (la *LL1Analyzer) look3(stopState ATNState, ctx *PredictionContext, look *IntervalSet, lookBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], calledRuleStack *BitSet, seeThruPreds, addEOF bool, t1 *RuleTransition) { - +func (la *LL1Analyzer) look3(stopState ATNState, ctx *PredictionContext, look *IntervalSet, lookBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], + calledRuleStack *BitSet, seeThruPreds, addEOF bool, t1 *RuleTransition) { + newContext := SingletonBasePredictionContextCreate(ctx, t1.followState.GetStateNumber()) - + defer func() { calledRuleStack.remove(t1.getTarget().GetRuleIndex()) }() - + calledRuleStack.add(t1.getTarget().GetRuleIndex()) la.look1(t1.getTarget(), stopState, newContext, look, lookBusy, calledRuleStack, seeThruPreds, addEOF) - + } diff --git a/runtime/Go/antlr/v4/nostatistics.go b/runtime/Go/antlr/v4/nostatistics.go new file mode 100644 index 0000000000..923c7b52c4 --- /dev/null +++ b/runtime/Go/antlr/v4/nostatistics.go @@ -0,0 +1,47 @@ +//go:build !antlr.stats + +package antlr + +// This file is compiled when the build configuration antlr.stats is not enabled. +// which then allows the compiler to optimize out all the code that is not used. +const collectStats = false + +// goRunStats is a dummy struct used when build configuration antlr.stats is not enabled. +type goRunStats struct { +} + +var Statistics = &goRunStats{} + +func (s *goRunStats) AddJStatRec(_ *JStatRec) { + // Do nothing - compiler will optimize this out (hopefully) +} + +func (s *goRunStats) CollectionAnomalies() { + // Do nothing - compiler will optimize this out (hopefully) +} + +func (s *goRunStats) Reset() { + // Do nothing - compiler will optimize this out (hopefully) +} + +func (s *goRunStats) Report(dir string, prefix string) error { + // Do nothing - compiler will optimize this out (hopefully) + return nil +} + +func (s *goRunStats) Analyze() { + // Do nothing - compiler will optimize this out (hopefully) +} + +type statsOption func(*goRunStats) error + +func (s *goRunStats) Configure(options ...statsOption) error { + // Do nothing - compiler will optimize this out (hopefully) + return nil +} + +func WithTopN(topN int) statsOption { + return func(s *goRunStats) error { + return nil + } +} diff --git a/runtime/Go/antlr/v4/parser.go b/runtime/Go/antlr/v4/parser.go index 7d9184eb93..fb57ac15db 100644 --- a/runtime/Go/antlr/v4/parser.go +++ b/runtime/Go/antlr/v4/parser.go @@ -667,7 +667,7 @@ func (p *BaseParser) GetDFAStrings() string { func (p *BaseParser) DumpDFA() { seenOne := false for _, dfa := range p.Interpreter.decisionToDFA { - if dfa.states.Len() > 0 { + if dfa.Len() > 0 { if seenOne { fmt.Println() } diff --git a/runtime/Go/antlr/v4/parser_atn_simulator.go b/runtime/Go/antlr/v4/parser_atn_simulator.go index df74767faa..04668fdceb 100644 --- a/runtime/Go/antlr/v4/parser_atn_simulator.go +++ b/runtime/Go/antlr/v4/parser_atn_simulator.go @@ -10,13 +10,29 @@ import ( "strings" ) -var ( - ParserATNSimulatorDebug = false - ParserATNSimulatorTraceATNSim = false - ParserATNSimulatorDFADebug = false - ParserATNSimulatorRetryDebug = false - TurnOffLRLoopEntryBranchOpt = false -) +var () + +// ClosureBusy is a store of ATNConfigs and is a tiny abstraction layer over +// a standard JStore so that we can use Lazy instantiation of the JStore, mostly +// to avoid polluting the stats module with a ton of JStore instances with nothing in them. +type ClosureBusy struct { + bMap *JStore[*ATNConfig, Comparator[*ATNConfig]] + desc string +} + +// NewClosureBusy creates a new ClosureBusy instance used to avoid infinite recursion for right-recursive rules +func NewClosureBusy(desc string) *ClosureBusy { + return &ClosureBusy{ + desc: desc, + } +} + +func (c *ClosureBusy) Put(config *ATNConfig) (*ATNConfig, bool) { + if c.bMap == nil { + c.bMap = NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ClosureBusyCollection, c.desc) + } + return c.bMap.Put(config) +} type ParserATNSimulator struct { BaseATNSimulator @@ -75,7 +91,7 @@ func (p *ParserATNSimulator) reset() { //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStream, decision int, outerContext ParserRuleContext) int { - if ParserATNSimulatorDebug || ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("adaptivePredict decision " + strconv.Itoa(decision) + " exec LA(1)==" + p.getLookaheadName(input) + " line " + strconv.Itoa(input.LT(1).GetLine()) + ":" + @@ -92,7 +108,15 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre defer func() { p.dfa = nil - p.mergeCache = nil // wack cache after each prediction + p.mergeCache = nil // whack cache after each prediction + // Do not attempt to run a GC now that we're done with the cache as makes the + // GC overhead terrible for badly formed grammars and has little effect on well formed + // grammars. + // I have made some extra effort to try and reduce memory pressure by reusing allocations when + // possible. However, it can only have a limited effect. The real solution is to encourage grammar + // authors to think more carefully about their grammar and to use the new antlr.stats tag to inspect + // what is happening at runtime, along with using the error listener to report ambiguities. + input.Seek(index) input.Release(m) }() @@ -117,7 +141,7 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre if outerContext == nil { outerContext = ParserRuleContextEmpty } - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("predictATN decision " + strconv.Itoa(dfa.decision) + " exec LA(1)==" + p.getLookaheadName(input) + ", outerContext=" + outerContext.String(p.parser.GetRuleNames(), nil)) @@ -148,7 +172,7 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre alt, re := p.execATN(dfa, s0, input, index, outerContext) parser.SetError(re) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("DFA after predictATN: " + dfa.String(p.parser.GetLiteralNames(), nil)) } return alt @@ -191,7 +215,7 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, startIndex int, outerContext ParserRuleContext) (int, RecognitionException) { - if ParserATNSimulatorDebug || ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("execATN decision " + strconv.Itoa(dfa.decision) + ", DFA state " + s0.String() + ", LA(1)==" + p.getLookaheadName(input) + @@ -200,7 +224,7 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, previousD := s0 - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("s0 = " + s0.String()) } t := input.LA(1) @@ -232,7 +256,7 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, // IF PREDS, MIGHT RESOLVE TO SINGLE ALT => SLL (or syntax error) conflictingAlts := D.configs.conflictingAlts if D.predicates != nil { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("DFA state has preds in DFA sim LL fail-over") } conflictIndex := input.Index() @@ -241,7 +265,7 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, } conflictingAlts = p.evalSemanticContext(D.predicates, outerContext, true) if conflictingAlts.length() == 1 { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("Full LL avoided") } return conflictingAlts.minValue(), nil @@ -252,7 +276,7 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, input.Seek(conflictIndex) } } - if ParserATNSimulatorDFADebug { + if runtimeConfig.parserATNSimulatorDFADebug { fmt.Println("ctx sensitive state " + outerContext.String(nil, nil) + " in " + D.String()) } fullCtx := true @@ -337,7 +361,7 @@ func (p *ParserATNSimulator) computeTargetState(dfa *DFA, previousD *DFAState, t predictedAlt := p.getUniqueAlt(reach) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { altSubSets := PredictionModegetConflictingAltSubsets(reach) fmt.Println("SLL altSubSets=" + fmt.Sprint(altSubSets) + ", previous=" + previousD.configs.String() + @@ -395,7 +419,7 @@ func (p *ParserATNSimulator) predicateDFAState(dfaState *DFAState, decisionState //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *ATNConfigSet, input TokenStream, startIndex int, outerContext ParserRuleContext) (int, RecognitionException) { - if ParserATNSimulatorDebug || ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("execATNWithFullContext " + s0.String()) } @@ -427,7 +451,7 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *A return alt, p.noViableAlt(input, outerContext, previous, startIndex) } altSubSets := PredictionModegetConflictingAltSubsets(reach) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("LL altSubSets=" + fmt.Sprint(altSubSets) + ", predict=" + strconv.Itoa(PredictionModegetUniqueAlt(altSubSets)) + ", resolvesToJustOneViableAlt=" + fmt.Sprint(PredictionModeresolvesToJustOneViableAlt(altSubSets))) @@ -503,7 +527,7 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *A //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullCtx bool) *ATNConfigSet { if p.mergeCache == nil { - p.mergeCache = NewJPCMap() + p.mergeCache = NewJPCMap(ReachSetCollection, "Merge cache for computeReachSet()") } intermediate := NewATNConfigSet(fullCtx) @@ -521,14 +545,14 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC // First figure out where we can reach on input t for _, c := range closure.configs { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("testing " + p.GetTokenName(t) + " at " + c.String()) } if _, ok := c.GetState().(*RuleStopState); ok { if fullCtx || t == TokenEOF { skippedStopStates = append(skippedStopStates, c) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("added " + c.String() + " to SkippedStopStates") } } @@ -540,7 +564,7 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC if target != nil { cfg := NewATNConfig4(c, target) intermediate.Add(cfg, p.mergeCache) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("added " + cfg.String() + " to intermediate") } } @@ -577,7 +601,7 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC // if reach == nil { reach = NewATNConfigSet(fullCtx) - closureBusy := NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst) + closureBusy := NewClosureBusy("ParserATNSimulator.computeReachSet() make a closureBusy") treatEOFAsEpsilon := t == TokenEOF amount := len(intermediate.configs) for k := 0; k < amount; k++ { @@ -618,7 +642,7 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC } } - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("computeReachSet " + closure.String() + " -> " + reach.String()) } @@ -672,7 +696,7 @@ func (p *ParserATNSimulator) computeStartState(a ATNState, ctx RuleContext, full // always at least the implicit call to start rule initialContext := predictionContextFromRuleContext(p.atn, ctx) configs := NewATNConfigSet(fullCtx) - if ParserATNSimulatorDebug || ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("computeStartState from ATN state " + a.String() + " initialContext=" + initialContext.String()) } @@ -680,7 +704,7 @@ func (p *ParserATNSimulator) computeStartState(a ATNState, ctx RuleContext, full for i := 0; i < len(a.GetTransitions()); i++ { target := a.GetTransitions()[i].getTarget() c := NewATNConfig6(target, i+1, initialContext) - closureBusy := NewJStore[*ATNConfig, Comparator[*ATNConfig]](atnConfCompInst) + closureBusy := NewClosureBusy("ParserATNSimulator.computeStartState() make a closureBusy") p.closure(c, configs, closureBusy, true, fullCtx, false) } return configs @@ -802,7 +826,7 @@ func (p *ParserATNSimulator) getPredsForAmbigAlts(ambigAlts *BitSet, configs *AT if nPredAlts == 0 { altToPred = nil } - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("getPredsForAmbigAlts result " + fmt.Sprint(altToPred)) } return altToPred @@ -931,7 +955,7 @@ func (p *ParserATNSimulator) splitAccordingToSemanticValidity(configs *ATNConfig // evalSemanticContext looks through a list of predicate/alt pairs, returning alts for the // pairs that win. A [SemanticContextNone] predicate indicates an alt containing an -// un-predicated config which behaves as "always true." If !complete +// un-predicated runtimeConfig which behaves as "always true." If !complete // then we stop at the first predicate that evaluates to true. This // includes pairs with nil predicates. // @@ -949,11 +973,11 @@ func (p *ParserATNSimulator) evalSemanticContext(predPredictions []*PredPredicti } predicateEvaluationResult := pair.pred.evaluate(p.parser, outerContext) - if ParserATNSimulatorDebug || ParserATNSimulatorDFADebug { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorDFADebug { fmt.Println("eval pred " + pair.String() + "=" + fmt.Sprint(predicateEvaluationResult)) } if predicateEvaluationResult { - if ParserATNSimulatorDebug || ParserATNSimulatorDFADebug { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorDFADebug { fmt.Println("PREDICT " + fmt.Sprint(pair.alt)) } predictions.add(pair.alt) @@ -965,15 +989,15 @@ func (p *ParserATNSimulator) evalSemanticContext(predPredictions []*PredPredicti return predictions } -func (p *ParserATNSimulator) closure(config *ATNConfig, configs *ATNConfigSet, closureBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], collectPredicates, fullCtx, treatEOFAsEpsilon bool) { +func (p *ParserATNSimulator) closure(config *ATNConfig, configs *ATNConfigSet, closureBusy *ClosureBusy, collectPredicates, fullCtx, treatEOFAsEpsilon bool) { initialDepth := 0 p.closureCheckingStopState(config, configs, closureBusy, collectPredicates, fullCtx, initialDepth, treatEOFAsEpsilon) } //goland:noinspection GoBoolExpressions -func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs *ATNConfigSet, closureBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) { - if ParserATNSimulatorTraceATNSim { +func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs *ATNConfigSet, closureBusy *ClosureBusy, collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("closure(" + config.String() + ")") } @@ -989,7 +1013,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs continue } else { // we have no context info, just chase follow links (if greedy) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("FALLING off rule " + p.getRuleName(config.GetState().GetRuleIndex())) } p.closureWork(config, configs, closureBusy, collectPredicates, fullCtx, depth, treatEOFAsEpsilon) @@ -1013,7 +1037,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs return } else { // else if we have no context info, just chase follow links (if greedy) - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("FALLING off rule " + p.getRuleName(config.GetState().GetRuleIndex())) } } @@ -1024,7 +1048,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs // Do the actual work of walking epsilon edges // //goland:noinspection GoBoolExpressions -func (p *ParserATNSimulator) closureWork(config *ATNConfig, configs *ATNConfigSet, closureBusy *JStore[*ATNConfig, Comparator[*ATNConfig]], collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) { +func (p *ParserATNSimulator) closureWork(config *ATNConfig, configs *ATNConfigSet, closureBusy *ClosureBusy, collectPredicates, fullCtx bool, depth int, treatEOFAsEpsilon bool) { state := config.GetState() // optimization if !state.GetEpsilonOnlyTransitions() { @@ -1067,7 +1091,7 @@ func (p *ParserATNSimulator) closureWork(config *ATNConfig, configs *ATNConfigSe configs.dipsIntoOuterContext = true // TODO: can remove? only care when we add to set per middle of this method newDepth-- - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("dips into outer ctx: " + c.String()) } } else { @@ -1093,7 +1117,7 @@ func (p *ParserATNSimulator) closureWork(config *ATNConfig, configs *ATNConfigSe //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) canDropLoopEntryEdgeInLeftRecursiveRule(config *ATNConfig) bool { - if TurnOffLRLoopEntryBranchOpt { + if !runtimeConfig.lRLoopEntryBranchOpt { return false } @@ -1219,7 +1243,7 @@ func (p *ParserATNSimulator) getEpsilonTarget(config *ATNConfig, t Transition, c //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) actionTransition(config *ATNConfig, t *ActionTransition) *ATNConfig { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("ACTION edge " + strconv.Itoa(t.ruleIndex) + ":" + strconv.Itoa(t.actionIndex)) } return NewATNConfig4(config, t.getTarget()) @@ -1229,7 +1253,7 @@ func (p *ParserATNSimulator) actionTransition(config *ATNConfig, t *ActionTransi func (p *ParserATNSimulator) precedenceTransition(config *ATNConfig, pt *PrecedencePredicateTransition, collectPredicates, inContext, fullCtx bool) *ATNConfig { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("PRED (collectPredicates=" + fmt.Sprint(collectPredicates) + ") " + strconv.Itoa(pt.precedence) + ">=_p, ctx dependent=true") if p.parser != nil { @@ -1241,7 +1265,7 @@ func (p *ParserATNSimulator) precedenceTransition(config *ATNConfig, if fullCtx { // In full context mode, we can evaluate predicates on-the-fly // during closure, which dramatically reduces the size of - // the config sets. It also obviates the need to test predicates + // the runtimeConfig sets. It also obviates the need to test predicates // later during conflict resolution. currentPosition := p.input.Index() p.input.Seek(p.startIndex) @@ -1257,8 +1281,8 @@ func (p *ParserATNSimulator) precedenceTransition(config *ATNConfig, } else { c = NewATNConfig4(config, pt.getTarget()) } - if ParserATNSimulatorDebug { - fmt.Println("config from pred transition=" + c.String()) + if runtimeConfig.parserATNSimulatorDebug { + fmt.Println("runtimeConfig from pred transition=" + c.String()) } return c } @@ -1266,7 +1290,7 @@ func (p *ParserATNSimulator) precedenceTransition(config *ATNConfig, //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) predTransition(config *ATNConfig, pt *PredicateTransition, collectPredicates, inContext, fullCtx bool) *ATNConfig { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("PRED (collectPredicates=" + fmt.Sprint(collectPredicates) + ") " + strconv.Itoa(pt.ruleIndex) + ":" + strconv.Itoa(pt.predIndex) + ", ctx dependent=" + fmt.Sprint(pt.isCtxDependent)) if p.parser != nil { @@ -1294,7 +1318,7 @@ func (p *ParserATNSimulator) predTransition(config *ATNConfig, pt *PredicateTran } else { c = NewATNConfig4(config, pt.getTarget()) } - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("config from pred transition=" + c.String()) } return c @@ -1302,7 +1326,7 @@ func (p *ParserATNSimulator) predTransition(config *ATNConfig, pt *PredicateTran //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) ruleTransition(config *ATNConfig, t *RuleTransition) *ATNConfig { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("CALL rule " + p.getRuleName(t.getTarget().GetRuleIndex()) + ", ctx=" + config.GetContext().String()) } returnState := t.followState @@ -1465,7 +1489,7 @@ func (p *ParserATNSimulator) getUniqueAlt(configs *ATNConfigSet) int { // //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) addDFAEdge(dfa *DFA, from *DFAState, t int, to *DFAState) *DFAState { - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("EDGE " + from.String() + " -> " + to.String() + " upon " + p.GetTokenName(t)) } if to == nil { @@ -1484,7 +1508,7 @@ func (p *ParserATNSimulator) addDFAEdge(dfa *DFA, from *DFAState, t int, to *DFA from.setIthEdge(t+1, to) // connect p.atn.edgeMu.Unlock() - if ParserATNSimulatorDebug { + if runtimeConfig.parserATNSimulatorDebug { var names []string if p.parser != nil { names = p.parser.GetLiteralNames() @@ -1508,24 +1532,27 @@ func (p *ParserATNSimulator) addDFAState(dfa *DFA, d *DFAState) *DFAState { if d == ATNSimulatorError { return d } - existing, present := dfa.states.Get(d) + + existing, present := dfa.Get(d) if present { - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Print("addDFAState " + d.String() + " exists") } return existing } - // The state was not present, so update it with configs + // The state will be added if not already there or we will be given back the existing state struct + // if it is present. // - d.stateNumber = dfa.states.Len() + d.stateNumber = dfa.Len() if !d.configs.readOnly { d.configs.OptimizeConfigs(&p.BaseATNSimulator) d.configs.readOnly = true d.configs.configLookup = nil } - dfa.states.Put(d) - if ParserATNSimulatorTraceATNSim { + dfa.Put(d) + + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("addDFAState new " + d.String()) } @@ -1534,7 +1561,7 @@ func (p *ParserATNSimulator) addDFAState(dfa *DFA, d *DFAState) *DFAState { //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) ReportAttemptingFullContext(dfa *DFA, conflictingAlts *BitSet, configs *ATNConfigSet, startIndex, stopIndex int) { - if ParserATNSimulatorDebug || ParserATNSimulatorRetryDebug { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorRetryDebug { interval := NewInterval(startIndex, stopIndex+1) fmt.Println("ReportAttemptingFullContext decision=" + strconv.Itoa(dfa.decision) + ":" + configs.String() + ", input=" + p.parser.GetTokenStream().GetTextFromInterval(interval)) @@ -1546,7 +1573,7 @@ func (p *ParserATNSimulator) ReportAttemptingFullContext(dfa *DFA, conflictingAl //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) ReportContextSensitivity(dfa *DFA, prediction int, configs *ATNConfigSet, startIndex, stopIndex int) { - if ParserATNSimulatorDebug || ParserATNSimulatorRetryDebug { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorRetryDebug { interval := NewInterval(startIndex, stopIndex+1) fmt.Println("ReportContextSensitivity decision=" + strconv.Itoa(dfa.decision) + ":" + configs.String() + ", input=" + p.parser.GetTokenStream().GetTextFromInterval(interval)) @@ -1564,7 +1591,7 @@ func (p *ParserATNSimulator) ReportContextSensitivity(dfa *DFA, prediction int, //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) ReportAmbiguity(dfa *DFA, _ *DFAState, startIndex, stopIndex int, exact bool, ambigAlts *BitSet, configs *ATNConfigSet) { - if ParserATNSimulatorDebug || ParserATNSimulatorRetryDebug { + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorRetryDebug { interval := NewInterval(startIndex, stopIndex+1) fmt.Println("ReportAmbiguity " + ambigAlts.String() + ":" + configs.String() + ", input=" + p.parser.GetTokenStream().GetTextFromInterval(interval)) diff --git a/runtime/Go/antlr/v4/parser_rule_context.go b/runtime/Go/antlr/v4/parser_rule_context.go index 38d524a9c7..78b3b1c9f7 100644 --- a/runtime/Go/antlr/v4/parser_rule_context.go +++ b/runtime/Go/antlr/v4/parser_rule_context.go @@ -11,28 +11,28 @@ import ( type ParserRuleContext interface { RuleContext - + SetException(RecognitionException) - + AddTokenNode(token Token) *TerminalNodeImpl AddErrorNode(badToken Token) *ErrorNodeImpl - + EnterRule(listener ParseTreeListener) ExitRule(listener ParseTreeListener) - + SetStart(Token) GetStart() Token - + SetStop(Token) GetStop() Token - + AddChild(child RuleContext) RuleContext RemoveLastChild() } type BaseParserRuleContext struct { *BaseRuleContext - + start, stop Token exception RecognitionException children []Tree @@ -40,9 +40,9 @@ type BaseParserRuleContext struct { func NewBaseParserRuleContext(parent ParserRuleContext, invokingStateNumber int) *BaseParserRuleContext { prc := new(BaseParserRuleContext) - + prc.BaseRuleContext = NewBaseRuleContext(parent, invokingStateNumber) - + prc.RuleIndex = -1 // * If we are debugging or building a parse tree for a Visitor, // we need to track all of the tokens and rule invocations associated @@ -56,7 +56,7 @@ func NewBaseParserRuleContext(parent ParserRuleContext, invokingStateNumber int) // The exception that forced prc rule to return. If the rule successfully // completed, prc is {@code nil}. prc.exception = nil - + return prc } @@ -81,12 +81,12 @@ func (prc *BaseParserRuleContext) GetText() string { if prc.GetChildCount() == 0 { return "" } - + var s string for _, child := range prc.children { s += child.(ParseTree).GetText() } - + return s } @@ -131,12 +131,12 @@ func (prc *BaseParserRuleContext) RemoveLastChild() { } func (prc *BaseParserRuleContext) AddTokenNode(token Token) *TerminalNodeImpl { - + node := NewTerminalNodeImpl(token) prc.addTerminalNodeChild(node) node.parentCtx = prc return node - + } func (prc *BaseParserRuleContext) AddErrorNode(badToken Token) *ErrorNodeImpl { @@ -150,7 +150,7 @@ func (prc *BaseParserRuleContext) GetChild(i int) Tree { if prc.children != nil && len(prc.children) >= i { return prc.children[i] } - + return nil } @@ -158,18 +158,18 @@ func (prc *BaseParserRuleContext) GetChildOfType(i int, childType reflect.Type) if childType == nil { return prc.GetChild(i).(RuleContext) } - + for j := 0; j < len(prc.children); j++ { child := prc.children[j] if reflect.TypeOf(child) == childType { if i == 0 { return child.(RuleContext) } - + i-- } } - + return nil } @@ -202,7 +202,7 @@ func (prc *BaseParserRuleContext) GetStop() Token { } func (prc *BaseParserRuleContext) GetToken(ttype int, i int) TerminalNode { - + for j := 0; j < len(prc.children); j++ { child := prc.children[j] if c2, ok := child.(TerminalNode); ok { @@ -210,7 +210,7 @@ func (prc *BaseParserRuleContext) GetToken(ttype int, i int) TerminalNode { if i == 0 { return c2 } - + i-- } } @@ -222,9 +222,9 @@ func (prc *BaseParserRuleContext) GetTokens(ttype int) []TerminalNode { if prc.children == nil { return make([]TerminalNode, 0) } - + tokens := make([]TerminalNode, 0) - + for j := 0; j < len(prc.children); j++ { child := prc.children[j] if tchild, ok := child.(TerminalNode); ok { @@ -233,7 +233,7 @@ func (prc *BaseParserRuleContext) GetTokens(ttype int) []TerminalNode { } } } - + return tokens } @@ -245,12 +245,12 @@ func (prc *BaseParserRuleContext) getChild(ctxType reflect.Type, i int) RuleCont if prc.children == nil || i < 0 || i >= len(prc.children) { return nil } - + j := -1 // what element have we found with ctxType? for _, o := range prc.children { - + childType := reflect.TypeOf(o) - + if childType.Implements(ctxType) { j++ if j == i { @@ -272,12 +272,12 @@ func (prc *BaseParserRuleContext) GetTypedRuleContexts(ctxType reflect.Type) []R if prc.children == nil { return make([]RuleContext, 0) } - + contexts := make([]RuleContext, 0) - + for _, child := range prc.children { childType := reflect.TypeOf(child) - + if childType.ConvertibleTo(ctxType) { contexts = append(contexts, child.(RuleContext)) } @@ -289,7 +289,7 @@ func (prc *BaseParserRuleContext) GetChildCount() int { if prc.children == nil { return 0 } - + return len(prc.children) } @@ -297,7 +297,7 @@ func (prc *BaseParserRuleContext) GetSourceInterval() Interval { if prc.start == nil || prc.stop == nil { return TreeInvalidInterval } - + return NewInterval(prc.start.GetTokenIndex(), prc.stop.GetTokenIndex()) } @@ -308,7 +308,7 @@ func (prc *BaseParserRuleContext) GetSourceInterval() Interval { // func (prc *BaseParserRuleContext) String(ruleNames []string, stop RuleContext) string { - + var p ParserRuleContext = prc s := "[" for p != nil && p != stop { @@ -352,12 +352,12 @@ type BaseInterpreterRuleContext struct { //goland:noinspection GoUnusedExportedFunction func NewBaseInterpreterRuleContext(parent BaseInterpreterRuleContext, invokingStateNumber, ruleIndex int) *BaseInterpreterRuleContext { - + prc := new(BaseInterpreterRuleContext) - + prc.BaseParserRuleContext = NewBaseParserRuleContext(parent, invokingStateNumber) - + prc.RuleIndex = ruleIndex - + return prc } diff --git a/runtime/Go/antlr/v4/prediction_context.go b/runtime/Go/antlr/v4/prediction_context.go index 5a6ae188c0..8830da7501 100644 --- a/runtime/Go/antlr/v4/prediction_context.go +++ b/runtime/Go/antlr/v4/prediction_context.go @@ -34,6 +34,7 @@ const ( ) // TODO: JI These are meant to be atomics - this does not seem to match the Java runtime here +// //goland:noinspection GoUnusedGlobalVariable var ( BasePredictionContextglobalNodeCount = 1 @@ -59,19 +60,18 @@ type PredictionContext struct { } func NewEmptyPredictionContext() *PredictionContext { - return &PredictionContext{ - cachedHash: calculateEmptyHash(), - pcType: PredictionContextEmpty, - returnState: BasePredictionContextEmptyReturnState, - } + nep := &PredictionContext{} + nep.cachedHash = calculateEmptyHash() + nep.pcType = PredictionContextEmpty + nep.returnState = BasePredictionContextEmptyReturnState + return nep } func NewBaseSingletonPredictionContext(parent *PredictionContext, returnState int) *PredictionContext { - pc := &PredictionContext{ - pcType: PredictionContextSingleton, - returnState: returnState, - parentCtx: parent, - } + pc := &PredictionContext{} + pc.pcType = PredictionContextSingleton + pc.returnState = returnState + pc.parentCtx = parent if parent != nil { pc.cachedHash = calculateHash(parent, returnState) } else { @@ -102,12 +102,12 @@ func NewArrayPredictionContext(parents []*PredictionContext, returnStates []int) } hash = murmurFinish(hash, len(parents)<<1) - return &PredictionContext{ - cachedHash: hash, - pcType: PredictionContextArray, - parents: parents, - returnStates: returnStates, - } + nec := &PredictionContext{} + nec.cachedHash = hash + nec.pcType = PredictionContextArray + nec.parents = parents + nec.returnStates = returnStates + return nec } func (p *PredictionContext) Hash() int { @@ -391,7 +391,7 @@ func mergeSingletons(a, b *PredictionContext, rootIsWildcard bool, mergeCache *J return previous } previous, present = mergeCache.Get(b, a) - if previous != nil { + if present { return previous } } @@ -547,14 +547,14 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa if mergeCache != nil { previous, present := mergeCache.Get(a, b) if present { - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("mergeArrays a=" + a.String() + ",b=" + b.String() + " -> previous") } return previous } previous, present = mergeCache.Get(b, a) if present { - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("mergeArrays a=" + a.String() + ",b=" + b.String() + " -> previous") } return previous @@ -635,7 +635,7 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa if mergeCache != nil { mergeCache.Put(a, b, a) } - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("mergeArrays a=" + a.String() + ",b=" + b.String() + " -> a") } return a @@ -644,40 +644,41 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa if mergeCache != nil { mergeCache.Put(a, b, b) } - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("mergeArrays a=" + a.String() + ",b=" + b.String() + " -> b") } return b } - combineCommonParents(mergedParents) + combineCommonParents(&mergedParents) if mergeCache != nil { mergeCache.Put(a, b, M) } - if ParserATNSimulatorTraceATNSim { + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("mergeArrays a=" + a.String() + ",b=" + b.String() + " -> " + M.String()) } return M } // Make pass over all M parents and merge any Equals() ones. -// Note: This is not used in Go as we are not using pointers in the slice anyway, but I have kept it for reference -// and if we ever need to use pointers in the slice. +// Note that we pass a pointer to the slice as we want to modify it in place. +// //goland:noinspection GoUnusedFunction -func combineCommonParents(parents []*PredictionContext) { - uniqueParents := NewJStore[*PredictionContext, Comparator[*PredictionContext]](pContextEqInst) +func combineCommonParents(parents *[]*PredictionContext) { + uniqueParents := NewJStore[*PredictionContext, Comparator[*PredictionContext]](pContextEqInst, PredictionContextCollection, "combineCommonParents for PredictionContext") - for p := 0; p < len(parents); p++ { - parent := parents[p] + for p := 0; p < len(*parents); p++ { + parent := (*parents)[p] _, _ = uniqueParents.Put(parent) } - for q := 0; q < len(parents); q++ { - pc, _ := uniqueParents.Get(parents[q]) - parents[q] = pc + for q := 0; q < len(*parents); q++ { + pc, _ := uniqueParents.Get((*parents)[q]) + (*parents)[q] = pc } } -func getCachedBasePredictionContext(context *PredictionContext, contextCache *PredictionContextCache, visited *JMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]]) *PredictionContext { +//func getCachedBasePredictionContext(context *PredictionContext, contextCache *PredictionContextCache, visited *JMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]]) *PredictionContext { +func getCachedBasePredictionContext(context *PredictionContext, contextCache *PredictionContextCache, visited *VisitRecord) *PredictionContext { if context.isEmpty() { return context } @@ -685,6 +686,7 @@ func getCachedBasePredictionContext(context *PredictionContext, contextCache *Pr if present { return existing } + existing, present = contextCache.Get(context) if present { visited.Put(context, existing) diff --git a/runtime/Go/antlr/v4/prediction_context_cache.go b/runtime/Go/antlr/v4/prediction_context_cache.go index 4b5e34f0e3..5ea527ac96 100644 --- a/runtime/Go/antlr/v4/prediction_context_cache.go +++ b/runtime/Go/antlr/v4/prediction_context_cache.go @@ -1,6 +1,10 @@ package antlr -var BasePredictionContextEMPTY = NewEmptyPredictionContext() +var BasePredictionContextEMPTY = &PredictionContext{ + cachedHash: calculateEmptyHash(), + pcType: PredictionContextEmpty, + returnState: BasePredictionContextEmptyReturnState, +} // PredictionContextCache is Used to cache [PredictionContext] objects. It is used for the shared // context cash associated with contexts in DFA states. This cache @@ -11,7 +15,7 @@ type PredictionContextCache struct { func NewPredictionContextCache() *PredictionContextCache { return &PredictionContextCache{ - cache: NewJMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]](pContextEqInst), + cache: NewJMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]](pContextEqInst, PredictionContextCacheCollection, "NewPredictionContextCache()"), } } diff --git a/runtime/Go/antlr/v4/prediction_mode.go b/runtime/Go/antlr/v4/prediction_mode.go index deb675b43d..3f85a6a520 100644 --- a/runtime/Go/antlr/v4/prediction_mode.go +++ b/runtime/Go/antlr/v4/prediction_mode.go @@ -29,7 +29,7 @@ const ( // behavior for syntactically-incorrect inputs. // PredictionModeSLL = 0 - + // PredictionModeLL represents the LL(*) prediction mode. // This prediction mode allows the current parser // context to be used for resolving SLL conflicts that occur during @@ -47,7 +47,7 @@ const ( // behavior for syntactically-incorrect inputs. // PredictionModeLL = 1 - + // PredictionModeLLExactAmbigDetection represents the LL(*) prediction mode // with exact ambiguity detection. // @@ -166,7 +166,7 @@ const ( // the configurations to strip out all the predicates so that a standard // [ATNConfigSet] will merge everything ignoring predicates. func PredictionModehasSLLConflictTerminatingPrediction(mode int, configs *ATNConfigSet) bool { - + // Configs in rule stop states indicate reaching the end of the decision // rule (local context) or end of start rule (full context). If all // configs meet this condition, then none of the configurations is able @@ -175,7 +175,7 @@ func PredictionModehasSLLConflictTerminatingPrediction(mode int, configs *ATNCon if PredictionModeallConfigsInRuleStopStates(configs) { return true } - + // pure SLL mode parsing if mode == PredictionModeSLL { // Don't bother with combining configs from different semantic @@ -185,7 +185,7 @@ func PredictionModehasSLLConflictTerminatingPrediction(mode int, configs *ATNCon // dup configs, tossing out semantic predicates dup := NewATNConfigSet(false) for _, c := range configs.configs { - + // NewATNConfig({semanticContext:}, c) c = NewATNConfig2(c, SemanticContextNone) dup.Add(c, nil) @@ -222,7 +222,7 @@ func PredictionModehasConfigInRuleStopState(configs *ATNConfigSet) bool { // the func returns true if all configurations in configs are in a // [RuleStopState] func PredictionModeallConfigsInRuleStopStates(configs *ATNConfigSet) bool { - + for _, c := range configs.configs { if _, ok := c.GetState().(*RuleStopState); !ok { return false @@ -430,7 +430,7 @@ func PredictionModehasConflictingAltSet(altsets []*BitSet) bool { // The func returns true if every member of altsets is equal to the others. func PredictionModeallSubsetsEqual(altsets []*BitSet) bool { var first *BitSet - + for i := 0; i < len(altsets); i++ { alts := altsets[i] if first == nil { @@ -439,7 +439,7 @@ func PredictionModeallSubsetsEqual(altsets []*BitSet) bool { return false } } - + return true } @@ -453,7 +453,7 @@ func PredictionModegetUniqueAlt(altsets []*BitSet) int { if all.length() == 1 { return all.minValue() } - + return ATNInvalidAltNumber } @@ -473,10 +473,10 @@ func PredictionModeGetAlts(altsets []*BitSet) *BitSet { // for each configuration c in configs: // map[c] U= c.ATNConfig.alt // map hash/equals uses s and x, not alt and not pred func PredictionModegetConflictingAltSubsets(configs *ATNConfigSet) []*BitSet { - configToAlts := NewJMap[*ATNConfig, *BitSet, *ATNAltConfigComparator[*ATNConfig]](atnAltCfgEqInst) - + configToAlts := NewJMap[*ATNConfig, *BitSet, *ATNAltConfigComparator[*ATNConfig]](atnAltCfgEqInst, AltSetCollection, "PredictionModegetConflictingAltSubsets()") + for _, c := range configs.configs { - + alts, ok := configToAlts.Get(c) if !ok { alts = NewBitSet() @@ -484,7 +484,7 @@ func PredictionModegetConflictingAltSubsets(configs *ATNConfigSet) []*BitSet { } alts.add(c.GetAlt()) } - + return configToAlts.Values() } @@ -494,7 +494,7 @@ func PredictionModegetConflictingAltSubsets(configs *ATNConfigSet) []*BitSet { // map[c.ATNConfig.state] U= c.ATNConfig.alt} func PredictionModeGetStateToAltMap(configs *ATNConfigSet) *AltDict { m := NewAltDict() - + for _, c := range configs.configs { alts := m.Get(c.GetState().String()) if alts == nil { @@ -522,7 +522,7 @@ func PredictionModehasStateAssociatedWithOneAlt(configs *ATNConfigSet) bool { // TODO: JI - Review this code - it does not seem to do the same thing as the Java code - maybe because [BitSet] is not like the Java utils BitSet func PredictionModegetSingleViableAlt(altsets []*BitSet) int { result := ATNInvalidAltNumber - + for i := 0; i < len(altsets); i++ { alts := altsets[i] minAlt := alts.minValue() diff --git a/runtime/Go/antlr/v4/recognizer.go b/runtime/Go/antlr/v4/recognizer.go index cd899f6e87..2e0b504fb3 100644 --- a/runtime/Go/antlr/v4/recognizer.go +++ b/runtime/Go/antlr/v4/recognizer.go @@ -7,7 +7,7 @@ package antlr import ( "fmt" "strings" - + "strconv" ) @@ -15,10 +15,10 @@ type Recognizer interface { GetLiteralNames() []string GetSymbolicNames() []string GetRuleNames() []string - + Sempred(RuleContext, int, int) bool Precpred(RuleContext, int) bool - + GetState() int SetState(int) Action(RuleContext, int, int) @@ -34,7 +34,7 @@ type Recognizer interface { type BaseRecognizer struct { listeners []ErrorListener state int - + RuleNames []string LiteralNames []string SymbolicNames []string @@ -130,7 +130,7 @@ func (b *BaseRecognizer) SetState(v int) { // // TODO: JI This is not yet implemented in the Go runtime. Maybe not needed. func (b *BaseRecognizer) GetRuleIndexMap() map[string]int { - + panic("Method not defined!") // var ruleNames = b.GetRuleNames() // if (ruleNames==nil) { @@ -220,7 +220,7 @@ func (b *BaseRecognizer) GetTokenErrorDisplay(t Token) string { s = strings.Replace(s, "\t", "\\t", -1) s = strings.Replace(s, "\n", "\\n", -1) s = strings.Replace(s, "\r", "\\r", -1) - + return "'" + s + "'" } diff --git a/runtime/Go/antlr/v4/semantic_context.go b/runtime/Go/antlr/v4/semantic_context.go index 503294ff79..68cb9061eb 100644 --- a/runtime/Go/antlr/v4/semantic_context.go +++ b/runtime/Go/antlr/v4/semantic_context.go @@ -197,7 +197,7 @@ type AND struct { func NewAND(a, b SemanticContext) *AND { - operands := NewJStore[SemanticContext, Comparator[SemanticContext]](semctxEqInst) + operands := NewJStore[SemanticContext, Comparator[SemanticContext]](semctxEqInst, SemanticContextCollection, "NewAND() operands") if aa, ok := a.(*AND); ok { for _, o := range aa.opnds { operands.Put(o) @@ -229,9 +229,7 @@ func NewAND(a, b SemanticContext) *AND { vs := operands.Values() opnds := make([]SemanticContext, len(vs)) - for i, v := range vs { - opnds[i] = v.(SemanticContext) - } + copy(opnds, vs) and := new(AND) and.opnds = opnds @@ -348,7 +346,7 @@ type OR struct { func NewOR(a, b SemanticContext) *OR { - operands := NewJStore[SemanticContext, Comparator[SemanticContext]](semctxEqInst) + operands := NewJStore[SemanticContext, Comparator[SemanticContext]](semctxEqInst, SemanticContextCollection, "NewOR() operands") if aa, ok := a.(*OR); ok { for _, o := range aa.opnds { operands.Put(o) @@ -381,9 +379,7 @@ func NewOR(a, b SemanticContext) *OR { vs := operands.Values() opnds := make([]SemanticContext, len(vs)) - for i, v := range vs { - opnds[i] = v.(SemanticContext) - } + copy(opnds, vs) o := new(OR) o.opnds = opnds diff --git a/runtime/Go/antlr/v4/statistics.go b/runtime/Go/antlr/v4/statistics.go new file mode 100644 index 0000000000..4c038f0e2c --- /dev/null +++ b/runtime/Go/antlr/v4/statistics.go @@ -0,0 +1,281 @@ +//go:build antlr.stats + +package antlr + +import ( + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "sync" +) + +// This file allows the user to collect statistics about the runtime of the ANTLR runtime. It is not enabled by default +// and so incurs no time penalty. To enable it, you must build the runtime with the antlr.stats build tag. +// + +// Tells various components to collect statistics - because it is only true when this file is included, it will +// allow the compiler to completely eliminate all the code that is only used when collecting statistics. +const collectStats = true + +// goRunStats is a collection of all the various data the ANTLR runtime has collected about a particular run. +// It is exported so that it can be used by others to look for things that are not already looked for in the +// runtime statistics. +type goRunStats struct { + + // jStats is a slice of all the [JStatRec] records that have been created, which is one for EVERY collection created + // during a run. It is exported so that it can be used by others to look for things that are not already looked for + // within this package. + // + jStats []*JStatRec + jStatsLock sync.RWMutex + topN int + topNByMax []*JStatRec + topNByUsed []*JStatRec + unusedCollections map[CollectionSource]int + counts map[CollectionSource]int +} + +const ( + collectionsFile = "collections" +) + +var ( + Statistics = &goRunStats{ + topN: 10, + } +) + +type statsOption func(*goRunStats) error + +// Configure allows the statistics system to be configured as the user wants and override the defaults +func (s *goRunStats) Configure(options ...statsOption) error { + for _, option := range options { + err := option(s) + if err != nil { + return err + } + } + return nil +} + +// WithTopN sets the number of things to list in the report when we are concerned with the top N things. +// +// For example, if you want to see the top 20 collections by size, you can do: +// +// antlr.Statistics.Configure(antlr.WithTopN(20)) +func WithTopN(topN int) statsOption { + return func(s *goRunStats) error { + s.topN = topN + return nil + } +} + +// Analyze looks through all the statistical records and computes all the outputs that might be useful to the user. +// +// The function gathers and analyzes a number of statistics about any particular run of +// an ANTLR generated recognizer. In the vast majority of cases, the statistics are only +// useful to maintainers of ANTLR itself, but they can be useful to users as well. They may be +// especially useful in tracking down bugs or performance problems when an ANTLR user could +// supply the output from this package, but cannot supply the grammar file(s) they are using, even +// privately to the maintainers. +// +// The statistics are gathered by the runtime itself, and are not gathered by the parser or lexer, but the user +// must call this function their selves to analyze the statistics. This is because none of the infrastructure is +// extant unless the calling program is built with the antlr.stats tag like so: +// +// go build -tags antlr.stats . +// +// When a program is built with the antlr.stats tag, the Statistics object is created and available outside +// the package. The user can then call the [Statistics.Analyze] function to analyze the statistics and then call the +// [Statistics.Report] function to report the statistics. +// +// Please forward any questions about this package to the ANTLR discussion groups on GitHub or send to them to +// me [Jim Idle] directly at jimi@idle.ws +// +// [Jim Idle]: https:://github.com/jim-idle +func (s *goRunStats) Analyze() { + + // Look for anything that looks strange and record it in our local maps etc for the report to present it + // + s.CollectionAnomalies() + s.TopNCollections() +} + +// TopNCollections looks through all the statistical records and gathers the top ten collections by size. +func (s *goRunStats) TopNCollections() { + + // Let's sort the stat records by MaxSize + // + sort.Slice(s.jStats, func(i, j int) bool { + return s.jStats[i].MaxSize > s.jStats[j].MaxSize + }) + + for i := 0; i < len(s.jStats) && i < s.topN; i++ { + s.topNByMax = append(s.topNByMax, s.jStats[i]) + } + + // Sort by the number of times used + // + sort.Slice(s.jStats, func(i, j int) bool { + return s.jStats[i].Gets+s.jStats[i].Puts > s.jStats[j].Gets+s.jStats[j].Puts + }) + for i := 0; i < len(s.jStats) && i < s.topN; i++ { + s.topNByUsed = append(s.topNByUsed, s.jStats[i]) + } +} + +// Report dumps a markdown formatted report of all the statistics collected during a run to the given dir output +// path, which should represent a directory. Generated files will be prefixed with the given prefix and will be +// given a type name such as `anomalies` and a time stamp such as `2021-09-01T12:34:56` and a .md suffix. +func (s *goRunStats) Report(dir string, prefix string) error { + + isDir, err := isDirectory(dir) + switch { + case err != nil: + return err + case !isDir: + return fmt.Errorf("output directory `%s` is not a directory", dir) + } + s.reportCollections(dir, prefix) + + // Clean out any old data in case the user forgets + // + s.Reset() + return nil +} + +func (s *goRunStats) Reset() { + s.jStats = nil + s.topNByUsed = nil + s.topNByMax = nil +} + +func (s *goRunStats) reportCollections(dir, prefix string) { + cname := filepath.Join(dir, ".asciidoctor") + // If the file doesn't exist, create it, or append to the file + f, err := os.OpenFile(cname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatal(err) + } + _, _ = f.WriteString(`// .asciidoctorconfig +++++ + +++++`) + _ = f.Close() + + fname := filepath.Join(dir, prefix+"_"+"_"+collectionsFile+"_"+".adoc") + // If the file doesn't exist, create it, or append to the file + f, err = os.OpenFile(fname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Fatal(err) + } + defer func(f *os.File) { + err := f.Close() + if err != nil { + log.Fatal(err) + } + }(f) + _, _ = f.WriteString("= Collections for " + prefix + "\n\n") + + _, _ = f.WriteString("== Summary\n") + + if s.unusedCollections != nil { + _, _ = f.WriteString("=== Unused Collections\n") + _, _ = f.WriteString("Unused collections incur a penalty for allocation that makes them a candidate for either\n") + _, _ = f.WriteString(" removal or optimization. If you are using a collection that is not used, you should\n") + _, _ = f.WriteString(" consider removing it. If you are using a collection that is used, but not very often,\n") + _, _ = f.WriteString(" you should consider using lazy initialization to defer the allocation until it is\n") + _, _ = f.WriteString(" actually needed.\n\n") + + _, _ = f.WriteString("\n.Unused collections\n") + _, _ = f.WriteString(`[cols="<3,>1"]` + "\n\n") + _, _ = f.WriteString("|===\n") + _, _ = f.WriteString("| Type | Count\n") + + for k, v := range s.unusedCollections { + _, _ = f.WriteString("| " + CollectionDescriptors[k].SybolicName + " | " + strconv.Itoa(v) + "\n") + } + f.WriteString("|===\n\n") + } + + _, _ = f.WriteString("\n.Summary of Collections\n") + _, _ = f.WriteString(`[cols="<3,>1"]` + "\n\n") + _, _ = f.WriteString("|===\n") + _, _ = f.WriteString("| Type | Count\n") + for k, v := range s.counts { + _, _ = f.WriteString("| " + CollectionDescriptors[k].SybolicName + " | " + strconv.Itoa(v) + "\n") + } + _, _ = f.WriteString("| Total | " + strconv.Itoa(len(s.jStats)) + "\n") + _, _ = f.WriteString("|===\n\n") + + _, _ = f.WriteString("\n.Summary of Top " + strconv.Itoa(s.topN) + " Collections by MaxSize\n") + _, _ = f.WriteString(`[cols="<1,<3,>1,>1,>1,>1"]` + "\n\n") + _, _ = f.WriteString("|===\n") + _, _ = f.WriteString("| Source | Description | MaxSize | EndSize | Puts | Gets\n") + for _, c := range s.topNByMax { + _, _ = f.WriteString("| " + CollectionDescriptors[c.Source].SybolicName + "\n") + _, _ = f.WriteString("| " + c.Description + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.MaxSize) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.CurSize) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.Puts) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.Gets) + "\n") + _, _ = f.WriteString("\n") + } + _, _ = f.WriteString("|===\n\n") + + _, _ = f.WriteString("\n.Summary of Top " + strconv.Itoa(s.topN) + " Collections by Access\n") + _, _ = f.WriteString(`[cols="<1,<3,>1,>1,>1,>1,>1"]` + "\n\n") + _, _ = f.WriteString("|===\n") + _, _ = f.WriteString("| Source | Description | MaxSize | EndSize | Puts | Gets | P+G\n") + for _, c := range s.topNByUsed { + _, _ = f.WriteString("| " + CollectionDescriptors[c.Source].SybolicName + "\n") + _, _ = f.WriteString("| " + c.Description + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.MaxSize) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.CurSize) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.Puts) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.Gets) + "\n") + _, _ = f.WriteString("| " + strconv.Itoa(c.Gets+c.Puts) + "\n") + _, _ = f.WriteString("\n") + } + _, _ = f.WriteString("|===\n\n") +} + +// AddJStatRec adds a [JStatRec] record to the [goRunStats] collection when build runtimeConfig antlr.stats is enabled. +func (s *goRunStats) AddJStatRec(rec *JStatRec) { + s.jStatsLock.Lock() + defer s.jStatsLock.Unlock() + s.jStats = append(s.jStats, rec) +} + +// CollectionAnomalies looks through all the statistical records and gathers any anomalies that have been found. +func (s *goRunStats) CollectionAnomalies() { + s.jStatsLock.RLock() + defer s.jStatsLock.RUnlock() + s.counts = make(map[CollectionSource]int, len(s.jStats)) + for _, c := range s.jStats { + + // Accumlate raw counts + // + s.counts[c.Source]++ + + // Look for allocated but unused collections and count them + if c.MaxSize == 0 && c.Puts == 0 { + if s.unusedCollections == nil { + s.unusedCollections = make(map[CollectionSource]int) + } + s.unusedCollections[c.Source]++ + } + if c.MaxSize > 6000 { + fmt.Println("Collection ", c.Description, "accumulated a max size of ", c.MaxSize, " - this is probably too large and indicates a poorly formed grammar") + } + } + +} diff --git a/runtime/Go/antlr/v4/stats_data.go b/runtime/Go/antlr/v4/stats_data.go new file mode 100644 index 0000000000..4d9eb94e5f --- /dev/null +++ b/runtime/Go/antlr/v4/stats_data.go @@ -0,0 +1,23 @@ +package antlr + +// A JStatRec is a record of a particular use of a [JStore], [JMap] or JPCMap] collection. Typically, it will be +// used to look for unused collections that wre allocated anyway, problems with hash bucket clashes, and anomalies +// such as huge numbers of Gets with no entries found GetNoEnt. You can refer to the CollectionAnomalies() function +// for ideas on what can be gleaned from these statistics about collections. +type JStatRec struct { + Source CollectionSource + MaxSize int + CurSize int + Gets int + GetHits int + GetMisses int + GetHashConflicts int + GetNoEnt int + Puts int + PutHits int + PutMisses int + PutHashConflicts int + MaxSlotSize int + Description string + CreateStack []byte +} diff --git a/runtime/Go/antlr/v4/testing_api_test.go b/runtime/Go/antlr/v4/testing_api_test.go deleted file mode 100644 index eed3ce34be..0000000000 --- a/runtime/Go/antlr/v4/testing_api_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package antlr - -import ( - "testing" -) - -func next(t *testing.T, lexer *LexerB, want string) { - var token = lexer.NextToken() - var got = token.String() - if got != want { - t.Errorf("got %q, wanted %q", got, want) - } -} - -func TestString(t *testing.T) { - str := NewInputStream("a b c 1 2 3") - lexer := NewLexerB(str) - next(t, lexer, "[@-1,0:0='a',<1>,1:0]") - next(t, lexer, "[@-1,1:1=' ',<7>,1:1]") - next(t, lexer, "[@-1,2:2='b',<1>,1:2]") - next(t, lexer, "[@-1,3:3=' ',<7>,1:3]") - next(t, lexer, "[@-1,4:4='c',<1>,1:4]") - next(t, lexer, "[@-1,5:5=' ',<7>,1:5]") - next(t, lexer, "[@-1,6:6='1',<2>,1:6]") - next(t, lexer, "[@-1,7:7=' ',<7>,1:7]") - next(t, lexer, "[@-1,8:8='2',<2>,1:8]") - next(t, lexer, "[@-1,9:9=' ',<7>,1:9]") - next(t, lexer, "[@-1,10:10='3',<2>,1:10]") - next(t, lexer, "[@-1,11:10='',<-1>,1:11]") -} diff --git a/runtime/Go/antlr/v4/testing_assert_test.go b/runtime/Go/antlr/v4/testing_assert_test.go deleted file mode 100644 index 4a402a34f3..0000000000 --- a/runtime/Go/antlr/v4/testing_assert_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. -// Use of this file is governed by the BSD 3-clause license that -// can be found in the LICENSE.txt file in the project root. - -// These assert functions are borrowed from https://github.com/stretchr/testify/ (MIT License) - -package antlr - -import ( - "fmt" - "reflect" - "testing" -) - -type assert struct { - t *testing.T -} - -func assertNew(t *testing.T) *assert { - return &assert{ - t: t, - } -} - -func (a *assert) Equal(expected, actual interface{}) bool { - if !objectsAreEqual(expected, actual) { - return a.Fail(fmt.Sprintf("Not equal:\n"+ - "expected: %#v\n"+ - " actual: %#v\n", expected, actual)) - } - return true -} - -func objectsAreEqual(expected, actual interface{}) bool { - if expected == nil || actual == nil { - return expected == actual - } - return reflect.DeepEqual(expected, actual) -} - -func (a *assert) Nil(object interface{}) bool { - if isNil(object) { - return true - } - return a.Fail(fmt.Sprintf("Expected nil, but got: %#v", object)) -} - -func (a *assert) NotNil(object interface{}) bool { - if !isNil(object) { - return true - } - return a.Fail("Expected value not to be nil.") -} - -// isNil checks if a specified object is nil or not, without Failing. -func isNil(object interface{}) bool { - if object == nil { - return true - } - - value := reflect.ValueOf(object) - kind := value.Kind() - if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { - return true - } - - return false -} - -func (a *assert) Panics(f func()) bool { - if funcDidPanic, panicValue := didPanic(f); !funcDidPanic { - return a.Fail(fmt.Sprintf("func %p should panic\n\r\tPanic value:\t%v", f, panicValue)) - } - - return true -} - -// Fail reports a failure through -func (a *assert) Fail(failureMessage string) bool { - a.t.Errorf("%s", failureMessage) - return false -} - -// didPanic returns true if the function passed to it panics. Otherwise, it returns false. -func didPanic(f func()) (bool, interface{}) { - didPanic := false - var message interface{} - func() { - defer func() { - if message = recover(); message != nil { - didPanic = true - } - }() - // call the target function - f() - }() - return didPanic, message -} diff --git a/runtime/Go/antlr/v4/testing_lexer_b_test.go b/runtime/Go/antlr/v4/testing_lexer_b_test.go deleted file mode 100644 index f3e22d7a69..0000000000 --- a/runtime/Go/antlr/v4/testing_lexer_b_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. -// Use of this file is governed by the BSD 3-clause license that -// can be found in the LICENSE.txt file in the project root. - -package antlr - -/* -LexerB is a lexer for testing purpose. - -This file is generated from this grammar. - -lexer grammar LexerB; - -ID : 'a'..'z'+; -INT : '0'..'9'+; -SEMI : ';'; -ASSIGN : '='; -PLUS : '+'; -MULT : '*'; -WS : ' '+; -*/ - -import ( - "fmt" - "sync" - "unicode" -) - -// Suppress unused import error -var _ = fmt.Printf -var _ = sync.Once{} -var _ = unicode.IsLetter - -type LexerB struct { - *BaseLexer - channelNames []string - modeNames []string - // TODO: EOF string -} - -var lexerbLexerStaticData struct { - once sync.Once - serializedATN []int32 - channelNames []string - modeNames []string - literalNames []string - symbolicNames []string - ruleNames []string - predictionContextCache *PredictionContextCache - atn *ATN - decisionToDFA []*DFA -} - -func lexerbLexerInit() { - staticData := &lexerbLexerStaticData - staticData.channelNames = []string{ - "DEFAULT_TOKEN_CHANNEL", "HIDDEN", - } - staticData.modeNames = []string{ - "DEFAULT_MODE", - } - staticData.literalNames = []string{ - "", "", "", "';'", "'='", "'+'", "'*'", - } - staticData.symbolicNames = []string{ - "", "ID", "INT", "SEMI", "ASSIGN", "PLUS", "MULT", "WS", - } - staticData.ruleNames = []string{ - "ID", "INT", "SEMI", "ASSIGN", "PLUS", "MULT", "WS", - } - staticData.predictionContextCache = NewPredictionContextCache() - staticData.serializedATN = []int32{ - 4, 0, 7, 38, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, - 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 1, 0, 4, 0, 17, 8, 0, 11, 0, 12, 0, 18, - 1, 1, 4, 1, 22, 8, 1, 11, 1, 12, 1, 23, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, - 4, 1, 5, 1, 5, 1, 6, 4, 6, 35, 8, 6, 11, 6, 12, 6, 36, 0, 0, 7, 1, 1, 3, - 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 1, 0, 0, 40, 0, 1, 1, 0, 0, 0, 0, 3, - 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 11, - 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 1, 16, 1, 0, 0, 0, 3, 21, 1, 0, 0, 0, 5, - 25, 1, 0, 0, 0, 7, 27, 1, 0, 0, 0, 9, 29, 1, 0, 0, 0, 11, 31, 1, 0, 0, - 0, 13, 34, 1, 0, 0, 0, 15, 17, 2, 97, 122, 0, 16, 15, 1, 0, 0, 0, 17, 18, - 1, 0, 0, 0, 18, 16, 1, 0, 0, 0, 18, 19, 1, 0, 0, 0, 19, 2, 1, 0, 0, 0, - 20, 22, 2, 48, 57, 0, 21, 20, 1, 0, 0, 0, 22, 23, 1, 0, 0, 0, 23, 21, 1, - 0, 0, 0, 23, 24, 1, 0, 0, 0, 24, 4, 1, 0, 0, 0, 25, 26, 5, 59, 0, 0, 26, - 6, 1, 0, 0, 0, 27, 28, 5, 61, 0, 0, 28, 8, 1, 0, 0, 0, 29, 30, 5, 43, 0, - 0, 30, 10, 1, 0, 0, 0, 31, 32, 5, 42, 0, 0, 32, 12, 1, 0, 0, 0, 33, 35, - 5, 32, 0, 0, 34, 33, 1, 0, 0, 0, 35, 36, 1, 0, 0, 0, 36, 34, 1, 0, 0, 0, - 36, 37, 1, 0, 0, 0, 37, 14, 1, 0, 0, 0, 4, 0, 18, 23, 36, 0, - } - deserializer := NewATNDeserializer(nil) - staticData.atn = deserializer.Deserialize(staticData.serializedATN) - atn := staticData.atn - staticData.decisionToDFA = make([]*DFA, len(atn.DecisionToState)) - decisionToDFA := staticData.decisionToDFA - for index, state := range atn.DecisionToState { - decisionToDFA[index] = NewDFA(state, index) - } -} - -// LexerBInit initializes any static state used to implement LexerB. By default, the -// static state used to implement the lexer is lazily initialized during the first call to -// NewLexerB(). You can call this function if you wish to initialize the static state ahead -// of time. -func LexerBInit() { - staticData := &lexerbLexerStaticData - staticData.once.Do(lexerbLexerInit) -} - -// NewLexerB produces a new lexer instance for the optional input antlr.CharStream. -func NewLexerB(input CharStream) *LexerB { - LexerBInit() - l := new(LexerB) - - l.BaseLexer = NewBaseLexer(input) - staticData := &lexerbLexerStaticData - l.Interpreter = NewLexerATNSimulator(l, staticData.atn, staticData.decisionToDFA, staticData.predictionContextCache) - l.channelNames = staticData.channelNames - l.modeNames = staticData.modeNames - l.RuleNames = staticData.ruleNames - l.LiteralNames = staticData.literalNames - l.SymbolicNames = staticData.symbolicNames - l.GrammarFileName = "LexerB.g4" - // TODO: l.EOF = antlr.TokenEOF - - return l -} - -// LexerB tokens. -// -//goland:noinspection GoUnusedConst -const ( - LexerBID = 1 - LexerBINT = 2 - LexerBSEMI = 3 - LexerBASSIGN = 4 - LexerBPLUS = 5 - LexerBMULT = 6 - LexerBWS = 7 -) diff --git a/runtime/Go/antlr/v4/testing_util_test.go b/runtime/Go/antlr/v4/testing_util_test.go deleted file mode 100644 index dd73703220..0000000000 --- a/runtime/Go/antlr/v4/testing_util_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package antlr - -import ( - "fmt" - "strings" -) - -// newTestCommonToken create common token with tokenType, text and channel -// notice: test purpose only -func newTestCommonToken(tokenType int, text string, channel int) *CommonToken { - t := new(CommonToken) - t.tokenType = tokenType - t.channel = channel - t.text = text - t.line = 0 - t.column = -1 - return t -} - -// tokensToString returns []Tokens string -// notice: test purpose only -func tokensToString(tokens []Token) string { - buf := make([]string, len(tokens)) - for i, token := range tokens { - buf[i] = fmt.Sprintf("%v", token) - } - - return "[" + strings.Join(buf, ", ") + "]" -} diff --git a/runtime/Go/antlr/v4/token.go b/runtime/Go/antlr/v4/token.go index 78c2c396cd..9670efb829 100644 --- a/runtime/Go/antlr/v4/token.go +++ b/runtime/Go/antlr/v4/token.go @@ -26,16 +26,16 @@ type Token interface { GetStop() int GetLine() int GetColumn() int - + GetText() string SetText(s string) - + GetTokenIndex() int SetTokenIndex(v int) - + GetTokenSource() TokenSource GetInputStream() CharStream - + String() string } @@ -55,15 +55,15 @@ type BaseToken struct { const ( TokenInvalidType = 0 - // TokenEpsilon - during lookahead operations, this "token" signifies we hit the rule end [ATN] state + // TokenEpsilon - during lookahead operations, this "token" signifies we hit the rule end [ATN] state // and did not follow it despite needing to. TokenEpsilon = -2 - + TokenMinUserTokenType = 1 TokenEOF = -1 - // TokenDefaultChannel is the default channel upon which tokens are sent to the parser. + // TokenDefaultChannel is the default channel upon which tokens are sent to the parser. // // All tokens go to the parser (unless [Skip] is called in the lexer rule) // on a particular "channel". The parser tunes to a particular channel @@ -125,7 +125,7 @@ type CommonToken struct { } func NewCommonToken(source *TokenSourceCharStreamPair, tokenType, channel, start, stop int) *CommonToken { - + t := &CommonToken{ BaseToken: BaseToken{ source: source, @@ -136,7 +136,7 @@ func NewCommonToken(source *TokenSourceCharStreamPair, tokenType, channel, start tokenIndex: -1, }, } - + if t.source.tokenSource != nil { t.line = source.tokenSource.GetLine() t.column = source.tokenSource.GetCharPositionInLine() @@ -199,14 +199,14 @@ func (c *CommonToken) String() string { } else { txt = "" } - + var ch string if c.channel > 0 { ch = ",channel=" + strconv.Itoa(c.channel) } else { ch = "" } - + return "[@" + strconv.Itoa(c.tokenIndex) + "," + strconv.Itoa(c.start) + ":" + strconv.Itoa(c.stop) + "='" + txt + "',<" + strconv.Itoa(c.tokenType) + ">" + ch + "," + strconv.Itoa(c.line) + ":" + strconv.Itoa(c.column) + "]" diff --git a/runtime/Go/antlr/v4/tokenstream_rewriter.go b/runtime/Go/antlr/v4/tokenstream_rewriter.go index 9d0c97283a..8f682fa010 100644 --- a/runtime/Go/antlr/v4/tokenstream_rewriter.go +++ b/runtime/Go/antlr/v4/tokenstream_rewriter.go @@ -94,7 +94,7 @@ const ( // Define the rewrite operation hierarchy type RewriteOperation interface { - + // Execute the rewrite operation by possibly adding to the buffer. // Return the index of the next token to operate on. Execute(buffer *bytes.Buffer) int @@ -174,7 +174,7 @@ func (op *BaseRewriteOperation) String() string { op.tokens.Get(op.GetIndex()), op.text, ) - + } type InsertBeforeOp struct { @@ -448,7 +448,7 @@ func (tsr *TokenStreamRewriter) GetText(programName string, interval Interval) s // ensure start/end are in range stop = min(stop, tsr.tokens.Size()-1) start = max(start, 0) - if rewrites == nil || len(rewrites) == 0 { + if len(rewrites) == 0 { return tsr.tokens.GetTextFromInterval(interval) // no instructions to execute } buf := bytes.Buffer{} diff --git a/runtime/Go/antlr/v4/tokenstream_rewriter_test.go b/runtime/Go/antlr/v4/tokenstream_rewriter_test.go deleted file mode 100644 index 5acfbb2aef..0000000000 --- a/runtime/Go/antlr/v4/tokenstream_rewriter_test.go +++ /dev/null @@ -1,420 +0,0 @@ -// Copyright (c) 2012-2022 The ANTLR Project. All rights reserved. -// Use of this file is governed by the BSD 3-clause license that -// can be found in the LICENSE.txt file in the project root. -package antlr - -import ( - "fmt" - "strings" - "sync" - "testing" - "unicode" -) - -/* Assume the following grammar for this test. - -lexer grammar LexerA; -A : 'a'; -B : 'b'; -C : 'c'; - -*/ - -func TestInsertBeforeIndex0(t *testing.T) { - input := NewInputStream("abc") - lexer := NewLexerA(input) - stream := NewCommonTokenStream(lexer, 0) - stream.Fill() - tokens := NewTokenStreamRewriter(stream) - tokens.InsertBeforeDefault(0, "0") - result := tokens.GetTextDefault() - if result != "0abc" { - t.Errorf("test failed, got %s", result) - } -} - -func prepareRewriter(str string) *TokenStreamRewriter { - input := NewInputStream(str) - lexer := NewLexerA(input) - stream := NewCommonTokenStream(lexer, 0) - stream.Fill() - return NewTokenStreamRewriter(stream) -} - -type LexerTest struct { - input string - expected string - description string - expectedException []string - ops func(*TokenStreamRewriter) -} - -func NewLexerTest(input, expected, desc string, ops func(*TokenStreamRewriter)) LexerTest { - return LexerTest{input: input, expected: expected, description: desc, ops: ops} -} - -func NewLexerExceptionTest(input string, expectedErr []string, desc string, ops func(*TokenStreamRewriter)) LexerTest { - return LexerTest{input: input, expectedException: expectedErr, description: desc, ops: ops} -} - -func panicTester(t *testing.T, expectedMsg []string, r *TokenStreamRewriter) { - defer func() { - r := recover() - if r == nil { - t.Errorf("Panic is expected, but finished normally") - } else { - sE := r.(string) - for _, e := range expectedMsg { - if !strings.Contains(sE, e) { - t.Errorf("Element [%s] is not in error message: [%s]", e, sE) - } - } - } - }() - r.GetTextDefault() -} - -//goland:noinspection ALL -func TestLexerA(t *testing.T) { - tests := []LexerTest{ - NewLexerTest("abc", "0abc", "InsertBeforeIndex0", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "0") - }), - NewLexerTest("abc", "abcx", "InsertAfterLastIndex", - func(r *TokenStreamRewriter) { - r.InsertAfterDefault(2, "x") - }), - NewLexerTest("abc", "axbxc", "2InsertBeforeAfterMiddleIndex", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(1, "x") - r.InsertAfterDefault(1, "x") - }), - NewLexerTest("abc", "xbc", "ReplaceIndex0", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(0, "x") - }), - NewLexerTest("abc", "abx", "ReplaceLastIndex", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(2, "x") - }), - NewLexerTest("abc", "axc", "ReplaceMiddleIndex", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(1, "x") - }), - NewLexerTest("abc", "ayc", "2ReplaceMiddleIndex", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(1, "x") - r.ReplaceDefaultPos(1, "y") - }), - NewLexerTest("abc", "_ayc", "2ReplaceMiddleIndex1InsertBefore", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "_") - r.ReplaceDefaultPos(1, "x") - r.ReplaceDefaultPos(1, "y") - }), - NewLexerTest("abc", "ac", "ReplaceThenDeleteMiddleIndex", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(1, "x") - r.DeleteDefaultPos(1) - }), - NewLexerExceptionTest("abc", []string{"insert op", "within boundaries of previous"}, - "InsertInPriorReplace", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(0, 2, "x") - r.InsertBeforeDefault(1, "0") - }), - NewLexerTest("abc", "0xbc", "InsertThenReplaceSameIndex", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "0") - r.ReplaceDefaultPos(0, "x") - }), - NewLexerTest("abc", "ayxbc", "2InsertMiddleIndex", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(1, "x") - r.InsertBeforeDefault(1, "y") - }), - NewLexerTest("abc", "yxzbc", "2InsertThenReplaceIndex0", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "x") - r.InsertBeforeDefault(0, "y") - r.ReplaceDefaultPos(0, "z") - }), - NewLexerTest("abc", "abyx", "ReplaceThenInsertBeforeLastIndex", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(2, "x") - r.InsertBeforeDefault(2, "y") - }), - NewLexerTest("abc", "abyx", "InsertThenReplaceLastIndex", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(2, "y") - r.ReplaceDefaultPos(2, "x") - }), - NewLexerTest("abc", "abxy", "ReplaceThenInsertAfterLastIndex", - func(r *TokenStreamRewriter) { - r.ReplaceDefaultPos(2, "x") - r.InsertAfterDefault(2, "y") - }), - NewLexerTest("abcccba", "abyxba", "ReplaceThenInsertAtLeftEdge", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "x") - r.InsertBeforeDefault(2, "y") - }), - NewLexerTest("abcccba", "abyxba", "ReplaceThenInsertAtLeftEdge", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "x") - r.InsertBeforeDefault(2, "y") - }), - NewLexerExceptionTest("abcccba", - []string{"insert op", "InsertBeforeOp", "within boundaries of previous", "ReplaceOp"}, - "ReplaceRangeThenInsertAtRightEdge", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "x") - r.InsertBeforeDefault(4, "y") - }), - NewLexerTest("abcccba", "abxyba", "ReplaceRangeThenInsertAfterRightEdge", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "x") - r.InsertAfterDefault(4, "y") - }), - NewLexerTest("abcccba", "x", "ReplaceAll", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(0, 6, "x") - }), - NewLexerTest("abcccba", "abxyzba", "ReplaceSubsetThenFetch", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "xyz") - }), - NewLexerExceptionTest("abcccba", - []string{"replace op boundaries of", "ReplaceOp", "overlap with previous"}, - "ReplaceThenReplaceSuperset", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "xyz") - r.ReplaceDefault(3, 5, "foo") - }), - NewLexerExceptionTest("abcccba", - []string{"replace op boundaries of", "ReplaceOp", "overlap with previous"}, - "ReplaceThenReplaceLowerIndexedSuperset", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 4, "xyz") - r.ReplaceDefault(1, 3, "foo") - }), - NewLexerTest("abcba", "fooa", "ReplaceSingleMiddleThenOverlappingSuperset", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 2, "xyz") - r.ReplaceDefault(0, 3, "foo") - }), - NewLexerTest("abc", "yxabc", "CombineInserts", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "x") - r.InsertBeforeDefault(0, "y") - }), - NewLexerTest("abc", "yazxbc", "Combine3Inserts", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(1, "x") - r.InsertBeforeDefault(0, "y") - r.InsertBeforeDefault(1, "z") - }), - NewLexerTest("abc", "zfoo", "CombineInsertOnLeftWithReplace", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(0, 2, "foo") - r.InsertBeforeDefault(0, "z") - }), - NewLexerTest("abc", "z", "CombineInsertOnLeftWithDelete", - func(r *TokenStreamRewriter) { - r.DeleteDefault(0, 2) - r.InsertBeforeDefault(0, "z") - }), - NewLexerTest("abc", "zaxbyc", "DisjointInserts", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(1, "x") - r.InsertBeforeDefault(2, "y") - r.InsertBeforeDefault(0, "z") - }), - NewLexerTest("abcc", "bar", "OverlappingReplace", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(1, 2, "foo") - r.ReplaceDefault(0, 3, "bar") - }), - NewLexerExceptionTest("abcc", - []string{"replace op boundaries of", "ReplaceOp", "overlap with previous"}, - "OverlappingReplace2", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(0, 3, "bar") - r.ReplaceDefault(1, 2, "foo") - }), - NewLexerTest("abcc", "barc", "OverlappingReplace3", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(1, 2, "foo") - r.ReplaceDefault(0, 2, "bar") - }), - NewLexerTest("abcc", "abar", "OverlappingReplace4", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(1, 2, "foo") - r.ReplaceDefault(1, 3, "bar") - }), - NewLexerTest("abcc", "afooc", "DropIdenticalReplace", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(1, 2, "foo") - r.ReplaceDefault(1, 2, "foo") - }), - NewLexerTest("abc", "afoofoo", "DropPrevCoveredInsert", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(1, "foo") - r.ReplaceDefault(1, 2, "foo") - }), - NewLexerTest("abcc", "axbfoo", "LeaveAloneDisjointInsert", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(1, "x") - r.ReplaceDefault(2, 3, "foo") - }), - NewLexerTest("abcc", "axbfoo", "LeaveAloneDisjointInsert2", - func(r *TokenStreamRewriter) { - r.ReplaceDefault(2, 3, "foo") - r.InsertBeforeDefault(1, "x") - }), - NewLexerTest("abc", "aby", "InsertBeforeTokenThenDeleteThatToken", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(2, "y") - r.DeleteDefaultPos(2) - }), - NewLexerTest("aa", "aa", "DistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "") - r.InsertAfterDefault(0, "") - r.InsertBeforeDefault(1, "") - r.InsertAfterDefault(1, "") - }), - NewLexerTest("aa", "

a

a", "DistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "

") - r.InsertBeforeDefault(0, "") - r.InsertAfterDefault(0, "

") - r.InsertAfterDefault(0, "") - r.InsertBeforeDefault(1, "") - r.InsertAfterDefault(1, "") - }), - NewLexerTest("ab", "

a

!b", "DistinguishBetweenInsertAfterAndInsertBeforeToPreserverOrder2", - func(r *TokenStreamRewriter) { - r.InsertBeforeDefault(0, "

") - r.InsertBeforeDefault(0, "") - r.InsertBeforeDefault(0, "

") - r.InsertAfterDefault(0, "

") - r.InsertAfterDefault(0, "") - r.InsertAfterDefault(0, "
") - r.InsertBeforeDefault(1, "!") - }), - } - - for _, c := range tests { - t.Run(c.description, func(t *testing.T) { - rewriter := prepareRewriter(c.input) - c.ops(rewriter) - if len(c.expectedException) > 0 { - panicTester(t, c.expectedException, rewriter) - } else { - result := rewriter.GetTextDefault() - if result != c.expected { - t.Errorf("Expected:%s | Result: %s", c.expected, result) - } - } - }) - } -} - -// Suppress unused import error -var _ = fmt.Printf -var _ = sync.Once{} -var _ = unicode.IsLetter - -type LexerA struct { - *BaseLexer - channelNames []string - modeNames []string - // TODO: EOF string -} - -var lexeraLexerStaticData struct { - once sync.Once - serializedATN []int32 - channelNames []string - modeNames []string - literalNames []string - symbolicNames []string - ruleNames []string - predictionContextCache *PredictionContextCache - atn *ATN - decisionToDFA []*DFA -} - -func lexeraLexerInit() { - staticData := &lexeraLexerStaticData - staticData.channelNames = []string{ - "DEFAULT_TOKEN_CHANNEL", "HIDDEN", - } - staticData.modeNames = []string{ - "DEFAULT_MODE", - } - staticData.literalNames = []string{ - "", "'a'", "'b'", "'c'", - } - staticData.symbolicNames = []string{ - "", "A", "B", "C", - } - staticData.ruleNames = []string{ - "A", "B", "C", - } - staticData.predictionContextCache = NewPredictionContextCache() - staticData.serializedATN = []int32{ - 4, 0, 3, 13, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 1, 0, 1, 0, 1, - 1, 1, 1, 1, 2, 1, 2, 0, 0, 3, 1, 1, 3, 2, 5, 3, 1, 0, 0, 12, 0, 1, 1, 0, - 0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 1, 7, 1, 0, 0, 0, 3, 9, 1, 0, - 0, 0, 5, 11, 1, 0, 0, 0, 7, 8, 5, 97, 0, 0, 8, 2, 1, 0, 0, 0, 9, 10, 5, - 98, 0, 0, 10, 4, 1, 0, 0, 0, 11, 12, 5, 99, 0, 0, 12, 6, 1, 0, 0, 0, 1, - 0, 0, - } - deserializer := NewATNDeserializer(nil) - staticData.atn = deserializer.Deserialize(staticData.serializedATN) - atn := staticData.atn - staticData.decisionToDFA = make([]*DFA, len(atn.DecisionToState)) - decisionToDFA := staticData.decisionToDFA - for index, state := range atn.DecisionToState { - decisionToDFA[index] = NewDFA(state, index) - } -} - -// LexerAInit initializes any static state used to implement LexerA. By default, the -// static state used to implement the lexer is lazily initialized during the first call to -// NewLexerA(). You can call this function if you wish to initialize the static state ahead -// of time. -func LexerAInit() { - staticData := &lexeraLexerStaticData - staticData.once.Do(lexeraLexerInit) -} - -// NewLexerA produces a new lexer instance for the optional input antlr.CharStream. -func NewLexerA(input CharStream) *LexerA { - LexerAInit() - l := new(LexerA) - l.BaseLexer = NewBaseLexer(input) - staticData := &lexeraLexerStaticData - l.Interpreter = NewLexerATNSimulator(l, staticData.atn, staticData.decisionToDFA, staticData.predictionContextCache) - l.channelNames = staticData.channelNames - l.modeNames = staticData.modeNames - l.RuleNames = staticData.ruleNames - l.LiteralNames = staticData.literalNames - l.SymbolicNames = staticData.symbolicNames - l.GrammarFileName = "LexerA.g4" - // TODO: l.EOF = antlr.TokenEOF - - return l -} - -// LexerA tokens. -// -//goland:noinspection GoUnusedConst -const ( - LexerAA = 1 - LexerAB = 2 - LexerAC = 3 -) diff --git a/runtime/Go/antlr/v4/transition.go b/runtime/Go/antlr/v4/transition.go index 62976688a6..313b0fc127 100644 --- a/runtime/Go/antlr/v4/transition.go +++ b/runtime/Go/antlr/v4/transition.go @@ -37,18 +37,18 @@ type BaseTransition struct { } func NewBaseTransition(target ATNState) *BaseTransition { - + if target == nil { panic("target cannot be nil.") } - + t := new(BaseTransition) - + t.target = target // Are we epsilon, action, sempred? t.isEpsilon = false t.intervalSet = nil - + return t } @@ -339,7 +339,7 @@ func NewSetTransition(target ATNState, set *IntervalSet) *SetTransition { }, } - if set != nil { + if set != nil { t.intervalSet = set } else { t.intervalSet = NewIntervalSet() @@ -375,7 +375,7 @@ func NewNotSetTransition(target ATNState, set *IntervalSet) *NotSetTransition { t.intervalSet = NewIntervalSet() t.intervalSet.addOne(TokenInvalidType) } - + return t } diff --git a/runtime/Go/antlr/v4/tree.go b/runtime/Go/antlr/v4/tree.go index 9f882ba154..6c17164381 100644 --- a/runtime/Go/antlr/v4/tree.go +++ b/runtime/Go/antlr/v4/tree.go @@ -33,7 +33,7 @@ type ParseTree interface { type RuleNode interface { ParseTree - + GetRuleContext() RuleContext GetBaseRuleContext() *BaseRuleContext } @@ -45,7 +45,7 @@ type TerminalNode interface { type ErrorNode interface { TerminalNode - + errorNode() } @@ -111,10 +111,10 @@ var _ TerminalNode = &TerminalNodeImpl{} func NewTerminalNodeImpl(symbol Token) *TerminalNodeImpl { tn := new(TerminalNodeImpl) - + tn.parentCtx = nil tn.symbol = symbol - + return tn } @@ -170,7 +170,7 @@ func (t *TerminalNodeImpl) String() string { if t.symbol.GetTokenType() == TokenEOF { return "" } - + return t.symbol.GetText() } @@ -261,7 +261,7 @@ func (i *IterativeParseTreeWalker) Walk(listener ParseTreeListener, t Tree) { var indexStack []int currentNode := t currentIndex := 0 - + for currentNode != nil { // pre-order visit switch tt := currentNode.(type) { @@ -280,7 +280,7 @@ func (i *IterativeParseTreeWalker) Walk(listener ParseTreeListener, t Tree) { currentNode = currentNode.GetChild(0) continue } - + for { // post-order visit if ruleNode, ok := currentNode.(RuleNode); ok { diff --git a/runtime/Go/antlr/v4/utils.go b/runtime/Go/antlr/v4/utils.go index 23e4e45210..733d7df9dc 100644 --- a/runtime/Go/antlr/v4/utils.go +++ b/runtime/Go/antlr/v4/utils.go @@ -9,8 +9,10 @@ import ( "errors" "fmt" "math/bits" + "os" "strconv" "strings" + "syscall" ) func intMin(a, b int) int { @@ -47,22 +49,6 @@ func (s *IntStack) Push(e int) { *s = append(*s, e) } -func standardEqualsFunction(a Collectable[any], b Collectable[any]) bool { - - return a.Equals(b) -} - -func standardHashFunction(a interface{}) int { - if h, ok := a.(hasher); ok { - return h.Hash() - } - panic("Not 'Hasher'") -} - -type hasher interface { - Hash() int -} - const bitsPerWord = 64 func indexForBit(bit int) int { @@ -159,27 +145,27 @@ func (b *BitSet) equals(other interface{}) bool { if !ok { return false } - + if b == otherBitSet { return true } - + // We only compare set bits, so we cannot rely on the two slices having the same size. Its // possible for two BitSets to have different slice lengths but the same set bits. So we only // compare the relevant words and ignore the trailing zeros. bLen := b.minLen() otherLen := otherBitSet.minLen() - + if bLen != otherLen { return false } - + for i := 0; i < bLen; i++ { if b.data[i] != otherBitSet.data[i] { return false } } - + return true } @@ -202,7 +188,7 @@ func (b *BitSet) length() int { func (b *BitSet) String() string { vals := make([]string, 0, b.length()) - + for i, v := range b.data { for v != 0 { n := bits.TrailingZeros64(v) @@ -210,7 +196,7 @@ func (b *BitSet) String() string { v &= ^(uint64(1) << n) } } - + return "{" + strings.Join(vals, ", ") + "}" } @@ -245,7 +231,7 @@ func (a *AltDict) values() []interface{} { } func EscapeWhitespace(s string, escapeSpaces bool) string { - + s = strings.Replace(s, "\t", "\\t", -1) s = strings.Replace(s, "\n", "\\n", -1) s = strings.Replace(s, "\r", "\\r", -1) @@ -258,29 +244,29 @@ func EscapeWhitespace(s string, escapeSpaces bool) string { //goland:noinspection GoUnusedExportedFunction func TerminalNodeToStringArray(sa []TerminalNode) []string { st := make([]string, len(sa)) - + for i, s := range sa { st[i] = fmt.Sprintf("%v", s) } - + return st } //goland:noinspection GoUnusedExportedFunction func PrintArrayJavaStyle(sa []string) string { var buffer bytes.Buffer - + buffer.WriteString("[") - + for i, s := range sa { buffer.WriteString(s) if i != len(sa)-1 { buffer.WriteString(", ") } } - + buffer.WriteString("]") - + return buffer.String() } @@ -296,12 +282,12 @@ func murmurUpdate(h int, value int) int { const r2 uint32 = 13 const m uint32 = 5 const n uint32 = 0xE6546B64 - + k := uint32(value) k *= c1 k = (k << r1) | (k >> (32 - r1)) k *= c2 - + hash := uint32(h) ^ k hash = (hash << r2) | (hash >> (32 - r2)) hash = hash*m + n @@ -316,6 +302,27 @@ func murmurFinish(h int, numberOfWords int) int { hash ^= hash >> 13 hash *= 0xc2b2ae35 hash ^= hash >> 16 - + return int(hash) } + +func isDirectory(dir string) (bool, error) { + fileInfo, err := os.Stat(dir) + if err != nil { + switch { + case errors.Is(err, syscall.ENOENT): + // The given directory does not exist, so we will try to create it + // + err = os.MkdirAll(dir, 0755) + if err != nil { + return false, err + } + + return true, nil + case err != nil: + return false, err + default: + } + } + return fileInfo.IsDir(), err +} diff --git a/runtime/Go/antlr/v4/utils_set.go b/runtime/Go/antlr/v4/utils_set.go deleted file mode 100644 index 9c361e078c..0000000000 --- a/runtime/Go/antlr/v4/utils_set.go +++ /dev/null @@ -1,236 +0,0 @@ -package antlr - -import "math" - -const ( - _initalCapacity = 16 - _initalBucketCapacity = 8 - _loadFactor = 0.75 -) - -type Set interface { - Add(value interface{}) (added interface{}) - Len() int - Get(value interface{}) (found interface{}) - Contains(value interface{}) bool - Values() []interface{} - Each(f func(interface{}) bool) -} - -type array2DHashSet struct { - buckets [][]Collectable[any] - hashcodeFunction func(interface{}) int - equalsFunction func(Collectable[any], Collectable[any]) bool - - n int // How many elements in set - threshold int // when to expand - - currentPrime int // jump by 4 primes each expand or whatever - initialBucketCapacity int -} - -func (as *array2DHashSet) Each(f func(interface{}) bool) { - if as.Len() < 1 { - return - } - - for _, bucket := range as.buckets { - for _, o := range bucket { - if o == nil { - break - } - if !f(o) { - return - } - } - } -} - -func (as *array2DHashSet) Values() []interface{} { - if as.Len() < 1 { - return nil - } - - values := make([]interface{}, 0, as.Len()) - as.Each(func(i interface{}) bool { - values = append(values, i) - return true - }) - return values -} - -func (as *array2DHashSet) Contains(value Collectable[any]) bool { - return as.Get(value) != nil -} - -func (as *array2DHashSet) Add(value Collectable[any]) interface{} { - if as.n > as.threshold { - as.expand() - } - return as.innerAdd(value) -} - -func (as *array2DHashSet) expand() { - old := as.buckets - - as.currentPrime += 4 - - var ( - newCapacity = len(as.buckets) << 1 - newTable = as.createBuckets(newCapacity) - newBucketLengths = make([]int, len(newTable)) - ) - - as.buckets = newTable - as.threshold = int(float64(newCapacity) * _loadFactor) - - for _, bucket := range old { - if bucket == nil { - continue - } - - for _, o := range bucket { - if o == nil { - break - } - - b := as.getBuckets(o) - bucketLength := newBucketLengths[b] - var newBucket []Collectable[any] - if bucketLength == 0 { - // new bucket - newBucket = as.createBucket(as.initialBucketCapacity) - newTable[b] = newBucket - } else { - newBucket = newTable[b] - if bucketLength == len(newBucket) { - // expand - newBucketCopy := make([]Collectable[any], len(newBucket)<<1) - copy(newBucketCopy[:bucketLength], newBucket) - newBucket = newBucketCopy - newTable[b] = newBucket - } - } - - newBucket[bucketLength] = o - newBucketLengths[b]++ - } - } -} - -func (as *array2DHashSet) Len() int { - return as.n -} - -func (as *array2DHashSet) Get(o Collectable[any]) interface{} { - if o == nil { - return nil - } - - b := as.getBuckets(o) - bucket := as.buckets[b] - if bucket == nil { // no bucket - return nil - } - - for _, e := range bucket { - if e == nil { - return nil // empty slot; not there - } - if as.equalsFunction(e, o) { - return e - } - } - - return nil -} - -func (as *array2DHashSet) innerAdd(o Collectable[any]) interface{} { - b := as.getBuckets(o) - - bucket := as.buckets[b] - - // new bucket - if bucket == nil { - bucket = as.createBucket(as.initialBucketCapacity) - bucket[0] = o - - as.buckets[b] = bucket - as.n++ - return o - } - - // look for it in bucket - for i := 0; i < len(bucket); i++ { - existing := bucket[i] - if existing == nil { // empty slot; not there, add. - bucket[i] = o - as.n++ - return o - } - - if as.equalsFunction(existing, o) { // found existing, quit - return existing - } - } - - // full bucket, expand and add to end - oldLength := len(bucket) - bucketCopy := make([]Collectable[any], oldLength<<1) - copy(bucketCopy[:oldLength], bucket) - bucket = bucketCopy - as.buckets[b] = bucket - bucket[oldLength] = o - as.n++ - return o -} - -func (as *array2DHashSet) getBuckets(value Collectable[any]) int { - hash := as.hashcodeFunction(value) - return hash & (len(as.buckets) - 1) -} - -func (as *array2DHashSet) createBuckets(cap int) [][]Collectable[any] { - return make([][]Collectable[any], cap) -} - -func (as *array2DHashSet) createBucket(cap int) []Collectable[any] { - return make([]Collectable[any], cap) -} - -func newArray2DHashSetWithCap( - hashcodeFunction func(interface{}) int, - equalsFunction func(Collectable[any], Collectable[any]) bool, - initCap int, - initBucketCap int, -) *array2DHashSet { - if hashcodeFunction == nil { - hashcodeFunction = standardHashFunction - } - - if equalsFunction == nil { - equalsFunction = standardEqualsFunction - } - - ret := &array2DHashSet{ - hashcodeFunction: hashcodeFunction, - equalsFunction: equalsFunction, - - n: 0, - threshold: int(math.Floor(_initalCapacity * _loadFactor)), - - currentPrime: 1, - initialBucketCapacity: initBucketCap, - } - - ret.buckets = ret.createBuckets(initCap) - return ret -} - -//goland:noinspection GoUnusedExportedFunction,GoUnusedFunction -func newArray2DHashSet( - hashcodeFunction func(interface{}) int, - equalsFunction func(Collectable[any], Collectable[any]) bool, -) *array2DHashSet { - return newArray2DHashSetWithCap(hashcodeFunction, equalsFunction, _initalCapacity, _initalBucketCapacity) -} diff --git a/runtime/Go/antlr/v4/utils_test.go b/runtime/Go/antlr/v4/utils_test.go deleted file mode 100644 index 17bb240daf..0000000000 --- a/runtime/Go/antlr/v4/utils_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package antlr - -import "testing" - -func testBitSet(t *testing.T, bs *BitSet, str string, length int, contains []int, minValue int, minLen int) { - t.Helper() - if got, want := bs.String(), str; got != want { - t.Errorf("%+v.String() = %q, want %q", bs, got, want) - } - if got, want := bs.length(), length; got != want { - t.Errorf("%+v.Length() = %q, want %q", bs, got, want) - } - for i := 0; i < len(bs.data)*bitsPerWord; i++ { - var want bool - for _, val := range contains { - if i == val { - want = true - break - } - } - if got := bs.contains(i); got != want { - t.Errorf("%+v.contains(%v) = %v, want %v", bs, i, got, want) - } - } - if got, want := bs.minValue(), minValue; got != want { - t.Errorf("%+v.minValue() = %v, want %v", bs, got, want) - } - if got, want := bs.minLen(), minLen; got != want { - t.Errorf("%+v.minLen() = %v, want %v", bs, got, want) - } -} - -//goland:noinspection GoBoolExpressions -func TestBitSet(t *testing.T) { - bs1 := NewBitSet() - testBitSet(t, bs1, "{}", 0, []int{}, 2147483647, 0) - bs1.add(0) - testBitSet(t, bs1, "{0}", 1, []int{0}, 0, 1) - bs1.add(63) - testBitSet(t, bs1, "{0, 63}", 2, []int{0, 63}, 0, 1) - bs1.remove(0) - testBitSet(t, bs1, "{63}", 1, []int{63}, 63, 1) - bs1.add(20) - testBitSet(t, bs1, "{20, 63}", 2, []int{20, 63}, 20, 1) - bs1.clear(63) - testBitSet(t, bs1, "{20}", 1, []int{20}, 20, 1) - bs2 := NewBitSet() - bs2.add(64) - bs1.or(bs2) - testBitSet(t, bs1, "{20, 64}", 2, []int{20, 64}, 20, 2) - bs1.remove(20) - testBitSet(t, bs1, "{64}", 1, []int{64}, 64, 2) - bs3 := NewBitSet() - bs3.add(63) - bs1.or(bs3) - testBitSet(t, bs1, "{63, 64}", 2, []int{63, 64}, 63, 2) - bs1.clear(64) - bs4 := NewBitSet() - bs4.or(bs1) - if got, want := bs4.equals(bs1), true; got != want { - t.Errorf("%+v.equals(%+v) = %v, want %v", bs4, bs1, got, want) - } -}