Skip to content

Feature/memory #4223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
parrt merged 5 commits into from
Apr 11, 2023
Merged

Feature/memory #4223

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ff11b13
feat: Clean out redundant code and stupid code, implement lazy init o…
jimidle Mar 29, 2023
ee65310
feat: Adds memory optimizations and compile time optional statistics
jimidle Apr 8, 2023
0e1c91a
feat: back out of lazy initilization for config sets - it needs anoth…
jimidle Apr 11, 2023
546ede9
feat: Used dedicated map for visit cache - improves memory and speed …
jimidle Apr 11, 2023
7f21978
fix: ensure Statistics.Reset() zeros out top collections by usage on …
jimidle Apr 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 45 additions & 75 deletions runtime/Go/antlr/v4/atn_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ const (
// path(s) to the root is the rule invocation(s) chain used to arrive in the
// state. The semantic context is the tree of semantic predicates encountered
// before reaching an ATN state.
//
type ATNConfig struct {
precedenceFilterSuppressed bool
state ATNState
Expand All @@ -31,18 +30,6 @@ type ATNConfig struct {
passedThroughNonGreedyDecision bool
}

//goland:noinspection GoUnusedExportedFunction
func NewATNConfig7(old *ATNConfig) *ATNConfig { // TODO: Dup - maybe delete this
return &ATNConfig{
state: old.state,
alt: old.alt,
context: old.context,
semanticContext: old.semanticContext,
reachesIntoOuterContext: old.reachesIntoOuterContext,
cType: old.cType,
}
}

// NewATNConfig6 creates a new ATNConfig instance given a state, alt and context only
func NewATNConfig6(state ATNState, alt int, context *PredictionContext) *ATNConfig {
return NewATNConfig5(state, alt, context, SemanticContextNone)
Expand All @@ -54,13 +41,13 @@ func NewATNConfig5(state ATNState, alt int, context *PredictionContext, semantic
panic("semanticContext cannot be nil") // TODO: Necessary?
}

return &ATNConfig{
state: state,
alt: alt,
context: context,
semanticContext: semanticContext,
cType: parserConfig,
}
pac := &ATNConfig{}
pac.state = state
pac.alt = alt
pac.context = context
pac.semanticContext = semanticContext
pac.cType = parserConfig
return pac
}

// NewATNConfig4 creates a new ATNConfig instance given an existing config, and a state only
Expand Down Expand Up @@ -89,11 +76,9 @@ func NewATNConfig(c *ATNConfig, state ATNState, context *PredictionContext, sema
if semanticContext == nil {
panic("semanticContext cannot be nil") // TODO: Remove this - probably put here for some bug that is now fixed
}
b := &ATNConfig{
cType: parserConfig,
}
b := &ATNConfig{}
b.InitATNConfig(c, state, c.GetAlt(), context, semanticContext)

b.cType = parserConfig
return b
}

Expand Down Expand Up @@ -250,112 +235,97 @@ func (a *ATNConfig) String() string {
}

func NewLexerATNConfig6(state ATNState, alt int, context *PredictionContext) *ATNConfig {
return &ATNConfig{
state: state,
alt: alt,
context: context,
semanticContext: SemanticContextNone,
cType: lexerConfig,
}
}

func NewLexerATNConfig5(state ATNState, alt int, context *PredictionContext, lexerActionExecutor *LexerActionExecutor) *ATNConfig {
return &ATNConfig{
state: state,
alt: alt,
context: context,
semanticContext: SemanticContextNone,
lexerActionExecutor: lexerActionExecutor,
cType: lexerConfig,
}
lac := &ATNConfig{}
lac.state = state
lac.alt = alt
lac.context = context
lac.semanticContext = SemanticContextNone
lac.cType = lexerConfig
return lac
}

func NewLexerATNConfig4(c *ATNConfig, state ATNState) *ATNConfig {
lac := &ATNConfig{
lexerActionExecutor: c.lexerActionExecutor,
passedThroughNonGreedyDecision: checkNonGreedyDecision(c, state),
}
lac := &ATNConfig{}
lac.lexerActionExecutor = c.lexerActionExecutor
lac.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state)
lac.InitATNConfig(c, state, c.GetAlt(), c.GetContext(), c.GetSemanticContext())
lac.cType = lexerConfig
return lac
}

func NewLexerATNConfig3(c *ATNConfig, state ATNState, lexerActionExecutor *LexerActionExecutor) *ATNConfig {
lac := &ATNConfig{
lexerActionExecutor: lexerActionExecutor,
passedThroughNonGreedyDecision: checkNonGreedyDecision(c, state),
}
lac := &ATNConfig{}
lac.lexerActionExecutor = lexerActionExecutor
lac.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state)
lac.InitATNConfig(c, state, c.GetAlt(), c.GetContext(), c.GetSemanticContext())
lac.cType = lexerConfig
return lac
}

func NewLexerATNConfig2(c *ATNConfig, state ATNState, context *PredictionContext) *ATNConfig {
lac := &ATNConfig{
lexerActionExecutor: c.lexerActionExecutor,
passedThroughNonGreedyDecision: checkNonGreedyDecision(c, state),
}
lac := &ATNConfig{}
lac.lexerActionExecutor = c.lexerActionExecutor
lac.passedThroughNonGreedyDecision = checkNonGreedyDecision(c, state)
lac.InitATNConfig(c, state, c.GetAlt(), context, c.GetSemanticContext())
lac.cType = lexerConfig
return lac
}

//goland:noinspection GoUnusedExportedFunction
func NewLexerATNConfig1(state ATNState, alt int, context *PredictionContext) *ATNConfig {
lac := &ATNConfig{
state: state,
alt: alt,
context: context,
semanticContext: SemanticContextNone,
cType: lexerConfig,
}
lac := &ATNConfig{}
lac.state = state
lac.alt = alt
lac.context = context
lac.semanticContext = SemanticContextNone
lac.cType = lexerConfig
return lac
}

// LHash is the default hash function for Lexer ATNConfig objects, it can be used directly or via
// the default comparator [ObjEqComparator].
func (l *ATNConfig) LHash() int {
func (a *ATNConfig) LHash() int {
var f int
if l.passedThroughNonGreedyDecision {
if a.passedThroughNonGreedyDecision {
f = 1
} else {
f = 0
}
h := murmurInit(7)
h = murmurUpdate(h, l.state.GetStateNumber())
h = murmurUpdate(h, l.alt)
h = murmurUpdate(h, l.context.Hash())
h = murmurUpdate(h, l.semanticContext.Hash())
h = murmurUpdate(h, a.state.GetStateNumber())
h = murmurUpdate(h, a.alt)
h = murmurUpdate(h, a.context.Hash())
h = murmurUpdate(h, a.semanticContext.Hash())
h = murmurUpdate(h, f)
h = murmurUpdate(h, l.lexerActionExecutor.Hash())
h = murmurUpdate(h, a.lexerActionExecutor.Hash())
h = murmurFinish(h, 6)
return h
}

// LEquals is the default comparison function for Lexer ATNConfig objects, it can be used directly or via
// the default comparator [ObjEqComparator].
func (l *ATNConfig) LEquals(other Collectable[*ATNConfig]) bool {
func (a *ATNConfig) LEquals(other Collectable[*ATNConfig]) bool {
var otherT, ok = other.(*ATNConfig)
if !ok {
return false
} else if l == otherT {
} else if a == otherT {
return true
} else if l.passedThroughNonGreedyDecision != otherT.passedThroughNonGreedyDecision {
} else if a.passedThroughNonGreedyDecision != otherT.passedThroughNonGreedyDecision {
return false
}

switch {
case l.lexerActionExecutor == nil && otherT.lexerActionExecutor == nil:
case a.lexerActionExecutor == nil && otherT.lexerActionExecutor == nil:
return true
case l.lexerActionExecutor != nil && otherT.lexerActionExecutor != nil:
if !l.lexerActionExecutor.Equals(otherT.lexerActionExecutor) {
case a.lexerActionExecutor != nil && otherT.lexerActionExecutor != nil:
if !a.lexerActionExecutor.Equals(otherT.lexerActionExecutor) {
return false
}
default:
return false // One but not both, are nil
}

return l.PEquals(otherT)
return a.PEquals(otherT)
}

func checkNonGreedyDecision(source *ATNConfig, target ATNState) bool {
Expand Down
36 changes: 15 additions & 21 deletions runtime/Go/antlr/v4/atn_config_set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type ATNConfigSet struct {
// read-only because a set becomes a DFA state.
configLookup *JStore[*ATNConfig, Comparator[*ATNConfig]]

// configs is the added elements.
// configs is the added elements that did not match an existing key in configLookup
configs []*ATNConfig

// TODO: These fields make me pretty uncomfortable, but it is nice to pack up
Expand Down Expand Up @@ -68,7 +68,7 @@ func (b *ATNConfigSet) Alts() *BitSet {
func NewATNConfigSet(fullCtx bool) *ATNConfigSet {
return &ATNConfigSet{
cachedHash: -1,
configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst),
configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst, ATNConfigLookupCollection, "NewATNConfigSet()"),
fullCtx: fullCtx,
}
}
Expand Down Expand Up @@ -127,7 +127,7 @@ func (b *ATNConfigSet) GetStates() *JStore[ATNState, Comparator[ATNState]] {

// states uses the standard comparator and Hash() provided by the ATNState instance
//
states := NewJStore[ATNState, Comparator[ATNState]](aStateEqInst)
states := NewJStore[ATNState, Comparator[ATNState]](aStateEqInst, ATNStateCollection, "ATNConfigSet.GetStates()")

for i := 0; i < len(b.configs); i++ {
states.Put(b.configs[i].GetState())
Expand Down Expand Up @@ -155,13 +155,13 @@ func (b *ATNConfigSet) OptimizeConfigs(interpreter *BaseATNSimulator) {
panic("set is read-only")
}

if b.configLookup.Len() == 0 {
// Empty indicate no optimization is possible
if b.configLookup == nil || b.configLookup.Len() == 0 {
return
}

for i := 0; i < len(b.configs); i++ {
config := b.configs[i]

config.SetContext(interpreter.getCachedContext(config.GetContext()))
}
}
Expand All @@ -174,12 +174,8 @@ func (b *ATNConfigSet) AddAll(coll []*ATNConfig) bool {
return false
}

// Compare is a hack function - O(n squared) at worst - just to verify that adding [DFA] states to the known
// set works, so long as comparison of [ATNConfigSet] works. For that to work, we
// need to make sure that the set of ATNConfigs in two sets are equivalent. The configs are
// only equal if they are in the same order too as Java uses ArrayList.equals(), which requires
// the same order.
//
// Compare The configs are only equal if they are in the same order and their Equals function returns true.
// Java uses ArrayList.equals(), which requires the same order.
func (b *ATNConfigSet) Compare(bs *ATNConfigSet) bool {
if len(b.configs) != len(bs.configs) {
return false
Expand Down Expand Up @@ -238,19 +234,17 @@ func (b *ATNConfigSet) hashCodeConfigs() int {
}

func (b *ATNConfigSet) Contains(item *ATNConfig) bool {
if b.configLookup == nil {
if b.readOnly {
panic("not implemented for read-only sets")
}

if b.configLookup == nil {
return false
}
return b.configLookup.Contains(item)
}

func (b *ATNConfigSet) ContainsFast(item *ATNConfig) bool {
if b.configLookup == nil {
panic("not implemented for read-only sets")
}

return b.configLookup.Contains(item) // TODO: containsFast is not implemented for Set
return b.Contains(item)
}

func (b *ATNConfigSet) Clear() {
Expand All @@ -259,11 +253,11 @@ func (b *ATNConfigSet) Clear() {
}
b.configs = make([]*ATNConfig, 0)
b.cachedHash = -1
b.configLookup = NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst)
b.configLookup = NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfCompInst, ATNConfigLookupCollection, "NewATNConfigSet()")
}

func (b *ATNConfigSet) String() string {

s := "["

for i, c := range b.configs {
Expand Down Expand Up @@ -301,7 +295,7 @@ func NewOrderedATNConfigSet() *ATNConfigSet {
return &ATNConfigSet{
cachedHash: -1,
// This set uses the standard Hash() and Equals() from ATNConfig
configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst),
configLookup: NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ATNConfigCollection, "ATNConfigSet.NewOrderedATNConfigSet()"),
fullCtx: false,
}
}
Loading