-
-
Notifications
You must be signed in to change notification settings - Fork 6.2k
Add doctor commands to interrogate levelDB queues #18732
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Closed
Closed
Changes from 3 commits
Commits
Show all changes
24 commits
Select commit
Hold shift + click to select a range
b06b088
Ensure that conflict status is reset if there are no conflicts
zeripath db1931c
Add some more trace and debug logging to process and testpatch
zeripath bd066ca
Add doctor commands for looking at the level DB
zeripath 2bce719
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath 1475c62
As per review
zeripath 93d8275
refix process manager trace ended
zeripath 53232c4
No revive - we want this to stutter
zeripath 1500a0d
Update modules/doctor/queue.go
zeripath a9ae5c3
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath 4e02994
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath a639e44
remove trace logging from process manager
zeripath 2821ddd
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath bd5ca04
use constants for queues
zeripath bbe2915
Merge branch 'main' into reset-conflict-status
6543 bdfa176
Merge branch 'main' into reset-conflict-status
zeripath 4680195
placate linter
zeripath 760c257
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath fc7301d
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath 2e9cdfb
placate the bloody linter
zeripath ff707de
replacate lint.
zeripath 9e8f61b
Merge remote-tracking branch 'origin/main' into reset-conflict-status
zeripath 900bb10
remove mismerged files
zeripath 99c2b4b
add actions job queue name
zeripath 71e16d6
restore licenses deleted by mistake
zeripath File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,216 @@ | ||
| // Copyright 2022 The Gitea Authors. All rights reserved. | ||
| // Use of this source code is governed by a MIT-style | ||
| // license that can be found in the LICENSE file. | ||
|
|
||
| package doctor | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "code.gitea.io/gitea/modules/log" | ||
| "code.gitea.io/gitea/modules/nosql" | ||
| "code.gitea.io/gitea/modules/queue" | ||
| "code.gitea.io/gitea/modules/setting" | ||
| "gitea.com/lunny/levelqueue" | ||
| ) | ||
|
|
||
| var uniqueQueueNames = []string{ | ||
zeripath marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| "code_indexer", | ||
| "repo_stats_update", | ||
| "mirror", | ||
| "pr_patch_checker", | ||
| "repo-archive", | ||
| } | ||
|
|
||
| var queueNames = []string{ | ||
| "issue_indexer", | ||
| "notification-service", | ||
| "mail", | ||
| "push_update", | ||
| "task", | ||
| } | ||
|
|
||
| var levelqueueTypes = []string{ | ||
| string(queue.PersistableChannelQueueType), | ||
| string(queue.PersistableChannelUniqueQueueType), | ||
| string(queue.LevelQueueType), | ||
| string(queue.LevelUniqueQueueType), | ||
| } | ||
|
|
||
| func checkUniqueQueues(ctx context.Context, logger log.Logger, autofix bool) error { | ||
| for _, name := range uniqueQueueNames { | ||
| q := setting.GetQueueSettings(name) | ||
| if q.Type == "" { | ||
| q.Type = string(queue.PersistableChannelQueueType) | ||
| } | ||
| found := false | ||
| for _, typ := range levelqueueTypes { | ||
| if typ == q.Type { | ||
| found = true | ||
| break | ||
| } | ||
| } | ||
| if !found { | ||
| logger.Info("Queue: %s\nType: %s\nNo LevelDB", q.Name, q.Type) | ||
zeripath marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| continue | ||
| } | ||
|
|
||
| connection := q.ConnectionString | ||
| if connection == "" { | ||
| connection = q.DataDir | ||
| } | ||
|
|
||
| db, err := nosql.GetManager().GetLevelDB(connection) | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to open DB connection %s: %v", q.Name, connection, err) | ||
zeripath marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| return err | ||
| } | ||
| defer db.Close() | ||
|
|
||
| prefix := q.Name | ||
|
|
||
| iQueue, err := levelqueue.NewQueue(db, []byte(prefix), false) | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to open Queue component: %v", q.Name, err) | ||
| return err | ||
| } | ||
|
|
||
| iSet, err := levelqueue.NewSet(db, []byte(prefix+"-unique"), false) | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to open Set component: %v", q.Name, err) | ||
| return err | ||
| } | ||
|
|
||
| qLen := iQueue.Len() | ||
| sMembers, err := iSet.Members() | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to get members of Set component: %v", q.Name, err) | ||
| return err | ||
| } | ||
| sLen := len(sMembers) | ||
|
|
||
| if int(qLen) == sLen { | ||
| if qLen == 0 { | ||
| logger.Info("Queue: %s\nType: %s\nLevelDB: %s", q.Name, q.Type, "empty") | ||
zeripath marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| } else { | ||
| logger.Info("Queue: %s\nType: %s\nLevelDB contains: %d entries", q.Name, q.Type, qLen) | ||
| } | ||
| continue | ||
| } | ||
| logger.Warn("Queue: %s\nType: %s\nContains different numbers of elements in Queue component %d to Set component %d", q.Name, q.Type, qLen, sLen) | ||
| if !autofix { | ||
| continue | ||
| } | ||
|
|
||
| // Empty out the old set members | ||
| for _, member := range sMembers { | ||
| _, err := iSet.Remove(member) | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to remove Set member %s: %v", q.Name, string(member), err) | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| // Now iterate across the queue | ||
| for i := int64(0); i < qLen; i++ { | ||
| // Pop from the left | ||
| qData, err := iQueue.LPop() | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to LPop out: %v", q.Name, err) | ||
| return err | ||
| } | ||
| // And add to the right | ||
| err = iQueue.RPush(qData) | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to RPush back: %v", q.Name, err) | ||
| return err | ||
| } | ||
| // And add back to the set | ||
| _, err = iSet.Add(qData) | ||
| if err != nil { | ||
| logger.Error("Queue: %s\nUnable to add back in to Set: %v", q.Name, err) | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func queueListDB(ctx context.Context, logger log.Logger, autofix bool) error { | ||
| connections := []string{} | ||
|
|
||
| for _, name := range append(uniqueQueueNames, queueNames...) { | ||
| q := setting.GetQueueSettings(name) | ||
| if q.Type == "" { | ||
| q.Type = string(queue.PersistableChannelQueueType) | ||
| } | ||
| found := false | ||
| for _, typ := range levelqueueTypes { | ||
| if typ == q.Type { | ||
| found = true | ||
| break | ||
| } | ||
| } | ||
| if !found { | ||
| continue | ||
| } | ||
| if q.ConnectionString != "" { | ||
| found := false | ||
| for _, connection := range connections { | ||
| if connection == q.ConnectionString { | ||
| found = true | ||
zeripath marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| } | ||
| } | ||
| if !found { | ||
| connections = append(connections, q.ConnectionString) | ||
| } | ||
| continue | ||
| } | ||
| found = false | ||
| for _, connection := range connections { | ||
| if connection == q.DataDir { | ||
| found = true | ||
zeripath marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| } | ||
| } | ||
| if !found { | ||
| connections = append(connections, q.DataDir) | ||
| } | ||
| } | ||
|
|
||
| for _, connection := range connections { | ||
| logger.Info("LevelDB: %s", connection) | ||
| db, err := nosql.GetManager().GetLevelDB(connection) | ||
| if err != nil { | ||
| logger.Error("Connection: %s Unable to open DB: %v", connection, err) | ||
| return err | ||
| } | ||
| defer db.Close() | ||
| iter := db.NewIterator(nil, nil) | ||
| for iter.Next() { | ||
| logger.Info("%s\n%s", log.NewColoredIDValue(string(iter.Key())), string(iter.Value())) | ||
| } | ||
| iter.Release() | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func init() { | ||
| Register(&Check{ | ||
| Title: "Check if there are corrupt level uniquequeues", | ||
| Name: "uniquequeues-corrupt", | ||
| IsDefault: false, | ||
| Run: checkUniqueQueues, | ||
| AbortIfFailed: false, | ||
| SkipDatabaseInitialization: false, | ||
| Priority: 1, | ||
| }) | ||
| Register(&Check{ | ||
| Title: "List all entries in leveldb", | ||
| Name: "queues-listdb", | ||
| IsDefault: false, | ||
| Run: queueListDB, | ||
| AbortIfFailed: false, | ||
| SkipDatabaseInitialization: false, | ||
| Priority: 1, | ||
| }) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.