Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(database/gredis): add Scan method for incremental key retrieval #3451

Merged
merged 20 commits into from
Apr 11, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions contrib/nosql/redis/redis_group_generic.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,33 @@ func (r GroupGeneric) Keys(ctx context.Context, pattern string) ([]string, error
return v.Strings(), err
}

// Scan executes a single iteration of the SCAN command, returning a subset of keys matching the pattern along with the next cursor position.
// This method provides more efficient and safer way to iterate over large datasets compared to KEYS command.
//
// Users are responsible for controlling the iteration by managing the cursor.
//
// The `count` optional parameter advises Redis on the number of keys to return. While it's not a strict limit, it guides the operation's granularity.
//
// https://redis.io/commands/scan/
func (r GroupGeneric) Scan(ctx context.Context, cursor uint64, option ...gredis.ScanOption) (uint64, []string, error) {
var usedOption interface{}
if len(option) > 0 {
usedOption = option[0].ToUsedOption()
}

v, err := r.Operation.Do(ctx, "Scan", mustMergeOptionToArgs(
[]interface{}{cursor}, usedOption,
)...)
if err != nil {
return 0, nil, err
}

nextCursor := gconv.Uint64(v.Slice()[0])
keys := gconv.SliceStr(v.Slice()[1])

return nextCursor, keys, nil
}

// FlushDB delete all the keys of the currently selected DB. This command never fails.
//
// https://redis.io/commands/flushdb/
Expand Down
83 changes: 83 additions & 0 deletions contrib/nosql/redis/redis_z_group_generic_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,89 @@ func Test_GroupGeneric_Keys(t *testing.T) {
})
}

func Test_GroupGeneric_Scan(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
defer redis.FlushDB(ctx)

err := redis.GroupString().MSet(ctx, map[string]interface{}{
"firstname": "Jack",
"lastname": "Stuntman",
"age": 35,
"nickname": "Jumper",
})
t.AssertNil(err)

performScan := func(cursor uint64, option ...gredis.ScanOption) ([]string, error) {
var allKeys = []string{}
for {
var nextCursor uint64
var keys []string
var err error

if option != nil {
nextCursor, keys, err = redis.Scan(ctx, cursor, option[0])
} else {
nextCursor, keys, err = redis.Scan(ctx, cursor)
}
if err != nil {
return nil, err
}

allKeys = append(allKeys, keys...)
if nextCursor == 0 {
break
}
cursor = nextCursor
}
return allKeys, nil
}

// Test scanning for keys with `*name*` pattern
optWithName := gredis.ScanOption{Match: "*name*", Count: 10}
keysWithName, err := performScan(0, optWithName)
t.AssertNil(err)
t.AssertGE(len(keysWithName), 3)
t.AssertIN(keysWithName, []string{"lastname", "firstname", "nickname"})

// Test scanning with a pattern that matches exactly one key
optWithAge := gredis.ScanOption{Match: "a??", Count: 10}
keysWithAge, err := performScan(0, optWithAge)
t.AssertNil(err)
t.AssertEQ(len(keysWithAge), 1)
t.AssertEQ(keysWithAge, []string{"age"})

// Test scanning for all keys
optWithAll := gredis.ScanOption{Match: "*", Count: 10}
all, err := performScan(0, optWithAll)
t.AssertNil(err)
t.AssertGE(len(all), 4)
t.AssertIN(all, []string{"lastname", "firstname", "age", "nickname"})

// Test empty pattern
optWithEmptyPattern := gredis.ScanOption{Match: ""}
emptyPatternKeys, err := performScan(0, optWithEmptyPattern)
t.AssertNil(err)
t.AssertEQ(len(emptyPatternKeys), 4)

// Test pattern with no matches
optWithNoMatch := gredis.ScanOption{Match: "xyz*", Count: 10}
noMatchKeys, err := performScan(0, optWithNoMatch)
t.AssertNil(err)
t.AssertEQ(len(noMatchKeys), 0)

// Test scanning for keys with invalid count value
optWithInvalidCount := gredis.ScanOption{Count: -1}
_, err = performScan(0, optWithInvalidCount)
t.AssertNQ(err, nil)

// Test scanning for all keys without options
allWithoutOpt, err := performScan(0)
t.AssertNil(err)
t.AssertGE(len(allWithoutOpt), 4)
t.AssertIN(all, []string{"lastname", "firstname", "age", "nickname"})
})
}

func Test_GroupGeneric_FlushDB(t *testing.T) {
gtest.C(t, func(t *gtest.T) {
defer redis.FlushDB(ctx)
Expand Down
32 changes: 32 additions & 0 deletions database/gredis/gredis_redis_group_generic.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ type IGroupGeneric interface {
RandomKey(ctx context.Context) (string, error)
DBSize(ctx context.Context) (int64, error)
Keys(ctx context.Context, pattern string) ([]string, error)
Scan(ctx context.Context, cursor uint64, option ...ScanOption) (uint64, []string, error)
FlushDB(ctx context.Context, option ...FlushOp) error
FlushAll(ctx context.Context, option ...FlushOp) error
Expire(ctx context.Context, key string, seconds int64, option ...ExpireOption) (int64, error)
Expand Down Expand Up @@ -60,3 +61,34 @@ type ExpireOption struct {
GT bool // GT -- Set expiry only when the new expiry is greater than current one
LT bool // LT -- Set expiry only when the new expiry is less than current one
}

// ScanOption provides options for function Scan.
type ScanOption struct {
Match string // Match -- Specifies a glob-style pattern for filtering keys.
Count int // Count -- Suggests the number of keys to return per scan.
Type string // Type -- Filters keys by their data type. Valid types are "string", "list", "set", "zset", "hash", and "stream".
}

// doScanOption is the internal representation of ScanOption.
type doScanOption struct {
Match *string
Count *int
Type *string
}

// ToUsedOption converts fields in ScanOption with zero values to nil. Only fields with values are retained.
func (scanOpt *ScanOption) ToUsedOption() doScanOption {
var usedOption doScanOption

if scanOpt.Match != "" {
usedOption.Match = &scanOpt.Match
}
if scanOpt.Count != 0 {
usedOption.Count = &scanOpt.Count
}
if scanOpt.Type != "" {
usedOption.Type = &scanOpt.Type
}

return usedOption
}
2 changes: 1 addition & 1 deletion util/gvalid/gvalid_register.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func RegisterRule(rule string, f RuleFunc) {
if customRuleFuncMap[rule] != nil {
intlog.PrintFunc(context.TODO(), func() string {
return fmt.Sprintf(
`rule "%s" is overwrotten by function "%s"`,
`rule "%s" is overwritten by function "%s"`,
rule, runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name(),
)
})
Expand Down
Loading