diff --git a/README.md b/README.md index 33ec9b9..c63a010 100644 --- a/README.md +++ b/README.md @@ -80,29 +80,29 @@ func ExampleClient() { | [FT.ADD](https://oss.redislabs.com/redisearch/Commands.html#ftadd) | [IndexOptions](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.IndexOptions) | | [FT.ADDHASH](https://oss.redislabs.com/redisearch/Commands.html#ftaddhash) | N/A | | [FT.ALTER](https://oss.redislabs.com/redisearch/Commands.html#ftalter) | N/A | -| [FT.ALIASADD](https://oss.redislabs.com/redisearch/Commands.html#ftaliasadd) | N/A | -| [FT.ALIASUPDATE](https://oss.redislabs.com/redisearch/Commands.html#ftaliasupdate) | N/A | -| [FT.ALIASDEL](https://oss.redislabs.com/redisearch/Commands.html#ftaliasdel) | N/A | +| [FT.ALIASADD](https://oss.redislabs.com/redisearch/Commands.html#ftaliasadd) | [AliasAdd](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AliasAdd) | +| [FT.ALIASUPDATE](https://oss.redislabs.com/redisearch/Commands.html#ftaliasupdate) | [AliasUpdate](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AliasUpdate) | +| [FT.ALIASDEL](https://oss.redislabs.com/redisearch/Commands.html#ftaliasdel) | [AliasDel](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.AliasDel) | | [FT.INFO](https://oss.redislabs.com/redisearch/Commands.html#ftinfo) | [Info](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Info) | | [FT.SEARCH](https://oss.redislabs.com/redisearch/Commands.html#ftsearch) | [Search](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Search) | | [FT.AGGREGATE](https://oss.redislabs.com/redisearch/Commands.html#ftaggregate) | [Aggregate](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Aggregate) | | [FT.CURSOR](https://oss.redislabs.com/redisearch/Aggregations.html#cursor_api) | [Aggregate](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Aggregate) + (*WithCursor option set to True) | | [FT.EXPLAIN](https://oss.redislabs.com/redisearch/Commands.html#ftexplain) | [Explain](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Explain) | | [FT.DEL](https://oss.redislabs.com/redisearch/Commands.html#ftdel) | [Delete](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Delete) | -| [FT.GET](https://oss.redislabs.com/redisearch/Commands.html#ftget) | N/A | -| [FT.MGET](https://oss.redislabs.com/redisearch/Commands.html#ftmget) | N/A | +| [FT.GET](https://oss.redislabs.com/redisearch/Commands.html#ftget) | [Get](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Get) | +| [FT.MGET](https://oss.redislabs.com/redisearch/Commands.html#ftmget) | [MultiGet](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Multi) | | [FT.DROP](https://oss.redislabs.com/redisearch/Commands.html#ftdrop) | [Drop](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.Drop) | | [FT.TAGVALS](https://oss.redislabs.com/redisearch/Commands.html#fttagvals) | N/A | -| [FT.SUGADD](https://oss.redislabs.com/redisearch/Commands.html#ftsugadd) | N/A | -| [FT.SUGGET](https://oss.redislabs.com/redisearch/Commands.html#ftsugget) | N/A | -| [FT.SUGDEL](https://oss.redislabs.com/redisearch/Commands.html#ftsugdel) | N/A | -| [FT.SUGLEN](https://oss.redislabs.com/redisearch/Commands.html#ftsuglen) | N/A | +| [FT.SUGADD](https://oss.redislabs.com/redisearch/Commands.html#ftsugadd) | [AddTerms](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.AddTerms) | +| [FT.SUGGET](https://oss.redislabs.com/redisearch/Commands.html#ftsugget) | [SuggestOpts](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.SuggestOpts) | +| [FT.SUGDEL](https://oss.redislabs.com/redisearch/Commands.html#ftsugdel) | [DeleteTerms](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.DeleteTerms) | +| [FT.SUGLEN](https://oss.redislabs.com/redisearch/Commands.html#ftsuglen) | [Autocompleter.Length](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Autocompleter.Length) | | [FT.SYNADD](https://oss.redislabs.com/redisearch/Commands.html#ftsynadd) | N/A | | [FT.SYNUPDATE](https://oss.redislabs.com/redisearch/Commands.html#ftsynupdate) | N/A | | [FT.SYNDUMP](https://oss.redislabs.com/redisearch/Commands.html#ftsyndump) | N/A | | [FT.SPELLCHECK](https://oss.redislabs.com/redisearch/Commands.html#ftspellcheck) | [SpellCheck](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.SpellCheck) | -| [FT.DICTADD](https://oss.redislabs.com/redisearch/Commands.html#ftdictadd) | N/A | -| [FT.DICTDEL](https://oss.redislabs.com/redisearch/Commands.html#ftdictdel) | N/A | -| [FT.DICTDUMP](https://oss.redislabs.com/redisearch/Commands.html#ftdictdump) | N/A | +| [FT.DICTADD](https://oss.redislabs.com/redisearch/Commands.html#ftdictadd) | [DictAdd](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.DictAdd) | +| [FT.DICTDEL](https://oss.redislabs.com/redisearch/Commands.html#ftdictdel) | [DictDel](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.DictDel) | +| [FT.DICTDUMP](https://oss.redislabs.com/redisearch/Commands.html#ftdictdump) | [DictDump](https://godoc.org/github.com/RediSearch/redisearch-go/redisearch#Client.DictDump) | | [FT.CONFIG](https://oss.redislabs.com/redisearch/Commands.html#ftconfig) | N/A | diff --git a/redisearch/aggregate_test.go b/redisearch/aggregate_test.go index b485ba8..823cb8b 100644 --- a/redisearch/aggregate_test.go +++ b/redisearch/aggregate_test.go @@ -79,7 +79,7 @@ func AddValues(c *Client) { } } -func init() { +func Init() { /* load test data */ c := createClient("docs-games-idx1") @@ -96,7 +96,7 @@ func init() { AddValues(c) } func TestAggregateGroupBy(t *testing.T) { - + Init() c := createClient("docs-games-idx1") q1 := NewAggregateQuery(). @@ -111,7 +111,7 @@ func TestAggregateGroupBy(t *testing.T) { } func TestAggregateMinMax(t *testing.T) { - + Init() c := createClient("docs-games-idx1") q1 := NewAggregateQuery().SetQuery(NewQuery("sony")). @@ -143,7 +143,7 @@ func TestAggregateMinMax(t *testing.T) { } func TestAggregateCountDistinct(t *testing.T) { - + Init() c := createClient("docs-games-idx1") q1 := NewAggregateQuery(). @@ -158,7 +158,7 @@ func TestAggregateCountDistinct(t *testing.T) { } func TestAggregateFilter(t *testing.T) { - + Init() c := createClient("docs-games-idx1") q1 := NewAggregateQuery(). @@ -246,7 +246,7 @@ func TestProjection_Serialize(t *testing.T) { Alias: tt.fields.Alias, } if got := p.Serialize(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Serialize() = %v, want %v", got, tt.want) + t.Errorf("serialize() = %v, want %v", got, tt.want) } }) } @@ -275,7 +275,7 @@ func TestCursor_Serialize(t *testing.T) { MaxIdle: tt.fields.MaxIdle, } if got := c.Serialize(); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Serialize() = %v, want %v", got, tt.want) + t.Errorf("serialize() = %v, want %v", got, tt.want) } }) } diff --git a/redisearch/autocomplete.go b/redisearch/autocomplete.go index f2ed18f..13e6df6 100644 --- a/redisearch/autocomplete.go +++ b/redisearch/autocomplete.go @@ -7,8 +7,13 @@ import ( // Autocompleter implements a redisearch auto-completer API type Autocompleter struct { - pool *redis.Pool name string + pool *redis.Pool +} + +// NewAutocompleter creates a new Autocompleter with the given pool and key name +func NewAutocompleterFromPool(pool *redis.Pool, name string) *Autocompleter { + return &Autocompleter{name: name, pool: pool} } // NewAutocompleter creates a new Autocompleter with the given host and key name @@ -23,7 +28,6 @@ func NewAutocompleter(addr, name string) *Autocompleter { // Delete deletes the Autocompleter key for this AC func (a *Autocompleter) Delete() error { - conn := a.pool.Get() defer conn.Close() @@ -33,7 +37,6 @@ func (a *Autocompleter) Delete() error { // AddTerms pushes new term suggestions to the index func (a *Autocompleter) AddTerms(terms ...Suggestion) error { - conn := a.pool.Get() defer conn.Close() @@ -62,49 +65,85 @@ func (a *Autocompleter) AddTerms(terms ...Suggestion) error { return nil } +// AddTerms pushes new term suggestions to the index +func (a *Autocompleter) DeleteTerms(terms ...Suggestion) error { + conn := a.pool.Get() + defer conn.Close() + + i := 0 + for _, term := range terms { + + args := redis.Args{a.name, term.Term} + if err := conn.Send("FT.SUGDEL", args...); err != nil { + return err + } + i++ + } + if err := conn.Flush(); err != nil { + return err + } + for i > 0 { + if _, err := conn.Receive(); err != nil { + return err + } + i-- + } + return nil +} + +// AddTerms pushes new term suggestions to the index +func (a *Autocompleter) Length() (len int64, err error) { + conn := a.pool.Get() + defer conn.Close() + len, err = redis.Int64(conn.Do("FT.SUGLEN", a.name)) + return +} + // Suggest gets completion suggestions from the Autocompleter dictionary to the given prefix. // If fuzzy is set, we also complete for prefixes that are in 1 Levenshten distance from the // given prefix // // Deprecated: Please use SuggestOpts() instead -func (a *Autocompleter) Suggest(prefix string, num int, fuzzy bool) ([]Suggestion, error) { +func (a *Autocompleter) Suggest(prefix string, num int, fuzzy bool) (ret []Suggestion, err error) { conn := a.pool.Get() defer conn.Close() - args := redis.Args{a.name, prefix, "MAX", num, "WITHSCORES"} - if fuzzy { - args = append(args, "FUZZY") - } + seropts := DefaultSuggestOptions + seropts.Num = num + seropts.Fuzzy = fuzzy + args, inc := a.Serialize(prefix, seropts) + vals, err := redis.Strings(conn.Do("FT.SUGGET", args...)) if err != nil { return nil, err } - ret := make([]Suggestion, 0, len(vals)/2) - for i := 0; i < len(vals); i += 2 { - - score, err := strconv.ParseFloat(vals[i+1], 64) - if err != nil { - continue - } - ret = append(ret, Suggestion{Term: vals[i], Score: score}) - - } - - return ret, nil + ret = ProcessSugGetVals(vals, inc, true, false) + return } // SuggestOpts gets completion suggestions from the Autocompleter dictionary to the given prefix. // SuggestOptions are passed allowing you specify if the returned values contain a payload, and scores. -// If SuggestOptions.Fuzzy is set, we also complete for prefixes that are in 1 Levenshten distance from the +// If SuggestOptions.Fuzzy is set, we also complete for prefixes that are in 1 Levenshtein distance from the // given prefix -func (a *Autocompleter) SuggestOpts(prefix string, opts SuggestOptions) ([]Suggestion, error) { +func (a *Autocompleter) SuggestOpts(prefix string, opts SuggestOptions) (ret []Suggestion, err error) { conn := a.pool.Get() defer conn.Close() - inc := 1 + args, inc := a.Serialize(prefix, opts) + vals, err := redis.Strings(conn.Do("FT.SUGGET", args...)) + if err != nil { + return nil, err + } + + ret = ProcessSugGetVals(vals, inc, opts.WithScores, opts.WithPayloads) + return +} + +func (a *Autocompleter) Serialize(prefix string, opts SuggestOptions) (redis.Args, int) { + inc := 1 args := redis.Args{a.name, prefix, "MAX", opts.Num} if opts.Fuzzy { args = append(args, "FUZZY") @@ -117,29 +156,26 @@ func (a *Autocompleter) SuggestOpts(prefix string, opts SuggestOptions) ([]Sugge args = append(args, "WITHPAYLOADS") inc++ } - vals, err := redis.Strings(conn.Do("FT.SUGGET", args...)) - if err != nil { - return nil, err - } + return args, inc +} - ret := make([]Suggestion, 0, len(vals)/inc) +func ProcessSugGetVals(vals []string, inc int, WithScores, WithPayloads bool) (ret []Suggestion) { + ret = make([]Suggestion, 0, len(vals)/inc) for i := 0; i < len(vals); i += inc { suggestion := Suggestion{Term: vals[i]} - if opts.WithScores { + if WithScores { score, err := strconv.ParseFloat(vals[i+1], 64) if err != nil { continue } suggestion.Score = score } - if opts.WithPayloads { + if WithPayloads { suggestion.Payload = vals[i+(inc-1)] } ret = append(ret, suggestion) } - - return ret, nil - + return } diff --git a/redisearch/autocomplete_test.go b/redisearch/autocomplete_test.go new file mode 100644 index 0000000..e2b4ba4 --- /dev/null +++ b/redisearch/autocomplete_test.go @@ -0,0 +1,101 @@ +package redisearch_test + +import ( + "fmt" + "github.com/RediSearch/redisearch-go/redisearch" + "github.com/gomodule/redigo/redis" + "github.com/stretchr/testify/assert" + "os" + "reflect" + "testing" +) + +func createAutocompleter(dictName string) *redisearch.Autocompleter { + value, exists := os.LookupEnv("REDISEARCH_TEST_HOST") + host := "localhost:6379" + if exists && value != "" { + host = value + } + return redisearch.NewAutocompleter(host, dictName) +} + +func TestAutocompleter_Serialize(t *testing.T) { + fuzzy := redisearch.DefaultSuggestOptions + fuzzy.Fuzzy = true + withscores := redisearch.DefaultSuggestOptions + withscores.WithScores = true + withpayloads := redisearch.DefaultSuggestOptions + withpayloads.WithPayloads = true + all := redisearch.DefaultSuggestOptions + all.Fuzzy = true + all.WithScores = true + all.WithPayloads = true + + type fields struct { + name string + } + type args struct { + prefix string + opts redisearch.SuggestOptions + } + tests := []struct { + name string + fields fields + args args + want redis.Args + want1 int + }{ + {"default options", fields{"key1"}, args{"ab", redisearch.DefaultSuggestOptions,}, redis.Args{"key1", "ab", "MAX", 5}, 1}, + {"FUZZY", fields{"key1"}, args{"ab", fuzzy,}, redis.Args{"key1", "ab", "MAX", 5, "FUZZY"}, 1}, + {"WITHSCORES", fields{"key1"}, args{"ab", withscores,}, redis.Args{"key1", "ab", "MAX", 5, "WITHSCORES"}, 2}, + {"WITHPAYLOADS", fields{"key1"}, args{"ab", withpayloads,}, redis.Args{"key1", "ab", "MAX", 5, "WITHPAYLOADS"}, 2}, + {"all", fields{"key1"}, args{"ab", all,}, redis.Args{"key1", "ab", "MAX", 5, "FUZZY", "WITHSCORES", "WITHPAYLOADS"}, 3}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := redisearch.NewAutocompleterFromPool(nil, tt.fields.name) + got, got1 := a.Serialize(tt.args.prefix, tt.args.opts) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("serialize() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("serialize() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func TestSuggest(t *testing.T) { + a := createAutocompleter("testing") + + // Add Terms to the Autocompleter + terms := make([]redisearch.Suggestion, 10) + for i := 0; i < 10; i++ { + terms[i] = redisearch.Suggestion{Term: fmt.Sprintf("foo %d", i), + Score: 1.0, Payload: fmt.Sprintf("bar %d", i)} + } + err := a.AddTerms(terms...) + assert.Nil(t, err) + suglen, err := a.Length() + assert.Nil(t, err) + assert.Equal(t, int64(10), suglen) + // Retrieve Terms From Autocompleter - Without Payloads / Scores + suggestions, err := a.SuggestOpts("f", redisearch.SuggestOptions{Num: 10}) + assert.Nil(t, err) + assert.Equal(t, 10, len(suggestions)) + for _, suggestion := range suggestions { + assert.Contains(t, suggestion.Term, "foo") + assert.Equal(t, suggestion.Payload, "") + assert.Zero(t, suggestion.Score) + } + + // Retrieve Terms From Autocompleter - With Payloads & Scores + suggestions, err = a.SuggestOpts("f", redisearch.SuggestOptions{Num: 10, WithScores: true, WithPayloads: true}) + assert.Nil(t, err) + assert.Equal(t, 10, len(suggestions)) + for _, suggestion := range suggestions { + assert.Contains(t, suggestion.Term, "foo") + assert.Contains(t, suggestion.Payload, "bar") + assert.NotZero(t, suggestion.Score) + } +} diff --git a/redisearch/client.go b/redisearch/client.go index f7a0465..5f774b2 100644 --- a/redisearch/client.go +++ b/redisearch/client.go @@ -2,38 +2,13 @@ package redisearch import ( "errors" - "fmt" - "github.com/gomodule/redigo/redis" "log" "reflect" "strconv" "strings" -) - -// Options are flags passed to the the abstract Index call, which receives them as interface{}, allowing -// for implementation specific options -type Options struct { - - // If set, we will not save the documents contents, just index them, for fetching ids only - NoSave bool - - NoFieldFlags bool - NoFrequencies bool - - NoOffsetVectors bool - - Stopwords []string -} - -// DefaultOptions represents the default options -var DefaultOptions = Options{ - NoSave: false, - NoFieldFlags: false, - NoFrequencies: false, - NoOffsetVectors: false, - Stopwords: nil, -} + "github.com/gomodule/redigo/redis" +) // Client is an interface to redisearch's redis commands type Client struct { @@ -64,235 +39,20 @@ func NewClient(addr, name string) *Client { } // CreateIndex configues the index and creates it on redis -func (i *Client) CreateIndex(s *Schema) error { +func (i *Client) CreateIndex(s *Schema) (err error) { args := redis.Args{i.name} // Set flags based on options - if s.Options.NoFieldFlags { - args = append(args, "NOFIELDS") - } - if s.Options.NoFrequencies { - args = append(args, "NOFREQS") - } - if s.Options.NoOffsetVectors { - args = append(args, "NOOFFSETS") - } - if s.Options.Stopwords != nil { - args = args.Add("STOPWORDS", len(s.Options.Stopwords)) - if len(s.Options.Stopwords) > 0 { - args = args.AddFlat(s.Options.Stopwords) - } - } - - args = append(args, "SCHEMA") - for _, f := range s.Fields { - - switch f.Type { - case TextField: - - args = append(args, f.Name, "TEXT") - if f.Options != nil { - opts, ok := f.Options.(TextFieldOptions) - if !ok { - return errors.New("Invalid text field options type") - } - - if opts.Weight != 0 && opts.Weight != 1 { - args = append(args, "WEIGHT", opts.Weight) - } - if opts.NoStem { - args = append(args, "NOSTEM") - } - - if opts.Sortable { - args = append(args, "SORTABLE") - } - - if opts.NoIndex { - args = append(args, "NOINDEX") - } - } - - case NumericField: - args = append(args, f.Name, "NUMERIC") - if f.Options != nil { - opts, ok := f.Options.(NumericFieldOptions) - if !ok { - return errors.New("Invalid numeric field options type") - } - - if opts.Sortable { - args = append(args, "SORTABLE") - } - if opts.NoIndex { - args = append(args, "NOINDEX") - } - } - case TagField: - args = append(args, f.Name, "TAG") - if f.Options != nil { - opts, ok := f.Options.(TagFieldOptions) - if !ok { - return errors.New("Invalid tag field options type") - } - if opts.Separator != 0 { - args = append(args, "SEPARATOR", fmt.Sprintf("%c", opts.Separator)) - - } - if opts.Sortable { - args = append(args, "SORTABLE") - } - if opts.NoIndex { - args = append(args, "NOINDEX") - } - } - default: - return fmt.Errorf("Unsupported field type %v", f.Type) - } - + args, err = SerializeSchema(s, args) + if err != nil { + return } conn := i.pool.Get() defer conn.Close() - _, err := conn.Do("FT.CREATE", args...) + _, err = conn.Do("FT.CREATE", args...) return err } -// IndexingOptions represent the options for indexing a single document -type IndexingOptions struct { - Language string - NoSave bool - Replace bool - Partial bool - ReplaceCondition string -} - -// DefaultIndexingOptions are the default options for document indexing -var DefaultIndexingOptions = IndexingOptions{ - Language: "", - NoSave: false, - Replace: false, - Partial: false, - ReplaceCondition: "", -} - -// IndexOptions indexes multiple documents on the index, with optional Options passed to options -func (i *Client) IndexOptions(opts IndexingOptions, docs ...Document) error { - - conn := i.pool.Get() - defer conn.Close() - - n := 0 - var merr MultiError - - for ii, doc := range docs { - args := make(redis.Args, 0, 6+len(doc.Properties)) - args = append(args, i.name, doc.Id, doc.Score) - // apply options - if opts.NoSave { - args = append(args, "NOSAVE") - } - if opts.Language != "" { - args = append(args, "LANGUAGE", opts.Language) - } - - if opts.Partial { - opts.Replace = true - } - - if opts.Replace { - args = append(args, "REPLACE") - if opts.Partial { - args = append(args, "PARTIAL") - } - if opts.ReplaceCondition != "" { - args = append(args, "IF", opts.ReplaceCondition) - } - } - - if doc.Payload != nil { - args = args.Add("PAYLOAD", doc.Payload) - } - - args = append(args, "FIELDS") - - for k, f := range doc.Properties { - args = append(args, k, f) - } - - if err := conn.Send("FT.ADD", args...); err != nil { - if merr == nil { - merr = NewMultiError(len(docs)) - } - merr[ii] = err - - return merr - } - n++ - } - - if err := conn.Flush(); err != nil { - return err - } - - for n > 0 { - if _, err := conn.Receive(); err != nil { - if merr == nil { - merr = NewMultiError(len(docs)) - } - merr[n-1] = err - } - n-- - } - - if merr == nil { - return nil - } - - return merr -} - -// convert the result from a redis query to a proper Document object -func loadDocument(arr []interface{}, idIdx, scoreIdx, payloadIdx, fieldsIdx int) (Document, error) { - - var score float64 = 1 - var err error - if scoreIdx > 0 { - if score, err = strconv.ParseFloat(string(arr[idIdx+scoreIdx].([]byte)), 64); err != nil { - return Document{}, fmt.Errorf("Could not parse score: %s", err) - } - } - - doc := NewDocument(string(arr[idIdx].([]byte)), float32(score)) - - if payloadIdx > 0 { - doc.Payload, _ = arr[idIdx+payloadIdx].([]byte) - } - - if fieldsIdx > 0 { - lst := arr[idIdx+fieldsIdx].([]interface{}) - for i := 0; i < len(lst); i += 2 { - var prop string - switch lst[i].(type) { - case []byte: - prop = string(lst[i].([]byte)) - default: - prop = lst[i].(string) - } - - var val interface{} - switch v := lst[i+1].(type) { - case []byte: - val = string(v) - default: - val = v - } - doc = doc.Set(prop, val) - } - } - - return doc, nil -} - // Index indexes a list of documents with the default options func (i *Client) Index(docs ...Document) error { return i.IndexOptions(DefaultIndexingOptions, docs...) @@ -349,6 +109,62 @@ func (i *Client) Search(q *Query) (docs []Document, total int, err error) { return } +// Adds an alias to an index. +func (i *Client) AliasAdd(name string) (err error) { + conn := i.pool.Get() + defer conn.Close() + args := redis.Args{name}.Add(i.name) + _, err = redis.String(conn.Do("FT.ALIASADD", args...)) + return +} + +// Deletes an alias to an index. +func (i *Client) AliasDel(name string) (err error) { + conn := i.pool.Get() + defer conn.Close() + args := redis.Args{name} + _, err = redis.String(conn.Do("FT.ALIASDEL", args...)) + return +} + +// Deletes an alias to an index. +func (i *Client) AliasUpdate(name string) (err error) { + conn := i.pool.Get() + defer conn.Close() + args := redis.Args{name}.Add(i.name) + _, err = redis.String(conn.Do("FT.ALIASUPDATE", args...)) + return +} + +// Adds terms to a dictionary. +func (i *Client) DictAdd(dictionaryName string, terms []string) (newTerms int, err error) { + conn := i.pool.Get() + defer conn.Close() + newTerms = 0 + args := redis.Args{dictionaryName}.AddFlat(terms) + newTerms, err = redis.Int(conn.Do("FT.DICTADD", args...)) + return +} + +// Deletes terms from a dictionary +func (i *Client) DictDel(dictionaryName string, terms []string) (deletedTerms int, err error) { + conn := i.pool.Get() + defer conn.Close() + deletedTerms = 0 + args := redis.Args{dictionaryName}.AddFlat(terms) + deletedTerms, err = redis.Int(conn.Do("FT.DICTDEL", args...)) + return +} + +// Dumps all terms in the given dictionary. +func (i *Client) DictDump(dictionaryName string) (terms []string, err error) { + conn := i.pool.Get() + defer conn.Close() + args := redis.Args{dictionaryName} + terms, err = redis.Strings(conn.Do("FT.DICTDUMP", args...)) + return +} + // SpellCheck performs spelling correction on a query, returning suggestions for misspelled terms, // the total number of results, or an error if something went wrong func (i *Client) SpellCheck(q *Query, s *SpellCheckOptions) (suggs []MisspelledTerm, total int, err error) { @@ -429,6 +245,68 @@ func (i *Client) Aggregate(q *AggregateQuery) (aggregateReply [][]string, total return } +// Get - Returns the full contents of a document +func (i *Client) Get(docId string) (doc *Document, err error) { + doc = nil + conn := i.pool.Get() + defer conn.Close() + var reply interface{} + args := redis.Args{i.name, docId} + reply, err = conn.Do("FT.GET", args...) + if reply != nil { + var array_reply []interface{} + array_reply, err = redis.Values(reply, err) + if err != nil { + return + } + if len(array_reply) > 0 { + document := NewDocument(docId, 1) + document.loadFields(array_reply) + doc = &document + } + } + return +} + +// MultiGet - Returns the full contents of multiple documents. +// Returns an array with exactly the same number of elements as the number of keys sent to the command. +// Each element in it is either an Document or nil if it was not found. +func (i *Client) MultiGet(documentIds []string) (docs []*Document, err error) { + docs = make([]*Document, len(documentIds)) + conn := i.pool.Get() + defer conn.Close() + var reply interface{} + args := redis.Args{i.name}.AddFlat(documentIds) + reply, err = conn.Do("FT.MGET", args...) + if reply != nil { + var array_reply []interface{} + array_reply, err = redis.Values(reply, err) + if err != nil { + return + } + for i := 0; i < len(array_reply); i++ { + + if array_reply[i] != nil { + var innerArray []interface{} + innerArray, err = redis.Values(array_reply[i], nil) + if err != nil { + return + } + if len(array_reply) > 0 { + document := NewDocument(documentIds[i], 1) + document.loadFields(innerArray) + docs[i] = &document + } + } else { + docs[i] = nil + } + + } + + } + return +} + // Explain Return a textual string explaining the query func (i *Client) Explain(q *Query) (string, error) { conn := i.pool.Get() @@ -464,24 +342,6 @@ func (i *Client) Delete(docId string, deleteDocument bool) (err error) { return } -// IndexInfo - Structure showing information about an existing index -type IndexInfo struct { - Schema Schema - Name string `redis:"index_name"` - DocCount uint64 `redis:"num_docs"` - RecordCount uint64 `redis:"num_records"` - TermCount uint64 `redis:"num_terms"` - MaxDocID uint64 `redis:"max_doc_id"` - InvertedIndexSizeMB float64 `redis:"inverted_sz_mb"` - OffsetVectorSizeMB float64 `redis:"offset_vector_sz_mb"` - DocTableSizeMB float64 `redis:"doc_table_size_mb"` - KeyTableSizeMB float64 `redis:"key_table_size_mb"` - RecordsPerDocAvg float64 `redis:"records_per_doc_avg"` - BytesPerRecordAvg float64 `redis:"bytes_per_record_avg"` - OffsetsPerTermAvg float64 `redis:"offsets_per_term_avg"` - OffsetBitsPerTermAvg float64 `redis:"offset_bits_per_record_avg"` -} - func (info *IndexInfo) setTarget(key string, value interface{}) error { v := reflect.ValueOf(info).Elem() for i := 0; i < v.NumField(); i++ { diff --git a/redisearch/client_test.go b/redisearch/client_test.go index 0bee919..384e15d 100644 --- a/redisearch/client_test.go +++ b/redisearch/client_test.go @@ -1,22 +1,14 @@ -package redisearch_test +package redisearch import ( "fmt" - "github.com/RediSearch/redisearch-go/redisearch" + "github.com/stretchr/testify/assert" "log" "os" + "reflect" "testing" ) -func createBenchClient(indexName string) *redisearch.Client { - value, exists := os.LookupEnv("REDISEARCH_TEST_HOST") - host := "localhost:6379" - if exists && value != "" { - host = value - } - return redisearch.NewClient(host, indexName) -} - func init() { /* load test data */ value, exists := os.LookupEnv("REDISEARCH_RDB_LOADED") @@ -25,34 +17,34 @@ func init() { requiresDatagen = false } if requiresDatagen { - c := createBenchClient("bench.ft.aggregate") + c := createClient("bench.ft.aggregate") - sc := redisearch.NewSchema(redisearch.DefaultOptions). - AddField(redisearch.NewTextField("foo")) + sc := NewSchema(DefaultOptions). + AddField(NewTextField("foo")) c.Drop() if err := c.CreateIndex(sc); err != nil { log.Fatal(err) } ndocs := 10000 - docs := make([]redisearch.Document, ndocs) + docs := make([]Document, ndocs) for i := 0; i < ndocs; i++ { - docs[i] = redisearch.NewDocument(fmt.Sprintf("doc%d", i), 1).Set("foo", "hello world") + docs[i] = NewDocument(fmt.Sprintf("doc%d", i), 1).Set("foo", "hello world") } - if err := c.IndexOptions(redisearch.DefaultIndexingOptions, docs...); err != nil { + if err := c.IndexOptions(DefaultIndexingOptions, docs...); err != nil { log.Fatal(err) } } } -func benchmarkAggregate(c *redisearch.Client, q *redisearch.AggregateQuery, b *testing.B) { +func benchmarkAggregate(c *Client, q *AggregateQuery, b *testing.B) { for n := 0; n < b.N; n++ { c.Aggregate(q) } } -func benchmarkAggregateCursor(c *redisearch.Client, q *redisearch.AggregateQuery, b *testing.B) { +func benchmarkAggregateCursor(c *Client, q *AggregateQuery, b *testing.B) { for n := 0; n < b.N; n++ { c.Aggregate(q) for q.CursorHasResults() { @@ -62,18 +54,425 @@ func benchmarkAggregateCursor(c *redisearch.Client, q *redisearch.AggregateQuery } func BenchmarkAgg_1(b *testing.B) { - c := createBenchClient("bench.ft.aggregate") - q := redisearch.NewAggregateQuery(). - SetQuery(redisearch.NewQuery("*")) + c := createClient("bench.ft.aggregate") + q := NewAggregateQuery(). + SetQuery(NewQuery("*")) b.ResetTimer() benchmarkAggregate(c, q, b) } func BenchmarkAggCursor_1(b *testing.B) { - c := createBenchClient("bench.ft.aggregate") - q := redisearch.NewAggregateQuery(). - SetQuery(redisearch.NewQuery("*")). - SetCursor(redisearch.NewCursor()) + c := createClient("bench.ft.aggregate") + q := NewAggregateQuery(). + SetQuery(NewQuery("*")). + SetCursor(NewCursor()) b.ResetTimer() benchmarkAggregateCursor(c, q, b) } + +func TestClient_Get(t *testing.T) { + + c := createClient("test-get") + c.Drop() + + sc := NewSchema(DefaultOptions). + AddField(NewTextField("foo")) + + if err := c.CreateIndex(sc); err != nil { + t.Fatal(err) + } + + docs := make([]Document, 10) + docPointers := make([]*Document, 10) + docIds := make([]string, 10) + for i := 0; i < 10; i++ { + docIds[i] = fmt.Sprintf("doc-get-%d", i) + docs[i] = NewDocument(docIds[i], 1).Set("foo", "Hello world") + docPointers[i] = &docs[i] + } + err := c.Index(docs...) + assert.Nil(t, err) + + type fields struct { + pool ConnPool + name string + } + type args struct { + docId string + } + tests := []struct { + name string + fields fields + args args + wantDoc *Document + wantErr bool + }{ + {"dont-exist", fields{pool: c.pool, name: c.name}, args{"dont-exist"}, nil, false}, + {"doc-get-1", fields{pool: c.pool, name: c.name}, args{"doc-get-1"}, &docs[1], false}, + {"doc-get-2", fields{pool: c.pool, name: c.name}, args{"doc-get-2"}, &docs[2], false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + gotDoc, err := i.Get(tt.args.docId) + if (err != nil) != tt.wantErr { + t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotDoc != nil { + if !reflect.DeepEqual(gotDoc, tt.wantDoc) { + t.Errorf("Get() gotDoc = %v, want %v", gotDoc, tt.wantDoc) + } + } + + }) + } +} + +func TestClient_MultiGet(t *testing.T) { + + c := createClient("test-get") + c.Drop() + + sc := NewSchema(DefaultOptions). + AddField(NewTextField("foo")) + + if err := c.CreateIndex(sc); err != nil { + t.Fatal(err) + } + + docs := make([]Document, 10) + docPointers := make([]*Document, 10) + docIds := make([]string, 10) + for i := 0; i < 10; i++ { + docIds[i] = fmt.Sprintf("doc-get-%d", i) + docs[i] = NewDocument(docIds[i], 1).Set("foo", "Hello world") + docPointers[i] = &docs[i] + } + err := c.Index(docs...) + assert.Nil(t, err) + + type fields struct { + pool ConnPool + name string + } + type args struct { + documentIds []string + } + tests := []struct { + name string + fields fields + args args + wantDocs []*Document + wantErr bool + }{ + {"dont-exist", fields{pool: c.pool, name: c.name}, args{[]string{"dont-exist"}}, []*Document{nil}, false}, + {"doc2", fields{pool: c.pool, name: c.name}, args{[]string{"doc-get-3"}}, []*Document{&docs[3]}, false}, + {"doc1", fields{pool: c.pool, name: c.name}, args{[]string{"doc-get-1"}}, []*Document{&docs[1]}, false}, + {"doc1-and-other-dont-exist", fields{pool: c.pool, name: c.name}, args{[]string{"doc-get-1", "dontexist"}}, []*Document{&docs[1], nil}, false}, + {"dont-exist-and-doc1", fields{pool: c.pool, name: c.name}, args{[]string{"dontexist", "doc-get-1"}}, []*Document{nil, &docs[1]}, false}, + {"alldocs", fields{pool: c.pool, name: c.name}, args{docIds}, docPointers, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + gotDocs, err := i.MultiGet(tt.args.documentIds) + if (err != nil) != tt.wantErr { + t.Errorf("MultiGet() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotDocs, tt.wantDocs) { + t.Errorf("MultiGet() gotDocs = %v, want %v", gotDocs, tt.wantDocs) + } + }) + } +} + +func TestClient_DictAdd(t *testing.T) { + c := createClient("test-get") + _, err := c.pool.Get().Do("FLUSHALL") + assert.Nil(t, err) + + type fields struct { + pool ConnPool + name string + } + type args struct { + dictionaryName string + terms []string + } + tests := []struct { + name string + fields fields + args args + wantNewTerms int + wantErr bool + }{ + {"empty-error", fields{pool: c.pool, name: c.name}, args{"dict1", []string{},}, 0, true}, + {"1-term", fields{pool: c.pool, name: c.name}, args{"dict1", []string{"term1"},}, 1, false}, + {"2nd-time-term", fields{pool: c.pool, name: c.name}, args{"dict1", []string{"term1"},}, 0, false}, + {"multi-term", fields{pool: c.pool, name: c.name}, args{"dict1", []string{"t1", "t2", "t3", "t4", "t5"},}, 5, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + gotNewTerms, err := i.DictAdd(tt.args.dictionaryName, tt.args.terms) + if (err != nil) != tt.wantErr { + t.Errorf("DictAdd() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotNewTerms != tt.wantNewTerms { + t.Errorf("DictAdd() gotNewTerms = %v, want %v", gotNewTerms, tt.wantNewTerms) + } + }) + } +} + +func TestClient_DictDel(t *testing.T) { + + c := createClient("test-get") + _, err := c.pool.Get().Do("FLUSHALL") + assert.Nil(t, err) + + terms := make([]string, 10) + for i := 0; i < 10; i++ { + terms[i] = fmt.Sprintf("term%d", i) + } + + c.DictAdd("dict1", terms) + + type fields struct { + pool ConnPool + name string + } + type args struct { + dictionaryName string + terms []string + } + tests := []struct { + name string + fields fields + args args + wantDeletedTerms int + wantErr bool + }{ + {"empty-error", fields{pool: c.pool, name: c.name}, args{"dict1", []string{},}, 0, true}, + {"1-term", fields{pool: c.pool, name: c.name}, args{"dict1", []string{"term1"},}, 1, false}, + {"2nd-time-term", fields{pool: c.pool, name: c.name}, args{"dict1", []string{"term1"},}, 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + gotDeletedTerms, err := i.DictDel(tt.args.dictionaryName, tt.args.terms) + if (err != nil) != tt.wantErr { + t.Errorf("DictDel() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotDeletedTerms != tt.wantDeletedTerms { + t.Errorf("DictDel() gotDeletedTerms = %v, want %v", gotDeletedTerms, tt.wantDeletedTerms) + } + }) + } +} + +func TestClient_DictDump(t *testing.T) { + c := createClient("test-get") + _, err := c.pool.Get().Do("FLUSHALL") + assert.Nil(t, err) + + terms1 := make([]string, 10) + for i := 0; i < 10; i++ { + terms1[i] = fmt.Sprintf("term%d", i) + } + c.DictAdd("dict1", terms1) + + type fields struct { + pool ConnPool + name string + } + type args struct { + dictionaryName string + } + tests := []struct { + name string + fields fields + args args + wantTerms []string + wantErr bool + }{ + {"empty-error", fields{pool: c.pool, name: c.name}, args{"dontexist"}, []string{}, true}, + {"dict1", fields{pool: c.pool, name: c.name}, args{"dict1"}, terms1, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + gotTerms, err := i.DictDump(tt.args.dictionaryName) + if (err != nil) != tt.wantErr { + t.Errorf("DictDump() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotTerms, tt.wantTerms) && !tt.wantErr { + t.Errorf("DictDump() gotTerms = %v, want %v", gotTerms, tt.wantTerms) + } + }) + } +} + +func TestClient_AliasAdd(t *testing.T) { + c := createClient("testalias") + c1_unexistingIndex := createClient("testaliasadd-dontexist") + + sc := NewSchema(DefaultOptions). + AddField(NewTextField("foo")). + AddField(NewTextField("bar")) + c.Drop() + assert.Nil(t, c.CreateIndex(sc)) + + docs := make([]Document, 100) + for i := 0; i < 100; i++ { + docs[i] = NewDocument(fmt.Sprintf("doc--alias-add-%d", i), 1).Set("foo", "hello world").Set("bar", "hello world foo bar baz") + } + err := c.Index(docs...) + + assert.Nil(t, err) + + type fields struct { + pool ConnPool + name string + } + type args struct { + name string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + {"unexisting-index", fields{pool: c1_unexistingIndex.pool, name: c1_unexistingIndex.name}, args{"dont-exist"}, true}, + {"alias-ok", fields{pool: c.pool, name: c.name}, args{"testalias"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + if err := i.AliasAdd(tt.args.name); (err != nil) != tt.wantErr { + t.Errorf("AliasAdd() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestClient_AliasDel(t *testing.T) { + c := createClient("testaliasdel") + c1_unexistingIndex := createClient("testaliasdel-dontexist") + + sc := NewSchema(DefaultOptions). + AddField(NewTextField("foo")). + AddField(NewTextField("bar")) + c.Drop() + err := c.CreateIndex(sc) + assert.Nil(t, err) + + docs := make([]Document, 100) + for i := 0; i < 100; i++ { + docs[i] = NewDocument(fmt.Sprintf("doc-alias-del-%d", i), 1).Set("foo", "hello world").Set("bar", "hello world foo bar baz") + } + err = c.Index(docs...) + + assert.Nil(t, err) + err = c.AliasAdd("aliasdel1") + assert.Nil(t, err) + + type fields struct { + pool ConnPool + name string + } + type args struct { + name string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + {"unexisting-index", fields{pool: c1_unexistingIndex.pool, name: c1_unexistingIndex.name}, args{"dont-exist"}, true}, + {"aliasdel1", fields{pool: c.pool, name: c.name}, args{"aliasdel1"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + if err := i.AliasDel(tt.args.name); (err != nil) != tt.wantErr { + t.Errorf("AliasDel() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestClient_AliasUpdate(t *testing.T) { + c := createClient("testaliasupdateindex") + + sc := NewSchema(DefaultOptions). + AddField(NewTextField("foo")). + AddField(NewTextField("bar")) + c.Drop() + err := c.CreateIndex(sc) + assert.Nil(t, err) + + docs := make([]Document, 100) + for i := 0; i < 100; i++ { + docs[i] = NewDocument(fmt.Sprintf("doc-alias-del-%d", i), 1).Set("foo", "hello world").Set("bar", "hello world foo bar baz") + } + err = c.Index(docs...) + + assert.Nil(t, err) + err = c.AliasAdd("aliasupdate") + assert.Nil(t, err) + type fields struct { + pool ConnPool + name string + } + type args struct { + name string + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + {"aliasupdate", fields{pool: c.pool, name: c.name}, args{"aliasupdate"}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := &Client{ + pool: tt.fields.pool, + name: tt.fields.name, + } + if err := i.AliasUpdate(tt.args.name); (err != nil) != tt.wantErr { + t.Errorf("AliasUpdate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/redisearch/document.go b/redisearch/document.go index bb02ed2..87ddee0 100644 --- a/redisearch/document.go +++ b/redisearch/document.go @@ -1,7 +1,9 @@ package redisearch import ( + "fmt" "sort" + "strconv" "strings" ) @@ -18,6 +20,36 @@ type Document struct { Properties map[string]interface{} } + +// IndexingOptions represent the options for indexing a single document +type IndexingOptions struct { + + // If set, we use a stemmer for the supplied language during indexing. If set to "", we Default to English. + Language string + + // If set to true, we will not save the actual document in the database and only index it. + NoSave bool + + // If set, we will do an UPSERT style insertion - and delete an older version of the document if it exists. + Replace bool + + // (only applicable with Replace): If set, you do not have to specify all fields for reindexing. + Partial bool + + // Applicable only in conjunction with Replace and optionally Partial + // Update the document only if a boolean expression applies to the document before the update + ReplaceCondition string +} + +// DefaultIndexingOptions are the default options for document indexing +var DefaultIndexingOptions = IndexingOptions{ + Language: "", + NoSave: false, + Replace: false, + Partial: false, + ReplaceCondition: "", +} + // NewDocument creates a document with the specific id and score func NewDocument(id string, score float32) Document { return Document{ @@ -54,6 +86,55 @@ func EscapeTextFileString(value string) (string) { return value } +// convert the result from a redis query to a proper Document object +func loadDocument(arr []interface{}, idIdx, scoreIdx, payloadIdx, fieldsIdx int) (Document, error) { + + var score float64 = 1 + var err error + if scoreIdx > 0 { + if score, err = strconv.ParseFloat(string(arr[idIdx+scoreIdx].([]byte)), 64); err != nil { + return Document{}, fmt.Errorf("Could not parse score: %s", err) + } + } + + doc := NewDocument(string(arr[idIdx].([]byte)), float32(score)) + + if payloadIdx > 0 { + doc.Payload, _ = arr[idIdx+payloadIdx].([]byte) + } + + if fieldsIdx > 0 { + lst := arr[idIdx+fieldsIdx].([]interface{}) + doc.loadFields(lst) + } + + return doc, nil +} + + +// SetPayload Sets the document payload +func (d *Document) loadFields(lst []interface{}) *Document{ + for i := 0; i < len(lst); i += 2 { + var prop string + switch lst[i].(type) { + case []byte: + prop = string(lst[i].([]byte)) + default: + prop = lst[i].(string) + } + + var val interface{} + switch v := lst[i+1].(type) { + case []byte: + val = string(v) + default: + val = v + } + *d = d.Set(prop,val) + } + return d +} + // DocumentList is used to sort documents by descending score type DocumentList []Document diff --git a/redisearch/pool.go b/redisearch/pool.go index 7847477..e2cfb66 100644 --- a/redisearch/pool.go +++ b/redisearch/pool.go @@ -53,12 +53,11 @@ func (p *MultiHostPool) Get() redis.Conn { // TODO: Add timeouts. and 2 separate pools for indexing and querying, with different timeouts return redis.Dial("tcp", host) }, maxConns) - pool.TestOnBorrow = func(c redis.Conn, t time.Time) error { - if time.Since(t).Seconds() > 1 { - _, err := c.Do("PING") - return err + pool.TestOnBorrow = func(c redis.Conn, t time.Time) (err error) { + if time.Since(t) > time.Second { + _, err = c.Do("PING") } - return nil + return err } p.pools[host] = pool diff --git a/redisearch/pool_test.go b/redisearch/pool_test.go new file mode 100644 index 0000000..12fb46a --- /dev/null +++ b/redisearch/pool_test.go @@ -0,0 +1,33 @@ +package redisearch_test + +import ( + "github.com/RediSearch/redisearch-go/redisearch" + "os" + "testing" +) + +func TestNewMultiHostPool(t *testing.T) { + value, exists := os.LookupEnv("REDISEARCH_TEST_HOST") + host := "localhost:6379" + if exists && value != "" { + host = value + } + type args struct { + hosts []string + } + tests := []struct { + name string + args args + }{ + {"multihost same address", args{[]string{host,},},}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := redisearch.NewMultiHostPool(tt.args.hosts) + conn := got.Get() + if conn == nil { + t.Errorf("NewMultiHostPool() = got nil connection") + } + }) + } +} diff --git a/redisearch/query.go b/redisearch/query.go index f583de5..c320a54 100644 --- a/redisearch/query.go +++ b/redisearch/query.go @@ -1,6 +1,11 @@ package redisearch -import "github.com/gomodule/redigo/redis" +import ( + "errors" + "fmt" + + "github.com/gomodule/redigo/redis" +) // Flag is a type for query flags type Flag uint64 @@ -294,3 +299,189 @@ func (q *Query) SummarizeOptions(opts SummaryOptions) *Query { q.SummarizeOpts = &opts return q } + +func SerializeSchema(s *Schema, args redis.Args) (redis.Args, error) { + if s.Options.NoFieldFlags { + args = append(args, "NOFIELDS") + } + if s.Options.NoFrequencies { + args = append(args, "NOFREQS") + } + if s.Options.NoOffsetVectors { + args = append(args, "NOOFFSETS") + } + if s.Options.Stopwords != nil { + args = args.Add("STOPWORDS", len(s.Options.Stopwords)) + if len(s.Options.Stopwords) > 0 { + args = args.AddFlat(s.Options.Stopwords) + } + } + + args = append(args, "SCHEMA") + for _, f := range s.Fields { + + switch f.Type { + case TextField: + + args = append(args, f.Name, "TEXT") + if f.Options != nil { + opts, ok := f.Options.(TextFieldOptions) + if !ok { + return nil, errors.New("Invalid text field options type") + } + + if opts.Weight != 0 && opts.Weight != 1 { + args = append(args, "WEIGHT", opts.Weight) + } + if opts.NoStem { + args = append(args, "NOSTEM") + } + + if opts.Sortable { + args = append(args, "SORTABLE") + } + + if opts.NoIndex { + args = append(args, "NOINDEX") + } + } + + case NumericField: + args = append(args, f.Name, "NUMERIC") + if f.Options != nil { + opts, ok := f.Options.(NumericFieldOptions) + if !ok { + return nil, errors.New("Invalid numeric field options type") + } + + if opts.Sortable { + args = append(args, "SORTABLE") + } + if opts.NoIndex { + args = append(args, "NOINDEX") + } + } + case TagField: + args = append(args, f.Name, "TAG") + if f.Options != nil { + opts, ok := f.Options.(TagFieldOptions) + if !ok { + return nil, errors.New("Invalid tag field options type") + } + if opts.Separator != 0 { + args = append(args, "SEPARATOR", fmt.Sprintf("%c", opts.Separator)) + + } + if opts.Sortable { + args = append(args, "SORTABLE") + } + if opts.NoIndex { + args = append(args, "NOINDEX") + } + } + default: + return nil, fmt.Errorf("Unsupported field type %v", f.Type) + } + + } + return args, nil +} + +// IndexOptions indexes multiple documents on the index, with optional Options passed to options +func (i *Client) IndexOptions(opts IndexingOptions, docs ...Document) error { + + conn := i.pool.Get() + defer conn.Close() + + n := 0 + var merr MultiError + + for ii, doc := range docs { + args := make(redis.Args, 0, 6+len(doc.Properties)) + args = append(args, i.name, doc.Id, doc.Score) + args = SerializeIndexingOptions(opts, args) + + if doc.Payload != nil { + args = args.Add("PAYLOAD", doc.Payload) + } + + args = append(args, "FIELDS") + + for k, f := range doc.Properties { + args = append(args, k, f) + } + + if err := conn.Send("FT.ADD", args...); err != nil { + if merr == nil { + merr = NewMultiError(len(docs)) + } + merr[ii] = err + + return merr + } + n++ + } + + if err := conn.Flush(); err != nil { + return err + } + + for n > 0 { + if _, err := conn.Receive(); err != nil { + if merr == nil { + merr = NewMultiError(len(docs)) + } + merr[n-1] = err + } + n-- + } + + if merr == nil { + return nil + } + + return merr +} + +func SerializeIndexingOptions(opts IndexingOptions, args redis.Args) redis.Args { + // apply options + if opts.NoSave { + args = append(args, "NOSAVE") + } + if opts.Language != "" { + args = append(args, "LANGUAGE", opts.Language) + } + + if opts.Partial { + opts.Replace = true + } + + if opts.Replace { + args = append(args, "REPLACE") + if opts.Partial { + args = append(args, "PARTIAL") + } + if opts.ReplaceCondition != "" { + args = append(args, "IF", opts.ReplaceCondition) + } + } + return args +} + +// IndexInfo - Structure showing information about an existing index +type IndexInfo struct { + Schema Schema + Name string `redis:"index_name"` + DocCount uint64 `redis:"num_docs"` + RecordCount uint64 `redis:"num_records"` + TermCount uint64 `redis:"num_terms"` + MaxDocID uint64 `redis:"max_doc_id"` + InvertedIndexSizeMB float64 `redis:"inverted_sz_mb"` + OffsetVectorSizeMB float64 `redis:"offset_vector_sz_mb"` + DocTableSizeMB float64 `redis:"doc_table_size_mb"` + KeyTableSizeMB float64 `redis:"key_table_size_mb"` + RecordsPerDocAvg float64 `redis:"records_per_doc_avg"` + BytesPerRecordAvg float64 `redis:"bytes_per_record_avg"` + OffsetsPerTermAvg float64 `redis:"offsets_per_term_avg"` + OffsetBitsPerTermAvg float64 `redis:"offset_bits_per_record_avg"` +} diff --git a/redisearch/query_test.go b/redisearch/query_test.go index 2d8cc32..bdfda37 100644 --- a/redisearch/query_test.go +++ b/redisearch/query_test.go @@ -1,9 +1,10 @@ package redisearch import ( - "github.com/gomodule/redigo/redis" "reflect" "testing" + + "github.com/gomodule/redigo/redis" ) func TestPaging_serialize(t *testing.T) { @@ -35,6 +36,32 @@ func TestPaging_serialize(t *testing.T) { } } +func Test_serializeIndexingOptions(t *testing.T) { + type args struct { + opts IndexingOptions + args redis.Args + } + tests := []struct { + name string + args args + want redis.Args + }{ + {"default with args", args{DefaultIndexingOptions, redis.Args{"idx1", "doc1", 1.0}}, redis.Args{"idx1", "doc1", 1.0}}, + {"default", args{DefaultIndexingOptions, redis.Args{}}, redis.Args{}}, + {"replace full doc", args{IndexingOptions{Replace: true}, redis.Args{}}, redis.Args{"REPLACE"}}, + {"replace partial", args{IndexingOptions{Replace: true, Partial: true}, redis.Args{}}, redis.Args{"REPLACE", "PARTIAL"}}, + {"replace if", args{IndexingOptions{Replace: true, ReplaceCondition: "@timestamp < 23323234234"}, redis.Args{}}, redis.Args{"REPLACE", "IF", "@timestamp < 23323234234"}}, + {"replace partial if", args{IndexingOptions{Replace: true, Partial: true, ReplaceCondition: "@timestamp < 23323234234"}, redis.Args{}}, redis.Args{"REPLACE", "PARTIAL", "IF", "@timestamp < 23323234234"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SerializeIndexingOptions(tt.args.opts, tt.args.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("serializeIndexingOptions() = %v, want %v", got, tt.want) + } + }) + } +} + func TestQuery_serialize(t *testing.T) { var raw = "test_query" type fields struct { diff --git a/redisearch/redisearch_test.go b/redisearch/redisearch_test.go index d51bf23..199c38d 100644 --- a/redisearch/redisearch_test.go +++ b/redisearch/redisearch_test.go @@ -20,14 +20,6 @@ func createClient(indexName string) *redisearch.Client { return redisearch.NewClient(host, indexName) } -func createAutocompleter(indexName string) *redisearch.Autocompleter { - value, exists := os.LookupEnv("REDISEARCH_TEST_HOST") - host := "localhost:6379" - if exists && value != "" { - host = value - } - return redisearch.NewAutocompleter(host, indexName) -} func TestClient(t *testing.T) { @@ -218,7 +210,7 @@ func TestHighlight(t *testing.T) { c.Drop() } -func TestSammurize(t *testing.T) { +func TestSummarize(t *testing.T) { c := createClient("testung") sc := redisearch.NewSchema(redisearch.DefaultOptions). @@ -311,40 +303,6 @@ func TestTags(t *testing.T) { } -func TestSuggest(t *testing.T) { - - a := createAutocompleter("testing") - - // Add Terms to the Autocompleter - terms := make([]redisearch.Suggestion, 10) - for i := 0; i < 10; i++ { - terms[i] = redisearch.Suggestion{Term: fmt.Sprintf("foo %d", i), - Score: 1.0, Payload: fmt.Sprintf("bar %d", i)} - } - err := a.AddTerms(terms...) - assert.Nil(t, err) - - // Retrieve Terms From Autocompleter - Without Payloads / Scores - suggestions, err := a.SuggestOpts("f", redisearch.SuggestOptions{Num: 10}) - assert.Nil(t, err) - assert.Equal(t, 10, len(suggestions)) - for _, suggestion := range suggestions { - assert.Contains(t, suggestion.Term, "foo") - assert.Equal(t, suggestion.Payload, "") - assert.Zero(t, suggestion.Score) - } - - // Retrieve Terms From Autocompleter - With Payloads & Scores - suggestions, err = a.SuggestOpts("f", redisearch.SuggestOptions{Num: 10, WithScores: true, WithPayloads: true}) - assert.Nil(t, err) - assert.Equal(t, 10, len(suggestions)) - for _, suggestion := range suggestions { - assert.Contains(t, suggestion.Term, "foo") - assert.Contains(t, suggestion.Payload, "bar") - assert.NotZero(t, suggestion.Score) - } -} - func TestDelete(t *testing.T) { c := createClient("testung") diff --git a/redisearch/schema.go b/redisearch/schema.go index 7bad285..fc81f93 100644 --- a/redisearch/schema.go +++ b/redisearch/schema.go @@ -3,6 +3,44 @@ package redisearch // FieldType is an enumeration of field/property types type FieldType int +// Options are flags passed to the the abstract Index call, which receives them as interface{}, allowing +// for implementation specific options +type Options struct { + + // If set, we will not save the documents contents, just index them, for fetching ids only. + NoSave bool + + // If set, we avoid saving field bits for each term. + // This saves memory, but does not allow filtering by specific fields. + // This is an option that is applied and index level. + NoFieldFlags bool + + // If set, we avoid saving the term frequencies in the index. + // This saves memory but does not allow sorting based on the frequencies of a given term within the document. + // This is an option that is applied and index level. + NoFrequencies bool + + // If set, , we avoid saving the term offsets for documents. + // This saves memory but does not allow exact searches or highlighting. Implies NOHL + // This is an option that is applied and index level. + NoOffsetVectors bool + + // Set the index with a custom stop-words list, to be ignored during indexing and search time + // This is an option that is applied and index level. + // If the list is nil the default stop-words list is used. + // See https://oss.redislabs.com/redisearch/Stopwords.html#default_stop-word_list + Stopwords []string +} + +// DefaultOptions represents the default options +var DefaultOptions = Options{ + NoSave: false, + NoFieldFlags: false, + NoFrequencies: false, + NoOffsetVectors: false, + Stopwords: nil, +} + const ( // TextField full-text field TextField FieldType = iota diff --git a/redisearch/spellcheck_test.go b/redisearch/spellcheck_test.go index 1ad402c..7dfbe91 100644 --- a/redisearch/spellcheck_test.go +++ b/redisearch/spellcheck_test.go @@ -1,9 +1,10 @@ package redisearch import ( - "github.com/gomodule/redigo/redis" "reflect" "testing" + + "github.com/gomodule/redigo/redis" ) func TestMisspelledTerm_Len(t *testing.T) { @@ -16,7 +17,7 @@ func TestMisspelledTerm_Len(t *testing.T) { fields fields want int }{ - {"empty", fields{"empty", []MisspelledSuggestion{},}, 0,}, + {"empty", fields{"empty", []MisspelledSuggestion{}}, 0}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -46,8 +47,8 @@ func TestMisspelledTerm_Less(t *testing.T) { args args want bool }{ - {"double-value-list-true", fields{"double", []MisspelledSuggestion{NewMisspelledSuggestion("double", 0), NewMisspelledSuggestion("doublee", 0.1)},}, args{1, 0}, true,}, - {"double-value-list-false", fields{"double", []MisspelledSuggestion{NewMisspelledSuggestion("double", 0), NewMisspelledSuggestion("doublee", 0.1)},}, args{0, 1}, false,}, + {"double-value-list-true", fields{"double", []MisspelledSuggestion{NewMisspelledSuggestion("double", 0), NewMisspelledSuggestion("doublee", 0.1)}}, args{1, 0}, true}, + {"double-value-list-false", fields{"double", []MisspelledSuggestion{NewMisspelledSuggestion("double", 0), NewMisspelledSuggestion("doublee", 0.1)}}, args{0, 1}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -72,8 +73,8 @@ func TestMisspelledTerm_Sort(t *testing.T) { fields fields want []MisspelledSuggestion }{ - {"empty", fields{"empty", []MisspelledSuggestion{},}, []MisspelledSuggestion{},}, - {"double-value-list", fields{"double", []MisspelledSuggestion{NewMisspelledSuggestion("double", 0), NewMisspelledSuggestion("doublee", 0.1)},}, []MisspelledSuggestion{NewMisspelledSuggestion("doublee", 0.1), NewMisspelledSuggestion("double", 0)},}, + {"empty", fields{"empty", []MisspelledSuggestion{}}, []MisspelledSuggestion{}}, + {"double-value-list", fields{"double", []MisspelledSuggestion{NewMisspelledSuggestion("double", 0), NewMisspelledSuggestion("doublee", 0.1)}}, []MisspelledSuggestion{NewMisspelledSuggestion("doublee", 0.1), NewMisspelledSuggestion("double", 0)}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -104,9 +105,9 @@ func TestMisspelledTerm_Swap(t *testing.T) { args args want []MisspelledSuggestion }{ - {"empty-list", fields{"empty", []MisspelledSuggestion{},}, args{0, 1}, []MisspelledSuggestion{},}, - {"single-value-list", fields{"single", []MisspelledSuggestion{NewMisspelledSuggestion("first", 1)},}, args{0, 1}, []MisspelledSuggestion{NewMisspelledSuggestion("first", 1)},}, - {"double-value-list", fields{"doubl", []MisspelledSuggestion{NewMisspelledSuggestion("double", 1), NewMisspelledSuggestion("doublee", 0.1)},}, args{0, 1}, []MisspelledSuggestion{NewMisspelledSuggestion("doublee", 0.1), NewMisspelledSuggestion("double", 1)},}, + {"empty-list", fields{"empty", []MisspelledSuggestion{}}, args{0, 1}, []MisspelledSuggestion{}}, + {"single-value-list", fields{"single", []MisspelledSuggestion{NewMisspelledSuggestion("first", 1)}}, args{0, 1}, []MisspelledSuggestion{NewMisspelledSuggestion("first", 1)}}, + {"double-value-list", fields{"doubl", []MisspelledSuggestion{NewMisspelledSuggestion("double", 1), NewMisspelledSuggestion("doublee", 0.1)}}, args{0, 1}, []MisspelledSuggestion{NewMisspelledSuggestion("doublee", 0.1), NewMisspelledSuggestion("double", 1)}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -132,7 +133,7 @@ func TestNewMisspelledSuggestion(t *testing.T) { args args want MisspelledSuggestion }{ - {"simple", args{"term", 1}, MisspelledSuggestion{"term", 1},}, + {"simple", args{"term", 1}, MisspelledSuggestion{"term", 1}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -232,8 +233,8 @@ func TestSpellCheckOptions_AddExclusionDict(t *testing.T) { args args want []string }{ - {"empty", fields{1, []string{}, []string{}}, args{"dict1"}, []string{"dict1"},}, - {"one-prior", fields{1, []string{"dict1"}, []string{}}, args{"dict2"}, []string{"dict1", "dict2"},}, + {"empty", fields{1, []string{}, []string{}}, args{"dict1"}, []string{"dict1"}}, + {"one-prior", fields{1, []string{"dict1"}, []string{}}, args{"dict2"}, []string{"dict1", "dict2"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -264,8 +265,8 @@ func TestSpellCheckOptions_AddInclusionDict(t *testing.T) { args args want []string }{ - {"empty", fields{1, []string{}, []string{}}, args{"dict1"}, []string{"dict1"},}, - {"one-prior", fields{1, []string{}, []string{"dict1"}}, args{"dict2"}, []string{"dict1", "dict2"},}, + {"empty", fields{1, []string{}, []string{}}, args{"dict1"}, []string{"dict1"}}, + {"one-prior", fields{1, []string{}, []string{"dict1"}}, args{"dict2"}, []string{"dict1", "dict2"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -371,11 +372,11 @@ func Test_loadMisspelledTerm(t *testing.T) { wantMissT MisspelledTerm wantErr bool }{ - {"empty", args{[]interface{}{}, 1, 2,}, MisspelledTerm{}, false}, - {"missing term", args{[]interface{}{"TERM",}, 1, 2,}, MisspelledTerm{}, true}, - {"missing sugestion array", args{[]interface{}{"TERM", "hockye"}, 1, 2,}, MisspelledTerm{}, true}, - {"incorrect float", args{[]interface{}{"TERM", "hockye", []interface{}{[]interface{}{[]byte("INCORRECT"), []byte("hockey")}}}, 1, 2,}, MisspelledTerm{}, true}, - {"correct1", args{[]interface{}{"TERM", "hockye", []interface{}{[]interface{}{[]byte("1"), []byte("hockey")}}}, 1, 2,}, MisspelledTerm{"hockye", []MisspelledSuggestion{NewMisspelledSuggestion("hockey", 1.0)}}, false}, + {"empty", args{[]interface{}{}, 1, 2}, MisspelledTerm{}, false}, + {"missing term", args{[]interface{}{"TERM"}, 1, 2}, MisspelledTerm{}, true}, + {"missing sugestion array", args{[]interface{}{"TERM", "hockye"}, 1, 2}, MisspelledTerm{}, true}, + {"incorrect float", args{[]interface{}{"TERM", "hockye", []interface{}{[]interface{}{[]byte("INCORRECT"), []byte("hockey")}}}, 1, 2}, MisspelledTerm{}, true}, + {"correct1", args{[]interface{}{"TERM", "hockye", []interface{}{[]interface{}{[]byte("1"), []byte("hockey")}}}, 1, 2}, MisspelledTerm{"hockye", []MisspelledSuggestion{NewMisspelledSuggestion("hockey", 1.0)}}, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/redisearch/suggest.go b/redisearch/suggest.go index 9b4cf0c..ae48227 100644 --- a/redisearch/suggest.go +++ b/redisearch/suggest.go @@ -17,6 +17,14 @@ type SuggestOptions struct { WithScores bool } +// DefaultIndexingOptions are the default options for document indexing +var DefaultSuggestOptions = SuggestOptions{ + Num: 5, + Fuzzy: false, + WithPayloads: false, + WithScores: false, +} + // SuggestionList is a sortable list of suggestions returned from an engine type SuggestionList []Suggestion