Change index setting to contacts_index and rename some stuff for clarity

Author: rowanseymour

Diff for: cmd/rp-indexer/main.go

@@ -55,7 +55,7 @@ func main() {
 	}
 
 	idxrs := []indexers.Indexer{
-		indexers.NewContactIndexer(cfg.ElasticURL, cfg.Index, 500),
+		indexers.NewContactIndexer(cfg.ElasticURL, cfg.ContactsIndex, 500),
 	}
 
 	if cfg.Rebuild {

Diff for: config.go

@@ -3,34 +3,35 @@ package indexer
 import "os"
 
 type Config struct {
-	ElasticURL       string `help:"the url for our elastic search instance"`
-	DB               string `help:"the connection string for our database"`
-	Index            string `help:"the alias for our contact index"`
-	Poll             int    `help:"the number of seconds to wait between checking for updated contacts"`
-	Rebuild          bool   `help:"whether to rebuild the index, swapping it when complete, then exiting (default false)"`
-	Cleanup          bool   `help:"whether to remove old indexes after a rebuild"`
-	LogLevel         string `help:"the log level, one of error, warn, info, debug"`
-	SentryDSN        string `help:"the sentry configuration to log errors to, if any"`
-	ContactsShards   int    `help:"The number of shards to use for the contacts index"`
-	ContactsReplicas int    `help:"The number of replicas to use for the contacts index"`
-
+	ElasticURL      string `help:"the url for our elastic search instance"`
+	DB              string `help:"the connection string for our database"`
+	Poll            int    `help:"the number of seconds to wait between checking for database updates"`
+	Rebuild         bool   `help:"whether to rebuild the index, swapping it when complete, then exiting (default false)"`
+	Cleanup         bool   `help:"whether to remove old indexes after a rebuild"`
+	LogLevel        string `help:"the log level, one of error, warn, info, debug"`
+	SentryDSN       string `help:"the sentry configuration to log errors to, if any"`
 	LibratoUsername string `help:"the username that will be used to authenticate to Librato"`
 	LibratoToken    string `help:"the token that will be used to authenticate to Librato"`
 	InstanceName    string `help:"the unique name of this instance used for analytics"`
+
+	ContactsIndex    string `help:"the alias to use for the contact index"`
+	ContactsShards   int    `help:"the number of shards to use for the contacts index"`
+	ContactsReplicas int    `help:"the number of replicas to use for the contacts index"`
 }
 
 func NewDefaultConfig() *Config {
 	hostname, _ := os.Hostname()
 
 	return &Config{
-		ElasticURL:       "http://localhost:9200",
-		DB:               "postgres://localhost/temba?sslmode=disable",
-		Index:            "contacts",
-		Poll:             5,
-		Rebuild:          false,
-		Cleanup:          false,
-		LogLevel:         "info",
-		InstanceName:     hostname,
+		ElasticURL:   "http://localhost:9200",
+		DB:           "postgres://localhost/temba?sslmode=disable",
+		Poll:         5,
+		Rebuild:      false,
+		Cleanup:      false,
+		LogLevel:     "info",
+		InstanceName: hostname,
+
+		ContactsIndex:    "contacts",
 		ContactsShards:   2,
 		ContactsReplicas: 1,
 	}

Diff for: indexers/base.go

@@ -33,7 +33,7 @@ type Indexer interface {
 	Stats() Stats
 }
 
-type ElasticSettings struct {
+type IndexDefinition struct {
 	Settings struct {
 		Index struct {
 			NumberOfShards       int `json:"number_of_shards"`
@@ -111,7 +111,7 @@ func (i *baseIndexer) FindIndexes() []string {
 // that index to `contacts`.
 //
 // If the day-specific name already exists, we append a .1 or .2 to the name.
-func (i *baseIndexer) createNewIndex(indexSettings ElasticSettings) (string, error) {
+func (i *baseIndexer) createNewIndex(def *IndexDefinition) (string, error) {
 	// create our day-specific name
 	index := fmt.Sprintf("%s_%s", i.name, time.Now().Format("2006_01_02"))
 	idx := 0
@@ -133,11 +133,9 @@ func (i *baseIndexer) createNewIndex(indexSettings ElasticSettings) (string, err
 	}
 
 	// create the new index
-	settings, err := json.Marshal(indexSettings)
-	if err != nil {
-		return "", err
-	}
-	_, err = utils.MakeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.elasticURL, index), settings, nil)
+	settings := jsonx.MustMarshal(def)
+
+	_, err := utils.MakeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.elasticURL, index), settings, nil)
 	if err != nil {
 		return "", err
 	}

Diff for: indexers/base_test.go

@@ -3,7 +3,6 @@ package indexers_test
 import (
 	"context"
 	"database/sql"
-	"io/ioutil"
 	"log"
 	"os"
 	"sort"
@@ -22,7 +21,7 @@ const elasticURL = "http://localhost:9200"
 const aliasName = "indexer_test"
 
 func setup(t *testing.T) (*sql.DB, *elastic.Client) {
-	testDB, err := ioutil.ReadFile("../testdb.sql")
+	testDB, err := os.ReadFile("../testdb.sql")
 	require.NoError(t, err)
 
 	db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")

Diff for: indexers/contacts.go

@@ -4,17 +4,16 @@ import (
 	"bytes"
 	"database/sql"
 	_ "embed"
-	"encoding/json"
 	"fmt"
+	"time"
+
+	"github.com/nyaruka/gocommon/jsonx"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-	"time"
 )
 
-//go:embed contacts.settings.json
-var contactsSettingsFile []byte
-
-var contactsSettings ElasticSettings
+//go:embed contacts.index.json
+var contactsIndexDefinition []byte
 
 // ContactIndexer is an indexer for contacts
 type ContactIndexer struct {
@@ -32,7 +31,7 @@ func NewContactIndexer(elasticURL, name string, batchSize int) *ContactIndexer {
 }
 
 // Index indexes modified contacts and returns the name of the concrete index
-func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool, shards int, replicas int) (string, error) {
+func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool, shards, replicas int) (string, error) {
 	var err error
 
 	// find our physical index
@@ -48,13 +47,13 @@ func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool, shards int, re
 
 	// doesn't exist or we are rebuilding, create it
 	if physicalIndex == "" || rebuild {
-		err = json.Unmarshal(contactsSettingsFile, &contactsSettings)
-		if err != nil {
-			return "", errors.Wrap(err, "error unmarshalling embeded contacts.settings.json file")
-		}
-		contactsSettings.Settings.Index.NumberOfShards = shards
-		contactsSettings.Settings.Index.NumberOfReplicas = replicas
-		physicalIndex, err = i.createNewIndex(contactsSettings)
+		def := &IndexDefinition{}
+		jsonx.MustUnmarshal(contactsIndexDefinition, def)
+
+		def.Settings.Index.NumberOfShards = shards
+		def.Settings.Index.NumberOfReplicas = replicas
+
+		physicalIndex, err = i.createNewIndex(def)
 		if err != nil {
 			return "", errors.Wrap(err, "error creating new index")
 		}

Diff for: indexers/contacts.settings.json renamed to indexers/contacts.index.json

@@ -1,8 +1,8 @@
 {
     "settings": {
         "index": {
-            "number_of_shards": 2,
-            "number_of_replicas": 1,
+            "number_of_shards": -1,
+            "number_of_replicas": -1,
             "routing_partition_size": 1
         },
         "analysis": {