From be02daf67790cfd149f510c77d229c9928b2cdb4 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 14 Feb 2022 13:51:04 -0500
Subject: [PATCH 01/48] Move a little more functionality into contacts package

---
 .github/workflows/ci.yml | 84 ++++++++++++++++++++--------------------
 .gitignore               |  4 +-
 contacts/query.go        | 72 ++++++++++++++++++++++++++++++++++
 contacts/settings.go     |  7 ++++
 deploy                   |  1 +
 indexer.go               | 71 ++-------------------------------
 6 files changed, 126 insertions(+), 113 deletions(-)
 create mode 100644 contacts/query.go
 create mode 100644 contacts/settings.go
 create mode 120000 deploy

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f591b51..5a4a1fd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,65 +1,65 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: '1.16.x'
-  es-version: '7.10.1'
+  go-version: "1.16.x"
+  es-version: "7.10.1"
 jobs:
   test:
     name: Test
     strategy:
       matrix:
-        pg-version: ['12', '13']
+        pg-version: ["12", "13"]
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v1
+      - name: Checkout code
+        uses: actions/checkout@v1
 
-    - name: Install ElasticSearch
-      uses: nyaruka/elasticsearch-action@master
-      with:
-        elastic version: ${{ env.es-version }}
+      - name: Install ElasticSearch
+        uses: nyaruka/elasticsearch-action@master
+        with:
+          elastic version: ${{ env.es-version }}
 
-    - name: Install PostgreSQL
-      uses: harmon758/postgresql-action@v1
-      with:
-        postgresql version: ${{ matrix.pg-version }}
-        postgresql db: elastic_test
-        postgresql user: temba
-        postgresql password: temba
+      - name: Install PostgreSQL
+        uses: harmon758/postgresql-action@v1
+        with:
+          postgresql version: ${{ matrix.pg-version }}
+          postgresql db: elastic_test
+          postgresql user: nyaruka
+          postgresql password: nyaruka
 
-    - name: Install Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: ${{ env.go-version }}
+      - name: Install Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ env.go-version }}
 
-    - name: Run tests
-      run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./...
+      - name: Run tests
+        run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./...
+
+      - name: Upload coverage
+        if: success()
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
 
-    - name: Upload coverage
-      if: success()
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-  
   release:
     name: Release
     needs: [test]
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v1
+      - name: Checkout code
+        uses: actions/checkout@v1
 
-    - name: Install Go
-      uses: actions/setup-go@v1
-      with:
-        go-version: ${{ env.go-version }}
+      - name: Install Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ env.go-version }}
 
-    - name: Publish release
-      uses: goreleaser/goreleaser-action@v1
-      with:
-        version: v0.147.2
-        args: release --rm-dist
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        fail_ci_if_error: true
+      - name: Publish release
+        uses: goreleaser/goreleaser-action@v1
+        with:
+          version: v0.147.2
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          fail_ci_if_error: true
diff --git a/.gitignore b/.gitignore
index 11d130f..1446c34 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,9 +5,7 @@
 *.dylib
 fabfile.py
 fabfile.pyc
-fabconfig.py
-fabconfig.pyc
-fabric
+deploy/
 rp-indexer
 
 # Test binary, build with `go test -c`
diff --git a/contacts/query.go b/contacts/query.go
new file mode 100644
index 0000000..19eac21
--- /dev/null
+++ b/contacts/query.go
@@ -0,0 +1,72 @@
+package contacts
+
+import (
+	"database/sql"
+	"time"
+)
+
+const sqlSelectModified = `
+SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
+  SELECT
+   id, org_id, uuid, name, language, status, ticket_count AS tickets, is_active, created_on, modified_on, last_seen_on,
+   EXTRACT(EPOCH FROM modified_on) * 1000000 as modified_on_mu,
+   (
+     SELECT array_to_json(array_agg(row_to_json(u)))
+     FROM (
+            SELECT scheme, path
+            FROM contacts_contacturn
+            WHERE contact_id = contacts_contact.id
+          ) u
+   ) as urns,
+   (
+     SELECT jsonb_agg(f.value)
+     FROM (
+                       select case
+                    when value ? 'ward'
+                      then jsonb_build_object(
+                        'ward_keyword', trim(substring(value ->> 'ward' from  '(?!.* > )([^>]+)'))
+                      )
+                    else '{}' :: jsonb
+                    end || district_value.value as value
+           FROM (
+                  select case
+                           when value ? 'district'
+                             then jsonb_build_object(
+                               'district_keyword', trim(substring(value ->> 'district' from  '(?!.* > )([^>]+)'))
+                             )
+                           else '{}' :: jsonb
+                           end || state_value.value as value
+                  FROM (
+
+                         select case
+                                  when value ? 'state'
+                                    then jsonb_build_object(
+                                      'state_keyword', trim(substring(value ->> 'state' from  '(?!.* > )([^>]+)'))
+                                    )
+                                  else '{}' :: jsonb
+                                  end ||
+                                jsonb_build_object('field', key) || value as value
+                         from jsonb_each(contacts_contact.fields)
+                       ) state_value
+                ) as district_value
+          ) as f
+   ) as fields,
+   (
+     SELECT array_to_json(array_agg(g.uuid))
+     FROM (
+            SELECT contacts_contactgroup.uuid
+            FROM contacts_contactgroup_contacts, contacts_contactgroup
+            WHERE contact_id = contacts_contact.id AND
+                  contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
+          ) g
+   ) as groups
+  FROM contacts_contact
+  WHERE modified_on >= $1
+  ORDER BY modified_on ASC
+  LIMIT 500000
+) t;
+`
+
+func FetchModified(db *sql.DB, lastModified time.Time) (*sql.Rows, error) {
+	return db.Query(sqlSelectModified, lastModified)
+}
diff --git a/contacts/settings.go b/contacts/settings.go
new file mode 100644
index 0000000..89dd5a9
--- /dev/null
+++ b/contacts/settings.go
@@ -0,0 +1,7 @@
+package contacts
+
+import _ "embed"
+
+// settings and mappings for our index
+//go:embed index_settings.json
+var IndexSettings string
diff --git a/deploy b/deploy
new file mode 120000
index 0000000..3ebc6a1
--- /dev/null
+++ b/deploy
@@ -0,0 +1 @@
+../utils/deploy
\ No newline at end of file
diff --git a/indexer.go b/indexer.go
index be30942..d80f04d 100644
--- a/indexer.go
+++ b/indexer.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/nyaruka/gocommon/httpx"
+	"github.com/nyaruka/rp-indexer/contacts"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -86,7 +87,7 @@ func CreateNewIndex(url string, alias string) (string, error) {
 
 	// initialize our index
 	createURL := fmt.Sprintf("%s/%s?include_type_name=true", url, physicalIndex)
-	_, err := MakeJSONRequest(http.MethodPut, createURL, indexSettings, nil)
+	_, err := MakeJSONRequest(http.MethodPut, createURL, contacts.IndexSettings, nil)
 	if err != nil {
 		return "", err
 	}
@@ -265,7 +266,7 @@ func IndexContacts(db *sql.DB, elasticURL string, index string, lastModified tim
 	start := time.Now()
 
 	for {
-		rows, err := db.Query(contactQuery, lastModified)
+		rows, err := contacts.FetchModified(db, lastModified)
 
 		queryCreated := 0
 		queryCount := 0
@@ -381,72 +382,6 @@ func MapIndexAlias(elasticURL string, alias string, newIndex string) error {
 	return err
 }
 
-const contactQuery = `
-SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
-  SELECT
-   id, org_id, uuid, name, language, status, ticket_count AS tickets, is_active, created_on, modified_on, last_seen_on,
-   EXTRACT(EPOCH FROM modified_on) * 1000000 as modified_on_mu,
-   (
-     SELECT array_to_json(array_agg(row_to_json(u)))
-     FROM (
-            SELECT scheme, path
-            FROM contacts_contacturn
-            WHERE contact_id = contacts_contact.id
-          ) u
-   ) as urns,
-   (
-     SELECT jsonb_agg(f.value)
-     FROM (
-                       select case
-                    when value ? 'ward'
-                      then jsonb_build_object(
-                        'ward_keyword', trim(substring(value ->> 'ward' from  '(?!.* > )([^>]+)'))
-                      )
-                    else '{}' :: jsonb
-                    end || district_value.value as value
-           FROM (
-                  select case
-                           when value ? 'district'
-                             then jsonb_build_object(
-                               'district_keyword', trim(substring(value ->> 'district' from  '(?!.* > )([^>]+)'))
-                             )
-                           else '{}' :: jsonb
-                           end || state_value.value as value
-                  FROM (
-
-                         select case
-                                  when value ? 'state'
-                                    then jsonb_build_object(
-                                      'state_keyword', trim(substring(value ->> 'state' from  '(?!.* > )([^>]+)'))
-                                    )
-                                  else '{}' :: jsonb
-                                  end ||
-                                jsonb_build_object('field', key) || value as value
-                         from jsonb_each(contacts_contact.fields)
-                       ) state_value
-                ) as district_value
-          ) as f
-   ) as fields,
-   (
-     SELECT array_to_json(array_agg(g.uuid))
-     FROM (
-            SELECT contacts_contactgroup.uuid
-            FROM contacts_contactgroup_contacts, contacts_contactgroup
-            WHERE contact_id = contacts_contact.id AND
-                  contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
-          ) g
-   ) as groups
-  FROM contacts_contact
-  WHERE modified_on >= $1
-  ORDER BY modified_on ASC
-  LIMIT 500000
-) t;
-`
-
-// settings and mappings for our index
-//go:embed contacts/index_settings.json
-var indexSettings string
-
 // gets our last modified contact
 const lastModifiedQuery = `{ "sort": [{ "modified_on_mu": "desc" }]}`
 

From 2d37e9ba6059b15a1782188487bae166ca18b332 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 14 Feb 2022 13:51:51 -0500
Subject: [PATCH 02/48] CI with go 1.17

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5a4a1fd..caaf34e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,7 +1,7 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: "1.16.x"
+  go-version: "1.17.x"
   es-version: "7.10.1"
 jobs:
   test:

From 14e00088506176fbcccd7b828b39a71189eb6ce0 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 14 Feb 2022 14:08:48 -0500
Subject: [PATCH 03/48] Move some elastic functionality into its own file

---
 contacts/settings.go |   7 +-
 elastic.go           | 150 +++++++++++++++++++++++++++++++++++++++
 indexer.go           | 162 +++----------------------------------------
 indexer_test.go      |   2 +-
 4 files changed, 165 insertions(+), 156 deletions(-)
 create mode 100644 elastic.go

diff --git a/contacts/settings.go b/contacts/settings.go
index 89dd5a9..4bb3608 100644
--- a/contacts/settings.go
+++ b/contacts/settings.go
@@ -1,7 +1,10 @@
 package contacts
 
-import _ "embed"
+import (
+	_ "embed"
+	"encoding/json"
+)
 
 // settings and mappings for our index
 //go:embed index_settings.json
-var IndexSettings string
+var IndexSettings json.RawMessage
diff --git a/elastic.go b/elastic.go
new file mode 100644
index 0000000..02f8a23
--- /dev/null
+++ b/elastic.go
@@ -0,0 +1,150 @@
+package indexer
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/nyaruka/gocommon/httpx"
+	log "github.com/sirupsen/logrus"
+)
+
+var retryConfig *httpx.RetryConfig
+
+func init() {
+	backoffs := make([]time.Duration, 5)
+	backoffs[0] = 1 * time.Second
+	for i := 1; i < len(backoffs); i++ {
+		backoffs[i] = backoffs[i-1] * 2
+	}
+
+	retryConfig = &httpx.RetryConfig{Backoffs: backoffs, ShouldRetry: shouldRetry}
+}
+
+func shouldRetry(request *http.Request, response *http.Response, withDelay time.Duration) bool {
+	// 429 Too Many Requests is recoverable. Sometimes the server puts
+	// a Retry-After response header to indicate when the server is
+	// available to start processing request from client.
+	if response.StatusCode == http.StatusTooManyRequests {
+		return true
+	}
+
+	// check for unexpected EOF
+	bodyBytes, err := ioutil.ReadAll(response.Body)
+	response.Body.Close()
+	if err != nil {
+		log.WithError(err).Error("error reading ES response, retrying")
+		return true
+	}
+
+	response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
+	return false
+}
+
+// MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct
+func MakeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) {
+	req, _ := http.NewRequest(method, url, bytes.NewReader(body))
+	req.Header.Add("Content-Type", "application/json")
+
+	resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
+
+	l := log.WithField("url", url).WithField("method", method).WithField("request", body)
+	if err != nil {
+		l.WithError(err).Error("error making ES request")
+		return resp, err
+	}
+	defer resp.Body.Close()
+
+	// if we have a body, try to decode it
+	jsonBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		l.WithError(err).Error("error reading ES response")
+		return resp, err
+	}
+
+	l = l.WithField("response", string(jsonBody)).WithField("status", resp.StatusCode)
+
+	// error if we got a non-200
+	if resp.StatusCode != http.StatusOK {
+		l.WithError(err).Error("error reaching ES")
+		return resp, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, jsonBody)
+	}
+
+	if jsonStruct == nil {
+		l.Debug("ES request successful")
+		return resp, nil
+	}
+
+	err = json.Unmarshal(jsonBody, jsonStruct)
+	if err != nil {
+		l.WithError(err).Error("error unmarshalling ES response")
+		return resp, err
+	}
+
+	l.Debug("ES request successful")
+	return resp, nil
+}
+
+// adds an alias for an index
+type addAliasCommand struct {
+	Add struct {
+		Index string `json:"index"`
+		Alias string `json:"alias"`
+	} `json:"add"`
+}
+
+// removes an alias for an index
+type removeAliasCommand struct {
+	Remove struct {
+		Index string `json:"index"`
+		Alias string `json:"alias"`
+	} `json:"remove"`
+}
+
+// our top level command for remapping aliases
+type aliasCommand struct {
+	Actions []interface{} `json:"actions"`
+}
+
+// our response for finding the most recent contact
+type queryResponse struct {
+	Hits struct {
+		Total struct {
+			Value int `json:"value"`
+		} `json:"total"`
+		Hits []struct {
+			Source struct {
+				ID         int64     `json:"id"`
+				ModifiedOn time.Time `json:"modified_on"`
+			} `json:"_source"`
+		} `json:"hits"`
+	} `json:"hits"`
+}
+
+// our response for indexing contacts
+type indexResponse struct {
+	Items []struct {
+		Index struct {
+			ID     string `json:"_id"`
+			Status int    `json:"status"`
+			Result string `json:"result"`
+		} `json:"index"`
+		Delete struct {
+			ID     string `json:"_id"`
+			Status int    `json:"status"`
+		} `json:"delete"`
+	} `json:"items"`
+}
+
+// our response for our index health
+type healthResponse struct {
+	Indices map[string]struct {
+		Status string `json:"status"`
+	} `json:"indices"`
+}
+
+// our response for figuring out the physical index for an alias
+type infoResponse map[string]interface{}
diff --git a/indexer.go b/indexer.go
index d80f04d..ce242f1 100644
--- a/indexer.go
+++ b/indexer.go
@@ -6,57 +6,17 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"net/http"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/rp-indexer/contacts"
 	log "github.com/sirupsen/logrus"
 )
 
 var batchSize = 500
 
-var retryConfig *httpx.RetryConfig
-
-func init() {
-	//setup httpx retry configuration
-	var retrycount = 5
-	var initialBackoff = 1 * time.Second
-	retryConfig = ElasticRetries(initialBackoff, retrycount)
-}
-
-func ElasticRetries(initialBackoff time.Duration, count int) *httpx.RetryConfig {
-	backoffs := make([]time.Duration, count)
-	backoffs[0] = initialBackoff
-	for i := 1; i < count; i++ {
-		backoffs[i] = backoffs[i-1] * 2
-	}
-	return &httpx.RetryConfig{Backoffs: backoffs, ShouldRetry: ShouldRetry}
-}
-func ShouldRetry(request *http.Request, response *http.Response, withDelay time.Duration) bool {
-
-	// 429 Too Many Requests is recoverable. Sometimes the server puts
-	// a Retry-After response header to indicate when the server is
-	// available to start processing request from client.
-	if response.StatusCode == http.StatusTooManyRequests {
-		return true
-	}
-
-	// check for unexpected EOF
-	bodyBytes, err := ioutil.ReadAll(response.Body)
-	response.Body.Close()
-	if err != nil {
-		log.WithError(err).Error("error reading ES response, retrying")
-		return true
-	}
-
-	response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
-	return false
-}
-
 // CreateNewIndex creates a new index for the passed in alias.
 //
 // Note that we do not create an index with the passed name, instead creating one
@@ -120,7 +80,7 @@ func GetLastModified(url string, index string) (time.Time, error) {
 // FindPhysicalIndexes finds all the physical indexes for the passed in alias
 func FindPhysicalIndexes(url string, alias string) []string {
 	indexResponse := infoResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, alias), "", &indexResponse)
+	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, alias), nil, &indexResponse)
 	indexes := make([]string, 0)
 
 	// error could mean a variety of things, but we'll figure that out later
@@ -150,7 +110,7 @@ func CleanupIndexes(url string, alias string) error {
 
 	// find all the current indexes
 	healthResponse := healthResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, "_cluster/health?level=indices"), "", &healthResponse)
+	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, "_cluster/health?level=indices"), nil, &healthResponse)
 	if err != nil {
 		return err
 	}
@@ -159,7 +119,7 @@ func CleanupIndexes(url string, alias string) error {
 	for key := range healthResponse.Indices {
 		if strings.HasPrefix(key, alias) && strings.Compare(key, currents[0]) < 0 {
 			log.WithField("index", key).Info("removing old index")
-			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", url, key), "", nil)
+			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", url, key), nil, nil)
 			if err != nil {
 				return err
 			}
@@ -169,51 +129,8 @@ func CleanupIndexes(url string, alias string) error {
 	return nil
 }
 
-// MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct
-func MakeJSONRequest(method string, url string, body string, jsonStruct interface{}) (*http.Response, error) {
-	req, _ := http.NewRequest(method, url, bytes.NewReader([]byte(body)))
-	req.Header.Add("Content-Type", "application/json")
-	resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
-
-	l := log.WithField("url", url).WithField("method", method).WithField("request", body)
-	if err != nil {
-		l.WithError(err).Error("error making ES request")
-		return resp, err
-	}
-	defer resp.Body.Close()
-
-	// if we have a body, try to decode it
-	jsonBody, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		l.WithError(err).Error("error reading ES response")
-		return resp, err
-	}
-
-	l = l.WithField("response", string(jsonBody)).WithField("status", resp.StatusCode)
-
-	// error if we got a non-200
-	if resp.StatusCode != http.StatusOK {
-		l.WithError(err).Error("error reaching ES")
-		return resp, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, jsonBody)
-	}
-
-	if jsonStruct == nil {
-		l.Debug("ES request successful")
-		return resp, nil
-	}
-
-	err = json.Unmarshal(jsonBody, jsonStruct)
-	if err != nil {
-		l.WithError(err).Error("error unmarshalling ES response")
-		return resp, err
-	}
-
-	l.Debug("ES request successful")
-	return resp, nil
-}
-
 // IndexBatch indexes the batch of contacts
-func IndexBatch(elasticURL string, index string, batch string) (int, int, error) {
+func IndexBatch(elasticURL string, index string, batch []byte) (int, int, error) {
 	response := indexResponse{}
 	indexURL := fmt.Sprintf("%s/%s/_bulk", elasticURL, index)
 
@@ -251,7 +168,7 @@ func IndexBatch(elasticURL string, index string, batch string) (int, int, error)
 
 // IndexContacts queries and indexes all contacts with a lastModified greater than or equal to the passed in time
 func IndexContacts(db *sql.DB, elasticURL string, index string, lastModified time.Time) (int, int, error) {
-	batch := strings.Builder{}
+	batch := &bytes.Buffer{}
 	createdCount, deletedCount, processedCount := 0, 0, 0
 
 	if index == "" {
@@ -305,7 +222,7 @@ func IndexContacts(db *sql.DB, elasticURL string, index string, lastModified tim
 
 			// write to elastic search in batches
 			if queryCount%batchSize == 0 {
-				created, deleted, err := IndexBatch(elasticURL, index, batch.String())
+				created, deleted, err := IndexBatch(elasticURL, index, batch.Bytes())
 				if err != nil {
 					return 0, 0, err
 				}
@@ -318,7 +235,7 @@ func IndexContacts(db *sql.DB, elasticURL string, index string, lastModified tim
 		}
 
 		if batch.Len() > 0 {
-			created, deleted, err := IndexBatch(elasticURL, index, batch.String())
+			created, deleted, err := IndexBatch(elasticURL, index, batch.Bytes())
 			if err != nil {
 				return 0, 0, err
 			}
@@ -378,76 +295,15 @@ func MapIndexAlias(elasticURL string, alias string, newIndex string) error {
 	if err != nil {
 		return err
 	}
-	_, err = MakeJSONRequest(http.MethodPost, aliasURL, string(aliasJSON), nil)
+	_, err = MakeJSONRequest(http.MethodPost, aliasURL, aliasJSON, nil)
 	return err
 }
 
 // gets our last modified contact
-const lastModifiedQuery = `{ "sort": [{ "modified_on_mu": "desc" }]}`
+var lastModifiedQuery = []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`)
 
 // indexes a contact
 const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
 
 // deletes a contact
 const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
-
-// adds an alias for an index
-type addAliasCommand struct {
-	Add struct {
-		Index string `json:"index"`
-		Alias string `json:"alias"`
-	} `json:"add"`
-}
-
-// removes an alias for an index
-type removeAliasCommand struct {
-	Remove struct {
-		Index string `json:"index"`
-		Alias string `json:"alias"`
-	} `json:"remove"`
-}
-
-// our top level command for remapping aliases
-type aliasCommand struct {
-	Actions []interface{} `json:"actions"`
-}
-
-// our response for finding the most recent contact
-type queryResponse struct {
-	Hits struct {
-		Total struct {
-			Value int `json:"value"`
-		} `json:"total"`
-		Hits []struct {
-			Source struct {
-				ID         int64     `json:"id"`
-				ModifiedOn time.Time `json:"modified_on"`
-			} `json:"_source"`
-		} `json:"hits"`
-	} `json:"hits"`
-}
-
-// our response for indexing contacts
-type indexResponse struct {
-	Items []struct {
-		Index struct {
-			ID     string `json:"_id"`
-			Status int    `json:"status"`
-			Result string `json:"result"`
-		} `json:"index"`
-		Delete struct {
-			ID     string `json:"_id"`
-			Status int    `json:"status"`
-		} `json:"delete"`
-	} `json:"items"`
-}
-
-// our response for our index health
-type healthResponse struct {
-	Indices map[string]struct {
-		Status string `json:"status"`
-	} `json:"indices"`
-}
-
-// our response for figuring out the physical index for an alias
-type infoResponse map[string]interface{}
diff --git a/indexer_test.go b/indexer_test.go
index 40b6296..cb43977 100644
--- a/indexer_test.go
+++ b/indexer_test.go
@@ -26,7 +26,7 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) {
 	testDB, err := ioutil.ReadFile("testdb.sql")
 	require.NoError(t, err)
 
-	db, err := sql.Open("postgres", "postgres://temba:temba@localhost:5432/elastic_test?sslmode=disable")
+	db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")
 	require.NoError(t, err)
 
 	_, err = db.Exec(string(testDB))

From 5aa540796197faea2470d3dea65b9ef4d320334b Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 14 Feb 2022 14:32:26 -0500
Subject: [PATCH 04/48] Clean up formatting of SQL

---
 contacts/query.go | 113 +++++++++++++++++++++++-----------------------
 1 file changed, 56 insertions(+), 57 deletions(-)

diff --git a/contacts/query.go b/contacts/query.go
index 19eac21..1584766 100644
--- a/contacts/query.go
+++ b/contacts/query.go
@@ -7,63 +7,62 @@ import (
 
 const sqlSelectModified = `
 SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
-  SELECT
-   id, org_id, uuid, name, language, status, ticket_count AS tickets, is_active, created_on, modified_on, last_seen_on,
-   EXTRACT(EPOCH FROM modified_on) * 1000000 as modified_on_mu,
-   (
-     SELECT array_to_json(array_agg(row_to_json(u)))
-     FROM (
-            SELECT scheme, path
-            FROM contacts_contacturn
-            WHERE contact_id = contacts_contact.id
-          ) u
-   ) as urns,
-   (
-     SELECT jsonb_agg(f.value)
-     FROM (
-                       select case
-                    when value ? 'ward'
-                      then jsonb_build_object(
-                        'ward_keyword', trim(substring(value ->> 'ward' from  '(?!.* > )([^>]+)'))
-                      )
-                    else '{}' :: jsonb
-                    end || district_value.value as value
-           FROM (
-                  select case
-                           when value ? 'district'
-                             then jsonb_build_object(
-                               'district_keyword', trim(substring(value ->> 'district' from  '(?!.* > )([^>]+)'))
-                             )
-                           else '{}' :: jsonb
-                           end || state_value.value as value
-                  FROM (
-
-                         select case
-                                  when value ? 'state'
-                                    then jsonb_build_object(
-                                      'state_keyword', trim(substring(value ->> 'state' from  '(?!.* > )([^>]+)'))
-                                    )
-                                  else '{}' :: jsonb
-                                  end ||
-                                jsonb_build_object('field', key) || value as value
-                         from jsonb_each(contacts_contact.fields)
-                       ) state_value
-                ) as district_value
-          ) as f
-   ) as fields,
-   (
-     SELECT array_to_json(array_agg(g.uuid))
-     FROM (
-            SELECT contacts_contactgroup.uuid
-            FROM contacts_contactgroup_contacts, contacts_contactgroup
-            WHERE contact_id = contacts_contact.id AND
-                  contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
-          ) g
-   ) as groups
-  FROM contacts_contact
-  WHERE modified_on >= $1
-  ORDER BY modified_on ASC
-  LIMIT 500000
+	SELECT
+		id,
+		org_id,
+		uuid,
+		name,
+		language,
+		status,
+		ticket_count AS tickets,
+		is_active,
+		created_on,
+		modified_on,
+		last_seen_on,
+		EXTRACT(EPOCH FROM modified_on) * 1000000 AS modified_on_mu,
+		(
+			SELECT array_to_json(array_agg(row_to_json(u)))
+			FROM (SELECT scheme, path FROM contacts_contacturn WHERE contact_id = contacts_contact.id) u
+		) AS urns,
+		(
+			SELECT jsonb_agg(f.value)
+			FROM (
+				SELECT 
+					CASE
+					WHEN value ? 'ward'
+					THEN jsonb_build_object('ward_keyword', trim(substring(value ->> 'ward' from  '(?!.* > )([^>]+)')))
+					ELSE '{}'::jsonb
+					END || district_value.value AS value
+				FROM (
+					SELECT 
+						CASE
+						WHEN value ? 'district'
+						THEN jsonb_build_object('district_keyword', trim(substring(value ->> 'district' from  '(?!.* > )([^>]+)')))
+						ELSE '{}'::jsonb
+						END || state_value.value as value
+					FROM (
+						SELECT 
+							CASE
+							WHEN value ? 'state'
+							THEN jsonb_build_object('state_keyword', trim(substring(value ->> 'state' from  '(?!.* > )([^>]+)')))
+							ELSE '{}' :: jsonb
+							END || jsonb_build_object('field', key) || value as value
+						FROM jsonb_each(contacts_contact.fields)
+					) state_value
+				) AS district_value
+			) AS f
+		) AS fields,
+		(
+			SELECT array_to_json(array_agg(g.uuid)) FROM (
+				SELECT contacts_contactgroup.uuid
+				FROM contacts_contactgroup_contacts, contacts_contactgroup
+				WHERE contact_id = contacts_contact.id AND contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
+			) g
+		) AS groups
+	FROM contacts_contact
+	WHERE modified_on >= $1
+	ORDER BY modified_on ASC
+	LIMIT 500000
 ) t;
 `
 

From 79a983c9be7e2fd80124caa4c1df513332cce8b3 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 14 Feb 2022 15:00:48 -0500
Subject: [PATCH 05/48] Index contact.current_flow_id as flow uuid

---
 contacts/index_settings.json |   3 +
 contacts/query.go            |   5 +-
 indexer_test.go              |   3 +
 testdb.sql                   | 123 ++++++++++++++++++-----------------
 4 files changed, 73 insertions(+), 61 deletions(-)

diff --git a/contacts/index_settings.json b/contacts/index_settings.json
index 9706745..75e8728 100644
--- a/contacts/index_settings.json
+++ b/contacts/index_settings.json
@@ -150,6 +150,9 @@
                 "status": {
                     "type": "keyword"
                 },
+                "flow": {
+                    "type": "keyword"
+                },
                 "tickets": {
                     "type": "integer"
                 },
diff --git a/contacts/query.go b/contacts/query.go
index 1584766..d909a02 100644
--- a/contacts/query.go
+++ b/contacts/query.go
@@ -58,7 +58,10 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 				FROM contacts_contactgroup_contacts, contacts_contactgroup
 				WHERE contact_id = contacts_contact.id AND contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
 			) g
-		) AS groups
+		) AS groups,
+		(
+			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
+		) AS flow
 	FROM contacts_contact
 	WHERE modified_on >= $1
 	ORDER BY modified_on ASC
diff --git a/indexer_test.go b/indexer_test.go
index cb43977..1a5af69 100644
--- a/indexer_test.go
+++ b/indexer_test.go
@@ -101,6 +101,9 @@ func TestIndexing(t *testing.T) {
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("tickets", 1), []int64{2, 3})
 	assertQuery(t, client, physicalName, elastic.NewRangeQuery("tickets").Gt(0), []int64{1, 2, 3})
 
+	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "6d3cf1eb-546e-4fb8-a5ca-69187648fbf6"), []int64{2, 3})
+	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4})
+
 	// created_on range query
 	assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8})
 
diff --git a/testdb.sql b/testdb.sql
index 99f379b..845d627 100644
--- a/testdb.sql
+++ b/testdb.sql
@@ -1,6 +1,13 @@
+DROP TABLE IF EXISTS flows_flow CASCADE;
+CREATE TABLE flows_flow (
+    id SERIAL PRIMARY KEY,
+    uuid character varying(36) NOT NULL,
+    name character varying(128) NOT NULL
+);
+
 DROP TABLE IF EXISTS contacts_contact CASCADE;
 CREATE TABLE contacts_contact (
-    id integer NOT NULL,
+    id SERIAL PRIMARY KEY,
     is_active boolean NOT NULL,
     status character varying(1) NOT NULL,
     created_by_id integer NOT NULL,
@@ -12,22 +19,14 @@ CREATE TABLE contacts_contact (
     name character varying(128),
     language character varying(3),
     uuid character varying(36) NOT NULL,
+    current_flow_id integer REFERENCES flows_flow(id),
     fields jsonb,
     ticket_count integer NOT NULL
 );
 
-CREATE SEQUENCE contacts_contact_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-ALTER SEQUENCE contacts_contact_id_seq OWNED BY contacts_contact.id;
-
 DROP TABLE IF EXISTS contacts_contacturn CASCADE;
 CREATE TABLE contacts_contacturn (
-    id integer NOT NULL,
+    id SERIAL PRIMARY KEY,
     contact_id integer,
     scheme character varying(128) NOT NULL,
     org_id integer NOT NULL,
@@ -39,46 +38,23 @@ CREATE TABLE contacts_contacturn (
     identity character varying(255) NOT NULL
 );
 
-CREATE SEQUENCE contacts_contacturn_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-ALTER SEQUENCE contacts_contacturn_id_seq OWNED BY contacts_contacturn.id;
-
 DROP TABLE IF EXISTS contacts_contactgroup CASCADE;
 CREATE TABLE contacts_contactgroup (
-    id integer NOT NULL,
+    id SERIAL PRIMARY KEY,
     uuid character varying(36) NOT NULL,
     name character varying(128) NOT NULL
 );
 
-CREATE SEQUENCE contacts_contactgroup_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-ALTER SEQUENCE contacts_contactgroup_id_seq OWNED BY contacts_contactgroup.id;
-
 DROP TABLE IF EXISTS contacts_contactgroup_contacts CASCADE;
 CREATE TABLE contacts_contactgroup_contacts (
-    id integer NOT NULL,
-    contactgroup_id integer NOT NULL,
-    contact_id integer NOT NULL
+    id SERIAL PRIMARY KEY,
+    contactgroup_id integer NOT NULL REFERENCES contacts_contactgroup(id),
+    contact_id integer NOT NULL REFERENCES contacts_contact(id)
 );
 
-CREATE SEQUENCE contacts_contactgroup_contacts_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-ALTER SEQUENCE contacts_contactgroup_contacts_id_seq OWNED BY contacts_contactgroup_contacts.id;
+INSERT INTO flows_flow(id, uuid, name) VALUES
+(1, '6d3cf1eb-546e-4fb8-a5ca-69187648fbf6', 'Favorites'),
+(2, '4eea8ff1-4fe2-4ce5-92a4-0870a499973a', 'Catch All');
 
 -- Fields:
 -- 17103bb1-1b48-4b70-92f7-1f6b73bd3488 - nickname (text)
@@ -88,42 +64,69 @@ ALTER SEQUENCE contacts_contactgroup_contacts_id_seq OWNED BY contacts_contactgr
 -- fcab2439-861c-4832-aa54-0c97f38f24ab - home_district (district)
 -- a551ade4-e5a0-4d83-b185-53b515ad2f2a - home_ward (ward)
 
-INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, last_seen_on, org_id, status, name, language, uuid, fields, ticket_count) VALUES
+INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, last_seen_on, org_id, status, name, language, uuid, fields, ticket_count, current_flow_id) VALUES
 (
-    1,  TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:11', 1, 'A', NULL, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', 
-    '{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}', 2
+    1,  
+    TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:11', 1, 'A', NULL, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', 
+    '{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}', 
+    2,
+    NULL
 ),
 (
-    2,  TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', '2020-08-03 13:11', 1, 'S', NULL, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209',
-    '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}', 1
+    2,  
+    TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', '2020-08-03 13:11', 1, 'S', NULL, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209',
+    '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}', 
+    1,
+    1
 ),
 (
-    3,  TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', '2018-05-04 21:11', 1, 'B', NULL, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', 
-    '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}', 1
+    3,  
+    TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', '2018-05-04 21:11', 1, 'B', NULL, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', 
+    '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}', 
+    1,
+    1
 ),
 (
-    4,  TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', '2015-12-31 23:59', 1, 'A', 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', 
-    '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}', 0
+    4,  
+    TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', '2015-12-31 23:59', 1, 'A', 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', 
+    '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}', 
+    0,
+    2
 ),
 (
-    5,  TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', '2020-08-04 21:11', 2, 'A', 'Ajodinabiff Dane', NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', 
-    '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }', 0
+    5,  
+    TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', '2020-08-04 21:11', 2, 'A', 'Ajodinabiff Dane', NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', 
+    '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }', 
+    0,
+    NULL
 ),
 (
-    6,  TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:00', 2, 'A', 'Joanne Stone', NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', 
-    '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }', 0
+    6,  
+    TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:00', 2, 'A', 'Joanne Stone', NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', 
+    '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }', 
+    0,
+    NULL
 ),
 (
-    7,  TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', NULL, 2, 'A', NULL, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', 
-    '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King-Côunty", "district": "USA > Washington > King-Côunty"} }', 0
+    7,  
+    TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', NULL, 2, 'A', NULL, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', 
+    '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King-Côunty", "district": "USA > Washington > King-Côunty"} }', 
+    0,
+    NULL
 ),
 (
-    8,  TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, 2, 'A', NULL, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', 
-    '{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King-Côunty > Central District", "ward": "USA > Washington > King-Côunty > Central District"} }', 0
+    8,  
+    TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, 2, 'A', NULL, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', 
+    '{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King-Côunty > Central District", "ward": "USA > Washington > King-Côunty > Central District"} }', 
+    0,
+    NULL
 ),
 (
-    9, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', NULL, 2, 'A', NULL, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', 
-    '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }', 0
+    9, 
+    TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', NULL, 2, 'A', NULL, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', 
+    '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }', 
+    0,
+    NULL
 );
 
 INSERT INTO contacts_contacturn(id, contact_id, scheme, org_id, priority, path, display, identity) VALUES

From 88d1e307a75ed88e9e9ba1e2f431d965aad03da0 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 14 Feb 2022 15:20:23 -0500
Subject: [PATCH 06/48] Update CHANGELOG.md for v7.1.0

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5287d0a..7626f84 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.1.0
+----------
+ * Index contact.current_flow_id as flow uuid
+ * CI with go 1.17
+
 v7.0.0
 ----------
  * Test on PG12 and 13

From f7ed2fa3f33cbe8ad855376f2fb08448dca57f89 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 7 Mar 2022 13:27:14 -0500
Subject: [PATCH 07/48] Tweak README

---
 README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 29e425f..aaaf73e 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,7 @@ your index occasionally to get rid of bloat.
 # Configuration
 
 Indexer uses a tiered configuration system, each option takes precendence over the ones above it:
+
  1. The configuration file
  2. Environment variables starting with `INDEXER_` 
  3. Command line parameters
@@ -40,7 +41,7 @@ environment variables and parameters and for more details on each option.
 
 For use with RapidPro, you will want to configure these settings:
 
- * `INDEXER_DB`: a URL connection string for your RapidPro database
+ * `INDEXER_DB`: a URL connection string for your RapidPro database or read replica
  * `INDEXER_ELASTIC_URL`: the URL for your ElasticSearch endpoint
  
 Recommended settings for error reporting:

From 664d933ae8baecbf3efaa3f592ec8285245ca4f8 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 7 Mar 2022 13:27:32 -0500
Subject: [PATCH 08/48] Update CHANGELOG.md for v7.2.0

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7626f84..457452e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+v7.2.0
+----------
+ * Tweak README
+
 v7.1.0
 ----------
  * Index contact.current_flow_id as flow uuid

From 89d77bd86f3b544b4b41a101f6e07a564adf133e Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 22 Mar 2022 14:50:03 -0500
Subject: [PATCH 09/48] Update golang.org/x/sys

---
 go.mod | 1 +
 go.sum | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 8e42fdc..c40cbc7 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@ require (
 	github.com/olivere/elastic/v7 v7.0.22
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.5.1
+	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
 )
 
 go 1.16
diff --git a/go.sum b/go.sum
index 16e5ac2..18f5213 100644
--- a/go.sum
+++ b/go.sum
@@ -84,8 +84,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

From 04d8945b8628246efb95d642d50b5911e69c427d Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 22 Mar 2022 14:58:33 -0500
Subject: [PATCH 10/48] Track history of flow ids on contacts

---
 contacts/index_settings.json |  3 +++
 contacts/query.go            |  8 ++++++--
 indexer_test.go              |  3 +++
 testdb.sql                   | 15 +++++++++++++++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/contacts/index_settings.json b/contacts/index_settings.json
index 75e8728..66db7aa 100644
--- a/contacts/index_settings.json
+++ b/contacts/index_settings.json
@@ -153,6 +153,9 @@
                 "flow": {
                     "type": "keyword"
                 },
+                "flow_history": {
+                    "type": "integer"
+                },
                 "tickets": {
                     "type": "integer"
                 },
diff --git a/contacts/query.go b/contacts/query.go
index d909a02..50dee78 100644
--- a/contacts/query.go
+++ b/contacts/query.go
@@ -61,11 +61,15 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 		) AS groups,
 		(
 			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
-		) AS flow
+		) AS flow,
+		(
+			SELECT array_to_json(array_agg(DISTINCT fr.flow_id))
+			FROM flows_flowrun fr WHERE fr.contact_id = contacts_contact.id
+		) AS flow_history
 	FROM contacts_contact
 	WHERE modified_on >= $1
 	ORDER BY modified_on ASC
-	LIMIT 500000
+	LIMIT 100000
 ) t;
 `
 
diff --git a/indexer_test.go b/indexer_test.go
index 1a5af69..8957363 100644
--- a/indexer_test.go
+++ b/indexer_test.go
@@ -104,6 +104,9 @@ func TestIndexing(t *testing.T) {
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "6d3cf1eb-546e-4fb8-a5ca-69187648fbf6"), []int64{2, 3})
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4})
 
+	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow_history", 1), []int64{1, 2})
+	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow_history", 2), []int64{1})
+
 	// created_on range query
 	assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8})
 
diff --git a/testdb.sql b/testdb.sql
index 845d627..46ec484 100644
--- a/testdb.sql
+++ b/testdb.sql
@@ -52,6 +52,15 @@ CREATE TABLE contacts_contactgroup_contacts (
     contact_id integer NOT NULL REFERENCES contacts_contact(id)
 );
 
+
+DROP TABLE IF EXISTS flows_flowrun CASCADE;
+CREATE TABLE flows_flowrun (
+    id SERIAL PRIMARY KEY,
+    uuid character varying(36) NOT NULL,
+    flow_id integer REFERENCES flows_flow(id),
+    contact_id integer REFERENCES contacts_contact(id)
+);
+
 INSERT INTO flows_flow(id, uuid, name) VALUES
 (1, '6d3cf1eb-546e-4fb8-a5ca-69187648fbf6', 'Favorites'),
 (2, '4eea8ff1-4fe2-4ce5-92a4-0870a499973a', 'Catch All');
@@ -152,3 +161,9 @@ INSERT INTO contacts_contactgroup_contacts(id, contact_id, contactgroup_id) VALU
 (1, 1, 1),
 (2, 1, 4),
 (3, 2, 4);
+
+INSERT INTO flows_flowrun(id, uuid, flow_id, contact_id) VALUES
+(1, '8b30ee61-e19d-427e-bb9f-4b8cd2c31d0c', 1, 1),
+(2, '94639979-155e-444d-95e9-a39dad64dbd5', 1, 1),
+(3, '74d918df-0e31-4547-98a9-5d765450e2ac', 2, 1),
+(4, '14fdf8fc-6e02-4759-b9be-cacc5991cd14', 1, 2);

From 28427467a54924eebae007b2729d3b2428a3d72b Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 22 Mar 2022 15:51:02 -0500
Subject: [PATCH 11/48] Add flow_id to index as well since we need to switch
 flows to use ids anyway

---
 contacts/index_settings.json | 3 +++
 contacts/query.go            | 1 +
 indexer_test.go              | 3 +++
 3 files changed, 7 insertions(+)

diff --git a/contacts/index_settings.json b/contacts/index_settings.json
index 66db7aa..5061272 100644
--- a/contacts/index_settings.json
+++ b/contacts/index_settings.json
@@ -153,6 +153,9 @@
                 "flow": {
                     "type": "keyword"
                 },
+                "flow_id": {
+                    "type": "integer"
+                },
                 "flow_history": {
                     "type": "integer"
                 },
diff --git a/contacts/query.go b/contacts/query.go
index 50dee78..6f18c61 100644
--- a/contacts/query.go
+++ b/contacts/query.go
@@ -62,6 +62,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 		(
 			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
 		) AS flow,
+		current_flow_id AS flow_id,
 		(
 			SELECT array_to_json(array_agg(DISTINCT fr.flow_id))
 			FROM flows_flowrun fr WHERE fr.contact_id = contacts_contact.id
diff --git a/indexer_test.go b/indexer_test.go
index 8957363..cfb0c72 100644
--- a/indexer_test.go
+++ b/indexer_test.go
@@ -104,6 +104,9 @@ func TestIndexing(t *testing.T) {
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "6d3cf1eb-546e-4fb8-a5ca-69187648fbf6"), []int64{2, 3})
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4})
 
+	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow_id", 1), []int64{2, 3})
+	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow_id", 2), []int64{4})
+
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow_history", 1), []int64{1, 2})
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow_history", 2), []int64{1})
 

From 93c72b05f1cdd9a043c05fd4a7abecfb1ee6da84 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 23 Mar 2022 10:30:30 -0500
Subject: [PATCH 12/48] Add Indexer interface and ContactIndexer implementation

---
 base.go                |  34 ++++++++++
 cmd/rp-indexer/main.go | 140 +++++++++++++++++++++++------------------
 go.mod                 |   1 +
 indexer.go             |   4 +-
 indexer_test.go        |   7 ++-
 5 files changed, 120 insertions(+), 66 deletions(-)
 create mode 100644 base.go

diff --git a/base.go b/base.go
new file mode 100644
index 0000000..67ff6b7
--- /dev/null
+++ b/base.go
@@ -0,0 +1,34 @@
+package indexer
+
+import (
+	"database/sql"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Indexer is base interface for indexers
+type Indexer interface {
+	Index(db *sql.DB) error
+}
+
+type BaseIndexer struct {
+	ElasticURL string
+	IndexName  string // e.g. contacts
+	Rebuild    bool   // whether indexer should rebuild entire index in one pass
+	Cleanup    bool   // whether after rebuilding, indexer should cleanup old indexes
+
+	// statistics
+	indexedTotal int64
+	deletedTotal int64
+	elapsedTotal time.Duration
+}
+
+// UpdateStats updates statistics for this indexer
+func (i *BaseIndexer) UpdateStats(indexed, deleted int, elapsed time.Duration) {
+	i.indexedTotal += int64(indexed)
+	i.deletedTotal += int64(deleted)
+	i.elapsedTotal += elapsed
+
+	logrus.WithField("index", i.IndexName).WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
+}
diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index 7129c06..323e43b 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -9,6 +9,8 @@ import (
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/ezconf"
 	indexer "github.com/nyaruka/rp-indexer"
+	"github.com/nyaruka/rp-indexer/contacts"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -54,7 +56,7 @@ func main() {
 		hook.StacktraceConfiguration.Skip = 4
 		hook.StacktraceConfiguration.Context = 5
 		if err != nil {
-			log.Fatalf("Invalid sentry DSN: '%s': %s", config.SentryDSN, err)
+			log.Fatalf("invalid sentry DSN: '%s': %s", config.SentryDSN, err)
 		}
 		log.StandardLogger().Hooks.Add(hook)
 	}
@@ -64,66 +66,20 @@ func main() {
 		log.Fatal(err)
 	}
 
-	for {
-		// find our physical index
-		physicalIndexes := indexer.FindPhysicalIndexes(config.ElasticURL, config.Index)
-		log.WithField("physicalIndexes", physicalIndexes).WithField("index", config.Index).Debug("found physical indexes")
-
-		physicalIndex := ""
-		if len(physicalIndexes) > 0 {
-			physicalIndex = physicalIndexes[0]
-		}
-
-		// whether we need to remap our alias after building
-		remapAlias := false
-
-		// doesn't exist or we are rebuilding, create it
-		if physicalIndex == "" || config.Rebuild {
-			physicalIndex, err = indexer.CreateNewIndex(config.ElasticURL, config.Index)
-			if err != nil {
-				logError(config.Rebuild, err, "error creating new index")
-				continue
-			}
-			log.WithField("index", config.Index).WithField("physicalIndex", physicalIndex).Info("created new physical index")
-			remapAlias = true
-		}
-
-		lastModified, err := indexer.GetLastModified(config.ElasticURL, physicalIndex)
-		if err != nil {
-			logError(config.Rebuild, err, "error finding last modified")
-			continue
-		}
+	ci := NewContactIndexer(config.ElasticURL, config.Index, config.Rebuild, config.Cleanup)
 
-		start := time.Now()
-		log.WithField("last_modified", lastModified).WithField("index", physicalIndex).Info("indexing contacts newer than last modified")
+	for {
+		err := ci.Index(db)
 
-		// now index our docs
-		indexed, deleted, err := indexer.IndexContacts(db, config.ElasticURL, physicalIndex, lastModified.Add(-5*time.Second))
 		if err != nil {
-			logError(config.Rebuild, err, "error indexing contacts")
-			continue
-		}
-		log.WithField("added", indexed).WithField("deleted", deleted).WithField("index", physicalIndex).WithField("elapsed", time.Now().Sub(start)).Info("completed indexing")
-
-		// if the index didn't previously exist or we are rebuilding, remap to our alias
-		if remapAlias {
-			err := indexer.MapIndexAlias(config.ElasticURL, config.Index, physicalIndex)
-			if err != nil {
-				logError(config.Rebuild, err, "error remapping alias")
-				continue
-			}
-			remapAlias = false
-		}
-
-		// cleanup our aliases if appropriate
-		if config.Cleanup {
-			err := indexer.CleanupIndexes(config.ElasticURL, config.Index)
-			if err != nil {
-				logError(config.Rebuild, err, "error cleaning up aliases")
-				continue
+			if config.Rebuild {
+				log.WithField("index", config.Index).WithError(err).Fatal("error during rebuilding")
+			} else {
+				log.WithField("index", config.Index).WithError(err).Error("error during indexing")
 			}
 		}
 
+		// if we were rebuilding then we're done
 		if config.Rebuild {
 			os.Exit(0)
 		}
@@ -133,11 +89,73 @@ func main() {
 	}
 }
 
-func logError(fatal bool, err error, msg string) {
-	if fatal {
-		log.WithError(err).Fatal(msg)
-	} else {
-		log.WithError(err).Error(msg)
-		time.Sleep(time.Second * 5)
+type ContactIndexer struct {
+	indexer.BaseIndexer
+}
+
+func NewContactIndexer(elasticURL, indexName string, rebuild, cleanup bool) indexer.Indexer {
+	return &ContactIndexer{
+		BaseIndexer: indexer.BaseIndexer{ElasticURL: elasticURL, IndexName: indexName, Rebuild: rebuild, Cleanup: cleanup},
+	}
+}
+
+func (i *ContactIndexer) Index(db *sql.DB) error {
+	var err error
+
+	// find our physical index
+	physicalIndexes := indexer.FindPhysicalIndexes(i.ElasticURL, i.IndexName)
+	log.WithField("physicalIndexes", physicalIndexes).WithField("index", i.IndexName).Debug("found physical indexes")
+
+	physicalIndex := ""
+	if len(physicalIndexes) > 0 {
+		physicalIndex = physicalIndexes[0]
 	}
+
+	// whether we need to remap our alias after building
+	remapAlias := false
+
+	// doesn't exist or we are rebuilding, create it
+	if physicalIndex == "" || i.Rebuild {
+		physicalIndex, err = indexer.CreateNewIndex(i.ElasticURL, i.IndexName, contacts.IndexSettings)
+		if err != nil {
+			return errors.Wrap(err, "error creating new index")
+		}
+		log.WithField("index", i.IndexName).WithField("physicalIndex", physicalIndex).Info("created new physical index")
+		remapAlias = true
+	}
+
+	lastModified, err := indexer.GetLastModified(i.ElasticURL, physicalIndex)
+	if err != nil {
+		return errors.Wrap(err, "error finding last modified")
+	}
+
+	log.WithField("last_modified", lastModified).WithField("index", physicalIndex).Info("indexing newer than last modified")
+
+	// now index our docs
+	start := time.Now()
+	indexed, deleted, err := indexer.IndexContacts(db, i.ElasticURL, physicalIndex, lastModified.Add(-5*time.Second))
+	if err != nil {
+		return errors.Wrap(err, "error indexing documents")
+	}
+
+	i.UpdateStats(indexed, deleted, time.Since(start))
+
+	// if the index didn't previously exist or we are rebuilding, remap to our alias
+	if remapAlias {
+		err := indexer.MapIndexAlias(i.ElasticURL, i.IndexName, physicalIndex)
+		if err != nil {
+			return errors.Wrap(err, "error remapping alias")
+		}
+		remapAlias = false
+	}
+
+	// cleanup our aliases if appropriate
+	if i.Cleanup {
+		err := indexer.CleanupIndexes(i.ElasticURL, i.IndexName)
+		if err != nil {
+			return errors.Wrap(err, "error cleaning up old indexes")
+		}
+	}
+
+	return nil
 }
diff --git a/go.mod b/go.mod
index c40cbc7..da1e7ab 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/nyaruka/ezconf v0.2.1
 	github.com/nyaruka/gocommon v1.3.0
 	github.com/olivere/elastic/v7 v7.0.22
+	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.5.1
 	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
diff --git a/indexer.go b/indexer.go
index ce242f1..5a3f5c1 100644
--- a/indexer.go
+++ b/indexer.go
@@ -24,7 +24,7 @@ var batchSize = 500
 // that index to `contacts`.
 //
 // If the day-specific name already exists, we append a .1 or .2 to the name.
-func CreateNewIndex(url string, alias string) (string, error) {
+func CreateNewIndex(url, alias string, settings json.RawMessage) (string, error) {
 	// create our day-specific name
 	physicalIndex := fmt.Sprintf("%s_%s", alias, time.Now().Format("2006_01_02"))
 	idx := 0
@@ -47,7 +47,7 @@ func CreateNewIndex(url string, alias string) (string, error) {
 
 	// initialize our index
 	createURL := fmt.Sprintf("%s/%s?include_type_name=true", url, physicalIndex)
-	_, err := MakeJSONRequest(http.MethodPut, createURL, contacts.IndexSettings, nil)
+	_, err := MakeJSONRequest(http.MethodPut, createURL, settings, nil)
 	if err != nil {
 		return "", err
 	}
diff --git a/indexer_test.go b/indexer_test.go
index 1a5af69..5c0cc10 100644
--- a/indexer_test.go
+++ b/indexer_test.go
@@ -13,6 +13,7 @@ import (
 	"time"
 
 	_ "github.com/lib/pq"
+	"github.com/nyaruka/rp-indexer/contacts"
 	"github.com/olivere/elastic/v7"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
@@ -58,11 +59,11 @@ func assertQuery(t *testing.T, client *elastic.Client, index string, query elast
 	}
 }
 
-func TestIndexing(t *testing.T) {
+func TestContacts(t *testing.T) {
 	batchSize = 4
 	db, client := setup(t)
 
-	physicalName, err := CreateNewIndex(elasticURL, indexName)
+	physicalName, err := CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
 	assert.NoError(t, err)
 
 	added, deleted, err := IndexContacts(db, elasticURL, physicalName, time.Time{})
@@ -253,7 +254,7 @@ func TestIndexing(t *testing.T) {
 	assert.Equal(t, physicalName, physical[0])
 
 	// rebuild again
-	newIndex, err := CreateNewIndex(elasticURL, indexName)
+	newIndex, err := CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
 	assert.NoError(t, err)
 
 	added, deleted, err = IndexContacts(db, elasticURL, newIndex, time.Time{})

From 133fbde3d860e15d27742f0d2264e8885c4786f5 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 23 Mar 2022 11:21:22 -0500
Subject: [PATCH 13/48] Rework dependencies so that contacts package import
 indexer

---
 base.go                                     |  17 +-
 base_test.go                                |  92 ++++++
 cmd/rp-indexer/main.go                      |  75 +----
 contacts/indexer.go                         | 235 +++++++++++++++
 indexer_test.go => contacts/indexer_test.go | 121 ++------
 contacts/settings.go                        |  10 -
 elastic.go                                  |   2 +-
 indexer.go                                  | 309 --------------------
 indexes.go                                  | 158 ++++++++++
 testdb_update.sql                           |   8 -
 10 files changed, 520 insertions(+), 507 deletions(-)
 create mode 100644 base_test.go
 create mode 100644 contacts/indexer.go
 rename indexer_test.go => contacts/indexer_test.go (79%)
 delete mode 100644 contacts/settings.go
 delete mode 100644 indexer.go
 create mode 100644 indexes.go
 delete mode 100644 testdb_update.sql

diff --git a/base.go b/base.go
index 67ff6b7..34f8e1e 100644
--- a/base.go
+++ b/base.go
@@ -9,14 +9,15 @@ import (
 
 // Indexer is base interface for indexers
 type Indexer interface {
+	Name() string
 	Index(db *sql.DB) error
 }
 
 type BaseIndexer struct {
+	name       string // e.g. contacts, used as based index name
 	ElasticURL string
-	IndexName  string // e.g. contacts
-	Rebuild    bool   // whether indexer should rebuild entire index in one pass
-	Cleanup    bool   // whether after rebuilding, indexer should cleanup old indexes
+	Rebuild    bool // whether indexer should rebuild entire index in one pass
+	Cleanup    bool // whether after rebuilding, indexer should cleanup old indexes
 
 	// statistics
 	indexedTotal int64
@@ -24,11 +25,19 @@ type BaseIndexer struct {
 	elapsedTotal time.Duration
 }
 
+func NewBaseIndexer(name, elasticURL string, rebuild, cleanup bool) BaseIndexer {
+	return BaseIndexer{name: name, ElasticURL: elasticURL, Rebuild: rebuild, Cleanup: cleanup}
+}
+
+func (i *BaseIndexer) Name() string {
+	return i.name
+}
+
 // UpdateStats updates statistics for this indexer
 func (i *BaseIndexer) UpdateStats(indexed, deleted int, elapsed time.Duration) {
 	i.indexedTotal += int64(indexed)
 	i.deletedTotal += int64(deleted)
 	i.elapsedTotal += elapsed
 
-	logrus.WithField("index", i.IndexName).WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
+	logrus.WithField("indexer", i.name).WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
 }
diff --git a/base_test.go b/base_test.go
new file mode 100644
index 0000000..f7edfbe
--- /dev/null
+++ b/base_test.go
@@ -0,0 +1,92 @@
+package indexer_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	indexer "github.com/nyaruka/rp-indexer"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRetryServer(t *testing.T) {
+	responseCounter := 0
+	responses := []func(w http.ResponseWriter, r *http.Request){
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Length", "5")
+		},
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Length", "1")
+		},
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Length", "1")
+		},
+		func(w http.ResponseWriter, r *http.Request) {
+			resp := `{
+				"took": 1,
+				"timed_out": false,
+				"_shards": {
+				  "total": 2,
+				  "successful": 2,
+				  "skipped": 0,
+				  "failed": 0
+				},
+				"hits": {
+				  "total": 1,
+				  "max_score": null,
+				  "hits": [
+					{
+					  "_index": "rp_elastic_test_2020_08_14_1",
+					  "_type": "_doc",
+					  "_id": "1",
+					  "_score": null,
+					  "_routing": "1",
+					  "_source": {
+						"id": 1,
+						"org_id": 1,
+						"uuid": "c7a2dd87-a80e-420b-8431-ca48d422e924",
+						"name": null,
+						"language": "eng",
+						"is_active": true,
+						"created_on": "2017-11-10T16:11:59.890662-05:00",
+						"modified_on": "2017-11-10T16:11:59.890662-05:00",
+						"last_seen_on": "2020-08-04T21:11:00-04:00",
+						"modified_on_mu": 1.510348319890662e15,
+						"urns": [
+						  {
+							"scheme": "tel",
+							"path": "+12067791111"
+						  },
+						  {
+							"scheme": "tel",
+							"path": "+12067792222"
+						  }
+						],
+						"fields": [
+						  {
+							"text": "the rock",
+							"field": "17103bb1-1b48-4b70-92f7-1f6b73bd3488"
+						  }
+						],
+						"groups": [
+						  "4ea0f313-2f62-4e57-bdf0-232b5191dd57",
+						  "529bac39-550a-4d6f-817c-1833f3449007"
+						]
+					  },
+					  "sort": [1]
+					}
+				  ]
+				}
+			  }`
+
+			w.Write([]byte(resp))
+		},
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		responses[responseCounter](w, r)
+		responseCounter++
+	}))
+	defer ts.Close()
+	indexer.FindPhysicalIndexes(ts.URL, "rp_elastic_test")
+	require.Equal(t, responseCounter, 4)
+}
diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index 323e43b..cbacc68 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -8,9 +8,7 @@ import (
 	"github.com/evalphobia/logrus_sentry"
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/ezconf"
-	indexer "github.com/nyaruka/rp-indexer"
 	"github.com/nyaruka/rp-indexer/contacts"
-	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -66,7 +64,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	ci := NewContactIndexer(config.ElasticURL, config.Index, config.Rebuild, config.Cleanup)
+	ci := contacts.NewIndexer(config.ElasticURL, config.Index, config.Rebuild, config.Cleanup)
 
 	for {
 		err := ci.Index(db)
@@ -88,74 +86,3 @@ func main() {
 		time.Sleep(time.Second * 5)
 	}
 }
-
-type ContactIndexer struct {
-	indexer.BaseIndexer
-}
-
-func NewContactIndexer(elasticURL, indexName string, rebuild, cleanup bool) indexer.Indexer {
-	return &ContactIndexer{
-		BaseIndexer: indexer.BaseIndexer{ElasticURL: elasticURL, IndexName: indexName, Rebuild: rebuild, Cleanup: cleanup},
-	}
-}
-
-func (i *ContactIndexer) Index(db *sql.DB) error {
-	var err error
-
-	// find our physical index
-	physicalIndexes := indexer.FindPhysicalIndexes(i.ElasticURL, i.IndexName)
-	log.WithField("physicalIndexes", physicalIndexes).WithField("index", i.IndexName).Debug("found physical indexes")
-
-	physicalIndex := ""
-	if len(physicalIndexes) > 0 {
-		physicalIndex = physicalIndexes[0]
-	}
-
-	// whether we need to remap our alias after building
-	remapAlias := false
-
-	// doesn't exist or we are rebuilding, create it
-	if physicalIndex == "" || i.Rebuild {
-		physicalIndex, err = indexer.CreateNewIndex(i.ElasticURL, i.IndexName, contacts.IndexSettings)
-		if err != nil {
-			return errors.Wrap(err, "error creating new index")
-		}
-		log.WithField("index", i.IndexName).WithField("physicalIndex", physicalIndex).Info("created new physical index")
-		remapAlias = true
-	}
-
-	lastModified, err := indexer.GetLastModified(i.ElasticURL, physicalIndex)
-	if err != nil {
-		return errors.Wrap(err, "error finding last modified")
-	}
-
-	log.WithField("last_modified", lastModified).WithField("index", physicalIndex).Info("indexing newer than last modified")
-
-	// now index our docs
-	start := time.Now()
-	indexed, deleted, err := indexer.IndexContacts(db, i.ElasticURL, physicalIndex, lastModified.Add(-5*time.Second))
-	if err != nil {
-		return errors.Wrap(err, "error indexing documents")
-	}
-
-	i.UpdateStats(indexed, deleted, time.Since(start))
-
-	// if the index didn't previously exist or we are rebuilding, remap to our alias
-	if remapAlias {
-		err := indexer.MapIndexAlias(i.ElasticURL, i.IndexName, physicalIndex)
-		if err != nil {
-			return errors.Wrap(err, "error remapping alias")
-		}
-		remapAlias = false
-	}
-
-	// cleanup our aliases if appropriate
-	if i.Cleanup {
-		err := indexer.CleanupIndexes(i.ElasticURL, i.IndexName)
-		if err != nil {
-			return errors.Wrap(err, "error cleaning up old indexes")
-		}
-	}
-
-	return nil
-}
diff --git a/contacts/indexer.go b/contacts/indexer.go
new file mode 100644
index 0000000..654556b
--- /dev/null
+++ b/contacts/indexer.go
@@ -0,0 +1,235 @@
+package contacts
+
+import (
+	"bytes"
+	"database/sql"
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	indexer "github.com/nyaruka/rp-indexer"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var BatchSize = 500
+
+// settings and mappings for our index
+//go:embed index_settings.json
+var IndexSettings json.RawMessage
+
+// indexes a contact
+const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
+
+// deletes a contact
+const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
+
+type ContactIndexer struct {
+	indexer.BaseIndexer
+}
+
+func NewIndexer(name, elasticURL string, rebuild, cleanup bool) indexer.Indexer {
+	return &ContactIndexer{
+		BaseIndexer: indexer.NewBaseIndexer(name, elasticURL, rebuild, cleanup),
+	}
+}
+
+func (i *ContactIndexer) Index(db *sql.DB) error {
+	var err error
+
+	// find our physical index
+	physicalIndexes := indexer.FindPhysicalIndexes(i.ElasticURL, i.Name())
+	logrus.WithField("physicalIndexes", physicalIndexes).WithField("indexer", i.Name()).Debug("found physical indexes")
+
+	physicalIndex := ""
+	if len(physicalIndexes) > 0 {
+		physicalIndex = physicalIndexes[0]
+	}
+
+	// whether we need to remap our alias after building
+	remapAlias := false
+
+	// doesn't exist or we are rebuilding, create it
+	if physicalIndex == "" || i.Rebuild {
+		physicalIndex, err = indexer.CreateNewIndex(i.ElasticURL, i.Name(), IndexSettings)
+		if err != nil {
+			return errors.Wrap(err, "error creating new index")
+		}
+		logrus.WithField("indexer", i.Name()).WithField("physicalIndex", physicalIndex).Info("created new physical index")
+		remapAlias = true
+	}
+
+	lastModified, err := indexer.GetLastModified(i.ElasticURL, physicalIndex)
+	if err != nil {
+		return errors.Wrap(err, "error finding last modified")
+	}
+
+	logrus.WithField("last_modified", lastModified).WithField("index", physicalIndex).Info("indexing newer than last modified")
+
+	// now index our docs
+	start := time.Now()
+	indexed, deleted, err := IndexModified(db, i.ElasticURL, physicalIndex, lastModified.Add(-5*time.Second))
+	if err != nil {
+		return errors.Wrap(err, "error indexing documents")
+	}
+
+	i.UpdateStats(indexed, deleted, time.Since(start))
+
+	// if the index didn't previously exist or we are rebuilding, remap to our alias
+	if remapAlias {
+		err := indexer.MapIndexAlias(i.ElasticURL, i.Name(), physicalIndex)
+		if err != nil {
+			return errors.Wrap(err, "error remapping alias")
+		}
+		remapAlias = false
+	}
+
+	// cleanup our aliases if appropriate
+	if i.Cleanup {
+		err := indexer.CleanupIndexes(i.ElasticURL, i.Name())
+		if err != nil {
+			return errors.Wrap(err, "error cleaning up old indexes")
+		}
+	}
+
+	return nil
+}
+
+// IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time
+func IndexModified(db *sql.DB, elasticURL string, index string, lastModified time.Time) (int, int, error) {
+	batch := &bytes.Buffer{}
+	createdCount, deletedCount, processedCount := 0, 0, 0
+
+	if index == "" {
+		return 0, 0, errors.New("empty index")
+	}
+
+	var modifiedOn time.Time
+	var contactJSON string
+	var id, orgID int64
+	var isActive bool
+
+	start := time.Now()
+
+	for {
+		rows, err := FetchModified(db, lastModified)
+
+		queryCreated := 0
+		queryCount := 0
+		queryModified := lastModified
+
+		// no more rows? return
+		if err == sql.ErrNoRows {
+			return 0, 0, nil
+		}
+		if err != nil {
+			return 0, 0, err
+		}
+		defer rows.Close()
+
+		for rows.Next() {
+			err = rows.Scan(&orgID, &id, &modifiedOn, &isActive, &contactJSON)
+			if err != nil {
+				return 0, 0, err
+			}
+
+			queryCount++
+			processedCount++
+			lastModified = modifiedOn
+
+			if isActive {
+				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Debug("modified contact")
+				batch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID))
+				batch.WriteString("\n")
+				batch.WriteString(contactJSON)
+				batch.WriteString("\n")
+			} else {
+				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Debug("deleted contact")
+				batch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID))
+				batch.WriteString("\n")
+			}
+
+			// write to elastic search in batches
+			if queryCount%BatchSize == 0 {
+				created, deleted, err := indexBatch(elasticURL, index, batch.Bytes())
+				if err != nil {
+					return 0, 0, err
+				}
+				batch.Reset()
+
+				queryCreated += created
+				createdCount += created
+				deletedCount += deleted
+			}
+		}
+
+		if batch.Len() > 0 {
+			created, deleted, err := indexBatch(elasticURL, index, batch.Bytes())
+			if err != nil {
+				return 0, 0, err
+			}
+
+			queryCreated += created
+			createdCount += created
+			deletedCount += deleted
+			batch.Reset()
+		}
+
+		// last modified stayed the same and we didn't add anything, seen it all, break out
+		if lastModified.Equal(queryModified) && queryCreated == 0 {
+			break
+		}
+
+		elapsed := time.Since(start)
+		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
+		logrus.WithFields(map[string]interface{}{
+			"rate":    int(rate),
+			"added":   createdCount,
+			"deleted": deletedCount,
+			"elapsed": elapsed,
+			"index":   index}).Info("updated contact index")
+
+		rows.Close()
+	}
+
+	return createdCount, deletedCount, nil
+}
+
+// indexes the batch of contacts
+func indexBatch(elasticURL string, index string, batch []byte) (int, int, error) {
+	response := indexer.IndexResponse{}
+	indexURL := fmt.Sprintf("%s/%s/_bulk", elasticURL, index)
+
+	_, err := indexer.MakeJSONRequest(http.MethodPut, indexURL, batch, &response)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	createdCount, deletedCount, conflictedCount := 0, 0, 0
+	for _, item := range response.Items {
+		if item.Index.ID != "" {
+			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("index response")
+			if item.Index.Status == 200 || item.Index.Status == 201 {
+				createdCount++
+			} else if item.Index.Status == 409 {
+				conflictedCount++
+			} else {
+				logrus.WithField("id", item.Index.ID).WithField("batch", batch).WithField("result", item.Index.Result).Error("error indexing contact")
+			}
+		} else if item.Delete.ID != "" {
+			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response")
+			if item.Delete.Status == 200 {
+				deletedCount++
+			} else if item.Delete.Status == 409 {
+				conflictedCount++
+			}
+		} else {
+			logrus.Error("unparsed item in response")
+		}
+	}
+	logrus.WithField("created", createdCount).WithField("deleted", deletedCount).WithField("conflicted", conflictedCount).Debug("indexed batch")
+
+	return createdCount, deletedCount, nil
+}
diff --git a/indexer_test.go b/contacts/indexer_test.go
similarity index 79%
rename from indexer_test.go
rename to contacts/indexer_test.go
index 5c0cc10..6e08a1c 100644
--- a/indexer_test.go
+++ b/contacts/indexer_test.go
@@ -1,4 +1,4 @@
-package indexer
+package contacts_test
 
 import (
 	"context"
@@ -7,12 +7,12 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"net/http/httptest"
 	"os"
 	"testing"
 	"time"
 
 	_ "github.com/lib/pq"
+	indexer "github.com/nyaruka/rp-indexer"
 	"github.com/nyaruka/rp-indexer/contacts"
 	"github.com/olivere/elastic/v7"
 	"github.com/sirupsen/logrus"
@@ -24,7 +24,7 @@ const elasticURL = "http://localhost:9200"
 const indexName = "rp_elastic_test"
 
 func setup(t *testing.T) (*sql.DB, *elastic.Client) {
-	testDB, err := ioutil.ReadFile("testdb.sql")
+	testDB, err := ioutil.ReadFile("../testdb.sql")
 	require.NoError(t, err)
 
 	db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")
@@ -36,7 +36,7 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) {
 	client, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
 	require.NoError(t, err)
 
-	existing := FindPhysicalIndexes(elasticURL, indexName)
+	existing := indexer.FindPhysicalIndexes(elasticURL, indexName)
 	for _, idx := range existing {
 		_, err = client.DeleteIndex(idx).Do(context.Background())
 		require.NoError(t, err)
@@ -59,14 +59,14 @@ func assertQuery(t *testing.T, client *elastic.Client, index string, query elast
 	}
 }
 
-func TestContacts(t *testing.T) {
-	batchSize = 4
+func TestIndexing(t *testing.T) {
+	contacts.BatchSize = 4
 	db, client := setup(t)
 
-	physicalName, err := CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
+	physicalName, err := indexer.CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
 	assert.NoError(t, err)
 
-	added, deleted, err := IndexContacts(db, elasticURL, physicalName, time.Time{})
+	added, deleted, err := contacts.IndexModified(db, elasticURL, physicalName, time.Time{})
 	assert.NoError(t, err)
 	assert.Equal(t, 9, added)
 	assert.Equal(t, 0, deleted)
@@ -237,12 +237,12 @@ func TestContacts(t *testing.T) {
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2})
 	assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{})
 
-	lastModified, err := GetLastModified(elasticURL, physicalName)
+	lastModified, err := indexer.GetLastModified(elasticURL, physicalName)
 	assert.NoError(t, err)
 	assert.Equal(t, time.Date(2017, 11, 10, 21, 11, 59, 890662000, time.UTC), lastModified.In(time.UTC))
 
 	// map our index over
-	err = MapIndexAlias(elasticURL, indexName, physicalName)
+	err = indexer.MapIndexAlias(elasticURL, indexName, physicalName)
 	assert.NoError(t, err)
 	time.Sleep(5 * time.Second)
 
@@ -250,20 +250,20 @@ func TestContacts(t *testing.T) {
 	assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{4})
 
 	// look up our mapping
-	physical := FindPhysicalIndexes(elasticURL, indexName)
+	physical := indexer.FindPhysicalIndexes(elasticURL, indexName)
 	assert.Equal(t, physicalName, physical[0])
 
 	// rebuild again
-	newIndex, err := CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
+	newIndex, err := indexer.CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
 	assert.NoError(t, err)
 
-	added, deleted, err = IndexContacts(db, elasticURL, newIndex, time.Time{})
+	added, deleted, err = contacts.IndexModified(db, elasticURL, newIndex, time.Time{})
 	assert.NoError(t, err)
 	assert.Equal(t, 9, added)
 	assert.Equal(t, 0, deleted)
 
 	// remap again
-	err = MapIndexAlias(elasticURL, indexName, newIndex)
+	err = indexer.MapIndexAlias(elasticURL, indexName, newIndex)
 	assert.NoError(t, err)
 	time.Sleep(5 * time.Second)
 
@@ -273,7 +273,7 @@ func TestContacts(t *testing.T) {
 	assert.Equal(t, resp.StatusCode, http.StatusOK)
 
 	// cleanup our indexes, will remove our original index
-	err = CleanupIndexes(elasticURL, indexName)
+	err = indexer.CleanupIndexes(elasticURL, indexName)
 	assert.NoError(t, err)
 
 	// old physical index should be gone
@@ -285,12 +285,13 @@ func TestContacts(t *testing.T) {
 	assertQuery(t, client, newIndex, elastic.NewMatchQuery("name", "john"), []int64{4})
 
 	// update our database, removing one contact, updating another
-	dbUpdate, err := ioutil.ReadFile("testdb_update.sql")
-	assert.NoError(t, err)
-	_, err = db.Exec(string(dbUpdate))
+	_, err = db.Exec(`
+	DELETE FROM contacts_contactgroup_contacts WHERE id = 3;
+	UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 2;
+	UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;`)
 	assert.NoError(t, err)
 
-	added, deleted, err = IndexContacts(db, elasticURL, indexName, lastModified)
+	added, deleted, err = contacts.IndexModified(db, elasticURL, indexName, lastModified)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, added)
 	assert.Equal(t, 1, deleted)
@@ -302,86 +303,4 @@ func TestContacts(t *testing.T) {
 
 	// 3 is no longer in our group
 	assertQuery(t, client, indexName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1})
-
-}
-func TestRetryServer(t *testing.T) {
-	responseCounter := 0
-	responses := []func(w http.ResponseWriter, r *http.Request){
-		func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Length", "5")
-		},
-		func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Length", "1")
-		},
-		func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Length", "1")
-		},
-		func(w http.ResponseWriter, r *http.Request) {
-			resp := `{
-				"took": 1,
-				"timed_out": false,
-				"_shards": {
-				  "total": 2,
-				  "successful": 2,
-				  "skipped": 0,
-				  "failed": 0
-				},
-				"hits": {
-				  "total": 1,
-				  "max_score": null,
-				  "hits": [
-					{
-					  "_index": "rp_elastic_test_2020_08_14_1",
-					  "_type": "_doc",
-					  "_id": "1",
-					  "_score": null,
-					  "_routing": "1",
-					  "_source": {
-						"id": 1,
-						"org_id": 1,
-						"uuid": "c7a2dd87-a80e-420b-8431-ca48d422e924",
-						"name": null,
-						"language": "eng",
-						"is_active": true,
-						"created_on": "2017-11-10T16:11:59.890662-05:00",
-						"modified_on": "2017-11-10T16:11:59.890662-05:00",
-						"last_seen_on": "2020-08-04T21:11:00-04:00",
-						"modified_on_mu": 1.510348319890662e15,
-						"urns": [
-						  {
-							"scheme": "tel",
-							"path": "+12067791111"
-						  },
-						  {
-							"scheme": "tel",
-							"path": "+12067792222"
-						  }
-						],
-						"fields": [
-						  {
-							"text": "the rock",
-							"field": "17103bb1-1b48-4b70-92f7-1f6b73bd3488"
-						  }
-						],
-						"groups": [
-						  "4ea0f313-2f62-4e57-bdf0-232b5191dd57",
-						  "529bac39-550a-4d6f-817c-1833f3449007"
-						]
-					  },
-					  "sort": [1]
-					}
-				  ]
-				}
-			  }`
-
-			w.Write([]byte(resp))
-		},
-	}
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		responses[responseCounter](w, r)
-		responseCounter++
-	}))
-	defer ts.Close()
-	FindPhysicalIndexes(ts.URL, "rp_elastic_test")
-	require.Equal(t, responseCounter, 4)
 }
diff --git a/contacts/settings.go b/contacts/settings.go
deleted file mode 100644
index 4bb3608..0000000
--- a/contacts/settings.go
+++ /dev/null
@@ -1,10 +0,0 @@
-package contacts
-
-import (
-	_ "embed"
-	"encoding/json"
-)
-
-// settings and mappings for our index
-//go:embed index_settings.json
-var IndexSettings json.RawMessage
diff --git a/elastic.go b/elastic.go
index 02f8a23..b9fbcf8 100644
--- a/elastic.go
+++ b/elastic.go
@@ -125,7 +125,7 @@ type queryResponse struct {
 }
 
 // our response for indexing contacts
-type indexResponse struct {
+type IndexResponse struct {
 	Items []struct {
 		Index struct {
 			ID     string `json:"_id"`
diff --git a/indexer.go b/indexer.go
deleted file mode 100644
index 5a3f5c1..0000000
--- a/indexer.go
+++ /dev/null
@@ -1,309 +0,0 @@
-package indexer
-
-import (
-	"bytes"
-	"database/sql"
-	_ "embed"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"sort"
-	"strings"
-	"time"
-
-	"github.com/nyaruka/rp-indexer/contacts"
-	log "github.com/sirupsen/logrus"
-)
-
-var batchSize = 500
-
-// CreateNewIndex creates a new index for the passed in alias.
-//
-// Note that we do not create an index with the passed name, instead creating one
-// based on the day, for example `contacts_2018_03_05`, then create an alias from
-// that index to `contacts`.
-//
-// If the day-specific name already exists, we append a .1 or .2 to the name.
-func CreateNewIndex(url, alias string, settings json.RawMessage) (string, error) {
-	// create our day-specific name
-	physicalIndex := fmt.Sprintf("%s_%s", alias, time.Now().Format("2006_01_02"))
-	idx := 0
-
-	// check if it exists
-	for {
-		resp, err := http.Get(fmt.Sprintf("%s/%s", url, physicalIndex))
-		if err != nil {
-			return "", err
-		}
-		// not found, great, move on
-		if resp.StatusCode == http.StatusNotFound {
-			break
-		}
-
-		// was found, increase our index and try again
-		idx++
-		physicalIndex = fmt.Sprintf("%s_%s_%d", alias, time.Now().Format("2006_01_02"), idx)
-	}
-
-	// initialize our index
-	createURL := fmt.Sprintf("%s/%s?include_type_name=true", url, physicalIndex)
-	_, err := MakeJSONRequest(http.MethodPut, createURL, settings, nil)
-	if err != nil {
-		return "", err
-	}
-
-	// all went well, return our physical index name
-	log.WithField("index", physicalIndex).Info("created index")
-	return physicalIndex, nil
-}
-
-// GetLastModified queries our index and finds the last modified contact, returning it
-func GetLastModified(url string, index string) (time.Time, error) {
-	lastModified := time.Time{}
-	if index == "" {
-		return lastModified, fmt.Errorf("empty index passed to GetLastModified")
-	}
-
-	// get the newest document on our index
-	queryResponse := queryResponse{}
-	_, err := MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/%s/_search", url, index), lastModifiedQuery, &queryResponse)
-	if err != nil {
-		return lastModified, err
-	}
-
-	if len(queryResponse.Hits.Hits) > 0 {
-		lastModified = queryResponse.Hits.Hits[0].Source.ModifiedOn
-	}
-	return lastModified, nil
-}
-
-// FindPhysicalIndexes finds all the physical indexes for the passed in alias
-func FindPhysicalIndexes(url string, alias string) []string {
-	indexResponse := infoResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, alias), nil, &indexResponse)
-	indexes := make([]string, 0)
-
-	// error could mean a variety of things, but we'll figure that out later
-	if err != nil {
-		return indexes
-	}
-
-	// our top level key is our physical index name
-	for key := range indexResponse {
-		indexes = append(indexes, key)
-	}
-
-	// reverse sort order should put our newest index first
-	sort.Sort(sort.Reverse(sort.StringSlice(indexes)))
-	return indexes
-}
-
-// CleanupIndexes removes all indexes that are older than the currently active index
-func CleanupIndexes(url string, alias string) error {
-	// find our current indexes
-	currents := FindPhysicalIndexes(url, alias)
-
-	// no current indexes? this a noop
-	if len(currents) == 0 {
-		return nil
-	}
-
-	// find all the current indexes
-	healthResponse := healthResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, "_cluster/health?level=indices"), nil, &healthResponse)
-	if err != nil {
-		return err
-	}
-
-	// for each active index, if it starts with our alias but is before our current index, remove it
-	for key := range healthResponse.Indices {
-		if strings.HasPrefix(key, alias) && strings.Compare(key, currents[0]) < 0 {
-			log.WithField("index", key).Info("removing old index")
-			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", url, key), nil, nil)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// IndexBatch indexes the batch of contacts
-func IndexBatch(elasticURL string, index string, batch []byte) (int, int, error) {
-	response := indexResponse{}
-	indexURL := fmt.Sprintf("%s/%s/_bulk", elasticURL, index)
-
-	_, err := MakeJSONRequest(http.MethodPut, indexURL, batch, &response)
-	if err != nil {
-		return 0, 0, err
-	}
-
-	createdCount, deletedCount, conflictedCount := 0, 0, 0
-	for _, item := range response.Items {
-		if item.Index.ID != "" {
-			log.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("index response")
-			if item.Index.Status == 200 || item.Index.Status == 201 {
-				createdCount++
-			} else if item.Index.Status == 409 {
-				conflictedCount++
-			} else {
-				log.WithField("id", item.Index.ID).WithField("batch", batch).WithField("result", item.Index.Result).Error("error indexing contact")
-			}
-		} else if item.Delete.ID != "" {
-			log.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response")
-			if item.Delete.Status == 200 {
-				deletedCount++
-			} else if item.Delete.Status == 409 {
-				conflictedCount++
-			}
-		} else {
-			log.Error("unparsed item in response")
-		}
-	}
-	log.WithField("created", createdCount).WithField("deleted", deletedCount).WithField("conflicted", conflictedCount).Debug("indexed batch")
-
-	return createdCount, deletedCount, nil
-}
-
-// IndexContacts queries and indexes all contacts with a lastModified greater than or equal to the passed in time
-func IndexContacts(db *sql.DB, elasticURL string, index string, lastModified time.Time) (int, int, error) {
-	batch := &bytes.Buffer{}
-	createdCount, deletedCount, processedCount := 0, 0, 0
-
-	if index == "" {
-		return createdCount, deletedCount, fmt.Errorf("empty index passed to IndexContacts")
-	}
-
-	var modifiedOn time.Time
-	var contactJSON string
-	var id, orgID int64
-	var isActive bool
-
-	start := time.Now()
-
-	for {
-		rows, err := contacts.FetchModified(db, lastModified)
-
-		queryCreated := 0
-		queryCount := 0
-		queryModified := lastModified
-
-		// no more rows? return
-		if err == sql.ErrNoRows {
-			return 0, 0, nil
-		}
-		if err != nil {
-			return 0, 0, err
-		}
-		defer rows.Close()
-
-		for rows.Next() {
-			err = rows.Scan(&orgID, &id, &modifiedOn, &isActive, &contactJSON)
-			if err != nil {
-				return 0, 0, err
-			}
-
-			queryCount++
-			processedCount++
-			lastModified = modifiedOn
-
-			if isActive {
-				log.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Debug("modified contact")
-				batch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID))
-				batch.WriteString("\n")
-				batch.WriteString(contactJSON)
-				batch.WriteString("\n")
-			} else {
-				log.WithField("id", id).WithField("modifiedOn", modifiedOn).Debug("deleted contact")
-				batch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID))
-				batch.WriteString("\n")
-			}
-
-			// write to elastic search in batches
-			if queryCount%batchSize == 0 {
-				created, deleted, err := IndexBatch(elasticURL, index, batch.Bytes())
-				if err != nil {
-					return 0, 0, err
-				}
-				batch.Reset()
-
-				queryCreated += created
-				createdCount += created
-				deletedCount += deleted
-			}
-		}
-
-		if batch.Len() > 0 {
-			created, deleted, err := IndexBatch(elasticURL, index, batch.Bytes())
-			if err != nil {
-				return 0, 0, err
-			}
-
-			queryCreated += created
-			createdCount += created
-			deletedCount += deleted
-			batch.Reset()
-		}
-
-		// last modified stayed the same and we didn't add anything, seen it all, break out
-		if lastModified.Equal(queryModified) && queryCreated == 0 {
-			break
-		}
-
-		elapsed := time.Since(start)
-		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
-		log.WithFields(map[string]interface{}{
-			"rate":    int(rate),
-			"added":   createdCount,
-			"deleted": deletedCount,
-			"elapsed": elapsed,
-			"index":   index}).Info("updated contact index")
-
-		rows.Close()
-	}
-
-	return createdCount, deletedCount, nil
-}
-
-// MapIndexAlias maps the passed in alias to the new physical index, optionally removing
-// existing aliases if they exit.
-func MapIndexAlias(elasticURL string, alias string, newIndex string) error {
-	commands := make([]interface{}, 0)
-
-	// find existing physical indexes
-	existing := FindPhysicalIndexes(elasticURL, alias)
-	for _, idx := range existing {
-		remove := removeAliasCommand{}
-		remove.Remove.Alias = alias
-		remove.Remove.Index = idx
-		commands = append(commands, remove)
-
-		log.WithField("index", idx).WithField("alias", alias).Info("removing old alias")
-	}
-
-	// add our new index
-	add := addAliasCommand{}
-	add.Add.Alias = alias
-	add.Add.Index = newIndex
-	commands = append(commands, add)
-
-	log.WithField("index", newIndex).WithField("alias", alias).Info("adding new alias")
-
-	aliasURL := fmt.Sprintf("%s/_aliases", elasticURL)
-	aliasJSON, err := json.Marshal(aliasCommand{Actions: commands})
-	if err != nil {
-		return err
-	}
-	_, err = MakeJSONRequest(http.MethodPost, aliasURL, aliasJSON, nil)
-	return err
-}
-
-// gets our last modified contact
-var lastModifiedQuery = []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`)
-
-// indexes a contact
-const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
-
-// deletes a contact
-const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
diff --git a/indexes.go b/indexes.go
new file mode 100644
index 0000000..f51c16b
--- /dev/null
+++ b/indexes.go
@@ -0,0 +1,158 @@
+package indexer
+
+import (
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// CreateNewIndex creates a new index for the passed in alias.
+//
+// Note that we do not create an index with the passed name, instead creating one
+// based on the day, for example `contacts_2018_03_05`, then create an alias from
+// that index to `contacts`.
+//
+// If the day-specific name already exists, we append a .1 or .2 to the name.
+func CreateNewIndex(url, alias string, settings json.RawMessage) (string, error) {
+	// create our day-specific name
+	physicalIndex := fmt.Sprintf("%s_%s", alias, time.Now().Format("2006_01_02"))
+	idx := 0
+
+	// check if it exists
+	for {
+		resp, err := http.Get(fmt.Sprintf("%s/%s", url, physicalIndex))
+		if err != nil {
+			return "", err
+		}
+		// not found, great, move on
+		if resp.StatusCode == http.StatusNotFound {
+			break
+		}
+
+		// was found, increase our index and try again
+		idx++
+		physicalIndex = fmt.Sprintf("%s_%s_%d", alias, time.Now().Format("2006_01_02"), idx)
+	}
+
+	// initialize our index
+	createURL := fmt.Sprintf("%s/%s?include_type_name=true", url, physicalIndex)
+	_, err := MakeJSONRequest(http.MethodPut, createURL, settings, nil)
+	if err != nil {
+		return "", err
+	}
+
+	// all went well, return our physical index name
+	log.WithField("index", physicalIndex).Info("created index")
+	return physicalIndex, nil
+}
+
+// GetLastModified queries an index and finds the last modified document, returning its modified time
+func GetLastModified(url string, index string) (time.Time, error) {
+	lastModified := time.Time{}
+	if index == "" {
+		return lastModified, fmt.Errorf("empty index passed to GetLastModified")
+	}
+
+	// get the newest document on our index
+	queryResponse := queryResponse{}
+	_, err := MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/%s/_search", url, index), []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`), &queryResponse)
+	if err != nil {
+		return lastModified, err
+	}
+
+	if len(queryResponse.Hits.Hits) > 0 {
+		lastModified = queryResponse.Hits.Hits[0].Source.ModifiedOn
+	}
+	return lastModified, nil
+}
+
+// FindPhysicalIndexes finds all the physical indexes for the passed in alias
+func FindPhysicalIndexes(url string, alias string) []string {
+	indexResponse := infoResponse{}
+	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, alias), nil, &indexResponse)
+	indexes := make([]string, 0)
+
+	// error could mean a variety of things, but we'll figure that out later
+	if err != nil {
+		return indexes
+	}
+
+	// our top level key is our physical index name
+	for key := range indexResponse {
+		indexes = append(indexes, key)
+	}
+
+	// reverse sort order should put our newest index first
+	sort.Sort(sort.Reverse(sort.StringSlice(indexes)))
+	return indexes
+}
+
+// CleanupIndexes removes all indexes that are older than the currently active index
+func CleanupIndexes(url string, alias string) error {
+	// find our current indexes
+	currents := FindPhysicalIndexes(url, alias)
+
+	// no current indexes? this a noop
+	if len(currents) == 0 {
+		return nil
+	}
+
+	// find all the current indexes
+	healthResponse := healthResponse{}
+	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, "_cluster/health?level=indices"), nil, &healthResponse)
+	if err != nil {
+		return err
+	}
+
+	// for each active index, if it starts with our alias but is before our current index, remove it
+	for key := range healthResponse.Indices {
+		if strings.HasPrefix(key, alias) && strings.Compare(key, currents[0]) < 0 {
+			log.WithField("index", key).Info("removing old index")
+			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", url, key), nil, nil)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// MapIndexAlias maps the passed in alias to the new physical index, optionally removing
+// existing aliases if they exit.
+func MapIndexAlias(elasticURL string, alias string, newIndex string) error {
+	commands := make([]interface{}, 0)
+
+	// find existing physical indexes
+	existing := FindPhysicalIndexes(elasticURL, alias)
+	for _, idx := range existing {
+		remove := removeAliasCommand{}
+		remove.Remove.Alias = alias
+		remove.Remove.Index = idx
+		commands = append(commands, remove)
+
+		log.WithField("index", idx).WithField("alias", alias).Info("removing old alias")
+	}
+
+	// add our new index
+	add := addAliasCommand{}
+	add.Add.Alias = alias
+	add.Add.Index = newIndex
+	commands = append(commands, add)
+
+	log.WithField("index", newIndex).WithField("alias", alias).Info("adding new alias")
+
+	aliasURL := fmt.Sprintf("%s/_aliases", elasticURL)
+	aliasJSON, err := json.Marshal(aliasCommand{Actions: commands})
+	if err != nil {
+		return err
+	}
+	_, err = MakeJSONRequest(http.MethodPost, aliasURL, aliasJSON, nil)
+	return err
+}
diff --git a/testdb_update.sql b/testdb_update.sql
deleted file mode 100644
index e551846..0000000
--- a/testdb_update.sql
+++ /dev/null
@@ -1,8 +0,0 @@
--- update one of our contacts
-DELETE FROM contacts_contactgroup_contacts WHERE id = 3;
-UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 2;
-
--- delete one of our others
-UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;
-
-

From 05b073f5b57610c79f244eae1fa022c8c28f60ef Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 23 Mar 2022 12:38:23 -0500
Subject: [PATCH 14/48] Move all generic elastic functionality into BaseIndexer

---
 base.go                  | 231 +++++++++++++++++++++++++++++++++++++--
 base_test.go             |   7 +-
 cmd/rp-indexer/main.go   |   4 +-
 contacts/indexer.go      |  78 ++++---------
 contacts/indexer_test.go |  26 +++--
 go.mod                   |  28 +++--
 go.sum                   |  39 +++++--
 elastic.go => http.go    |  61 -----------
 indexes.go               | 141 +++---------------------
 9 files changed, 330 insertions(+), 285 deletions(-)
 rename elastic.go => http.go (64%)

diff --git a/base.go b/base.go
index 34f8e1e..397baf9 100644
--- a/base.go
+++ b/base.go
@@ -2,22 +2,26 @@ package indexer
 
 import (
 	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"sort"
+	"strings"
 	"time"
 
+	"github.com/nyaruka/gocommon/jsonx"
 	"github.com/sirupsen/logrus"
 )
 
 // Indexer is base interface for indexers
 type Indexer interface {
 	Name() string
-	Index(db *sql.DB) error
+	Index(db *sql.DB, rebuild, cleanup bool) error
 }
 
 type BaseIndexer struct {
 	name       string // e.g. contacts, used as based index name
 	ElasticURL string
-	Rebuild    bool // whether indexer should rebuild entire index in one pass
-	Cleanup    bool // whether after rebuilding, indexer should cleanup old indexes
 
 	// statistics
 	indexedTotal int64
@@ -25,19 +29,232 @@ type BaseIndexer struct {
 	elapsedTotal time.Duration
 }
 
-func NewBaseIndexer(name, elasticURL string, rebuild, cleanup bool) BaseIndexer {
-	return BaseIndexer{name: name, ElasticURL: elasticURL, Rebuild: rebuild, Cleanup: cleanup}
+func NewBaseIndexer(name, elasticURL string) BaseIndexer {
+	return BaseIndexer{name: name, ElasticURL: elasticURL}
 }
 
 func (i *BaseIndexer) Name() string {
 	return i.name
 }
 
-// UpdateStats updates statistics for this indexer
-func (i *BaseIndexer) UpdateStats(indexed, deleted int, elapsed time.Duration) {
+func (i *BaseIndexer) Stats() (int64, int64, time.Duration) {
+	return i.indexedTotal, i.deletedTotal, i.elapsedTotal
+}
+
+// RecordComplete records a complete index and updates statistics
+func (i *BaseIndexer) RecordComplete(indexed, deleted int, elapsed time.Duration) {
 	i.indexedTotal += int64(indexed)
 	i.deletedTotal += int64(deleted)
 	i.elapsedTotal += elapsed
 
 	logrus.WithField("indexer", i.name).WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
 }
+
+// our response for figuring out the physical index for an alias
+type infoResponse map[string]interface{}
+
+// FindPhysicalIndexes finds all our physical indexes
+func (i *BaseIndexer) FindPhysicalIndexes() []string {
+	response := infoResponse{}
+	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.ElasticURL, i.name), nil, &response)
+	indexes := make([]string, 0)
+
+	// error could mean a variety of things, but we'll figure that out later
+	if err != nil {
+		return indexes
+	}
+
+	// our top level key is our physical index name
+	for key := range response {
+		indexes = append(indexes, key)
+	}
+
+	// reverse sort order should put our newest index first
+	sort.Sort(sort.Reverse(sort.StringSlice(indexes)))
+
+	logrus.WithField("indexer", i.name).WithField("indexes", indexes).Debug("found physical indexes")
+
+	return indexes
+}
+
+// CreateNewIndex creates a new index for the passed in alias.
+//
+// Note that we do not create an index with the passed name, instead creating one
+// based on the day, for example `contacts_2018_03_05`, then create an alias from
+// that index to `contacts`.
+//
+// If the day-specific name already exists, we append a .1 or .2 to the name.
+func (i *BaseIndexer) CreateNewIndex(settings json.RawMessage) (string, error) {
+	// create our day-specific name
+	index := fmt.Sprintf("%s_%s", i.name, time.Now().Format("2006_01_02"))
+	idx := 0
+
+	// check if it exists
+	for {
+		resp, err := http.Get(fmt.Sprintf("%s/%s", i.ElasticURL, index))
+		if err != nil {
+			return "", err
+		}
+		// not found, great, move on
+		if resp.StatusCode == http.StatusNotFound {
+			break
+		}
+
+		// was found, increase our index and try again
+		idx++
+		index = fmt.Sprintf("%s_%s_%d", i.name, time.Now().Format("2006_01_02"), idx)
+	}
+
+	// create the new index
+	_, err := MakeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.ElasticURL, index), settings, nil)
+	if err != nil {
+		return "", err
+	}
+
+	// all went well, return our physical index name
+	logrus.WithField("indexer", i.name).WithField("index", index).Info("created new index")
+
+	return index, nil
+}
+
+// our top level command for remapping aliases
+type aliasCommand struct {
+	Actions []interface{} `json:"actions"`
+}
+
+// adds an alias for an index
+type addAliasCommand struct {
+	Add struct {
+		Index string `json:"index"`
+		Alias string `json:"alias"`
+	} `json:"add"`
+}
+
+// removes an alias for an index
+type removeAliasCommand struct {
+	Remove struct {
+		Index string `json:"index"`
+		Alias string `json:"alias"`
+	} `json:"remove"`
+}
+
+// UpdateAlias maps the passed in alias to the new physical index, optionally removing
+// existing aliases if they exit.
+func (i *BaseIndexer) UpdateAlias(newIndex string) error {
+	commands := make([]interface{}, 0)
+
+	// find existing physical indexes
+	existing := i.FindPhysicalIndexes()
+	for _, idx := range existing {
+		remove := removeAliasCommand{}
+		remove.Remove.Alias = i.name
+		remove.Remove.Index = idx
+		commands = append(commands, remove)
+
+		logrus.WithField("indexer", i.name).WithField("index", idx).Debug("removing old alias")
+	}
+
+	// add our new index
+	add := addAliasCommand{}
+	add.Add.Alias = i.name
+	add.Add.Index = newIndex
+	commands = append(commands, add)
+
+	aliasJSON := jsonx.MustMarshal(aliasCommand{Actions: commands})
+
+	_, err := MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/_aliases", i.ElasticURL), aliasJSON, nil)
+
+	logrus.WithField("indexer", i.name).WithField("index", newIndex).Debug("adding new alias")
+
+	return err
+}
+
+// our response for our index health
+type healthResponse struct {
+	Indices map[string]struct {
+		Status string `json:"status"`
+	} `json:"indices"`
+}
+
+// CleanupIndexes removes all indexes that are older than the currently active index
+func (i *BaseIndexer) CleanupIndexes() error {
+	// find our current indexes
+	currents := i.FindPhysicalIndexes()
+
+	// no current indexes? this a noop
+	if len(currents) == 0 {
+		return nil
+	}
+
+	// find all the current indexes
+	healthResponse := healthResponse{}
+	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.ElasticURL, "_cluster/health?level=indices"), nil, &healthResponse)
+	if err != nil {
+		return err
+	}
+
+	// for each active index, if it starts with our alias but is before our current index, remove it
+	for key := range healthResponse.Indices {
+		if strings.HasPrefix(key, i.name) && strings.Compare(key, currents[0]) < 0 {
+			logrus.WithField("index", key).Info("removing old index")
+			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", i.ElasticURL, key), nil, nil)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// our response for indexing contacts
+type indexResponse struct {
+	Items []struct {
+		Index struct {
+			ID     string `json:"_id"`
+			Status int    `json:"status"`
+			Result string `json:"result"`
+		} `json:"index"`
+		Delete struct {
+			ID     string `json:"_id"`
+			Status int    `json:"status"`
+		} `json:"delete"`
+	} `json:"items"`
+}
+
+// indexes the batch of contacts
+func (i *BaseIndexer) IndexBatch(index string, batch []byte) (int, int, error) {
+	response := indexResponse{}
+	indexURL := fmt.Sprintf("%s/%s/_bulk", i.ElasticURL, index)
+
+	_, err := MakeJSONRequest(http.MethodPut, indexURL, batch, &response)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	createdCount, deletedCount, conflictedCount := 0, 0, 0
+	for _, item := range response.Items {
+		if item.Index.ID != "" {
+			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("index response")
+			if item.Index.Status == 200 || item.Index.Status == 201 {
+				createdCount++
+			} else if item.Index.Status == 409 {
+				conflictedCount++
+			} else {
+				logrus.WithField("id", item.Index.ID).WithField("batch", batch).WithField("result", item.Index.Result).Error("error indexing document")
+			}
+		} else if item.Delete.ID != "" {
+			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response")
+			if item.Delete.Status == 200 {
+				deletedCount++
+			} else if item.Delete.Status == 409 {
+				conflictedCount++
+			}
+		} else {
+			logrus.Error("unparsed item in response")
+		}
+	}
+	logrus.WithField("created", createdCount).WithField("deleted", deletedCount).WithField("conflicted", conflictedCount).Debug("indexed batch")
+
+	return createdCount, deletedCount, nil
+}
diff --git a/base_test.go b/base_test.go
index f7edfbe..e49be40 100644
--- a/base_test.go
+++ b/base_test.go
@@ -5,7 +5,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	indexer "github.com/nyaruka/rp-indexer"
+	"github.com/nyaruka/rp-indexer/contacts"
 	"github.com/stretchr/testify/require"
 )
 
@@ -87,6 +87,9 @@ func TestRetryServer(t *testing.T) {
 		responseCounter++
 	}))
 	defer ts.Close()
-	indexer.FindPhysicalIndexes(ts.URL, "rp_elastic_test")
+
+	ci := contacts.NewIndexer("rp_elastic_test", ts.URL)
+	ci.FindPhysicalIndexes()
+
 	require.Equal(t, responseCounter, 4)
 }
diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index cbacc68..bc4688e 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -64,10 +64,10 @@ func main() {
 		log.Fatal(err)
 	}
 
-	ci := contacts.NewIndexer(config.ElasticURL, config.Index, config.Rebuild, config.Cleanup)
+	ci := contacts.NewIndexer(config.ElasticURL, config.Index)
 
 	for {
-		err := ci.Index(db)
+		err := ci.Index(db, config.Rebuild, config.Cleanup)
 
 		if err != nil {
 			if config.Rebuild {
diff --git a/contacts/indexer.go b/contacts/indexer.go
index 654556b..6905cf5 100644
--- a/contacts/indexer.go
+++ b/contacts/indexer.go
@@ -6,7 +6,6 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"time"
 
 	indexer "github.com/nyaruka/rp-indexer"
@@ -16,7 +15,6 @@ import (
 
 var BatchSize = 500
 
-// settings and mappings for our index
 //go:embed index_settings.json
 var IndexSettings json.RawMessage
 
@@ -26,22 +24,23 @@ const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "v
 // deletes a contact
 const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
 
-type ContactIndexer struct {
+// ContactIndexer is an indexer for contacts
+type Indexer struct {
 	indexer.BaseIndexer
 }
 
-func NewIndexer(name, elasticURL string, rebuild, cleanup bool) indexer.Indexer {
-	return &ContactIndexer{
-		BaseIndexer: indexer.NewBaseIndexer(name, elasticURL, rebuild, cleanup),
+// NewIndexer creates a new contact indexer
+func NewIndexer(name, elasticURL string) *Indexer {
+	return &Indexer{
+		BaseIndexer: indexer.NewBaseIndexer(name, elasticURL),
 	}
 }
 
-func (i *ContactIndexer) Index(db *sql.DB) error {
+func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) error {
 	var err error
 
 	// find our physical index
-	physicalIndexes := indexer.FindPhysicalIndexes(i.ElasticURL, i.Name())
-	logrus.WithField("physicalIndexes", physicalIndexes).WithField("indexer", i.Name()).Debug("found physical indexes")
+	physicalIndexes := i.FindPhysicalIndexes()
 
 	physicalIndex := ""
 	if len(physicalIndexes) > 0 {
@@ -52,12 +51,12 @@ func (i *ContactIndexer) Index(db *sql.DB) error {
 	remapAlias := false
 
 	// doesn't exist or we are rebuilding, create it
-	if physicalIndex == "" || i.Rebuild {
-		physicalIndex, err = indexer.CreateNewIndex(i.ElasticURL, i.Name(), IndexSettings)
+	if physicalIndex == "" || rebuild {
+		physicalIndex, err = i.CreateNewIndex(IndexSettings)
 		if err != nil {
 			return errors.Wrap(err, "error creating new index")
 		}
-		logrus.WithField("indexer", i.Name()).WithField("physicalIndex", physicalIndex).Info("created new physical index")
+		logrus.WithField("indexer", i.Name()).WithField("index", physicalIndex).Info("created new physical index")
 		remapAlias = true
 	}
 
@@ -66,20 +65,20 @@ func (i *ContactIndexer) Index(db *sql.DB) error {
 		return errors.Wrap(err, "error finding last modified")
 	}
 
-	logrus.WithField("last_modified", lastModified).WithField("index", physicalIndex).Info("indexing newer than last modified")
+	logrus.WithField("indexer", i.Name()).WithField("index", physicalIndex).WithField("last_modified", lastModified).Info("indexing newer than last modified")
 
 	// now index our docs
 	start := time.Now()
-	indexed, deleted, err := IndexModified(db, i.ElasticURL, physicalIndex, lastModified.Add(-5*time.Second))
+	indexed, deleted, err := i.IndexModified(db, physicalIndex, lastModified.Add(-5*time.Second))
 	if err != nil {
 		return errors.Wrap(err, "error indexing documents")
 	}
 
-	i.UpdateStats(indexed, deleted, time.Since(start))
+	i.RecordComplete(indexed, deleted, time.Since(start))
 
 	// if the index didn't previously exist or we are rebuilding, remap to our alias
 	if remapAlias {
-		err := indexer.MapIndexAlias(i.ElasticURL, i.Name(), physicalIndex)
+		err := i.UpdateAlias(physicalIndex)
 		if err != nil {
 			return errors.Wrap(err, "error remapping alias")
 		}
@@ -87,8 +86,8 @@ func (i *ContactIndexer) Index(db *sql.DB) error {
 	}
 
 	// cleanup our aliases if appropriate
-	if i.Cleanup {
-		err := indexer.CleanupIndexes(i.ElasticURL, i.Name())
+	if cleanup {
+		err := i.CleanupIndexes()
 		if err != nil {
 			return errors.Wrap(err, "error cleaning up old indexes")
 		}
@@ -98,7 +97,7 @@ func (i *ContactIndexer) Index(db *sql.DB) error {
 }
 
 // IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time
-func IndexModified(db *sql.DB, elasticURL string, index string, lastModified time.Time) (int, int, error) {
+func (i *Indexer) IndexModified(db *sql.DB, index string, lastModified time.Time) (int, int, error) {
 	batch := &bytes.Buffer{}
 	createdCount, deletedCount, processedCount := 0, 0, 0
 
@@ -153,7 +152,7 @@ func IndexModified(db *sql.DB, elasticURL string, index string, lastModified tim
 
 			// write to elastic search in batches
 			if queryCount%BatchSize == 0 {
-				created, deleted, err := indexBatch(elasticURL, index, batch.Bytes())
+				created, deleted, err := i.IndexBatch(index, batch.Bytes())
 				if err != nil {
 					return 0, 0, err
 				}
@@ -166,7 +165,7 @@ func IndexModified(db *sql.DB, elasticURL string, index string, lastModified tim
 		}
 
 		if batch.Len() > 0 {
-			created, deleted, err := indexBatch(elasticURL, index, batch.Bytes())
+			created, deleted, err := i.IndexBatch(index, batch.Bytes())
 			if err != nil {
 				return 0, 0, err
 			}
@@ -196,40 +195,3 @@ func IndexModified(db *sql.DB, elasticURL string, index string, lastModified tim
 
 	return createdCount, deletedCount, nil
 }
-
-// indexes the batch of contacts
-func indexBatch(elasticURL string, index string, batch []byte) (int, int, error) {
-	response := indexer.IndexResponse{}
-	indexURL := fmt.Sprintf("%s/%s/_bulk", elasticURL, index)
-
-	_, err := indexer.MakeJSONRequest(http.MethodPut, indexURL, batch, &response)
-	if err != nil {
-		return 0, 0, err
-	}
-
-	createdCount, deletedCount, conflictedCount := 0, 0, 0
-	for _, item := range response.Items {
-		if item.Index.ID != "" {
-			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("index response")
-			if item.Index.Status == 200 || item.Index.Status == 201 {
-				createdCount++
-			} else if item.Index.Status == 409 {
-				conflictedCount++
-			} else {
-				logrus.WithField("id", item.Index.ID).WithField("batch", batch).WithField("result", item.Index.Result).Error("error indexing contact")
-			}
-		} else if item.Delete.ID != "" {
-			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response")
-			if item.Delete.Status == 200 {
-				deletedCount++
-			} else if item.Delete.Status == 409 {
-				conflictedCount++
-			}
-		} else {
-			logrus.Error("unparsed item in response")
-		}
-	}
-	logrus.WithField("created", createdCount).WithField("deleted", deletedCount).WithField("conflicted", conflictedCount).Debug("indexed batch")
-
-	return createdCount, deletedCount, nil
-}
diff --git a/contacts/indexer_test.go b/contacts/indexer_test.go
index 6e08a1c..a355617 100644
--- a/contacts/indexer_test.go
+++ b/contacts/indexer_test.go
@@ -36,7 +36,9 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) {
 	client, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
 	require.NoError(t, err)
 
-	existing := indexer.FindPhysicalIndexes(elasticURL, indexName)
+	ci := contacts.NewIndexer(indexName, elasticURL)
+
+	existing := ci.FindPhysicalIndexes()
 	for _, idx := range existing {
 		_, err = client.DeleteIndex(idx).Do(context.Background())
 		require.NoError(t, err)
@@ -59,14 +61,16 @@ func assertQuery(t *testing.T, client *elastic.Client, index string, query elast
 	}
 }
 
-func TestIndexing(t *testing.T) {
+func TestIndexer(t *testing.T) {
 	contacts.BatchSize = 4
 	db, client := setup(t)
 
-	physicalName, err := indexer.CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
+	ci := contacts.NewIndexer(indexName, elasticURL)
+
+	physicalName, err := ci.CreateNewIndex(contacts.IndexSettings)
 	assert.NoError(t, err)
 
-	added, deleted, err := contacts.IndexModified(db, elasticURL, physicalName, time.Time{})
+	added, deleted, err := ci.IndexModified(db, physicalName, time.Time{})
 	assert.NoError(t, err)
 	assert.Equal(t, 9, added)
 	assert.Equal(t, 0, deleted)
@@ -242,7 +246,7 @@ func TestIndexing(t *testing.T) {
 	assert.Equal(t, time.Date(2017, 11, 10, 21, 11, 59, 890662000, time.UTC), lastModified.In(time.UTC))
 
 	// map our index over
-	err = indexer.MapIndexAlias(elasticURL, indexName, physicalName)
+	err = ci.UpdateAlias(physicalName)
 	assert.NoError(t, err)
 	time.Sleep(5 * time.Second)
 
@@ -250,20 +254,20 @@ func TestIndexing(t *testing.T) {
 	assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{4})
 
 	// look up our mapping
-	physical := indexer.FindPhysicalIndexes(elasticURL, indexName)
+	physical := ci.FindPhysicalIndexes()
 	assert.Equal(t, physicalName, physical[0])
 
 	// rebuild again
-	newIndex, err := indexer.CreateNewIndex(elasticURL, indexName, contacts.IndexSettings)
+	newIndex, err := ci.CreateNewIndex(contacts.IndexSettings)
 	assert.NoError(t, err)
 
-	added, deleted, err = contacts.IndexModified(db, elasticURL, newIndex, time.Time{})
+	added, deleted, err = ci.IndexModified(db, newIndex, time.Time{})
 	assert.NoError(t, err)
 	assert.Equal(t, 9, added)
 	assert.Equal(t, 0, deleted)
 
 	// remap again
-	err = indexer.MapIndexAlias(elasticURL, indexName, newIndex)
+	err = ci.UpdateAlias(newIndex)
 	assert.NoError(t, err)
 	time.Sleep(5 * time.Second)
 
@@ -273,7 +277,7 @@ func TestIndexing(t *testing.T) {
 	assert.Equal(t, resp.StatusCode, http.StatusOK)
 
 	// cleanup our indexes, will remove our original index
-	err = indexer.CleanupIndexes(elasticURL, indexName)
+	err = ci.CleanupIndexes()
 	assert.NoError(t, err)
 
 	// old physical index should be gone
@@ -291,7 +295,7 @@ func TestIndexing(t *testing.T) {
 	UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;`)
 	assert.NoError(t, err)
 
-	added, deleted, err = contacts.IndexModified(db, elasticURL, indexName, lastModified)
+	added, deleted, err = ci.IndexModified(db, indexName, lastModified)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, added)
 	assert.Equal(t, 1, deleted)
diff --git a/go.mod b/go.mod
index da1e7ab..fcfb589 100644
--- a/go.mod
+++ b/go.mod
@@ -1,18 +1,32 @@
 module github.com/nyaruka/rp-indexer
 
 require (
-	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
 	github.com/evalphobia/logrus_sentry v0.4.5
-	github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2
+	github.com/lib/pq v1.10.4
 	github.com/nyaruka/ezconf v0.2.1
-	github.com/nyaruka/gocommon v1.3.0
+	github.com/nyaruka/gocommon v1.17.1
 	github.com/olivere/elastic/v7 v7.0.22
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.5.1
+	github.com/stretchr/testify v1.7.0
+)
+
+require (
+	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fatih/structs v1.0.0 // indirect
+	github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 // indirect
+	github.com/go-chi/chi v4.1.2+incompatible // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/naoina/toml v0.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/shopspring/decimal v1.2.0 // indirect
+	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
 	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
 
-go 1.16
+go 1.17
diff --git a/go.sum b/go.sum
index 18f5213..f78ba21 100644
--- a/go.sum
+++ b/go.sum
@@ -1,6 +1,7 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
+github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -15,34 +16,44 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 h1:md1zEr2oSVWYNfQj+6TL/nmAFf5gY3Tp44lzskzK9QU=
 github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
+github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
+github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
+github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2 h1:hRGSmZu7j271trc9sneMrpOW7GN5ngLm8YUZIPzf394=
-github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
+github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
+github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
-github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU=
-github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8MTEjxTC9M=
-github.com/nyaruka/phonenumbers v1.0.34/go.mod h1:GQ0cTHlrxPrhoLwyQ1blyN1hO794ygt6FTHWrFB5SSc=
+github.com/nyaruka/gocommon v1.17.1 h1:4bbNp+0/BIbne4VDiKOxh3kcbdvEu/WsrsZiG/VyRZ8=
+github.com/nyaruka/gocommon v1.17.1/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0=
+github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U=
 github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s=
 github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
@@ -61,8 +72,10 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -71,12 +84,12 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -85,10 +98,15 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
 golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -102,7 +120,10 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/elastic.go b/http.go
similarity index 64%
rename from elastic.go
rename to http.go
index b9fbcf8..d526984 100644
--- a/elastic.go
+++ b/http.go
@@ -87,64 +87,3 @@ func MakeJSONRequest(method string, url string, body []byte, jsonStruct interfac
 	l.Debug("ES request successful")
 	return resp, nil
 }
-
-// adds an alias for an index
-type addAliasCommand struct {
-	Add struct {
-		Index string `json:"index"`
-		Alias string `json:"alias"`
-	} `json:"add"`
-}
-
-// removes an alias for an index
-type removeAliasCommand struct {
-	Remove struct {
-		Index string `json:"index"`
-		Alias string `json:"alias"`
-	} `json:"remove"`
-}
-
-// our top level command for remapping aliases
-type aliasCommand struct {
-	Actions []interface{} `json:"actions"`
-}
-
-// our response for finding the most recent contact
-type queryResponse struct {
-	Hits struct {
-		Total struct {
-			Value int `json:"value"`
-		} `json:"total"`
-		Hits []struct {
-			Source struct {
-				ID         int64     `json:"id"`
-				ModifiedOn time.Time `json:"modified_on"`
-			} `json:"_source"`
-		} `json:"hits"`
-	} `json:"hits"`
-}
-
-// our response for indexing contacts
-type IndexResponse struct {
-	Items []struct {
-		Index struct {
-			ID     string `json:"_id"`
-			Status int    `json:"status"`
-			Result string `json:"result"`
-		} `json:"index"`
-		Delete struct {
-			ID     string `json:"_id"`
-			Status int    `json:"status"`
-		} `json:"delete"`
-	} `json:"items"`
-}
-
-// our response for our index health
-type healthResponse struct {
-	Indices map[string]struct {
-		Status string `json:"status"`
-	} `json:"indices"`
-}
-
-// our response for figuring out the physical index for an alias
-type infoResponse map[string]interface{}
diff --git a/indexes.go b/indexes.go
index f51c16b..f235a14 100644
--- a/indexes.go
+++ b/indexes.go
@@ -2,54 +2,24 @@ package indexer
 
 import (
 	_ "embed"
-	"encoding/json"
 	"fmt"
 	"net/http"
-	"sort"
-	"strings"
 	"time"
-
-	log "github.com/sirupsen/logrus"
 )
 
-// CreateNewIndex creates a new index for the passed in alias.
-//
-// Note that we do not create an index with the passed name, instead creating one
-// based on the day, for example `contacts_2018_03_05`, then create an alias from
-// that index to `contacts`.
-//
-// If the day-specific name already exists, we append a .1 or .2 to the name.
-func CreateNewIndex(url, alias string, settings json.RawMessage) (string, error) {
-	// create our day-specific name
-	physicalIndex := fmt.Sprintf("%s_%s", alias, time.Now().Format("2006_01_02"))
-	idx := 0
-
-	// check if it exists
-	for {
-		resp, err := http.Get(fmt.Sprintf("%s/%s", url, physicalIndex))
-		if err != nil {
-			return "", err
-		}
-		// not found, great, move on
-		if resp.StatusCode == http.StatusNotFound {
-			break
-		}
-
-		// was found, increase our index and try again
-		idx++
-		physicalIndex = fmt.Sprintf("%s_%s_%d", alias, time.Now().Format("2006_01_02"), idx)
-	}
-
-	// initialize our index
-	createURL := fmt.Sprintf("%s/%s?include_type_name=true", url, physicalIndex)
-	_, err := MakeJSONRequest(http.MethodPut, createURL, settings, nil)
-	if err != nil {
-		return "", err
-	}
-
-	// all went well, return our physical index name
-	log.WithField("index", physicalIndex).Info("created index")
-	return physicalIndex, nil
+// our response for finding the last modified document
+type queryResponse struct {
+	Hits struct {
+		Total struct {
+			Value int `json:"value"`
+		} `json:"total"`
+		Hits []struct {
+			Source struct {
+				ID         int64     `json:"id"`
+				ModifiedOn time.Time `json:"modified_on"`
+			} `json:"_source"`
+		} `json:"hits"`
+	} `json:"hits"`
 }
 
 // GetLastModified queries an index and finds the last modified document, returning its modified time
@@ -71,88 +41,3 @@ func GetLastModified(url string, index string) (time.Time, error) {
 	}
 	return lastModified, nil
 }
-
-// FindPhysicalIndexes finds all the physical indexes for the passed in alias
-func FindPhysicalIndexes(url string, alias string) []string {
-	indexResponse := infoResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, alias), nil, &indexResponse)
-	indexes := make([]string, 0)
-
-	// error could mean a variety of things, but we'll figure that out later
-	if err != nil {
-		return indexes
-	}
-
-	// our top level key is our physical index name
-	for key := range indexResponse {
-		indexes = append(indexes, key)
-	}
-
-	// reverse sort order should put our newest index first
-	sort.Sort(sort.Reverse(sort.StringSlice(indexes)))
-	return indexes
-}
-
-// CleanupIndexes removes all indexes that are older than the currently active index
-func CleanupIndexes(url string, alias string) error {
-	// find our current indexes
-	currents := FindPhysicalIndexes(url, alias)
-
-	// no current indexes? this a noop
-	if len(currents) == 0 {
-		return nil
-	}
-
-	// find all the current indexes
-	healthResponse := healthResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", url, "_cluster/health?level=indices"), nil, &healthResponse)
-	if err != nil {
-		return err
-	}
-
-	// for each active index, if it starts with our alias but is before our current index, remove it
-	for key := range healthResponse.Indices {
-		if strings.HasPrefix(key, alias) && strings.Compare(key, currents[0]) < 0 {
-			log.WithField("index", key).Info("removing old index")
-			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", url, key), nil, nil)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-// MapIndexAlias maps the passed in alias to the new physical index, optionally removing
-// existing aliases if they exit.
-func MapIndexAlias(elasticURL string, alias string, newIndex string) error {
-	commands := make([]interface{}, 0)
-
-	// find existing physical indexes
-	existing := FindPhysicalIndexes(elasticURL, alias)
-	for _, idx := range existing {
-		remove := removeAliasCommand{}
-		remove.Remove.Alias = alias
-		remove.Remove.Index = idx
-		commands = append(commands, remove)
-
-		log.WithField("index", idx).WithField("alias", alias).Info("removing old alias")
-	}
-
-	// add our new index
-	add := addAliasCommand{}
-	add.Add.Alias = alias
-	add.Add.Index = newIndex
-	commands = append(commands, add)
-
-	log.WithField("index", newIndex).WithField("alias", alias).Info("adding new alias")
-
-	aliasURL := fmt.Sprintf("%s/_aliases", elasticURL)
-	aliasJSON, err := json.Marshal(aliasCommand{Actions: commands})
-	if err != nil {
-		return err
-	}
-	_, err = MakeJSONRequest(http.MethodPost, aliasURL, aliasJSON, nil)
-	return err
-}

From a20bdf41184b07f31d7dd81800cd7417cf49b593 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 23 Mar 2022 15:58:03 -0500
Subject: [PATCH 15/48] Better testing

---
 base.go                  |  79 +++++--
 base_test.go             |   4 +-
 cmd/rp-indexer/main.go   |   4 +-
 contacts/indexer.go      |  58 +++--
 contacts/indexer_test.go | 497 +++++++++++++++++++++------------------
 http.go                  |   8 +-
 indexes.go               |  43 ----
 7 files changed, 357 insertions(+), 336 deletions(-)
 delete mode 100644 indexes.go

diff --git a/base.go b/base.go
index 397baf9..eeb3b73 100644
--- a/base.go
+++ b/base.go
@@ -16,12 +16,13 @@ import (
 // Indexer is base interface for indexers
 type Indexer interface {
 	Name() string
-	Index(db *sql.DB, rebuild, cleanup bool) error
+	Index(db *sql.DB, rebuild, cleanup bool) (string, error)
+	Stats() (int64, int64, time.Duration)
 }
 
 type BaseIndexer struct {
-	name       string // e.g. contacts, used as based index name
-	ElasticURL string
+	elasticURL string
+	name       string // e.g. contacts, used as the alias
 
 	// statistics
 	indexedTotal int64
@@ -29,14 +30,18 @@ type BaseIndexer struct {
 	elapsedTotal time.Duration
 }
 
-func NewBaseIndexer(name, elasticURL string) BaseIndexer {
-	return BaseIndexer{name: name, ElasticURL: elasticURL}
+func NewBaseIndexer(elasticURL, name string) BaseIndexer {
+	return BaseIndexer{elasticURL: elasticURL, name: name}
 }
 
 func (i *BaseIndexer) Name() string {
 	return i.name
 }
 
+func (i *BaseIndexer) Log() *logrus.Entry {
+	return logrus.WithField("indexer", i.name)
+}
+
 func (i *BaseIndexer) Stats() (int64, int64, time.Duration) {
 	return i.indexedTotal, i.deletedTotal, i.elapsedTotal
 }
@@ -47,16 +52,16 @@ func (i *BaseIndexer) RecordComplete(indexed, deleted int, elapsed time.Duration
 	i.deletedTotal += int64(deleted)
 	i.elapsedTotal += elapsed
 
-	logrus.WithField("indexer", i.name).WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
+	i.Log().WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
 }
 
 // our response for figuring out the physical index for an alias
 type infoResponse map[string]interface{}
 
-// FindPhysicalIndexes finds all our physical indexes
-func (i *BaseIndexer) FindPhysicalIndexes() []string {
+// FindIndexes finds all our physical indexes
+func (i *BaseIndexer) FindIndexes() []string {
 	response := infoResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.ElasticURL, i.name), nil, &response)
+	_, err := makeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.elasticURL, i.name), nil, &response)
 	indexes := make([]string, 0)
 
 	// error could mean a variety of things, but we'll figure that out later
@@ -72,7 +77,7 @@ func (i *BaseIndexer) FindPhysicalIndexes() []string {
 	// reverse sort order should put our newest index first
 	sort.Sort(sort.Reverse(sort.StringSlice(indexes)))
 
-	logrus.WithField("indexer", i.name).WithField("indexes", indexes).Debug("found physical indexes")
+	i.Log().WithField("indexes", indexes).Debug("found physical indexes")
 
 	return indexes
 }
@@ -91,7 +96,7 @@ func (i *BaseIndexer) CreateNewIndex(settings json.RawMessage) (string, error) {
 
 	// check if it exists
 	for {
-		resp, err := http.Get(fmt.Sprintf("%s/%s", i.ElasticURL, index))
+		resp, err := http.Get(fmt.Sprintf("%s/%s", i.elasticURL, index))
 		if err != nil {
 			return "", err
 		}
@@ -106,13 +111,13 @@ func (i *BaseIndexer) CreateNewIndex(settings json.RawMessage) (string, error) {
 	}
 
 	// create the new index
-	_, err := MakeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.ElasticURL, index), settings, nil)
+	_, err := makeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.elasticURL, index), settings, nil)
 	if err != nil {
 		return "", err
 	}
 
 	// all went well, return our physical index name
-	logrus.WithField("indexer", i.name).WithField("index", index).Info("created new index")
+	i.Log().WithField("index", index).Info("created new index")
 
 	return index, nil
 }
@@ -144,7 +149,7 @@ func (i *BaseIndexer) UpdateAlias(newIndex string) error {
 	commands := make([]interface{}, 0)
 
 	// find existing physical indexes
-	existing := i.FindPhysicalIndexes()
+	existing := i.FindIndexes()
 	for _, idx := range existing {
 		remove := removeAliasCommand{}
 		remove.Remove.Alias = i.name
@@ -162,9 +167,9 @@ func (i *BaseIndexer) UpdateAlias(newIndex string) error {
 
 	aliasJSON := jsonx.MustMarshal(aliasCommand{Actions: commands})
 
-	_, err := MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/_aliases", i.ElasticURL), aliasJSON, nil)
+	_, err := makeJSONRequest(http.MethodPost, fmt.Sprintf("%s/_aliases", i.elasticURL), aliasJSON, nil)
 
-	logrus.WithField("indexer", i.name).WithField("index", newIndex).Debug("adding new alias")
+	i.Log().WithField("index", newIndex).Info("updated alias")
 
 	return err
 }
@@ -179,7 +184,7 @@ type healthResponse struct {
 // CleanupIndexes removes all indexes that are older than the currently active index
 func (i *BaseIndexer) CleanupIndexes() error {
 	// find our current indexes
-	currents := i.FindPhysicalIndexes()
+	currents := i.FindIndexes()
 
 	// no current indexes? this a noop
 	if len(currents) == 0 {
@@ -188,7 +193,7 @@ func (i *BaseIndexer) CleanupIndexes() error {
 
 	// find all the current indexes
 	healthResponse := healthResponse{}
-	_, err := MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.ElasticURL, "_cluster/health?level=indices"), nil, &healthResponse)
+	_, err := makeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.elasticURL, "_cluster/health?level=indices"), nil, &healthResponse)
 	if err != nil {
 		return err
 	}
@@ -197,7 +202,7 @@ func (i *BaseIndexer) CleanupIndexes() error {
 	for key := range healthResponse.Indices {
 		if strings.HasPrefix(key, i.name) && strings.Compare(key, currents[0]) < 0 {
 			logrus.WithField("index", key).Info("removing old index")
-			_, err = MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", i.ElasticURL, key), nil, nil)
+			_, err = makeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", i.elasticURL, key), nil, nil)
 			if err != nil {
 				return err
 			}
@@ -225,9 +230,9 @@ type indexResponse struct {
 // indexes the batch of contacts
 func (i *BaseIndexer) IndexBatch(index string, batch []byte) (int, int, error) {
 	response := indexResponse{}
-	indexURL := fmt.Sprintf("%s/%s/_bulk", i.ElasticURL, index)
+	indexURL := fmt.Sprintf("%s/%s/_bulk", i.elasticURL, index)
 
-	_, err := MakeJSONRequest(http.MethodPut, indexURL, batch, &response)
+	_, err := makeJSONRequest(http.MethodPut, indexURL, batch, &response)
 	if err != nil {
 		return 0, 0, err
 	}
@@ -258,3 +263,35 @@ func (i *BaseIndexer) IndexBatch(index string, batch []byte) (int, int, error) {
 
 	return createdCount, deletedCount, nil
 }
+
+// our response for finding the last modified document
+type queryResponse struct {
+	Hits struct {
+		Total struct {
+			Value int `json:"value"`
+		} `json:"total"`
+		Hits []struct {
+			Source struct {
+				ID         int64     `json:"id"`
+				ModifiedOn time.Time `json:"modified_on"`
+			} `json:"_source"`
+		} `json:"hits"`
+	} `json:"hits"`
+}
+
+// GetLastModified queries a concrete index and finds the last modified document, returning its modified time
+func (i *BaseIndexer) GetLastModified(index string) (time.Time, error) {
+	lastModified := time.Time{}
+
+	// get the newest document on our index
+	queryResponse := queryResponse{}
+	_, err := makeJSONRequest(http.MethodPost, fmt.Sprintf("%s/%s/_search", i.elasticURL, index), []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`), &queryResponse)
+	if err != nil {
+		return lastModified, err
+	}
+
+	if len(queryResponse.Hits.Hits) > 0 {
+		lastModified = queryResponse.Hits.Hits[0].Source.ModifiedOn
+	}
+	return lastModified, nil
+}
diff --git a/base_test.go b/base_test.go
index e49be40..9327622 100644
--- a/base_test.go
+++ b/base_test.go
@@ -88,8 +88,8 @@ func TestRetryServer(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	ci := contacts.NewIndexer("rp_elastic_test", ts.URL)
-	ci.FindPhysicalIndexes()
+	ci := contacts.NewIndexer(ts.URL, "rp_elastic_test", 500)
+	ci.FindIndexes()
 
 	require.Equal(t, responseCounter, 4)
 }
diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index bc4688e..b50f89e 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -64,10 +64,10 @@ func main() {
 		log.Fatal(err)
 	}
 
-	ci := contacts.NewIndexer(config.ElasticURL, config.Index)
+	ci := contacts.NewIndexer(config.ElasticURL, config.Index, 500)
 
 	for {
-		err := ci.Index(db, config.Rebuild, config.Cleanup)
+		_, err := ci.Index(db, config.Rebuild, config.Cleanup)
 
 		if err != nil {
 			if config.Rebuild {
diff --git a/contacts/indexer.go b/contacts/indexer.go
index 6905cf5..df3f581 100644
--- a/contacts/indexer.go
+++ b/contacts/indexer.go
@@ -13,10 +13,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var BatchSize = 500
-
 //go:embed index_settings.json
-var IndexSettings json.RawMessage
+var indexSettings json.RawMessage
 
 // indexes a contact
 const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
@@ -27,20 +25,24 @@ const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d,
 // ContactIndexer is an indexer for contacts
 type Indexer struct {
 	indexer.BaseIndexer
+
+	batchSize int
 }
 
 // NewIndexer creates a new contact indexer
-func NewIndexer(name, elasticURL string) *Indexer {
+func NewIndexer(elasticURL, name string, batchSize int) *Indexer {
 	return &Indexer{
-		BaseIndexer: indexer.NewBaseIndexer(name, elasticURL),
+		BaseIndexer: indexer.NewBaseIndexer(elasticURL, name),
+		batchSize:   batchSize,
 	}
 }
 
-func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) error {
+// Index indexes modified contacts and returns the name of the concrete index
+func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 	var err error
 
 	// find our physical index
-	physicalIndexes := i.FindPhysicalIndexes()
+	physicalIndexes := i.FindIndexes()
 
 	physicalIndex := ""
 	if len(physicalIndexes) > 0 {
@@ -52,26 +54,26 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) error {
 
 	// doesn't exist or we are rebuilding, create it
 	if physicalIndex == "" || rebuild {
-		physicalIndex, err = i.CreateNewIndex(IndexSettings)
+		physicalIndex, err = i.CreateNewIndex(indexSettings)
 		if err != nil {
-			return errors.Wrap(err, "error creating new index")
+			return "", errors.Wrap(err, "error creating new index")
 		}
-		logrus.WithField("indexer", i.Name()).WithField("index", physicalIndex).Info("created new physical index")
+		i.Log().WithField("index", physicalIndex).Info("created new physical index")
 		remapAlias = true
 	}
 
-	lastModified, err := indexer.GetLastModified(i.ElasticURL, physicalIndex)
+	lastModified, err := i.GetLastModified(physicalIndex)
 	if err != nil {
-		return errors.Wrap(err, "error finding last modified")
+		return "", errors.Wrap(err, "error finding last modified")
 	}
 
-	logrus.WithField("indexer", i.Name()).WithField("index", physicalIndex).WithField("last_modified", lastModified).Info("indexing newer than last modified")
+	i.Log().WithField("index", physicalIndex).WithField("last_modified", lastModified).Info("indexing newer than last modified")
 
 	// now index our docs
 	start := time.Now()
-	indexed, deleted, err := i.IndexModified(db, physicalIndex, lastModified.Add(-5*time.Second))
+	indexed, deleted, err := i.indexModified(db, physicalIndex, lastModified.Add(-5*time.Second))
 	if err != nil {
-		return errors.Wrap(err, "error indexing documents")
+		return "", errors.Wrap(err, "error indexing documents")
 	}
 
 	i.RecordComplete(indexed, deleted, time.Since(start))
@@ -80,7 +82,7 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) error {
 	if remapAlias {
 		err := i.UpdateAlias(physicalIndex)
 		if err != nil {
-			return errors.Wrap(err, "error remapping alias")
+			return "", errors.Wrap(err, "error updating alias")
 		}
 		remapAlias = false
 	}
@@ -89,22 +91,18 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) error {
 	if cleanup {
 		err := i.CleanupIndexes()
 		if err != nil {
-			return errors.Wrap(err, "error cleaning up old indexes")
+			return "", errors.Wrap(err, "error cleaning up old indexes")
 		}
 	}
 
-	return nil
+	return physicalIndex, nil
 }
 
 // IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time
-func (i *Indexer) IndexModified(db *sql.DB, index string, lastModified time.Time) (int, int, error) {
+func (i *Indexer) indexModified(db *sql.DB, index string, lastModified time.Time) (int, int, error) {
 	batch := &bytes.Buffer{}
 	createdCount, deletedCount, processedCount := 0, 0, 0
 
-	if index == "" {
-		return 0, 0, errors.New("empty index")
-	}
-
 	var modifiedOn time.Time
 	var contactJSON string
 	var id, orgID int64
@@ -140,18 +138,20 @@ func (i *Indexer) IndexModified(db *sql.DB, index string, lastModified time.Time
 
 			if isActive {
 				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Debug("modified contact")
+
 				batch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID))
 				batch.WriteString("\n")
 				batch.WriteString(contactJSON)
 				batch.WriteString("\n")
 			} else {
 				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Debug("deleted contact")
+
 				batch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID))
 				batch.WriteString("\n")
 			}
 
 			// write to elastic search in batches
-			if queryCount%BatchSize == 0 {
+			if queryCount%i.batchSize == 0 {
 				created, deleted, err := i.IndexBatch(index, batch.Bytes())
 				if err != nil {
 					return 0, 0, err
@@ -181,16 +181,12 @@ func (i *Indexer) IndexModified(db *sql.DB, index string, lastModified time.Time
 			break
 		}
 
+		rows.Close()
+
 		elapsed := time.Since(start)
 		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
-		logrus.WithFields(map[string]interface{}{
-			"rate":    int(rate),
-			"added":   createdCount,
-			"deleted": deletedCount,
-			"elapsed": elapsed,
-			"index":   index}).Info("updated contact index")
 
-		rows.Close()
+		i.Log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed}).Info("indexed contact batch")
 	}
 
 	return createdCount, deletedCount, nil
diff --git a/contacts/indexer_test.go b/contacts/indexer_test.go
index a355617..57d6754 100644
--- a/contacts/indexer_test.go
+++ b/contacts/indexer_test.go
@@ -6,12 +6,15 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
-	"net/http"
 	"os"
+	"sort"
+	"strconv"
+	"strings"
 	"testing"
 	"time"
 
 	_ "github.com/lib/pq"
+	"github.com/nyaruka/gocommon/jsonx"
 	indexer "github.com/nyaruka/rp-indexer"
 	"github.com/nyaruka/rp-indexer/contacts"
 	"github.com/olivere/elastic/v7"
@@ -21,7 +24,7 @@ import (
 )
 
 const elasticURL = "http://localhost:9200"
-const indexName = "rp_elastic_test"
+const aliasName = "indexer_test"
 
 func setup(t *testing.T) (*sql.DB, *elastic.Client) {
 	testDB, err := ioutil.ReadFile("../testdb.sql")
@@ -33,278 +36,308 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) {
 	_, err = db.Exec(string(testDB))
 	require.NoError(t, err)
 
-	client, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
+	es, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
 	require.NoError(t, err)
 
-	ci := contacts.NewIndexer(indexName, elasticURL)
+	// delete all indexes with our alias prefix
+	existing, err := es.IndexNames()
+	require.NoError(t, err)
 
-	existing := ci.FindPhysicalIndexes()
-	for _, idx := range existing {
-		_, err = client.DeleteIndex(idx).Do(context.Background())
-		require.NoError(t, err)
+	for _, name := range existing {
+		if strings.HasPrefix(name, aliasName) {
+			_, err = es.DeleteIndex(name).Do(context.Background())
+			require.NoError(t, err)
+		}
 	}
 
 	logrus.SetLevel(logrus.DebugLevel)
 
-	return db, client
+	return db, es
 }
 
-func assertQuery(t *testing.T, client *elastic.Client, index string, query elastic.Query, hits []int64) {
-	results, err := client.Search().Index(index).Query(query).Sort("id", true).Pretty(true).Do(context.Background())
+func assertQuery(t *testing.T, client *elastic.Client, query elastic.Query, expected []int64, msgAndArgs ...interface{}) {
+	results, err := client.Search().Index(aliasName).Query(query).Sort("id", true).Pretty(true).Do(context.Background())
 	assert.NoError(t, err)
-	assert.Equal(t, int64(len(hits)), results.Hits.TotalHits.Value)
 
-	if int64(len(hits)) == results.Hits.TotalHits.Value {
-		for i, hit := range results.Hits.Hits {
-			assert.Equal(t, fmt.Sprintf("%d", hits[i]), hit.Id)
-		}
+	actual := make([]int64, len(results.Hits.Hits))
+	for h, hit := range results.Hits.Hits {
+		asInt, _ := strconv.Atoi(hit.Id)
+		actual[h] = int64(asInt)
 	}
-}
-
-func TestIndexer(t *testing.T) {
-	contacts.BatchSize = 4
-	db, client := setup(t)
-
-	ci := contacts.NewIndexer(indexName, elasticURL)
-
-	physicalName, err := ci.CreateNewIndex(contacts.IndexSettings)
-	assert.NoError(t, err)
-
-	added, deleted, err := ci.IndexModified(db, physicalName, time.Time{})
-	assert.NoError(t, err)
-	assert.Equal(t, 9, added)
-	assert.Equal(t, 0, deleted)
-
-	time.Sleep(2 * time.Second)
-
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JOHn"), []int64{4})
-
-	// prefix on name matches both john and joanne, but no ajodi
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JO"), []int64{4, 6})
-	assertQuery(t, client, physicalName, elastic.NewTermQuery("name.keyword", "JOHN DOE"), []int64{4})
-
-	// can search on both first and last name
-	boolQuery := elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("name", "john"),
-		elastic.NewMatchQuery("name", "doe"))
-	assertQuery(t, client, physicalName, boolQuery, []int64{4})
-
-	// can search on a long name
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "Ajodinabiff"), []int64{5})
-
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "eng"), []int64{1})
-
-	// test contact, not indexed
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "fra"), []int64{})
 
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("status", "B"), []int64{3})
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("status", "S"), []int64{2})
-
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("org_id", "1"), []int64{1, 2, 3, 4})
+	assert.Equal(t, expected, actual, msgAndArgs...)
+}
 
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("tickets", 2), []int64{1})
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("tickets", 1), []int64{2, 3})
-	assertQuery(t, client, physicalName, elastic.NewRangeQuery("tickets").Gt(0), []int64{1, 2, 3})
+func assertIndexesWithPrefix(t *testing.T, es *elastic.Client, prefix string, expected []string) {
+	all, err := es.IndexNames()
+	require.NoError(t, err)
 
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "6d3cf1eb-546e-4fb8-a5ca-69187648fbf6"), []int64{2, 3})
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4})
+	actual := []string{}
+	for _, name := range all {
+		if strings.HasPrefix(name, prefix) {
+			actual = append(actual, name)
+		}
+	}
+	sort.Strings(actual)
+	assert.Equal(t, expected, actual)
+}
 
-	// created_on range query
-	assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8})
+func assertIndexerStats(t *testing.T, ix indexer.Indexer, expectedIndexed, expectedDeleted int64) {
+	actualIndexed, actualDeleted, _ := ix.Stats()
+	assert.Equal(t, expectedIndexed, actualIndexed, "indexed mismatch")
+	assert.Equal(t, expectedDeleted, actualDeleted, "deleted mismatch")
+}
 
-	// last_seen_on range query
-	assertQuery(t, client, physicalName, elastic.NewRangeQuery("last_seen_on").Lt("2019-01-01"), []int64{3, 4})
+var queryTests = []struct {
+	query    elastic.Query
+	expected []int64
+}{
+	{elastic.NewMatchQuery("org_id", "1"), []int64{1, 2, 3, 4}},
+	{elastic.NewMatchQuery("name", "JOHn"), []int64{4}},
+	{elastic.NewTermQuery("name.keyword", "JOHN DOE"), []int64{4}},
+	{elastic.NewBoolQuery().Must(elastic.NewMatchQuery("name", "john"), elastic.NewMatchQuery("name", "doe")), []int64{4}}, // can search on both first and last name
+	{elastic.NewMatchQuery("name", "Ajodinabiff"), []int64{5}},                                                             // long name
+	{elastic.NewMatchQuery("language", "eng"), []int64{1}},
+	{elastic.NewMatchQuery("status", "B"), []int64{3}},
+	{elastic.NewMatchQuery("status", "S"), []int64{2}},
+	{elastic.NewMatchQuery("tickets", 2), []int64{1}},
+	{elastic.NewMatchQuery("tickets", 1), []int64{2, 3}},
+	{elastic.NewRangeQuery("tickets").Gt(0), []int64{1, 2, 3}},
+	{elastic.NewMatchQuery("flow", "6d3cf1eb-546e-4fb8-a5ca-69187648fbf6"), []int64{2, 3}},
+	{elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4}},
+	{elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8}},                   // created_on range
+	{elastic.NewRangeQuery("last_seen_on").Lt("2019-01-01"), []int64{3, 4}},                    // last_seen_on range
+	{elastic.NewExistsQuery("last_seen_on"), []int64{1, 2, 3, 4, 5, 6}},                        // last_seen_on is set
+	{elastic.NewBoolQuery().MustNot(elastic.NewExistsQuery("last_seen_on")), []int64{7, 8, 9}}, // last_seen_on is not set
+	{
+		elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("urns.scheme", "facebook"),
+			elastic.NewMatchQuery("urns.path.keyword", "1000001"),
+		)),
+		[]int64{8},
+	},
+	{ // urn substring
+		elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("urns.scheme", "tel"),
+			elastic.NewMatchPhraseQuery("urns.path", "779"),
+		)),
+		[]int64{1, 2, 3, 6},
+	},
+	{ // urn substring with more characters (77911)
+		elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("urns.scheme", "tel"),
+			elastic.NewMatchPhraseQuery("urns.path", "77911"),
+		)),
+		[]int64{1},
+	},
+	{ // urn substring with more characters (600055)
+		elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("urns.scheme", "tel"),
+			elastic.NewMatchPhraseQuery("urns.path", "600055"),
+		)),
+		[]int64{5},
+	},
+	{ // match a contact with multiple tel urns
+		elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("urns.scheme", "tel"),
+			elastic.NewMatchPhraseQuery("urns.path", "222"),
+		)),
+		[]int64{1},
+	},
+	{ // text field
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
+			elastic.NewMatchQuery("fields.text", "the rock")),
+		),
+		[]int64{1},
+	},
+	{ // people with no nickname
+		elastic.NewBoolQuery().MustNot(
+			elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+				elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
+				elastic.NewExistsQuery("fields.text")),
+			),
+		),
+		[]int64{2, 3, 4, 5, 6, 7, 8, 9},
+	},
+	{ // no tokenizing of field text
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
+			elastic.NewMatchQuery("fields.text", "rock"),
+		)),
+		[]int64{},
+	},
+	{ // number field range
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "05bca1cd-e322-4837-9595-86d0d85e5adb"),
+			elastic.NewRangeQuery("fields.number").Gt(10),
+		)),
+		[]int64{2},
+	},
+	{ // datetime field range
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "e0eac267-463a-4c00-9732-cab62df07b16"),
+			elastic.NewRangeQuery("fields.datetime").Lt(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)),
+		)),
+		[]int64{3},
+	},
+	{ // state field
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
+			elastic.NewMatchPhraseQuery("fields.state", "washington"),
+		)),
+		[]int64{5},
+	},
+	{
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
+			elastic.NewMatchQuery("fields.state_keyword", "  washington"),
+		)),
+		[]int64{5},
+	},
+	{ // doesn't include country
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
+			elastic.NewMatchQuery("fields.state_keyword", "usa"),
+		)),
+		[]int64{},
+	},
+	{
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
+			elastic.NewMatchPhraseQuery("fields.state", "usa"),
+		)),
+		[]int64{},
+	},
+	{ // district field
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
+			elastic.NewMatchPhraseQuery("fields.district", "king"),
+		)),
+		[]int64{7, 9},
+	},
+	{ // phrase matches all
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
+			elastic.NewMatchPhraseQuery("fields.district", "King-Côunty"),
+		)),
+		[]int64{7},
+	},
+	{
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
+			elastic.NewMatchQuery("fields.district_keyword", "King-Côunty"),
+		)),
+		[]int64{7},
+	},
+	{ // ward field
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
+			elastic.NewMatchPhraseQuery("fields.ward", "district"),
+		)),
+		[]int64{8},
+	},
+	{
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
+			elastic.NewMatchQuery("fields.ward_keyword", "central district"),
+		)),
+		[]int64{8},
+	},
+	{ // no substring though on keyword
+		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
+			elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
+			elastic.NewMatchQuery("fields.ward_keyword", "district"),
+		)),
+		[]int64{},
+	},
+	{elastic.NewMatchQuery("groups", "4ea0f313-2f62-4e57-bdf0-232b5191dd57"), []int64{1}},
+	{elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2}},
+	{elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{}},
+}
 
-	// last_seen_on is set / not set queries
-	assertQuery(t, client, physicalName, elastic.NewExistsQuery("last_seen_on"), []int64{1, 2, 3, 4, 5, 6})
-	assertQuery(t, client, physicalName, elastic.NewBoolQuery().MustNot(elastic.NewExistsQuery("last_seen_on")), []int64{7, 8, 9})
+func TestIndexer(t *testing.T) {
+	db, es := setup(t)
 
-	// urn query
-	query := elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("urns.scheme", "facebook"),
-		elastic.NewMatchQuery("urns.path.keyword", "1000001")))
-	assertQuery(t, client, physicalName, query, []int64{8})
+	ix1 := contacts.NewIndexer(elasticURL, aliasName, 4)
+	assert.Equal(t, "indexer_test", ix1.Name())
 
-	// urn substring query
-	query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("urns.scheme", "tel"),
-		elastic.NewMatchPhraseQuery("urns.path", "779")))
-	assertQuery(t, client, physicalName, query, []int64{1, 2, 3, 6})
+	expectedIndexName := fmt.Sprintf("indexer_test_%s", time.Now().Format("2006_01_02"))
 
-	// urn substring query with more characters (77911)
-	query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("urns.scheme", "tel"),
-		elastic.NewMatchPhraseQuery("urns.path", "77911")))
-	assertQuery(t, client, physicalName, query, []int64{1})
+	indexName, err := ix1.Index(db, false, false)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedIndexName, indexName)
 
-	// urn substring query with more characters (600055)
-	query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("urns.scheme", "tel"),
-		elastic.NewMatchPhraseQuery("urns.path", "600055")))
-	assertQuery(t, client, physicalName, query, []int64{5})
+	time.Sleep(1 * time.Second)
 
-	// match a contact with multiple tel urns
-	query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("urns.scheme", "tel"),
-		elastic.NewMatchPhraseQuery("urns.path", "222")))
-	assertQuery(t, client, physicalName, query, []int64{1})
+	assertIndexerStats(t, ix1, 9, 0)
+	assertIndexesWithPrefix(t, es, aliasName, []string{expectedIndexName})
 
-	// text query
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
-		elastic.NewMatchQuery("fields.text", "the rock")))
-	assertQuery(t, client, physicalName, query, []int64{1})
+	for _, tc := range queryTests {
+		src, _ := tc.query.Source()
+		assertQuery(t, es, tc.query, tc.expected, "query mismatch for %s", string(jsonx.MustMarshal(src)))
+	}
 
-	// people with no nickname
-	notQuery := elastic.NewBoolQuery().MustNot(
-		elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-			elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
-			elastic.NewExistsQuery("fields.text"))))
-	assertQuery(t, client, physicalName, notQuery, []int64{2, 3, 4, 5, 6, 7, 8, 9})
-
-	// no tokenizing of field text
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
-		elastic.NewMatchQuery("fields.text", "rock")))
-	assertQuery(t, client, physicalName, query, []int64{})
-
-	// number field range query
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "05bca1cd-e322-4837-9595-86d0d85e5adb"),
-		elastic.NewRangeQuery("fields.number").Gt(10)))
-	assertQuery(t, client, physicalName, query, []int64{2})
-
-	// datetime field range query
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "e0eac267-463a-4c00-9732-cab62df07b16"),
-		elastic.NewRangeQuery("fields.datetime").Lt(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))))
-	assertQuery(t, client, physicalName, query, []int64{3})
-
-	// state query
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
-		elastic.NewMatchPhraseQuery("fields.state", "washington")))
-	assertQuery(t, client, physicalName, query, []int64{5})
-
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
-		elastic.NewMatchQuery("fields.state_keyword", "  washington")))
-	assertQuery(t, client, physicalName, query, []int64{5})
-
-	// doesn't include country
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
-		elastic.NewMatchQuery("fields.state_keyword", "usa")))
-	assertQuery(t, client, physicalName, query, []int64{})
-
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
-		elastic.NewMatchPhraseQuery("fields.state", "usa")))
-	assertQuery(t, client, physicalName, query, []int64{})
-
-	// district query
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
-		elastic.NewMatchPhraseQuery("fields.district", "king")))
-	assertQuery(t, client, physicalName, query, []int64{7, 9})
-
-	// phrase matches all
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
-		elastic.NewMatchPhraseQuery("fields.district", "King-Côunty")))
-	assertQuery(t, client, physicalName, query, []int64{7})
-
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
-		elastic.NewMatchQuery("fields.district_keyword", "King-Côunty")))
-	assertQuery(t, client, physicalName, query, []int64{7})
-
-	// ward query
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
-		elastic.NewMatchPhraseQuery("fields.ward", "district")))
-	assertQuery(t, client, physicalName, query, []int64{8})
-
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
-		elastic.NewMatchQuery("fields.ward_keyword", "central district")))
-	assertQuery(t, client, physicalName, query, []int64{8})
-
-	// no substring though on keyword
-	query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
-		elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
-		elastic.NewMatchQuery("fields.ward_keyword", "district")))
-	assertQuery(t, client, physicalName, query, []int64{})
-
-	// group query
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4ea0f313-2f62-4e57-bdf0-232b5191dd57"), []int64{1})
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2})
-	assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{})
-
-	lastModified, err := indexer.GetLastModified(elasticURL, physicalName)
+	lastModified, err := ix1.GetLastModified(indexName)
 	assert.NoError(t, err)
 	assert.Equal(t, time.Date(2017, 11, 10, 21, 11, 59, 890662000, time.UTC), lastModified.In(time.UTC))
 
-	// map our index over
-	err = ci.UpdateAlias(physicalName)
+	// now make some contact changes, removing one contact, updating another
+	_, err = db.Exec(`
+	DELETE FROM contacts_contactgroup_contacts WHERE id = 3;
+	UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 2;
+	UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;`)
+	require.NoError(t, err)
+
+	// and index again...
+	indexName, err = ix1.Index(db, false, false)
 	assert.NoError(t, err)
-	time.Sleep(5 * time.Second)
+	assert.Equal(t, expectedIndexName, indexName) // same index used
+	assertIndexerStats(t, ix1, 10, 1)
 
-	// try a test query to check it worked
-	assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{4})
+	time.Sleep(1 * time.Second)
 
-	// look up our mapping
-	physical := ci.FindPhysicalIndexes()
-	assert.Equal(t, physicalName, physical[0])
+	assertIndexesWithPrefix(t, es, aliasName, []string{expectedIndexName})
 
-	// rebuild again
-	newIndex, err := ci.CreateNewIndex(contacts.IndexSettings)
-	assert.NoError(t, err)
+	// should only match new john, old john is gone
+	assertQuery(t, es, elastic.NewMatchQuery("name", "john"), []int64{2})
 
-	added, deleted, err = ci.IndexModified(db, newIndex, time.Time{})
-	assert.NoError(t, err)
-	assert.Equal(t, 9, added)
-	assert.Equal(t, 0, deleted)
+	// 3 is no longer in our group
+	assertQuery(t, es, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1})
 
-	// remap again
-	err = ci.UpdateAlias(newIndex)
-	assert.NoError(t, err)
-	time.Sleep(5 * time.Second)
+	// change John's name to Eric..
+	_, err = db.Exec(`
+	UPDATE contacts_contact SET name = 'Eric', modified_on = '2020-08-20 14:00:00+00' where id = 2;`)
+	require.NoError(t, err)
 
-	// old index still around
-	resp, err := http.Get(fmt.Sprintf("%s/%s", elasticURL, physicalName))
-	assert.NoError(t, err)
-	assert.Equal(t, resp.StatusCode, http.StatusOK)
+	// and simulate another indexer doing a parallel rebuild
+	ix2 := contacts.NewIndexer(elasticURL, aliasName, 4)
 
-	// cleanup our indexes, will remove our original index
-	err = ci.CleanupIndexes()
+	indexName2, err := ix2.Index(db, true, false)
 	assert.NoError(t, err)
+	assert.Equal(t, expectedIndexName+"_1", indexName2) // new index used
+	assertIndexerStats(t, ix2, 8, 0)
 
-	// old physical index should be gone
-	resp, err = http.Get(fmt.Sprintf("%s/%s", elasticURL, physicalName))
-	assert.NoError(t, err)
-	assert.Equal(t, resp.StatusCode, http.StatusNotFound)
+	time.Sleep(1 * time.Second)
 
-	// new index still works
-	assertQuery(t, client, newIndex, elastic.NewMatchQuery("name", "john"), []int64{4})
+	// check we have a new index but the old index is still around
+	assertIndexesWithPrefix(t, es, aliasName, []string{expectedIndexName, expectedIndexName + "_1"})
 
-	// update our database, removing one contact, updating another
-	_, err = db.Exec(`
-	DELETE FROM contacts_contactgroup_contacts WHERE id = 3;
-	UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 2;
-	UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;`)
-	assert.NoError(t, err)
+	// and the alias points to the new index
+	assertQuery(t, es, elastic.NewMatchQuery("name", "eric"), []int64{2})
 
-	added, deleted, err = ci.IndexModified(db, indexName, lastModified)
+	// simulate another indexer doing a parallel rebuild with cleanup
+	ix3 := contacts.NewIndexer(elasticURL, aliasName, 4)
+	indexName3, err := ix3.Index(db, true, true)
 	assert.NoError(t, err)
-	assert.Equal(t, 1, added)
-	assert.Equal(t, 1, deleted)
+	assert.Equal(t, expectedIndexName+"_2", indexName3) // new index used
+	assertIndexerStats(t, ix3, 8, 0)
 
-	time.Sleep(5 * time.Second)
+	// check we cleaned up indexes besides the new one
+	assertIndexesWithPrefix(t, es, aliasName, []string{expectedIndexName + "_2"})
 
-	// should only match new john, old john is gone
-	assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{2})
-
-	// 3 is no longer in our group
-	assertQuery(t, client, indexName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1})
+	// check that the original indexer now indexes against the new index
+	indexName, err = ix1.Index(db, false, false)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedIndexName+"_2", indexName)
 }
diff --git a/http.go b/http.go
index d526984..15277ee 100644
--- a/http.go
+++ b/http.go
@@ -44,11 +44,9 @@ func shouldRetry(request *http.Request, response *http.Response, withDelay time.
 	return false
 }
 
-// MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct
-func MakeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) {
-	req, _ := http.NewRequest(method, url, bytes.NewReader(body))
-	req.Header.Add("Content-Type", "application/json")
-
+// utility function to make a JSON request, optionally decoding the response into the passed in struct
+func makeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) {
+	req, _ := httpx.NewRequest(method, url, bytes.NewReader(body), map[string]string{"Content-Type": "application/json"})
 	resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
 
 	l := log.WithField("url", url).WithField("method", method).WithField("request", body)
diff --git a/indexes.go b/indexes.go
deleted file mode 100644
index f235a14..0000000
--- a/indexes.go
+++ /dev/null
@@ -1,43 +0,0 @@
-package indexer
-
-import (
-	_ "embed"
-	"fmt"
-	"net/http"
-	"time"
-)
-
-// our response for finding the last modified document
-type queryResponse struct {
-	Hits struct {
-		Total struct {
-			Value int `json:"value"`
-		} `json:"total"`
-		Hits []struct {
-			Source struct {
-				ID         int64     `json:"id"`
-				ModifiedOn time.Time `json:"modified_on"`
-			} `json:"_source"`
-		} `json:"hits"`
-	} `json:"hits"`
-}
-
-// GetLastModified queries an index and finds the last modified document, returning its modified time
-func GetLastModified(url string, index string) (time.Time, error) {
-	lastModified := time.Time{}
-	if index == "" {
-		return lastModified, fmt.Errorf("empty index passed to GetLastModified")
-	}
-
-	// get the newest document on our index
-	queryResponse := queryResponse{}
-	_, err := MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/%s/_search", url, index), []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`), &queryResponse)
-	if err != nil {
-		return lastModified, err
-	}
-
-	if len(queryResponse.Hits.Hits) > 0 {
-		lastModified = queryResponse.Hits.Hits[0].Source.ModifiedOn
-	}
-	return lastModified, nil
-}

From 83e8231d0f9a125b3ca2b3a16d225793c82e267a Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 24 Mar 2022 11:00:44 -0500
Subject: [PATCH 16/48] Move all indexers into same package

---
 base_test.go                                  |  95 ---------------
 cmd/rp-indexer/main.go                        |   4 +-
 contacts/query.go                             |  74 ------------
 base.go => indexers/base.go                   |  70 ++++++-----
 indexers/base_test.go                         |  84 +++++++++++++
 contacts/indexer.go => indexers/contacts.go   | 113 +++++++++++++-----
 .../contacts.settings.json                    |   0
 .../contacts_test.go                          |  91 ++------------
 http.go => utils/http.go                      |   6 +-
 utils/http_test.go                            |  40 +++++++
 10 files changed, 260 insertions(+), 317 deletions(-)
 delete mode 100644 base_test.go
 delete mode 100644 contacts/query.go
 rename base.go => indexers/base.go (72%)
 create mode 100644 indexers/base_test.go
 rename contacts/indexer.go => indexers/contacts.go (56%)
 rename contacts/index_settings.json => indexers/contacts.settings.json (100%)
 rename contacts/indexer_test.go => indexers/contacts_test.go (79%)
 rename http.go => utils/http.go (91%)
 create mode 100644 utils/http_test.go

diff --git a/base_test.go b/base_test.go
deleted file mode 100644
index 9327622..0000000
--- a/base_test.go
+++ /dev/null
@@ -1,95 +0,0 @@
-package indexer_test
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-
-	"github.com/nyaruka/rp-indexer/contacts"
-	"github.com/stretchr/testify/require"
-)
-
-func TestRetryServer(t *testing.T) {
-	responseCounter := 0
-	responses := []func(w http.ResponseWriter, r *http.Request){
-		func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Length", "5")
-		},
-		func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Length", "1")
-		},
-		func(w http.ResponseWriter, r *http.Request) {
-			w.Header().Set("Content-Length", "1")
-		},
-		func(w http.ResponseWriter, r *http.Request) {
-			resp := `{
-				"took": 1,
-				"timed_out": false,
-				"_shards": {
-				  "total": 2,
-				  "successful": 2,
-				  "skipped": 0,
-				  "failed": 0
-				},
-				"hits": {
-				  "total": 1,
-				  "max_score": null,
-				  "hits": [
-					{
-					  "_index": "rp_elastic_test_2020_08_14_1",
-					  "_type": "_doc",
-					  "_id": "1",
-					  "_score": null,
-					  "_routing": "1",
-					  "_source": {
-						"id": 1,
-						"org_id": 1,
-						"uuid": "c7a2dd87-a80e-420b-8431-ca48d422e924",
-						"name": null,
-						"language": "eng",
-						"is_active": true,
-						"created_on": "2017-11-10T16:11:59.890662-05:00",
-						"modified_on": "2017-11-10T16:11:59.890662-05:00",
-						"last_seen_on": "2020-08-04T21:11:00-04:00",
-						"modified_on_mu": 1.510348319890662e15,
-						"urns": [
-						  {
-							"scheme": "tel",
-							"path": "+12067791111"
-						  },
-						  {
-							"scheme": "tel",
-							"path": "+12067792222"
-						  }
-						],
-						"fields": [
-						  {
-							"text": "the rock",
-							"field": "17103bb1-1b48-4b70-92f7-1f6b73bd3488"
-						  }
-						],
-						"groups": [
-						  "4ea0f313-2f62-4e57-bdf0-232b5191dd57",
-						  "529bac39-550a-4d6f-817c-1833f3449007"
-						]
-					  },
-					  "sort": [1]
-					}
-				  ]
-				}
-			  }`
-
-			w.Write([]byte(resp))
-		},
-	}
-	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		responses[responseCounter](w, r)
-		responseCounter++
-	}))
-	defer ts.Close()
-
-	ci := contacts.NewIndexer(ts.URL, "rp_elastic_test", 500)
-	ci.FindIndexes()
-
-	require.Equal(t, responseCounter, 4)
-}
diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index b50f89e..021bea1 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -8,7 +8,7 @@ import (
 	"github.com/evalphobia/logrus_sentry"
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/ezconf"
-	"github.com/nyaruka/rp-indexer/contacts"
+	"github.com/nyaruka/rp-indexer/indexers"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -64,7 +64,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	ci := contacts.NewIndexer(config.ElasticURL, config.Index, 500)
+	ci := indexers.NewContactIndexer(config.ElasticURL, config.Index, 500)
 
 	for {
 		_, err := ci.Index(db, config.Rebuild, config.Cleanup)
diff --git a/contacts/query.go b/contacts/query.go
deleted file mode 100644
index d909a02..0000000
--- a/contacts/query.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package contacts
-
-import (
-	"database/sql"
-	"time"
-)
-
-const sqlSelectModified = `
-SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
-	SELECT
-		id,
-		org_id,
-		uuid,
-		name,
-		language,
-		status,
-		ticket_count AS tickets,
-		is_active,
-		created_on,
-		modified_on,
-		last_seen_on,
-		EXTRACT(EPOCH FROM modified_on) * 1000000 AS modified_on_mu,
-		(
-			SELECT array_to_json(array_agg(row_to_json(u)))
-			FROM (SELECT scheme, path FROM contacts_contacturn WHERE contact_id = contacts_contact.id) u
-		) AS urns,
-		(
-			SELECT jsonb_agg(f.value)
-			FROM (
-				SELECT 
-					CASE
-					WHEN value ? 'ward'
-					THEN jsonb_build_object('ward_keyword', trim(substring(value ->> 'ward' from  '(?!.* > )([^>]+)')))
-					ELSE '{}'::jsonb
-					END || district_value.value AS value
-				FROM (
-					SELECT 
-						CASE
-						WHEN value ? 'district'
-						THEN jsonb_build_object('district_keyword', trim(substring(value ->> 'district' from  '(?!.* > )([^>]+)')))
-						ELSE '{}'::jsonb
-						END || state_value.value as value
-					FROM (
-						SELECT 
-							CASE
-							WHEN value ? 'state'
-							THEN jsonb_build_object('state_keyword', trim(substring(value ->> 'state' from  '(?!.* > )([^>]+)')))
-							ELSE '{}' :: jsonb
-							END || jsonb_build_object('field', key) || value as value
-						FROM jsonb_each(contacts_contact.fields)
-					) state_value
-				) AS district_value
-			) AS f
-		) AS fields,
-		(
-			SELECT array_to_json(array_agg(g.uuid)) FROM (
-				SELECT contacts_contactgroup.uuid
-				FROM contacts_contactgroup_contacts, contacts_contactgroup
-				WHERE contact_id = contacts_contact.id AND contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
-			) g
-		) AS groups,
-		(
-			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
-		) AS flow
-	FROM contacts_contact
-	WHERE modified_on >= $1
-	ORDER BY modified_on ASC
-	LIMIT 500000
-) t;
-`
-
-func FetchModified(db *sql.DB, lastModified time.Time) (*sql.Rows, error) {
-	return db.Query(sqlSelectModified, lastModified)
-}
diff --git a/base.go b/indexers/base.go
similarity index 72%
rename from base.go
rename to indexers/base.go
index eeb3b73..808ddcb 100644
--- a/base.go
+++ b/indexers/base.go
@@ -1,4 +1,4 @@
-package indexer
+package indexers
 
 import (
 	"database/sql"
@@ -10,9 +10,16 @@ import (
 	"time"
 
 	"github.com/nyaruka/gocommon/jsonx"
+	"github.com/nyaruka/rp-indexer/utils"
 	"github.com/sirupsen/logrus"
 )
 
+// indexes a document
+const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
+
+// deletes a document
+const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
+
 // Indexer is base interface for indexers
 type Indexer interface {
 	Name() string
@@ -20,7 +27,7 @@ type Indexer interface {
 	Stats() (int64, int64, time.Duration)
 }
 
-type BaseIndexer struct {
+type baseIndexer struct {
 	elasticURL string
 	name       string // e.g. contacts, used as the alias
 
@@ -30,38 +37,38 @@ type BaseIndexer struct {
 	elapsedTotal time.Duration
 }
 
-func NewBaseIndexer(elasticURL, name string) BaseIndexer {
-	return BaseIndexer{elasticURL: elasticURL, name: name}
+func newBaseIndexer(elasticURL, name string) baseIndexer {
+	return baseIndexer{elasticURL: elasticURL, name: name}
 }
 
-func (i *BaseIndexer) Name() string {
+func (i *baseIndexer) Name() string {
 	return i.name
 }
 
-func (i *BaseIndexer) Log() *logrus.Entry {
-	return logrus.WithField("indexer", i.name)
+func (i *baseIndexer) Stats() (int64, int64, time.Duration) {
+	return i.indexedTotal, i.deletedTotal, i.elapsedTotal
 }
 
-func (i *BaseIndexer) Stats() (int64, int64, time.Duration) {
-	return i.indexedTotal, i.deletedTotal, i.elapsedTotal
+func (i *baseIndexer) log() *logrus.Entry {
+	return logrus.WithField("indexer", i.name)
 }
 
-// RecordComplete records a complete index and updates statistics
-func (i *BaseIndexer) RecordComplete(indexed, deleted int, elapsed time.Duration) {
+// records a complete index and updates statistics
+func (i *baseIndexer) recordComplete(indexed, deleted int, elapsed time.Duration) {
 	i.indexedTotal += int64(indexed)
 	i.deletedTotal += int64(deleted)
 	i.elapsedTotal += elapsed
 
-	i.Log().WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
+	i.log().WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
 }
 
 // our response for figuring out the physical index for an alias
 type infoResponse map[string]interface{}
 
 // FindIndexes finds all our physical indexes
-func (i *BaseIndexer) FindIndexes() []string {
+func (i *baseIndexer) FindIndexes() []string {
 	response := infoResponse{}
-	_, err := makeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.elasticURL, i.name), nil, &response)
+	_, err := utils.MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.elasticURL, i.name), nil, &response)
 	indexes := make([]string, 0)
 
 	// error could mean a variety of things, but we'll figure that out later
@@ -77,19 +84,19 @@ func (i *BaseIndexer) FindIndexes() []string {
 	// reverse sort order should put our newest index first
 	sort.Sort(sort.Reverse(sort.StringSlice(indexes)))
 
-	i.Log().WithField("indexes", indexes).Debug("found physical indexes")
+	i.log().WithField("indexes", indexes).Debug("found physical indexes")
 
 	return indexes
 }
 
-// CreateNewIndex creates a new index for the passed in alias.
+// creates a new index for the passed in alias.
 //
 // Note that we do not create an index with the passed name, instead creating one
 // based on the day, for example `contacts_2018_03_05`, then create an alias from
 // that index to `contacts`.
 //
 // If the day-specific name already exists, we append a .1 or .2 to the name.
-func (i *BaseIndexer) CreateNewIndex(settings json.RawMessage) (string, error) {
+func (i *baseIndexer) createNewIndex(settings json.RawMessage) (string, error) {
 	// create our day-specific name
 	index := fmt.Sprintf("%s_%s", i.name, time.Now().Format("2006_01_02"))
 	idx := 0
@@ -111,13 +118,13 @@ func (i *BaseIndexer) CreateNewIndex(settings json.RawMessage) (string, error) {
 	}
 
 	// create the new index
-	_, err := makeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.elasticURL, index), settings, nil)
+	_, err := utils.MakeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.elasticURL, index), settings, nil)
 	if err != nil {
 		return "", err
 	}
 
 	// all went well, return our physical index name
-	i.Log().WithField("index", index).Info("created new index")
+	i.log().WithField("index", index).Info("created new index")
 
 	return index, nil
 }
@@ -143,9 +150,8 @@ type removeAliasCommand struct {
 	} `json:"remove"`
 }
 
-// UpdateAlias maps the passed in alias to the new physical index, optionally removing
-// existing aliases if they exit.
-func (i *BaseIndexer) UpdateAlias(newIndex string) error {
+// maps this indexer's alias to the new physical index, removing existing aliases if they exist
+func (i *baseIndexer) updateAlias(newIndex string) error {
 	commands := make([]interface{}, 0)
 
 	// find existing physical indexes
@@ -167,9 +173,9 @@ func (i *BaseIndexer) UpdateAlias(newIndex string) error {
 
 	aliasJSON := jsonx.MustMarshal(aliasCommand{Actions: commands})
 
-	_, err := makeJSONRequest(http.MethodPost, fmt.Sprintf("%s/_aliases", i.elasticURL), aliasJSON, nil)
+	_, err := utils.MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/_aliases", i.elasticURL), aliasJSON, nil)
 
-	i.Log().WithField("index", newIndex).Info("updated alias")
+	i.log().WithField("index", newIndex).Info("updated alias")
 
 	return err
 }
@@ -181,8 +187,8 @@ type healthResponse struct {
 	} `json:"indices"`
 }
 
-// CleanupIndexes removes all indexes that are older than the currently active index
-func (i *BaseIndexer) CleanupIndexes() error {
+// removes all indexes that are older than the currently active index
+func (i *baseIndexer) cleanupIndexes() error {
 	// find our current indexes
 	currents := i.FindIndexes()
 
@@ -193,7 +199,7 @@ func (i *BaseIndexer) CleanupIndexes() error {
 
 	// find all the current indexes
 	healthResponse := healthResponse{}
-	_, err := makeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.elasticURL, "_cluster/health?level=indices"), nil, &healthResponse)
+	_, err := utils.MakeJSONRequest(http.MethodGet, fmt.Sprintf("%s/%s", i.elasticURL, "_cluster/health?level=indices"), nil, &healthResponse)
 	if err != nil {
 		return err
 	}
@@ -202,7 +208,7 @@ func (i *BaseIndexer) CleanupIndexes() error {
 	for key := range healthResponse.Indices {
 		if strings.HasPrefix(key, i.name) && strings.Compare(key, currents[0]) < 0 {
 			logrus.WithField("index", key).Info("removing old index")
-			_, err = makeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", i.elasticURL, key), nil, nil)
+			_, err = utils.MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", i.elasticURL, key), nil, nil)
 			if err != nil {
 				return err
 			}
@@ -228,11 +234,11 @@ type indexResponse struct {
 }
 
 // indexes the batch of contacts
-func (i *BaseIndexer) IndexBatch(index string, batch []byte) (int, int, error) {
+func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) {
 	response := indexResponse{}
 	indexURL := fmt.Sprintf("%s/%s/_bulk", i.elasticURL, index)
 
-	_, err := makeJSONRequest(http.MethodPut, indexURL, batch, &response)
+	_, err := utils.MakeJSONRequest(http.MethodPut, indexURL, batch, &response)
 	if err != nil {
 		return 0, 0, err
 	}
@@ -280,12 +286,12 @@ type queryResponse struct {
 }
 
 // GetLastModified queries a concrete index and finds the last modified document, returning its modified time
-func (i *BaseIndexer) GetLastModified(index string) (time.Time, error) {
+func (i *baseIndexer) GetLastModified(index string) (time.Time, error) {
 	lastModified := time.Time{}
 
 	// get the newest document on our index
 	queryResponse := queryResponse{}
-	_, err := makeJSONRequest(http.MethodPost, fmt.Sprintf("%s/%s/_search", i.elasticURL, index), []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`), &queryResponse)
+	_, err := utils.MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/%s/_search", i.elasticURL, index), []byte(`{ "sort": [{ "modified_on_mu": "desc" }]}`), &queryResponse)
 	if err != nil {
 		return lastModified, err
 	}
diff --git a/indexers/base_test.go b/indexers/base_test.go
new file mode 100644
index 0000000..c4a4d54
--- /dev/null
+++ b/indexers/base_test.go
@@ -0,0 +1,84 @@
+package indexers_test
+
+import (
+	"context"
+	"database/sql"
+	"io/ioutil"
+	"log"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/nyaruka/rp-indexer/indexers"
+	"github.com/olivere/elastic/v7"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const elasticURL = "http://localhost:9200"
+const aliasName = "indexer_test"
+
+func setup(t *testing.T) (*sql.DB, *elastic.Client) {
+	testDB, err := ioutil.ReadFile("../testdb.sql")
+	require.NoError(t, err)
+
+	db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")
+	require.NoError(t, err)
+
+	_, err = db.Exec(string(testDB))
+	require.NoError(t, err)
+
+	es, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
+	require.NoError(t, err)
+
+	// delete all indexes with our alias prefix
+	existing, err := es.IndexNames()
+	require.NoError(t, err)
+
+	for _, name := range existing {
+		if strings.HasPrefix(name, aliasName) {
+			_, err = es.DeleteIndex(name).Do(context.Background())
+			require.NoError(t, err)
+		}
+	}
+
+	logrus.SetLevel(logrus.DebugLevel)
+
+	return db, es
+}
+
+func assertQuery(t *testing.T, client *elastic.Client, query elastic.Query, expected []int64, msgAndArgs ...interface{}) {
+	results, err := client.Search().Index(aliasName).Query(query).Sort("id", true).Pretty(true).Do(context.Background())
+	assert.NoError(t, err)
+
+	actual := make([]int64, len(results.Hits.Hits))
+	for h, hit := range results.Hits.Hits {
+		asInt, _ := strconv.Atoi(hit.Id)
+		actual[h] = int64(asInt)
+	}
+
+	assert.Equal(t, expected, actual, msgAndArgs...)
+}
+
+func assertIndexesWithPrefix(t *testing.T, es *elastic.Client, prefix string, expected []string) {
+	all, err := es.IndexNames()
+	require.NoError(t, err)
+
+	actual := []string{}
+	for _, name := range all {
+		if strings.HasPrefix(name, prefix) {
+			actual = append(actual, name)
+		}
+	}
+	sort.Strings(actual)
+	assert.Equal(t, expected, actual)
+}
+
+func assertIndexerStats(t *testing.T, ix indexers.Indexer, expectedIndexed, expectedDeleted int64) {
+	actualIndexed, actualDeleted, _ := ix.Stats()
+	assert.Equal(t, expectedIndexed, actualIndexed, "indexed mismatch")
+	assert.Equal(t, expectedDeleted, actualDeleted, "deleted mismatch")
+}
diff --git a/contacts/indexer.go b/indexers/contacts.go
similarity index 56%
rename from contacts/indexer.go
rename to indexers/contacts.go
index df3f581..e25d198 100644
--- a/contacts/indexer.go
+++ b/indexers/contacts.go
@@ -1,4 +1,4 @@
-package contacts
+package indexers
 
 import (
 	"bytes"
@@ -8,37 +8,30 @@ import (
 	"fmt"
 	"time"
 
-	indexer "github.com/nyaruka/rp-indexer"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
-//go:embed index_settings.json
-var indexSettings json.RawMessage
-
-// indexes a contact
-const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
-
-// deletes a contact
-const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
+//go:embed contacts.settings.json
+var contactsSettings json.RawMessage
 
 // ContactIndexer is an indexer for contacts
-type Indexer struct {
-	indexer.BaseIndexer
+type ContactIndexer struct {
+	baseIndexer
 
 	batchSize int
 }
 
-// NewIndexer creates a new contact indexer
-func NewIndexer(elasticURL, name string, batchSize int) *Indexer {
-	return &Indexer{
-		BaseIndexer: indexer.NewBaseIndexer(elasticURL, name),
+// NewContactIndexer creates a new contact indexer
+func NewContactIndexer(elasticURL, name string, batchSize int) *ContactIndexer {
+	return &ContactIndexer{
+		baseIndexer: newBaseIndexer(elasticURL, name),
 		batchSize:   batchSize,
 	}
 }
 
 // Index indexes modified contacts and returns the name of the concrete index
-func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
+func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 	var err error
 
 	// find our physical index
@@ -54,11 +47,11 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 
 	// doesn't exist or we are rebuilding, create it
 	if physicalIndex == "" || rebuild {
-		physicalIndex, err = i.CreateNewIndex(indexSettings)
+		physicalIndex, err = i.createNewIndex(contactsSettings)
 		if err != nil {
 			return "", errors.Wrap(err, "error creating new index")
 		}
-		i.Log().WithField("index", physicalIndex).Info("created new physical index")
+		i.log().WithField("index", physicalIndex).Info("created new physical index")
 		remapAlias = true
 	}
 
@@ -67,7 +60,7 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 		return "", errors.Wrap(err, "error finding last modified")
 	}
 
-	i.Log().WithField("index", physicalIndex).WithField("last_modified", lastModified).Info("indexing newer than last modified")
+	i.log().WithField("index", physicalIndex).WithField("last_modified", lastModified).Info("indexing newer than last modified")
 
 	// now index our docs
 	start := time.Now()
@@ -76,11 +69,11 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 		return "", errors.Wrap(err, "error indexing documents")
 	}
 
-	i.RecordComplete(indexed, deleted, time.Since(start))
+	i.recordComplete(indexed, deleted, time.Since(start))
 
 	// if the index didn't previously exist or we are rebuilding, remap to our alias
 	if remapAlias {
-		err := i.UpdateAlias(physicalIndex)
+		err := i.updateAlias(physicalIndex)
 		if err != nil {
 			return "", errors.Wrap(err, "error updating alias")
 		}
@@ -89,7 +82,7 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 
 	// cleanup our aliases if appropriate
 	if cleanup {
-		err := i.CleanupIndexes()
+		err := i.cleanupIndexes()
 		if err != nil {
 			return "", errors.Wrap(err, "error cleaning up old indexes")
 		}
@@ -98,8 +91,72 @@ func (i *Indexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) {
 	return physicalIndex, nil
 }
 
+const sqlSelectModifiedContacts = `
+SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
+	SELECT
+		id,
+		org_id,
+		uuid,
+		name,
+		language,
+		status,
+		ticket_count AS tickets,
+		is_active,
+		created_on,
+		modified_on,
+		last_seen_on,
+		EXTRACT(EPOCH FROM modified_on) * 1000000 AS modified_on_mu,
+		(
+			SELECT array_to_json(array_agg(row_to_json(u)))
+			FROM (SELECT scheme, path FROM contacts_contacturn WHERE contact_id = contacts_contact.id) u
+		) AS urns,
+		(
+			SELECT jsonb_agg(f.value)
+			FROM (
+				SELECT 
+					CASE
+					WHEN value ? 'ward'
+					THEN jsonb_build_object('ward_keyword', trim(substring(value ->> 'ward' from  '(?!.* > )([^>]+)')))
+					ELSE '{}'::jsonb
+					END || district_value.value AS value
+				FROM (
+					SELECT 
+						CASE
+						WHEN value ? 'district'
+						THEN jsonb_build_object('district_keyword', trim(substring(value ->> 'district' from  '(?!.* > )([^>]+)')))
+						ELSE '{}'::jsonb
+						END || state_value.value as value
+					FROM (
+						SELECT 
+							CASE
+							WHEN value ? 'state'
+							THEN jsonb_build_object('state_keyword', trim(substring(value ->> 'state' from  '(?!.* > )([^>]+)')))
+							ELSE '{}' :: jsonb
+							END || jsonb_build_object('field', key) || value as value
+						FROM jsonb_each(contacts_contact.fields)
+					) state_value
+				) AS district_value
+			) AS f
+		) AS fields,
+		(
+			SELECT array_to_json(array_agg(g.uuid)) FROM (
+				SELECT contacts_contactgroup.uuid
+				FROM contacts_contactgroup_contacts, contacts_contactgroup
+				WHERE contact_id = contacts_contact.id AND contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
+			) g
+		) AS groups,
+		(
+			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
+		) AS flow
+	FROM contacts_contact
+	WHERE modified_on >= $1
+	ORDER BY modified_on ASC
+	LIMIT 500000
+) t;
+`
+
 // IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time
-func (i *Indexer) indexModified(db *sql.DB, index string, lastModified time.Time) (int, int, error) {
+func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified time.Time) (int, int, error) {
 	batch := &bytes.Buffer{}
 	createdCount, deletedCount, processedCount := 0, 0, 0
 
@@ -111,7 +168,7 @@ func (i *Indexer) indexModified(db *sql.DB, index string, lastModified time.Time
 	start := time.Now()
 
 	for {
-		rows, err := FetchModified(db, lastModified)
+		rows, err := db.Query(sqlSelectModifiedContacts, lastModified)
 
 		queryCreated := 0
 		queryCount := 0
@@ -152,7 +209,7 @@ func (i *Indexer) indexModified(db *sql.DB, index string, lastModified time.Time
 
 			// write to elastic search in batches
 			if queryCount%i.batchSize == 0 {
-				created, deleted, err := i.IndexBatch(index, batch.Bytes())
+				created, deleted, err := i.indexBatch(index, batch.Bytes())
 				if err != nil {
 					return 0, 0, err
 				}
@@ -165,7 +222,7 @@ func (i *Indexer) indexModified(db *sql.DB, index string, lastModified time.Time
 		}
 
 		if batch.Len() > 0 {
-			created, deleted, err := i.IndexBatch(index, batch.Bytes())
+			created, deleted, err := i.indexBatch(index, batch.Bytes())
 			if err != nil {
 				return 0, 0, err
 			}
@@ -186,7 +243,7 @@ func (i *Indexer) indexModified(db *sql.DB, index string, lastModified time.Time
 		elapsed := time.Since(start)
 		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
 
-		i.Log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed}).Info("indexed contact batch")
+		i.log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed}).Info("indexed contact batch")
 	}
 
 	return createdCount, deletedCount, nil
diff --git a/contacts/index_settings.json b/indexers/contacts.settings.json
similarity index 100%
rename from contacts/index_settings.json
rename to indexers/contacts.settings.json
diff --git a/contacts/indexer_test.go b/indexers/contacts_test.go
similarity index 79%
rename from contacts/indexer_test.go
rename to indexers/contacts_test.go
index 57d6754..8ef9916 100644
--- a/contacts/indexer_test.go
+++ b/indexers/contacts_test.go
@@ -1,94 +1,19 @@
-package contacts_test
+package indexers_test
 
 import (
-	"context"
-	"database/sql"
 	"fmt"
-	"io/ioutil"
-	"log"
-	"os"
-	"sort"
-	"strconv"
-	"strings"
 	"testing"
 	"time"
 
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/gocommon/jsonx"
-	indexer "github.com/nyaruka/rp-indexer"
-	"github.com/nyaruka/rp-indexer/contacts"
+	"github.com/nyaruka/rp-indexer/indexers"
 	"github.com/olivere/elastic/v7"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-const elasticURL = "http://localhost:9200"
-const aliasName = "indexer_test"
-
-func setup(t *testing.T) (*sql.DB, *elastic.Client) {
-	testDB, err := ioutil.ReadFile("../testdb.sql")
-	require.NoError(t, err)
-
-	db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")
-	require.NoError(t, err)
-
-	_, err = db.Exec(string(testDB))
-	require.NoError(t, err)
-
-	es, err := elastic.NewClient(elastic.SetURL(elasticURL), elastic.SetTraceLog(log.New(os.Stdout, "", log.LstdFlags)), elastic.SetSniff(false))
-	require.NoError(t, err)
-
-	// delete all indexes with our alias prefix
-	existing, err := es.IndexNames()
-	require.NoError(t, err)
-
-	for _, name := range existing {
-		if strings.HasPrefix(name, aliasName) {
-			_, err = es.DeleteIndex(name).Do(context.Background())
-			require.NoError(t, err)
-		}
-	}
-
-	logrus.SetLevel(logrus.DebugLevel)
-
-	return db, es
-}
-
-func assertQuery(t *testing.T, client *elastic.Client, query elastic.Query, expected []int64, msgAndArgs ...interface{}) {
-	results, err := client.Search().Index(aliasName).Query(query).Sort("id", true).Pretty(true).Do(context.Background())
-	assert.NoError(t, err)
-
-	actual := make([]int64, len(results.Hits.Hits))
-	for h, hit := range results.Hits.Hits {
-		asInt, _ := strconv.Atoi(hit.Id)
-		actual[h] = int64(asInt)
-	}
-
-	assert.Equal(t, expected, actual, msgAndArgs...)
-}
-
-func assertIndexesWithPrefix(t *testing.T, es *elastic.Client, prefix string, expected []string) {
-	all, err := es.IndexNames()
-	require.NoError(t, err)
-
-	actual := []string{}
-	for _, name := range all {
-		if strings.HasPrefix(name, prefix) {
-			actual = append(actual, name)
-		}
-	}
-	sort.Strings(actual)
-	assert.Equal(t, expected, actual)
-}
-
-func assertIndexerStats(t *testing.T, ix indexer.Indexer, expectedIndexed, expectedDeleted int64) {
-	actualIndexed, actualDeleted, _ := ix.Stats()
-	assert.Equal(t, expectedIndexed, actualIndexed, "indexed mismatch")
-	assert.Equal(t, expectedDeleted, actualDeleted, "deleted mismatch")
-}
-
-var queryTests = []struct {
+var contactQueryTests = []struct {
 	query    elastic.Query
 	expected []int64
 }{
@@ -256,10 +181,10 @@ var queryTests = []struct {
 	{elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{}},
 }
 
-func TestIndexer(t *testing.T) {
+func TestContacts(t *testing.T) {
 	db, es := setup(t)
 
-	ix1 := contacts.NewIndexer(elasticURL, aliasName, 4)
+	ix1 := indexers.NewContactIndexer(elasticURL, aliasName, 4)
 	assert.Equal(t, "indexer_test", ix1.Name())
 
 	expectedIndexName := fmt.Sprintf("indexer_test_%s", time.Now().Format("2006_01_02"))
@@ -273,7 +198,7 @@ func TestIndexer(t *testing.T) {
 	assertIndexerStats(t, ix1, 9, 0)
 	assertIndexesWithPrefix(t, es, aliasName, []string{expectedIndexName})
 
-	for _, tc := range queryTests {
+	for _, tc := range contactQueryTests {
 		src, _ := tc.query.Source()
 		assertQuery(t, es, tc.query, tc.expected, "query mismatch for %s", string(jsonx.MustMarshal(src)))
 	}
@@ -311,7 +236,7 @@ func TestIndexer(t *testing.T) {
 	require.NoError(t, err)
 
 	// and simulate another indexer doing a parallel rebuild
-	ix2 := contacts.NewIndexer(elasticURL, aliasName, 4)
+	ix2 := indexers.NewContactIndexer(elasticURL, aliasName, 4)
 
 	indexName2, err := ix2.Index(db, true, false)
 	assert.NoError(t, err)
@@ -327,7 +252,7 @@ func TestIndexer(t *testing.T) {
 	assertQuery(t, es, elastic.NewMatchQuery("name", "eric"), []int64{2})
 
 	// simulate another indexer doing a parallel rebuild with cleanup
-	ix3 := contacts.NewIndexer(elasticURL, aliasName, 4)
+	ix3 := indexers.NewContactIndexer(elasticURL, aliasName, 4)
 	indexName3, err := ix3.Index(db, true, true)
 	assert.NoError(t, err)
 	assert.Equal(t, expectedIndexName+"_2", indexName3) // new index used
diff --git a/http.go b/utils/http.go
similarity index 91%
rename from http.go
rename to utils/http.go
index 15277ee..a246b4a 100644
--- a/http.go
+++ b/utils/http.go
@@ -1,4 +1,4 @@
-package indexer
+package utils
 
 import (
 	"bytes"
@@ -44,8 +44,8 @@ func shouldRetry(request *http.Request, response *http.Response, withDelay time.
 	return false
 }
 
-// utility function to make a JSON request, optionally decoding the response into the passed in struct
-func makeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) {
+// MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct
+func MakeJSONRequest(method string, url string, body []byte, jsonStruct interface{}) (*http.Response, error) {
 	req, _ := httpx.NewRequest(method, url, bytes.NewReader(body), map[string]string{"Content-Type": "application/json"})
 	resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil)
 
diff --git a/utils/http_test.go b/utils/http_test.go
new file mode 100644
index 0000000..fd8dea7
--- /dev/null
+++ b/utils/http_test.go
@@ -0,0 +1,40 @@
+package utils_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/nyaruka/rp-indexer/utils"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRetryServer(t *testing.T) {
+	responseCounter := 0
+	responses := []func(w http.ResponseWriter, r *http.Request){
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Length", "5")
+		},
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Length", "1")
+		},
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Length", "1")
+		},
+		func(w http.ResponseWriter, r *http.Request) {
+			w.Write([]byte(`{"foo": 1}`))
+		},
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		responses[responseCounter](w, r)
+		responseCounter++
+	}))
+	defer ts.Close()
+
+	resp, err := utils.MakeJSONRequest("GET", ts.URL, nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, 200, resp.StatusCode)
+
+	require.Equal(t, responseCounter, 4)
+}

From 3a051b34afccc3d69ce72b2295505c497b14c849 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 24 Mar 2022 13:48:27 -0500
Subject: [PATCH 17/48] Rework cmd to use a daemon process

---
 cmd/rp-indexer/main.go | 83 ++++++++++++++++++++----------------------
 config.go              | 33 +++++++++++++++++
 daemon.go              | 75 ++++++++++++++++++++++++++++++++++++++
 go.mod                 |  1 +
 go.sum                 |  6 +++
 5 files changed, 154 insertions(+), 44 deletions(-)
 create mode 100644 config.go
 create mode 100644 daemon.go

diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index 021bea1..b959c05 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -3,86 +3,81 @@ package main
 import (
 	"database/sql"
 	"os"
-	"time"
+	"os/signal"
+	"syscall"
 
 	"github.com/evalphobia/logrus_sentry"
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/ezconf"
+	indexer "github.com/nyaruka/rp-indexer"
 	"github.com/nyaruka/rp-indexer/indexers"
 	log "github.com/sirupsen/logrus"
 )
 
-type config struct {
-	ElasticURL string `help:"the url for our elastic search instance"`
-	DB         string `help:"the connection string for our database"`
-	Index      string `help:"the alias for our contact index"`
-	Poll       int    `help:"the number of seconds to wait between checking for updated contacts"`
-	Rebuild    bool   `help:"whether to rebuild the index, swapping it when complete, then exiting (default false)"`
-	Cleanup    bool   `help:"whether to remove old indexes after a rebuild"`
-	LogLevel   string `help:"the log level, one of error, warn, info, debug"`
-	SentryDSN  string `help:"the sentry configuration to log errors to, if any"`
-}
-
 func main() {
-	config := config{
-		ElasticURL: "http://localhost:9200",
-		DB:         "postgres://localhost/temba?sslmode=disable",
-		Index:      "contacts",
-		Poll:       5,
-		Rebuild:    false,
-		Cleanup:    false,
-		LogLevel:   "info",
-	}
-	loader := ezconf.NewLoader(&config, "indexer", "Indexes RapidPro contacts to ElasticSearch", []string{"indexer.toml"})
+	cfg := indexer.NewDefaultConfig()
+	loader := ezconf.NewLoader(cfg, "indexer", "Indexes RapidPro contacts to ElasticSearch", []string{"indexer.toml"})
 	loader.MustLoad()
 
 	// configure our logger
 	log.SetOutput(os.Stdout)
 	log.SetFormatter(&log.TextFormatter{})
 
-	level, err := log.ParseLevel(config.LogLevel)
+	level, err := log.ParseLevel(cfg.LogLevel)
 	if err != nil {
 		log.Fatalf("Invalid log level '%s'", level)
 	}
 	log.SetLevel(level)
 
 	// if we have a DSN entry, try to initialize it
-	if config.SentryDSN != "" {
-		hook, err := logrus_sentry.NewSentryHook(config.SentryDSN, []log.Level{log.PanicLevel, log.FatalLevel, log.ErrorLevel})
+	if cfg.SentryDSN != "" {
+		hook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []log.Level{log.PanicLevel, log.FatalLevel, log.ErrorLevel})
 		hook.Timeout = 0
 		hook.StacktraceConfiguration.Enable = true
 		hook.StacktraceConfiguration.Skip = 4
 		hook.StacktraceConfiguration.Context = 5
 		if err != nil {
-			log.Fatalf("invalid sentry DSN: '%s': %s", config.SentryDSN, err)
+			log.Fatalf("invalid sentry DSN: '%s': %s", cfg.SentryDSN, err)
 		}
 		log.StandardLogger().Hooks.Add(hook)
 	}
 
-	db, err := sql.Open("postgres", config.DB)
+	db, err := sql.Open("postgres", cfg.DB)
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("unable to connect to database")
 	}
 
-	ci := indexers.NewContactIndexer(config.ElasticURL, config.Index, 500)
-
-	for {
-		_, err := ci.Index(db, config.Rebuild, config.Cleanup)
+	idxrs := []indexers.Indexer{
+		indexers.NewContactIndexer(cfg.ElasticURL, cfg.Index, 500),
+	}
 
-		if err != nil {
-			if config.Rebuild {
-				log.WithField("index", config.Index).WithError(err).Fatal("error during rebuilding")
-			} else {
-				log.WithField("index", config.Index).WithError(err).Error("error during indexing")
-			}
+	if cfg.Rebuild {
+		// if rebuilding, just do a complete index and quit. In future when we support multiple indexers,
+		// the rebuild argument can be become the name of the index to rebuild, e.g. --rebuild=contacts
+		idxr := idxrs[0]
+		if _, err := idxr.Index(db, true, cfg.Cleanup); err != nil {
+			log.WithField("indexer", idxr.Name()).WithError(err).Fatal("error during rebuilding")
 		}
+	} else {
+		d := indexer.NewDaemon(cfg, db, idxrs)
+		d.Start()
 
-		// if we were rebuilding then we're done
-		if config.Rebuild {
-			os.Exit(0)
-		}
+		handleSignals(d)
+	}
+}
 
-		// sleep a bit before starting again
-		time.Sleep(time.Second * 5)
+// handleSignals takes care of trapping quit, interrupt or terminate signals and doing the right thing
+func handleSignals(d *indexer.Daemon) {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+
+	for {
+		sig := <-sigs
+		switch sig {
+		case syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT:
+			log.WithField("signal", sig).Info("received exit signal, exiting")
+			d.Stop()
+			return
+		}
 	}
 }
diff --git a/config.go b/config.go
new file mode 100644
index 0000000..c83e755
--- /dev/null
+++ b/config.go
@@ -0,0 +1,33 @@
+package indexer
+
+import "os"
+
+type Config struct {
+	ElasticURL string `help:"the url for our elastic search instance"`
+	DB         string `help:"the connection string for our database"`
+	Index      string `help:"the alias for our contact index"`
+	Poll       int    `help:"the number of seconds to wait between checking for updated contacts"`
+	Rebuild    bool   `help:"whether to rebuild the index, swapping it when complete, then exiting (default false)"`
+	Cleanup    bool   `help:"whether to remove old indexes after a rebuild"`
+	LogLevel   string `help:"the log level, one of error, warn, info, debug"`
+	SentryDSN  string `help:"the sentry configuration to log errors to, if any"`
+
+	LibratoUsername string `help:"the username that will be used to authenticate to Librato"`
+	LibratoToken    string `help:"the token that will be used to authenticate to Librato"`
+	InstanceName    string `help:"the unique name of this instance used for analytics"`
+}
+
+func NewDefaultConfig() *Config {
+	hostname, _ := os.Hostname()
+
+	return &Config{
+		ElasticURL:   "http://localhost:9200",
+		DB:           "postgres://localhost/temba?sslmode=disable",
+		Index:        "contacts",
+		Poll:         5,
+		Rebuild:      false,
+		Cleanup:      false,
+		LogLevel:     "info",
+		InstanceName: hostname,
+	}
+}
diff --git a/daemon.go b/daemon.go
new file mode 100644
index 0000000..b3f6f05
--- /dev/null
+++ b/daemon.go
@@ -0,0 +1,75 @@
+package indexer
+
+import (
+	"database/sql"
+	"sync"
+	"time"
+
+	"github.com/nyaruka/librato"
+	"github.com/nyaruka/rp-indexer/indexers"
+	"github.com/sirupsen/logrus"
+)
+
+type Daemon struct {
+	cfg      *Config
+	db       *sql.DB
+	wg       *sync.WaitGroup
+	quit     chan bool
+	indexers []indexers.Indexer
+}
+
+// NewDaemon creates a new daemon to run the given indexers
+func NewDaemon(cfg *Config, db *sql.DB, ixs []indexers.Indexer) *Daemon {
+	return &Daemon{
+		cfg:      cfg,
+		db:       db,
+		wg:       &sync.WaitGroup{},
+		quit:     make(chan bool),
+		indexers: ixs,
+	}
+}
+
+// Start starts this daemon
+func (d *Daemon) Start() {
+	// if we have a librato token, configure it
+	if d.cfg.LibratoToken != "" {
+		librato.Configure(d.cfg.LibratoUsername, d.cfg.LibratoToken, d.cfg.InstanceName, time.Second, d.wg)
+		librato.Start()
+	}
+
+	for _, i := range d.indexers {
+		d.startIndexer(i, time.Second*5)
+	}
+}
+
+func (d *Daemon) startIndexer(indexer indexers.Indexer, interval time.Duration) {
+	d.wg.Add(1) // add ourselves to the wait group
+
+	go func() {
+		defer func() {
+			logrus.WithField("indexer", indexer.Name()).Info("indexer exiting")
+			d.wg.Done()
+		}()
+
+		for {
+			select {
+			case <-d.quit:
+				return
+			case <-time.After(interval):
+				_, err := indexer.Index(d.db, d.cfg.Rebuild, d.cfg.Cleanup)
+				if err != nil {
+					logrus.WithField("index", d.cfg.Index).WithError(err).Error("error during indexing")
+				}
+			}
+		}
+	}()
+}
+
+// Stop stops this daemon
+func (d *Daemon) Stop() {
+	logrus.Info("daemon stopping")
+	librato.Stop()
+
+	close(d.quit)
+	d.wg.Wait()
+}
diff --git a/go.mod b/go.mod
index fcfb589..fbfe75c 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ require (
 	github.com/lib/pq v1.10.4
 	github.com/nyaruka/ezconf v0.2.1
 	github.com/nyaruka/gocommon v1.17.1
+	github.com/nyaruka/librato v1.0.0
 	github.com/olivere/elastic/v7 v7.0.22
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
diff --git a/go.sum b/go.sum
index f78ba21..2dfdccb 100644
--- a/go.sum
+++ b/go.sum
@@ -36,6 +36,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
 github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
@@ -53,6 +54,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
 github.com/nyaruka/gocommon v1.17.1 h1:4bbNp+0/BIbne4VDiKOxh3kcbdvEu/WsrsZiG/VyRZ8=
 github.com/nyaruka/gocommon v1.17.1/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0=
+github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
+github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
 github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U=
 github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s=
 github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
@@ -63,12 +66,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
 github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -96,6 +101,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

From 05c06b07c6e198219a6625ef466812d3305252e7 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 24 Mar 2022 15:35:53 -0500
Subject: [PATCH 18/48] Add stats reporting cron task and optional librato
 config

---
 daemon.go             | 64 ++++++++++++++++++++++++++++++++++++++-----
 indexers/base.go      | 23 +++++++++-------
 indexers/base_test.go |  6 ++--
 indexers/contacts.go  |  4 +--
 4 files changed, 75 insertions(+), 22 deletions(-)

diff --git a/daemon.go b/daemon.go
index b3f6f05..80b317c 100644
--- a/daemon.go
+++ b/daemon.go
@@ -16,16 +16,19 @@ type Daemon struct {
 	wg       *sync.WaitGroup
 	quit     chan bool
 	indexers []indexers.Indexer
+
+	prevStats map[indexers.Indexer]indexers.Stats
 }
 
 // NewDaemon creates a new daemon to run the given indexers
 func NewDaemon(cfg *Config, db *sql.DB, ixs []indexers.Indexer) *Daemon {
 	return &Daemon{
-		cfg:      cfg,
-		db:       db,
-		wg:       &sync.WaitGroup{},
-		quit:     make(chan bool),
-		indexers: ixs,
+		cfg:       cfg,
+		db:        db,
+		wg:        &sync.WaitGroup{},
+		quit:      make(chan bool),
+		indexers:  ixs,
+		prevStats: make(map[indexers.Indexer]indexers.Stats, len(ixs)),
 	}
 }
 
@@ -40,14 +43,18 @@ func (d *Daemon) Start() {
 	for _, i := range d.indexers {
 		d.startIndexer(i, time.Second*5)
 	}
+
+	d.startStatsReporter(time.Minute)
 }
 
 func (d *Daemon) startIndexer(indexer indexers.Indexer, interval time.Duration) {
 	d.wg.Add(1) // add ourselves to the wait group
 
+	log := logrus.WithField("indexer", indexer.Name())
+
 	go func() {
 		defer func() {
-			logrus.WithField("indexer", indexer.Name()).Info("indexer exiting")
+			log.Info("indexer exiting")
 			d.wg.Done()
 		}()
 
@@ -58,13 +65,56 @@ func (d *Daemon) startIndexer(indexer indexers.Indexer, interval time.Duration)
 			case <-time.After(interval):
 				_, err := indexer.Index(d.db, d.cfg.Rebuild, d.cfg.Cleanup)
 				if err != nil {
-					logrus.WithField("index", d.cfg.Index).WithError(err).Error("error during indexing")
+					log.WithError(err).Error("error during indexing")
 				}
 			}
 		}
 	}()
 }
 
+func (d *Daemon) startStatsReporter(interval time.Duration) {
+	d.wg.Add(1) // add ourselves to the wait group
+
+	go func() {
+		defer func() {
+			logrus.Info("analytics exiting")
+			d.wg.Done()
+		}()
+
+		for {
+			select {
+			case <-d.quit:
+				return
+			case <-time.After(interval):
+				d.reportStats()
+			}
+		}
+	}()
+}
+
+func (d *Daemon) reportStats() {
+	metrics := make(map[string]float64, len(d.indexers)*2)
+
+	for _, ix := range d.indexers {
+		stats := ix.Stats()
+		prev := d.prevStats[ix]
+
+		metrics[ix.Name()+"_indexed"] = float64(stats.Indexed - prev.Indexed)
+		metrics[ix.Name()+"_deleted"] = float64(stats.Deleted - prev.Deleted)
+
+		d.prevStats[ix] = stats
+	}
+
+	log := logrus.NewEntry(logrus.StandardLogger())
+
+	for k, v := range metrics {
+		librato.Gauge("indexer."+k, v)
+		log = log.WithField(k, v)
+	}
+
+	log.Info("stats reported")
+}
+
 // Stop stops this daemon
 func (d *Daemon) Stop() {
 	logrus.Info("daemon stopping")
diff --git a/indexers/base.go b/indexers/base.go
index 808ddcb..b661784 100644
--- a/indexers/base.go
+++ b/indexers/base.go
@@ -20,21 +20,24 @@ const indexCommand = `{ "index": { "_id": %d, "_type": "_doc", "version": %d, "v
 // deletes a document
 const deleteCommand = `{ "delete" : { "_id": %d, "_type": "_doc", "version": %d, "version_type": "external", "routing": %d} }`
 
+type Stats struct {
+	Indexed int64         // total number of documents indexed
+	Deleted int64         // total number of documents deleted
+	Elapsed time.Duration // total time spent actually indexing
+}
+
 // Indexer is base interface for indexers
 type Indexer interface {
 	Name() string
 	Index(db *sql.DB, rebuild, cleanup bool) (string, error)
-	Stats() (int64, int64, time.Duration)
+	Stats() Stats
 }
 
 type baseIndexer struct {
 	elasticURL string
 	name       string // e.g. contacts, used as the alias
 
-	// statistics
-	indexedTotal int64
-	deletedTotal int64
-	elapsedTotal time.Duration
+	stats Stats
 }
 
 func newBaseIndexer(elasticURL, name string) baseIndexer {
@@ -45,8 +48,8 @@ func (i *baseIndexer) Name() string {
 	return i.name
 }
 
-func (i *baseIndexer) Stats() (int64, int64, time.Duration) {
-	return i.indexedTotal, i.deletedTotal, i.elapsedTotal
+func (i *baseIndexer) Stats() Stats {
+	return i.stats
 }
 
 func (i *baseIndexer) log() *logrus.Entry {
@@ -55,9 +58,9 @@ func (i *baseIndexer) log() *logrus.Entry {
 
 // records a complete index and updates statistics
 func (i *baseIndexer) recordComplete(indexed, deleted int, elapsed time.Duration) {
-	i.indexedTotal += int64(indexed)
-	i.deletedTotal += int64(deleted)
-	i.elapsedTotal += elapsed
+	i.stats.Indexed += int64(indexed)
+	i.stats.Deleted += int64(deleted)
+	i.stats.Elapsed += elapsed
 
 	i.log().WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing")
 }
diff --git a/indexers/base_test.go b/indexers/base_test.go
index c4a4d54..b473f22 100644
--- a/indexers/base_test.go
+++ b/indexers/base_test.go
@@ -78,7 +78,7 @@ func assertIndexesWithPrefix(t *testing.T, es *elastic.Client, prefix string, ex
 }
 
 func assertIndexerStats(t *testing.T, ix indexers.Indexer, expectedIndexed, expectedDeleted int64) {
-	actualIndexed, actualDeleted, _ := ix.Stats()
-	assert.Equal(t, expectedIndexed, actualIndexed, "indexed mismatch")
-	assert.Equal(t, expectedDeleted, actualDeleted, "deleted mismatch")
+	actual := ix.Stats()
+	assert.Equal(t, expectedIndexed, actual.Indexed, "indexed mismatch")
+	assert.Equal(t, expectedDeleted, actual.Deleted, "deleted mismatch")
 }
diff --git a/indexers/contacts.go b/indexers/contacts.go
index e25d198..c77230e 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -60,7 +60,7 @@ func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error
 		return "", errors.Wrap(err, "error finding last modified")
 	}
 
-	i.log().WithField("index", physicalIndex).WithField("last_modified", lastModified).Info("indexing newer than last modified")
+	i.log().WithField("index", physicalIndex).WithField("last_modified", lastModified).Debug("indexing newer than last modified")
 
 	// now index our docs
 	start := time.Now()
@@ -243,7 +243,7 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti
 		elapsed := time.Since(start)
 		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
 
-		i.log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed}).Info("indexed contact batch")
+		i.log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed}).Debug("indexed contact batch")
 	}
 
 	return createdCount, deletedCount, nil

From d4e4c5ad67ec149946789fe5b708500776f7480c Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 24 Mar 2022 15:48:32 -0500
Subject: [PATCH 19/48] Add rate to stats reporting

---
 daemon.go | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/daemon.go b/daemon.go
index 80b317c..19b78d5 100644
--- a/daemon.go
+++ b/daemon.go
@@ -99,8 +99,17 @@ func (d *Daemon) reportStats() {
 		stats := ix.Stats()
 		prev := d.prevStats[ix]
 
-		metrics[ix.Name()+"_indexed"] = float64(stats.Indexed - prev.Indexed)
-		metrics[ix.Name()+"_deleted"] = float64(stats.Deleted - prev.Deleted)
+		indexedInPeriod := stats.Indexed - prev.Indexed
+		deletedInPeriod := stats.Deleted - prev.Deleted
+		elapsedInPeriod := stats.Elapsed - prev.Elapsed
+		rateInPeriod := float64(0)
+		if indexedInPeriod > 0 && elapsedInPeriod > 0 {
+			rateInPeriod = float64(indexedInPeriod) / (float64(elapsedInPeriod) / float64(time.Second))
+		}
+
+		metrics[ix.Name()+"_indexed"] = float64(indexedInPeriod)
+		metrics[ix.Name()+"_deleted"] = float64(deletedInPeriod)
+		metrics[ix.Name()+"_rate"] = rateInPeriod
 
 		d.prevStats[ix] = stats
 	}

From 18e59911119311424c33639d8bd2bf17af896540 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 24 Mar 2022 16:46:16 -0500
Subject: [PATCH 20/48] Update CHANGELOG.md for v7.3.0

---
 CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 457452e..6c1782a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+v7.3.0
+----------
+ * Add stats reporting cron task and optional librato config
+ * Refactor to support different indexer types
+ * Update golang.org/x/sys
+
 v7.2.0
 ----------
  * Tweak README

From f0b2a686c81d0c6591cef1ba8c268256c9a518cf Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 29 Mar 2022 14:32:00 -0500
Subject: [PATCH 21/48] Poll interval is configurable

---
 cmd/rp-indexer/main.go |  3 ++-
 daemon.go              | 10 ++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index b959c05..69ff29a 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -5,6 +5,7 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/evalphobia/logrus_sentry"
 	_ "github.com/lib/pq"
@@ -59,7 +60,7 @@ func main() {
 			log.WithField("indexer", idxr.Name()).WithError(err).Fatal("error during rebuilding")
 		}
 	} else {
-		d := indexer.NewDaemon(cfg, db, idxrs)
+		d := indexer.NewDaemon(cfg, db, idxrs, time.Duration(cfg.Poll)*time.Second)
 		d.Start()
 
 		handleSignals(d)
diff --git a/daemon.go b/daemon.go
index 19b78d5..b57b7d1 100644
--- a/daemon.go
+++ b/daemon.go
@@ -16,18 +16,20 @@ type Daemon struct {
 	wg       *sync.WaitGroup
 	quit     chan bool
 	indexers []indexers.Indexer
+	poll     time.Duration
 
 	prevStats map[indexers.Indexer]indexers.Stats
 }
 
 // NewDaemon creates a new daemon to run the given indexers
-func NewDaemon(cfg *Config, db *sql.DB, ixs []indexers.Indexer) *Daemon {
+func NewDaemon(cfg *Config, db *sql.DB, ixs []indexers.Indexer, poll time.Duration) *Daemon {
 	return &Daemon{
 		cfg:       cfg,
 		db:        db,
 		wg:        &sync.WaitGroup{},
 		quit:      make(chan bool),
 		indexers:  ixs,
+		poll:      poll,
 		prevStats: make(map[indexers.Indexer]indexers.Stats, len(ixs)),
 	}
 }
@@ -41,13 +43,13 @@ func (d *Daemon) Start() {
 	}
 
 	for _, i := range d.indexers {
-		d.startIndexer(i, time.Second*5)
+		d.startIndexer(i)
 	}
 
 	d.startStatsReporter(time.Minute)
 }
 
-func (d *Daemon) startIndexer(indexer indexers.Indexer, interval time.Duration) {
+func (d *Daemon) startIndexer(indexer indexers.Indexer) {
 	d.wg.Add(1) // add ourselves to the wait group
 
 	log := logrus.WithField("indexer", indexer.Name())
@@ -62,7 +64,7 @@ func (d *Daemon) startIndexer(indexer indexers.Indexer, interval time.Duration)
 			select {
 			case <-d.quit:
 				return
-			case <-time.After(interval):
+			case <-time.After(d.poll):
 				_, err := indexer.Index(d.db, d.cfg.Rebuild, d.cfg.Cleanup)
 				if err != nil {
 					log.WithError(err).Error("error during indexing")

From af632fc47684fbcebbc1c59bbb53a2802a4aa282 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 30 Mar 2022 14:32:19 -0500
Subject: [PATCH 22/48] If indexing fails, log status code from elasticsearch

---
 indexers/base.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indexers/base.go b/indexers/base.go
index b661784..7e5e639 100644
--- a/indexers/base.go
+++ b/indexers/base.go
@@ -255,7 +255,7 @@ func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) {
 			} else if item.Index.Status == 409 {
 				conflictedCount++
 			} else {
-				logrus.WithField("id", item.Index.ID).WithField("batch", batch).WithField("result", item.Index.Result).Error("error indexing document")
+				logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).WithField("result", item.Index.Result).Error("error indexing document")
 			}
 		} else if item.Delete.ID != "" {
 			logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Debug("delete response")

From c99c739d6919d9a3e3602d2ac9a4d8e2e66b14d1 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 30 Mar 2022 14:41:05 -0500
Subject: [PATCH 23/48] Update CHANGELOG.md for v7.3.1

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6c1782a..7ae2c68 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.3.1
+----------
+ * If indexing fails, log status code from elasticsearch
+ * Poll interval is configurable
+
 v7.3.0
 ----------
  * Add stats reporting cron task and optional librato config

From bace1d5dda447753f4d448d806967284852053c9 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 31 Mar 2022 11:30:25 -0500
Subject: [PATCH 24/48] Tweak new field name, add tests

---
 indexers/contacts.go            | 5 ++---
 indexers/contacts.settings.json | 2 +-
 indexers/contacts_test.go       | 2 ++
 testdb.sql                      | 4 +++-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/indexers/contacts.go b/indexers/contacts.go
index 75259f3..c6f2d08 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -150,9 +150,8 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 		) AS flow,
 		current_flow_id AS flow_id,
 		(
-			SELECT array_to_json(array_agg(DISTINCT fr.flow_id))
-			FROM flows_flowrun fr WHERE fr.contact_id = contacts_contact.id
-		) AS flow_history
+			SELECT array_to_json(array_agg(DISTINCT fr.flow_id)) FROM flows_flowrun fr WHERE fr.contact_id = contacts_contact.id
+		) AS flow_history_ids
 	FROM contacts_contact
 	WHERE modified_on >= $1
 	ORDER BY modified_on ASC
diff --git a/indexers/contacts.settings.json b/indexers/contacts.settings.json
index 5061272..7a76a84 100644
--- a/indexers/contacts.settings.json
+++ b/indexers/contacts.settings.json
@@ -156,7 +156,7 @@
                 "flow_id": {
                     "type": "integer"
                 },
-                "flow_history": {
+                "flow_history_ids": {
                     "type": "integer"
                 },
                 "tickets": {
diff --git a/indexers/contacts_test.go b/indexers/contacts_test.go
index 848e417..071b9af 100644
--- a/indexers/contacts_test.go
+++ b/indexers/contacts_test.go
@@ -32,6 +32,8 @@ var contactQueryTests = []struct {
 	{elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4}},
 	{elastic.NewMatchQuery("flow_id", 1), []int64{2, 3}},
 	{elastic.NewMatchQuery("flow_id", 2), []int64{4}},
+	{elastic.NewMatchQuery("flow_history_ids", 1), []int64{1, 2, 3}},
+	{elastic.NewMatchQuery("flow_history_ids", 2), []int64{1, 2}},
 	{elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8}},                   // created_on range
 	{elastic.NewRangeQuery("last_seen_on").Lt("2019-01-01"), []int64{3, 4}},                    // last_seen_on range
 	{elastic.NewExistsQuery("last_seen_on"), []int64{1, 2, 3, 4, 5, 6}},                        // last_seen_on is set
diff --git a/testdb.sql b/testdb.sql
index 46ec484..8e72d88 100644
--- a/testdb.sql
+++ b/testdb.sql
@@ -166,4 +166,6 @@ INSERT INTO flows_flowrun(id, uuid, flow_id, contact_id) VALUES
 (1, '8b30ee61-e19d-427e-bb9f-4b8cd2c31d0c', 1, 1),
 (2, '94639979-155e-444d-95e9-a39dad64dbd5', 1, 1),
 (3, '74d918df-0e31-4547-98a9-5d765450e2ac', 2, 1),
-(4, '14fdf8fc-6e02-4759-b9be-cacc5991cd14', 1, 2);
+(4, '14fdf8fc-6e02-4759-b9be-cacc5991cd14', 1, 2),
+(5, '5171b4d8-e9bb-46c1-901e-53e13f3afe5d', 2, 2),
+(6, '4cc84e32-910f-41d6-865d-63fe25db4cde', 1, 3);

From fe5e3dfcbc7cfdc962718c005245000f331e60d8 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 31 Mar 2022 11:52:33 -0500
Subject: [PATCH 25/48] Update CHANGELOG.md for v7.3.3

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7ae2c68..88366a2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+v7.3.3
+----------
+ * Include flow id history as flow_history_ids and current flow id as flow_id 
+
 v7.3.1
 ----------
  * If indexing fails, log status code from elasticsearch

From 426d91d991603e4748538beaca56d16cb1e28565 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 4 Apr 2022 13:20:06 -0500
Subject: [PATCH 26/48] Add group_ids field to replace groups

---
 indexers/contacts.go            |  3 +++
 indexers/contacts.settings.json | 33 ++++++++++++++++++---------------
 indexers/contacts_test.go       |  3 +++
 3 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/indexers/contacts.go b/indexers/contacts.go
index c6f2d08..c9ea627 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -145,6 +145,9 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 				WHERE contact_id = contacts_contact.id AND contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
 			) g
 		) AS groups,
+		(
+			SELECT array_to_json(array_agg(gc.contactgroup_id)) FROM contacts_contactgroup_contacts gc WHERE gc.contact_id = contacts_contact.id
+		) AS group_ids,
 		(
 			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
 		) AS flow,
diff --git a/indexers/contacts.settings.json b/indexers/contacts.settings.json
index 7a76a84..2dcfdd7 100644
--- a/indexers/contacts.settings.json
+++ b/indexers/contacts.settings.json
@@ -79,6 +79,23 @@
                 "required": true
             },
             "properties": {
+                "uuid": {
+                    "type": "keyword"
+                },
+                "name": {
+                    "type": "text",
+                    "analyzer": "prefix",
+                    "search_analyzer": "name_search",
+                    "fields": {
+                        "keyword": {
+                            "type": "keyword",
+                            "normalizer": "lowercase"
+                        }
+                    }
+                },
+                "status": {
+                    "type": "keyword"
+                },
                 "fields": {
                     "type": "nested",
                     "properties": {
@@ -144,10 +161,7 @@
                 "groups": {
                     "type": "keyword"
                 },
-                "uuid": {
-                    "type": "keyword"
-                },
-                "status": {
+                "group_ids": {
                     "type": "keyword"
                 },
                 "flow": {
@@ -177,17 +191,6 @@
                 },
                 "last_seen_on": {
                     "type": "date"
-                },
-                "name": {
-                    "type": "text",
-                    "analyzer": "prefix",
-                    "search_analyzer": "name_search",
-                    "fields": {
-                        "keyword": {
-                            "type": "keyword",
-                            "normalizer": "lowercase"
-                        }
-                    }
                 }
             }
         }
diff --git a/indexers/contacts_test.go b/indexers/contacts_test.go
index 071b9af..2491d38 100644
--- a/indexers/contacts_test.go
+++ b/indexers/contacts_test.go
@@ -183,6 +183,9 @@ var contactQueryTests = []struct {
 	{elastic.NewMatchQuery("groups", "4ea0f313-2f62-4e57-bdf0-232b5191dd57"), []int64{1}},
 	{elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2}},
 	{elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{}},
+	{elastic.NewMatchQuery("group_ids", 1), []int64{1}},
+	{elastic.NewMatchQuery("group_ids", 4), []int64{1, 2}},
+	{elastic.NewMatchQuery("group_ids", 2), []int64{}},
 }
 
 func TestContacts(t *testing.T) {

From 5f1a7b731d336818f98ff49062f792c390f817c7 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 4 Apr 2022 13:31:02 -0500
Subject: [PATCH 27/48] Update CHANGELOG.md for v7.3.4

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 88366a2..115805a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+v7.3.4
+----------
+ * Add group_ids field to replace groups
+
 v7.3.3
 ----------
  * Include flow id history as flow_history_ids and current flow id as flow_id 

From 856d66e6f153d7961376b82abb06a7345fab7b84 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 5 Apr 2022 10:10:02 -0500
Subject: [PATCH 28/48] Lower batch size to 100000 and log batch progress
 during rebuilds

---
 indexers/contacts.go | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/indexers/contacts.go b/indexers/contacts.go
index c9ea627..5a8160f 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -64,7 +64,7 @@ func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error
 
 	// now index our docs
 	start := time.Now()
-	indexed, deleted, err := i.indexModified(db, physicalIndex, lastModified.Add(-5*time.Second))
+	indexed, deleted, err := i.indexModified(db, physicalIndex, lastModified.Add(-5*time.Second), rebuild)
 	if err != nil {
 		return "", errors.Wrap(err, "error indexing documents")
 	}
@@ -158,12 +158,12 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 	FROM contacts_contact
 	WHERE modified_on >= $1
 	ORDER BY modified_on ASC
-	LIMIT 500000
+	LIMIT 100000
 ) t;
 `
 
 // IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time
-func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified time.Time) (int, int, error) {
+func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified time.Time, rebuild bool) (int, int, error) {
 	batch := &bytes.Buffer{}
 	createdCount, deletedCount, processedCount := 0, 0, 0
 
@@ -250,7 +250,14 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti
 		elapsed := time.Since(start)
 		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
 
-		i.log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed}).Debug("indexed contact batch")
+		log := i.log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed})
+
+		// if we're rebuilding, always log batch progress
+		if rebuild {
+			log.Info("indexed contact batch")
+		} else {
+			log.Debug("indexed contact batch")
+		}
 	}
 
 	return createdCount, deletedCount, nil

From 9f52f9f02b1305890ae726f128332a9dfdc977e7 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 5 Apr 2022 10:14:52 -0500
Subject: [PATCH 29/48] Update CHANGELOG.md for v7.3.5

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 115805a..6e7835f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+v7.3.5
+----------
+ * Lower batch size to 100000 and log batch progress during rebuilds
+
 v7.3.4
 ----------
  * Add group_ids field to replace groups

From 1b239fd6d87c0689590ffa0037fa4ca2645e3057 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Tue, 5 Apr 2022 10:43:42 -0500
Subject: [PATCH 30/48] Drop the flow and groups fields which have been
 replaced by flow_id and group_ids

---
 indexers/contacts.go            | 10 ----------
 indexers/contacts.settings.json |  6 ------
 indexers/contacts_test.go       |  7 +------
 3 files changed, 1 insertion(+), 22 deletions(-)

diff --git a/indexers/contacts.go b/indexers/contacts.go
index 5a8160f..1237e2c 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -138,19 +138,9 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 				) AS district_value
 			) AS f
 		) AS fields,
-		(
-			SELECT array_to_json(array_agg(g.uuid)) FROM (
-				SELECT contacts_contactgroup.uuid
-				FROM contacts_contactgroup_contacts, contacts_contactgroup
-				WHERE contact_id = contacts_contact.id AND contacts_contactgroup_contacts.contactgroup_id = contacts_contactgroup.id
-			) g
-		) AS groups,
 		(
 			SELECT array_to_json(array_agg(gc.contactgroup_id)) FROM contacts_contactgroup_contacts gc WHERE gc.contact_id = contacts_contact.id
 		) AS group_ids,
-		(
-			SELECT f.uuid FROM flows_flow f WHERE f.id = contacts_contact.current_flow_id
-		) AS flow,
 		current_flow_id AS flow_id,
 		(
 			SELECT array_to_json(array_agg(DISTINCT fr.flow_id)) FROM flows_flowrun fr WHERE fr.contact_id = contacts_contact.id
diff --git a/indexers/contacts.settings.json b/indexers/contacts.settings.json
index 2dcfdd7..bde3278 100644
--- a/indexers/contacts.settings.json
+++ b/indexers/contacts.settings.json
@@ -158,15 +158,9 @@
                         }
                     }
                 },
-                "groups": {
-                    "type": "keyword"
-                },
                 "group_ids": {
                     "type": "keyword"
                 },
-                "flow": {
-                    "type": "keyword"
-                },
                 "flow_id": {
                     "type": "integer"
                 },
diff --git a/indexers/contacts_test.go b/indexers/contacts_test.go
index 2491d38..a6b34db 100644
--- a/indexers/contacts_test.go
+++ b/indexers/contacts_test.go
@@ -28,8 +28,6 @@ var contactQueryTests = []struct {
 	{elastic.NewMatchQuery("tickets", 2), []int64{1}},
 	{elastic.NewMatchQuery("tickets", 1), []int64{2, 3}},
 	{elastic.NewRangeQuery("tickets").Gt(0), []int64{1, 2, 3}},
-	{elastic.NewMatchQuery("flow", "6d3cf1eb-546e-4fb8-a5ca-69187648fbf6"), []int64{2, 3}},
-	{elastic.NewMatchQuery("flow", "4eea8ff1-4fe2-4ce5-92a4-0870a499973a"), []int64{4}},
 	{elastic.NewMatchQuery("flow_id", 1), []int64{2, 3}},
 	{elastic.NewMatchQuery("flow_id", 2), []int64{4}},
 	{elastic.NewMatchQuery("flow_history_ids", 1), []int64{1, 2, 3}},
@@ -180,9 +178,6 @@ var contactQueryTests = []struct {
 		)),
 		[]int64{},
 	},
-	{elastic.NewMatchQuery("groups", "4ea0f313-2f62-4e57-bdf0-232b5191dd57"), []int64{1}},
-	{elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2}},
-	{elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{}},
 	{elastic.NewMatchQuery("group_ids", 1), []int64{1}},
 	{elastic.NewMatchQuery("group_ids", 4), []int64{1, 2}},
 	{elastic.NewMatchQuery("group_ids", 2), []int64{}},
@@ -235,7 +230,7 @@ func TestContacts(t *testing.T) {
 	assertQuery(t, es, elastic.NewMatchQuery("name", "john"), []int64{2})
 
 	// 3 is no longer in our group
-	assertQuery(t, es, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1})
+	assertQuery(t, es, elastic.NewMatchQuery("group_ids", 4), []int64{1})
 
 	// change John's name to Eric..
 	_, err = db.Exec(`

From c00bed33bedc6493f6b283334fcf946a165bcc9c Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 6 Apr 2022 09:10:54 -0500
Subject: [PATCH 31/48] Restore batch size to 500000

---
 indexers/contacts.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/indexers/contacts.go b/indexers/contacts.go
index 5a8160f..3789b55 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -158,7 +158,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 	FROM contacts_contact
 	WHERE modified_on >= $1
 	ORDER BY modified_on ASC
-	LIMIT 100000
+	LIMIT 500000
 ) t;
 `
 

From 37d969b09833798cb1c1838e1ccaf89ad4ca69aa Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 6 Apr 2022 09:14:17 -0500
Subject: [PATCH 32/48] Update CHANGELOG.md for v7.3.5

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 115805a..7341669 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+v7.3.5
+----------
+ * Log batch progress during rebuilds
+
 v7.3.4
 ----------
  * Add group_ids field to replace groups

From c9da6b57d031884c69e57c3b47b20cf108d54a78 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 7 Apr 2022 09:32:04 -0500
Subject: [PATCH 33/48] Ignore malformed field value numbers

---
 indexers/contacts.settings.json | 3 ++-
 testdb.sql                      | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/indexers/contacts.settings.json b/indexers/contacts.settings.json
index 2dcfdd7..6778fac 100644
--- a/indexers/contacts.settings.json
+++ b/indexers/contacts.settings.json
@@ -108,7 +108,8 @@
                         },
                         "number": {
                             "type": "scaled_float",
-                            "scaling_factor": 10000
+                            "scaling_factor": 10000,
+                            "ignore_malformed": true
                         },
                         "datetime": {
                             "type": "date"
diff --git a/testdb.sql b/testdb.sql
index 8e72d88..c34ac70 100644
--- a/testdb.sql
+++ b/testdb.sql
@@ -98,7 +98,7 @@ INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_
 (
     4,  
     TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', '2015-12-31 23:59', 1, 'A', 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', 
-    '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}', 
+    '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888", "number": 8888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888888 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}', 
     0,
     2
 ),

From 28309be0cb6fd4bb2b078214569c23b6612bd4cc Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 7 Apr 2022 11:29:50 -0500
Subject: [PATCH 34/48] Update CHANGELOG.md for v7.3.6

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7341669..01a57cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.3.6
+----------
+ * Ignore malformed field value numbers
+ * Drop the flow and groups fields which have been replaced by flow_id and group_ids
+
 v7.3.5
 ----------
  * Log batch progress during rebuilds

From 29edf3f3736a1750bc9efff8e5bd6931d171ed2c Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 7 Apr 2022 15:24:55 -0500
Subject: [PATCH 35/48] Test with latest ES 7.17

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index caaf34e..a315567 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,7 +2,7 @@ name: CI
 on: [push, pull_request]
 env:
   go-version: "1.17.x"
-  es-version: "7.10.1"
+  es-version: "7.17.2"
 jobs:
   test:
     name: Test

From af3a0b9be9285becb457ed8b8646d019406248cf Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 11 Apr 2022 12:39:30 -0500
Subject: [PATCH 36/48] Better logging within batches during rebuilds

---
 indexers/contacts.go | 95 ++++++++++++++++++++++++++------------------
 1 file changed, 57 insertions(+), 38 deletions(-)

diff --git a/indexers/contacts.go b/indexers/contacts.go
index b08ec8f..2ba5e28 100644
--- a/indexers/contacts.go
+++ b/indexers/contacts.go
@@ -154,21 +154,39 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
 
 // IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time
 func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified time.Time, rebuild bool) (int, int, error) {
-	batch := &bytes.Buffer{}
-	createdCount, deletedCount, processedCount := 0, 0, 0
+	totalFetched, totalCreated, totalDeleted := 0, 0, 0
 
 	var modifiedOn time.Time
 	var contactJSON string
 	var id, orgID int64
 	var isActive bool
 
+	subBatch := &bytes.Buffer{}
 	start := time.Now()
 
 	for {
+		batchStart := time.Now()        // start time for this batch
+		batchFetched := 0               // contacts fetched in this batch
+		batchCreated := 0               // contacts created in ES
+		batchDeleted := 0               // contacts deleted in ES
+		batchESTime := time.Duration(0) // time spent indexing for this batch
+
+		indexSubBatch := func(b *bytes.Buffer) error {
+			t := time.Now()
+			created, deleted, err := i.indexBatch(index, b.Bytes())
+			if err != nil {
+				return err
+			}
+
+			batchESTime += time.Since(t)
+			batchCreated += created
+			batchDeleted += deleted
+			b.Reset()
+			return nil
+		}
+
 		rows, err := db.Query(sqlSelectModifiedContacts, lastModified)
 
-		queryCreated := 0
-		queryCount := 0
 		queryModified := lastModified
 
 		// no more rows? return
@@ -186,61 +204,57 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti
 				return 0, 0, err
 			}
 
-			queryCount++
-			processedCount++
+			batchFetched++
 			lastModified = modifiedOn
 
 			if isActive {
 				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Debug("modified contact")
 
-				batch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID))
-				batch.WriteString("\n")
-				batch.WriteString(contactJSON)
-				batch.WriteString("\n")
+				subBatch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID))
+				subBatch.WriteString("\n")
+				subBatch.WriteString(contactJSON)
+				subBatch.WriteString("\n")
 			} else {
 				logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Debug("deleted contact")
 
-				batch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID))
-				batch.WriteString("\n")
+				subBatch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID))
+				subBatch.WriteString("\n")
 			}
 
 			// write to elastic search in batches
-			if queryCount%i.batchSize == 0 {
-				created, deleted, err := i.indexBatch(index, batch.Bytes())
-				if err != nil {
+			if batchFetched%i.batchSize == 0 {
+				if err := indexSubBatch(subBatch); err != nil {
 					return 0, 0, err
 				}
-				batch.Reset()
-
-				queryCreated += created
-				createdCount += created
-				deletedCount += deleted
 			}
 		}
 
-		if batch.Len() > 0 {
-			created, deleted, err := i.indexBatch(index, batch.Bytes())
-			if err != nil {
+		if subBatch.Len() > 0 {
+			if err := indexSubBatch(subBatch); err != nil {
 				return 0, 0, err
 			}
-
-			queryCreated += created
-			createdCount += created
-			deletedCount += deleted
-			batch.Reset()
-		}
-
-		// last modified stayed the same and we didn't add anything, seen it all, break out
-		if lastModified.Equal(queryModified) && queryCreated == 0 {
-			break
 		}
 
 		rows.Close()
 
-		elapsed := time.Since(start)
-		rate := float32(processedCount) / (float32(elapsed) / float32(time.Second))
-
-		log := i.log().WithField("index", index).WithFields(logrus.Fields{"rate": int(rate), "added": createdCount, "deleted": deletedCount, "elapsed": elapsed})
+		totalFetched += batchFetched
+		totalCreated += batchCreated
+		totalDeleted += batchDeleted
+
+		totalTime := time.Since(start)
+		batchTime := time.Since(batchStart)
+		batchRate := int(float32(batchFetched) / (float32(batchTime) / float32(time.Second)))
+
+		log := i.log().WithField("index", index).WithFields(logrus.Fields{
+			"rate":             batchRate,
+			"batch_fetched":    batchFetched,
+			"batch_created":    batchCreated,
+			"batch_elapsed":    batchTime,
+			"batch_elapsed_es": batchESTime,
+			"total_fetched":    totalFetched,
+			"total_created":    totalCreated,
+			"total_elapsed":    totalTime,
+		})
 
 		// if we're rebuilding, always log batch progress
 		if rebuild {
@@ -248,7 +262,12 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti
 		} else {
 			log.Debug("indexed contact batch")
 		}
+
+		// last modified stayed the same and we didn't add anything, seen it all, break out
+		if lastModified.Equal(queryModified) && batchCreated == 0 {
+			break
+		}
 	}
 
-	return createdCount, deletedCount, nil
+	return totalCreated, totalDeleted, nil
 }

From 26521354d192dd0964773cfd3de7271026f1f4e5 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 11 Apr 2022 12:40:08 -0500
Subject: [PATCH 37/48] Update CHANGELOG.md for v7.3.7

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 01a57cf..1f99a8a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.3.7
+----------
+ * Better logging within batches during rebuilds
+ * Test with latest ES 7.17
+
 v7.3.6
 ----------
  * Ignore malformed field value numbers

From c8b3ee483faa2d7206161b759cff7cb7db89ca83 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 11 May 2022 09:37:56 -0500
Subject: [PATCH 38/48] Don't panic on connection failure to ES

---
 utils/http.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/utils/http.go b/utils/http.go
index a246b4a..152f499 100644
--- a/utils/http.go
+++ b/utils/http.go
@@ -25,6 +25,11 @@ func init() {
 }
 
 func shouldRetry(request *http.Request, response *http.Response, withDelay time.Duration) bool {
+	// no response is a connection timeout which we can retry
+	if response == nil {
+		return true
+	}
+
 	// 429 Too Many Requests is recoverable. Sometimes the server puts
 	// a Retry-After response header to indicate when the server is
 	// available to start processing request from client.

From 83ed02f49e8ae946339d8c8ef067630817572d2c Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 11 May 2022 09:45:37 -0500
Subject: [PATCH 39/48] Update dependencies and go version to 1.18

---
 .github/workflows/ci.yml |   2 +-
 go.mod                   |  31 +++++-----
 go.sum                   | 125 ++++++++++-----------------------------
 3 files changed, 49 insertions(+), 109 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a315567..df7e980 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,7 +1,7 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: "1.17.x"
+  go-version: "1.18.x"
   es-version: "7.17.2"
 jobs:
   test:
diff --git a/go.mod b/go.mod
index fbfe75c..6dcf8ab 100644
--- a/go.mod
+++ b/go.mod
@@ -1,33 +1,34 @@
 module github.com/nyaruka/rp-indexer
 
+go 1.18
+
 require (
-	github.com/evalphobia/logrus_sentry v0.4.5
-	github.com/lib/pq v1.10.4
+	github.com/evalphobia/logrus_sentry v0.8.2
+	github.com/lib/pq v1.10.5
 	github.com/nyaruka/ezconf v0.2.1
-	github.com/nyaruka/gocommon v1.17.1
+	github.com/nyaruka/gocommon v1.20.0
 	github.com/nyaruka/librato v1.0.0
-	github.com/olivere/elastic/v7 v7.0.22
+	github.com/olivere/elastic/v7 v7.0.32
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 )
 
 require (
-	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
+	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fatih/structs v1.0.0 // indirect
-	github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.0 // indirect
+	github.com/getsentry/raven-go v0.2.0 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/shopspring/decimal v1.2.0 // indirect
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
-	golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
+	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
-
-go 1.17
diff --git a/go.sum b/go.sum
index 2dfdccb..b7060e5 100644
--- a/go.sum
+++ b/go.sum
@@ -1,135 +1,74 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/aws/aws-sdk-go v1.35.20/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k=
-github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
-github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
-github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
+github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/evalphobia/logrus_sentry v0.4.5 h1:weRoBjojMYPp57TLDjPEkP58JVHHSiqNrxG+h3ODdPM=
-github.com/evalphobia/logrus_sentry v0.4.5/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc=
-github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU=
+github.com/evalphobia/logrus_sentry v0.8.2 h1:dotxHq+YLZsT1Bb45bB5UQbfCh3gM/nFFetyN46VoDQ=
+github.com/evalphobia/logrus_sentry v0.8.2/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc=
 github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
-github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 h1:md1zEr2oSVWYNfQj+6TL/nmAFf5gY3Tp44lzskzK9QU=
-github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro=
+github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
+github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
+github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
 github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
 github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
-github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
-github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
-github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
-github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
-github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
-github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
-github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
-github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk=
-github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
-github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
+github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
+github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
+github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
 github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
 github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
-github.com/nyaruka/gocommon v1.17.1 h1:4bbNp+0/BIbne4VDiKOxh3kcbdvEu/WsrsZiG/VyRZ8=
-github.com/nyaruka/gocommon v1.17.1/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0=
+github.com/nyaruka/gocommon v1.20.0 h1:qbxRsBBPvpfGbuBq08jlQGxa5t+UZX/YGV7+kR+/moM=
+github.com/nyaruka/gocommon v1.20.0/go.mod h1:JrQSLAPo9ezSy1AzsJ1zDr1HW0/eu+aipICJkN/+kpg=
 github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
 github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
-github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U=
-github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s=
-github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8=
-github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
+github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
-github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
+github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
-github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
-github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
+golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
-golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
-gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

From d10c3f269d2594c651c724296b4fe5f74b03edae Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Wed, 11 May 2022 09:52:57 -0500
Subject: [PATCH 40/48] Update CHANGELOG.md for v7.3.8

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1f99a8a..4b1a0e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.3.8
+----------
+ * Update dependencies and go version to 1.18
+ * Don't panic on connection failure to ES
+
 v7.3.7
 ----------
  * Better logging within batches during rebuilds

From 9da2b68716e3f896ca70d35f8485106545cfade6 Mon Sep 17 00:00:00 2001
From: Morris Mukiri <morrismukiri@gmail.com>
Date: Mon, 16 May 2022 11:44:59 +0300
Subject: [PATCH 41/48] Add arm64 as a build target

Ensure there are arm64 binaries in the release assets
---
 goreleaser.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/goreleaser.yml b/goreleaser.yml
index 62c2c86..58364d8 100644
--- a/goreleaser.yml
+++ b/goreleaser.yml
@@ -7,6 +7,7 @@ build:
     - linux
   goarch:
     - amd64
+    - arm64
 
 archives:
   - files:

From f337d3211f6deb651375d24b4f72fb175f1ee539 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 26 May 2022 13:59:22 -0500
Subject: [PATCH 42/48] Use analytics package from gocommon instead of librato
 directly

---
 daemon.go | 11 ++++++-----
 go.mod    |  4 ++--
 go.sum    |  4 ++--
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/daemon.go b/daemon.go
index b57b7d1..8831075 100644
--- a/daemon.go
+++ b/daemon.go
@@ -5,7 +5,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/nyaruka/librato"
+	"github.com/nyaruka/gocommon/analytics"
 	"github.com/nyaruka/rp-indexer/indexers"
 	"github.com/sirupsen/logrus"
 )
@@ -38,10 +38,11 @@ func NewDaemon(cfg *Config, db *sql.DB, ixs []indexers.Indexer, poll time.Durati
 func (d *Daemon) Start() {
 	// if we have a librato token, configure it
 	if d.cfg.LibratoToken != "" {
-		librato.Configure(d.cfg.LibratoUsername, d.cfg.LibratoToken, d.cfg.InstanceName, time.Second, d.wg)
-		librato.Start()
+		analytics.RegisterBackend(analytics.NewLibrato(d.cfg.LibratoUsername, d.cfg.LibratoToken, d.cfg.InstanceName, time.Second, d.wg))
 	}
 
+	analytics.Start()
+
 	for _, i := range d.indexers {
 		d.startIndexer(i)
 	}
@@ -119,7 +120,7 @@ func (d *Daemon) reportStats() {
 	log := logrus.NewEntry(logrus.StandardLogger())
 
 	for k, v := range metrics {
-		librato.Gauge("indexer."+k, v)
+		analytics.Gauge("indexer."+k, v)
 		log = log.WithField(k, v)
 	}
 
@@ -129,7 +130,7 @@ func (d *Daemon) reportStats() {
 // Stop stops this daemon
 func (d *Daemon) Stop() {
 	logrus.Info("daemon stopping")
-	librato.Stop()
+	analytics.Stop()
 
 	close(d.quit)
 	d.wg.Wait()
diff --git a/go.mod b/go.mod
index 6dcf8ab..b51636b 100644
--- a/go.mod
+++ b/go.mod
@@ -6,8 +6,7 @@ require (
 	github.com/evalphobia/logrus_sentry v0.8.2
 	github.com/lib/pq v1.10.5
 	github.com/nyaruka/ezconf v0.2.1
-	github.com/nyaruka/gocommon v1.20.0
-	github.com/nyaruka/librato v1.0.0
+	github.com/nyaruka/gocommon v1.21.0
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
@@ -26,6 +25,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1 // indirect
+	github.com/nyaruka/librato v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
 	golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
diff --git a/go.sum b/go.sum
index b7060e5..5a7e4fd 100644
--- a/go.sum
+++ b/go.sum
@@ -32,8 +32,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
-github.com/nyaruka/gocommon v1.20.0 h1:qbxRsBBPvpfGbuBq08jlQGxa5t+UZX/YGV7+kR+/moM=
-github.com/nyaruka/gocommon v1.20.0/go.mod h1:JrQSLAPo9ezSy1AzsJ1zDr1HW0/eu+aipICJkN/+kpg=
+github.com/nyaruka/gocommon v1.21.0 h1:nu7M2cdSPrkqUPdGsEeWX047+neo69H4x+4g/OKpoLM=
+github.com/nyaruka/gocommon v1.21.0/go.mod h1:cv9r6amof1gSktfPZROClZhLFzdSIH/N9KbW6Nny4g8=
 github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
 github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
 github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=

From 3a25a43b679afd866082cfb4d7b8ba566c942514 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Thu, 26 May 2022 14:36:40 -0500
Subject: [PATCH 43/48] Update CHANGELOG.md for v7.3.9

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4b1a0e4..366ac34 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.3.9
+----------
+ * Use analytics package from gocommon instead of librato directly
+ * Add arm64 as a build target
+
 v7.3.8
 ----------
  * Update dependencies and go version to 1.18

From 4b923c727939e31b8962797507ae6e8f40904fb9 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 27 Jun 2022 11:02:16 +0100
Subject: [PATCH 44/48] Log app version on startup

---
 cmd/rp-indexer/main.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index 69ff29a..c582fdb 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -15,11 +15,19 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+var (
+	// https://goreleaser.com/cookbooks/using-main.version
+	version = "dev"
+	date    = "unknown"
+)
+
 func main() {
 	cfg := indexer.NewDefaultConfig()
 	loader := ezconf.NewLoader(cfg, "indexer", "Indexes RapidPro contacts to ElasticSearch", []string{"indexer.toml"})
 	loader.MustLoad()
 
+	log.WithField("version", version).WithField("released", date).Info("starting indexer")
+
 	// configure our logger
 	log.SetOutput(os.Stdout)
 	log.SetFormatter(&log.TextFormatter{})

From 81d61a866271626fccbd625120c178378b523d48 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 27 Jun 2022 11:23:18 +0100
Subject: [PATCH 45/48] Update CHANGELOG.md for v7.3.10

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 366ac34..db329a3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+v7.3.10
+----------
+ * Log app version on startup
+
 v7.3.9
 ----------
  * Use analytics package from gocommon instead of librato directly

From 75fc168de27fa7d25ececc9d11e61e8c020f7462 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Mon, 27 Jun 2022 14:50:16 +0100
Subject: [PATCH 46/48] Tweak startup logging

---
 cmd/rp-indexer/main.go | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go
index c582fdb..49379c4 100644
--- a/cmd/rp-indexer/main.go
+++ b/cmd/rp-indexer/main.go
@@ -26,17 +26,15 @@ func main() {
 	loader := ezconf.NewLoader(cfg, "indexer", "Indexes RapidPro contacts to ElasticSearch", []string{"indexer.toml"})
 	loader.MustLoad()
 
-	log.WithField("version", version).WithField("released", date).Info("starting indexer")
-
-	// configure our logger
-	log.SetOutput(os.Stdout)
-	log.SetFormatter(&log.TextFormatter{})
-
 	level, err := log.ParseLevel(cfg.LogLevel)
 	if err != nil {
 		log.Fatalf("Invalid log level '%s'", level)
 	}
+
 	log.SetLevel(level)
+	log.SetOutput(os.Stdout)
+	log.SetFormatter(&log.TextFormatter{})
+	log.WithField("version", version).WithField("released", date).Info("starting indexer")
 
 	// if we have a DSN entry, try to initialize it
 	if cfg.SentryDSN != "" {

From 4af922ebb2148fdaa244395048f5856345dc30b2 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Fri, 8 Jul 2022 12:05:49 +0100
Subject: [PATCH 47/48] Update README

---
 README.md | 99 ++++++++++++++++++++++++++-----------------------------
 1 file changed, 47 insertions(+), 52 deletions(-)

diff --git a/README.md b/README.md
index aaaf73e..d79b474 100644
--- a/README.md
+++ b/README.md
@@ -1,43 +1,43 @@
-# RapidPro Indexer
+# Indexer
 
 [![Build Status](https://github.com/nyaruka/rp-indexer/workflows/CI/badge.svg)](https://github.com/nyaruka/rp-indexer/actions?query=workflow%3ACI) 
 [![codecov](https://codecov.io/gh/nyaruka/rp-indexer/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/rp-indexer) 
 [![Go Report Card](https://goreportcard.com/badge/github.com/nyaruka/rp-indexer)](https://goreportcard.com/report/github.com/nyaruka/rp-indexer)
 
-Simple service for indexing RapidPro contacts into ElasticSearch.
+Service for indexing RapidPro/TextIt contacts into Elasticsearch.
 
-# Deploying
+## Deploying
 
-As Indexer is a Go application, it compiles to a binary and that binary along with the config file is all
+As it is a Go application, it compiles to a binary and that binary along with the config file is all
 you need to run it on your server. You can find bundles for each platform in the
-[releases directory](https://github.com/nyaruka/rp-indexer/releases). You should only run a single indexer
+[releases directory](https://github.com/nyaruka/rp-indexer/releases). You should only run a single
 instance for a deployment.
 
-Indexer can run in two modes:
+It can run in two modes:
 
 1) the default mode, which simply queries the ElasticSearch database, finds the most recently
-modified contact, then on a schedule queries the `contacts_contact` table on the RapidPro
+modified contact, then on a schedule queries the `contacts_contact` table in the 
 database for contacts to add or delete. You should run this as a long running service which
-constantly keeps ElasticSearch in sync with your RapidPro contacts.
+constantly keeps ElasticSearch in sync with your contacts.
 
 2) a rebuild mode, started with `--rebuild`. This builds a brand new index from nothing, querying
 all contacts on RapidPro. Once complete, this switches out the alias for the contact index
 with the newly build index. This can be run on a cron (in parallel with the mode above) to rebuild
 your index occasionally to get rid of bloat.
 
-# Configuration
+## Configuration
 
-Indexer uses a tiered configuration system, each option takes precendence over the ones above it:
+The service uses a tiered configuration system, each option takes precendence over the ones above it:
 
  1. The configuration file
  2. Environment variables starting with `INDEXER_` 
  3. Command line parameters
 
-We recommend running Indexer with no changes to the configuration and no parameters, using only
+We recommend running it with no changes to the configuration and no parameters, using only
 environment variables to configure it. You can use `% rp-indexer --help` to see a list of the
 environment variables and parameters and for more details on each option.
 
-## RapidPro Configuration
+### RapidPro
 
 For use with RapidPro, you will want to configure these settings:
 
@@ -48,9 +48,42 @@ Recommended settings for error reporting:
 
  * `INDEXER_SENTRY_DSN`: The DSN to use when logging errors to Sentry
 
-# Development
+### Reference
 
-Once you've checked out the code, you can build Indexer with:
+These are the configuration options that can be provided as parameters or environment variables. If using environment 
+varibles, convert to uppercase, replace dashes with underscores and prefix the name with `INDEXER_`, e.g. `-log-level` 
+becomes `INDEXER_LOG_LEVEL`.
+
+```
+  -cleanup
+      whether to remove old indexes after a rebuild
+  -db string
+      the connection string for our database (default "postgres://localhost/rapidpro?sslmode=disable")
+  -debug-conf
+      print where config values are coming from
+  -elastic-url string
+      the url for our elastic search instance (default "http://localhost:9200")
+  -help
+      print usage information
+  -index string
+      the alias for our contact index (default "contacts")
+  -librato-username
+      the Librato username for metrics reporting
+  -librato-token
+      the Librato token for metrics reporting
+  -log-level string
+      the log level, one of error, warn, info, debug (default "info")
+  -poll int
+      the number of seconds to wait between checking for updated contacts (default 5)
+  -rebuild
+      whether to rebuild the index, swapping it when complete, then exiting (default false)
+  -sentry-dsn string
+      the sentry configuration to log errors to, if any
+```
+
+## Development
+
+Once you've checked out the code, you can build the service with:
 
 ```
 go build github.com/nyaruka/rp-indexer/cmd/rp-indexer
@@ -69,41 +102,3 @@ To run all of the tests:
 ```
 go test ./... -p=1
 ```
-
-# Usage
-
-```
-Indexes RapidPro contacts to ElasticSearch
-
-Usage of indexer:
-  -cleanup
-    	whether to remove old indexes after a rebuild
-  -db string
-    	the connection string for our database (default "postgres://localhost/rapidpro?sslmode=disable")
-  -debug-conf
-    	print where config values are coming from
-  -elastic-url string
-    	the url for our elastic search instance (default "http://localhost:9200")
-  -help
-    	print usage information
-  -index string
-    	the alias for our contact index (default "contacts")
-  -log-level string
-    	the log level, one of error, warn, info, debug (default "info")
-  -poll int
-    	the number of seconds to wait between checking for updated contacts (default 5)
-  -rebuild
-    	whether to rebuild the index, swapping it when complete, then exiting (default false)
-  -sentry-dsn string
-    	the sentry configuration to log errors to, if any
-
-Environment variables:
-                             INDEXER_CLEANUP - bool
-                                  INDEXER_DB - string
-                         INDEXER_ELASTIC_URL - string
-                               INDEXER_INDEX - string
-                           INDEXER_LOG_LEVEL - string
-                                INDEXER_POLL - int
-                             INDEXER_REBUILD - bool
-                          INDEXER_SENTRY_DSN - string
-                          ```

From 8211bd6208d8752c297c0f9d3eba099130176d51 Mon Sep 17 00:00:00 2001
From: Rowan Seymour <rowanseymour@gmail.com>
Date: Fri, 8 Jul 2022 12:08:31 +0100
Subject: [PATCH 48/48] Update CHANGELOG.md for v7.4.0

---
 CHANGELOG.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index db329a3..352510e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+v7.4.0
+----------
+ * Update README
+ * Tweak startup logging
+
 v7.3.10
 ----------
  * Log app version on startup