From db54150e71e81662b8d2c17d32f2ee50be6bdf5e Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 4 Jun 2021 12:35:53 -0500 Subject: [PATCH 1/9] Fix creating of location keyword fields when values have punctuation --- indexer.go | 6 +++--- indexer_test.go | 4 ++-- testdb.sql | 54 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/indexer.go b/indexer.go index 7d0ea67..8e62daa 100644 --- a/indexer.go +++ b/indexer.go @@ -399,7 +399,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( select case when value ? 'ward' then jsonb_build_object( - 'ward_keyword', trim(substring(value ->> 'ward' from '(?!.* > )([\w ]+)')) + 'ward_keyword', trim(substring(value ->> 'ward' from '(?!.* > )([^>]+)')) ) else '{}' :: jsonb end || district_value.value as value @@ -407,7 +407,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( select case when value ? 'district' then jsonb_build_object( - 'district_keyword', trim(substring(value ->> 'district' from '(?!.* > )([\w ]+)')) + 'district_keyword', trim(substring(value ->> 'district' from '(?!.* > )([^>]+)')) ) else '{}' :: jsonb end || state_value.value as value @@ -416,7 +416,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( select case when value ? 'state' then jsonb_build_object( - 'state_keyword', trim(substring(value ->> 'state' from '(?!.* > )([\w ]+)')) + 'state_keyword', trim(substring(value ->> 'state' from '(?!.* > )([^>]+)')) ) else '{}' :: jsonb end || diff --git a/indexer_test.go b/indexer_test.go index 41d2a8f..9ce9818 100644 --- a/indexer_test.go +++ b/indexer_test.go @@ -202,12 +202,12 @@ func TestIndexing(t *testing.T) { // phrase matches all query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"), - elastic.NewMatchPhraseQuery("fields.district", "King Côunty"))) + elastic.NewMatchPhraseQuery("fields.district", "King-Côunty"))) assertQuery(t, client, physicalName, query, []int64{7}) query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"), - elastic.NewMatchQuery("fields.district_keyword", "King Côunty"))) + elastic.NewMatchQuery("fields.district_keyword", "King-Côunty"))) assertQuery(t, client, physicalName, query, []int64{7}) // ward query diff --git a/testdb.sql b/testdb.sql index 76bb79c..5914f8b 100644 --- a/testdb.sql +++ b/testdb.sql @@ -88,24 +88,42 @@ ALTER SEQUENCE contacts_contactgroup_contacts_id_seq OWNED BY contacts_contactgr -- a551ade4-e5a0-4d83-b185-53b515ad2f2a - home_ward (ward) INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, last_seen_on, org_id, status, name, language, uuid, fields) VALUES -(1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:11', 1, 'A', NULL, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', -'{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}'), -(2, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', '2020-08-03 13:11', 1, 'S', NULL, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', -'{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}'), -(3, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', '2018-05-04 21:11', 1, 'B', NULL, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', -'{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}'), -(4, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', '2015-12-31 23:59', 1, 'A', 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', -'{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}'), -(5, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', '2020-08-04 21:11', 2, 'A', 'Ajodinabiff Dane', NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', -'{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }'), -(6, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:00', 2, 'A', 'Joanne Stone', NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', -'{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }'), -(7, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', NULL, 2, 'A', NULL, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', -'{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King Côunty", "district": "USA > Washington > King Côunty"} }'), -(8, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, 2, 'A', NULL, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', -'{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King Côunty > Central District", "ward": "USA > Washington > King Côunty > Central District"} }'), -(9, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', NULL, 2, 'A', NULL, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', -'{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }'); +( + 1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:11', 1, 'A', NULL, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', + '{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}' +), +( + 2, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', '2020-08-03 13:11', 1, 'S', NULL, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', + '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}' +), +( + 3, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', '2018-05-04 21:11', 1, 'B', NULL, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', + '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}' +), +( + 4, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', '2015-12-31 23:59', 1, 'A', 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', + '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}' + ), +( + 5, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', '2020-08-04 21:11', 2, 'A', 'Ajodinabiff Dane', NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', + '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }' +), +( + 6, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:00', 2, 'A', 'Joanne Stone', NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', + '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }' +), +( + 7, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', NULL, 2, 'A', NULL, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', + '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King-Côunty", "district": "USA > Washington > King-Côunty"} }' +), +( + 8, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, 2, 'A', NULL, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', + '{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King-Côunty > Central District", "ward": "USA > Washington > King-Côunty > Central District"} }' +), +( + 9, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', NULL, 2, 'A', NULL, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', + '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }' +); INSERT INTO contacts_contacturn(id, contact_id, scheme, org_id, priority, path, display, identity) VALUES (1, 1, 'tel', 1, 50, '+12067791111', NULL, 'tel:+12067791111'), From b47cd517f507eaf97584d2fe30c258e109164505 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 4 Jun 2021 13:30:27 -0500 Subject: [PATCH 2/9] Tweaks from linter --- indexer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer.go b/indexer.go index 8e62daa..9760bd6 100644 --- a/indexer.go +++ b/indexer.go @@ -68,7 +68,7 @@ func CreateNewIndex(url string, alias string) (string, error) { idx := 0 // check if it exists - for true { + for { resp, err := http.Get(fmt.Sprintf("%s/%s", url, physicalIndex)) if err != nil { return "", err @@ -332,7 +332,7 @@ func IndexContacts(db *sql.DB, elasticURL string, index string, lastModified tim break } - elapsed := time.Now().Sub(start) + elapsed := time.Since(start) rate := float32(processedCount) / (float32(elapsed) / float32(time.Second)) log.WithFields(map[string]interface{}{ "rate": int(rate), From d66e01ac85140252c9308e4595b232842bb6efbb Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 4 Jun 2021 13:44:58 -0500 Subject: [PATCH 3/9] Update CHANGELOG.md for v6.3.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9592de..23fdb43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v6.3.0 +---------- + * Fix creating of location keyword fields when values have punctuation + v6.2.0 ---------- * add rp-indexer to .gitignore From fa9d10cecc0a98adca41e4deedabbe3a96ef85b6 Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Tue, 6 Jul 2021 12:39:37 -0700 Subject: [PATCH 4/9] touch for 6.4 rev --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 77db83a..078dba7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# RapidPro Indexer +# RapidPro Indexer [![Build Status](https://github.com/nyaruka/rp-indexer/workflows/CI/badge.svg)](https://github.com/nyaruka/rp-indexer/actions?query=workflow%3ACI) [![codecov](https://codecov.io/gh/nyaruka/rp-indexer/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/rp-indexer) From d00236692a88260288ec17463e99e22e0fe3272a Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Tue, 6 Jul 2021 12:40:05 -0700 Subject: [PATCH 5/9] Update CHANGELOG.md for v6.4.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23fdb43..626ad79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v6.4.0 +---------- + * 6.4.0 candidate + v6.3.0 ---------- * Fix creating of location keyword fields when values have punctuation From db356722a1d0f405fe8c1549176df8d79144970d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 29 Jul 2021 09:50:33 -0500 Subject: [PATCH 6/9] Update to go 1.16, use embedded file for index settings, remove no longer used is_blocked and and is_stopped fields --- .github/workflows/ci.yml | 9 +- contacts/index_settings.json | 183 ++++++++++++++++++++++++++++++++++ go.mod | 7 +- go.sum | 65 ++---------- indexer.go | 188 +---------------------------------- indexer_test.go | 5 - 6 files changed, 201 insertions(+), 256 deletions(-) create mode 100644 contacts/index_settings.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 762223e..733f9e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,8 @@ name: CI on: [push, pull_request] +env: + go-version: '1.16.x' + es-version: '7.10.1' jobs: test: name: Test @@ -14,7 +17,7 @@ jobs: - name: Install ElasticSearch uses: nyaruka/elasticsearch-action@master with: - elastic version: '7.10.1' + elastic version: ${{ env.es-version }} - name: Install PostgreSQL uses: harmon758/postgresql-action@v1 @@ -27,7 +30,7 @@ jobs: - name: Install Go uses: actions/setup-go@v1 with: - go-version: 1.15.x + go-version: ${{ env.go-version }} - name: Run tests run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./... @@ -50,7 +53,7 @@ jobs: - name: Install Go uses: actions/setup-go@v1 with: - go-version: 1.15.x + go-version: ${{ env.go-version }} - name: Publish release uses: goreleaser/goreleaser-action@v1 diff --git a/contacts/index_settings.json b/contacts/index_settings.json new file mode 100644 index 0000000..8abbe5e --- /dev/null +++ b/contacts/index_settings.json @@ -0,0 +1,183 @@ +{ + "settings": { + "index": { + "number_of_shards": 2, + "number_of_replicas": 1, + "routing_partition_size": 1 + }, + "analysis": { + "analyzer": { + "trigrams": { + "type": "custom", + "tokenizer": "trigram", + "filter": [ + "lowercase" + ] + }, + "locations": { + "tokenizer": "location_tokenizer", + "filter": [ + "lowercase", + "word_delimiter" + ] + }, + "prefix": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "prefix_filter" + ] + }, + "name_search": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "max_length" + ] + } + }, + "tokenizer": { + "location_tokenizer": { + "type": "pattern", + "pattern": "(.* > )?([^>]+)", + "group": 2 + }, + "trigram": { + "type": "ngram", + "min_gram": 3, + "max_gram": 3 + } + }, + "normalizer": { + "lowercase": { + "type": "custom", + "char_filter": [], + "filter": [ + "lowercase", + "trim" + ] + } + }, + "filter": { + "prefix_filter": { + "type": "edge_ngram", + "min_gram": 2, + "max_gram": 8 + }, + "max_length": { + "type": "truncate", + "length": 8 + } + } + } + }, + "mappings": { + "_doc": { + "_routing": { + "required": true + }, + "properties": { + "fields": { + "type": "nested", + "properties": { + "field": { + "type": "keyword" + }, + "text": { + "type": "keyword", + "normalizer": "lowercase" + }, + "number": { + "type": "scaled_float", + "scaling_factor": 10000 + }, + "datetime": { + "type": "date" + }, + "state": { + "type": "text", + "analyzer": "locations" + }, + "state_keyword": { + "type": "keyword", + "normalizer": "lowercase" + }, + "district": { + "type": "text", + "analyzer": "locations" + }, + "district_keyword": { + "type": "keyword", + "normalizer": "lowercase" + }, + "ward": { + "type": "text", + "analyzer": "locations" + }, + "ward_keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "urns": { + "type": "nested", + "properties": { + "path": { + "type": "text", + "analyzer": "trigrams", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "scheme": { + "type": "keyword", + "normalizer": "lowercase" + } + } + }, + "groups": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + }, + "status": { + "type": "keyword" + }, + "language": { + "type": "keyword", + "normalizer": "lowercase" + }, + "modified_on": { + "type": "date" + }, + "created_on": { + "type": "date" + }, + "modified_on_mu": { + "type": "long" + }, + "last_seen_on": { + "type": "date" + }, + "name": { + "type": "text", + "analyzer": "prefix", + "search_analyzer": "name_search", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 2b8b84c..8e42fdc 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,8 @@ require ( github.com/nyaruka/ezconf v0.2.1 github.com/nyaruka/gocommon v1.3.0 github.com/olivere/elastic/v7 v7.0.22 - github.com/onsi/ginkgo v1.14.0 // indirect - github.com/sirupsen/logrus v1.0.5 + github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.5.1 - gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect - gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect ) -go 1.13 +go 1.16 diff --git a/go.sum b/go.sum index 5eecd13..16e5ac2 100644 --- a/go.sum +++ b/go.sum @@ -13,9 +13,6 @@ github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3 h1:md1zEr2oSVWYNfQj+6TL/nmAFf5gY3Tp44lzskzK9QU= github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -24,20 +21,9 @@ github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -52,8 +38,6 @@ github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hz github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU= @@ -61,13 +45,6 @@ github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8M github.com/nyaruka/phonenumbers v1.0.34/go.mod h1:GQ0cTHlrxPrhoLwyQ1blyN1hO794ygt6FTHWrFB5SSc= github.com/olivere/elastic/v7 v7.0.22 h1:esBA6JJwvYgfms0EVlH7Z+9J4oQ/WUADF2y/nCNDw7s= github.com/olivere/elastic/v7 v7.0.22/go.mod h1:VDexNy9NjmtAkrjNoI7tImv7FR4tf5zUA3ickqu5Pc8= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -75,20 +52,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.0.5 h1:8c8b5uO0zS4X6RPl/sd1ENwSkIc0/H2PaHxE3udaE8I= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v1.1.1/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -96,39 +71,27 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180921000356-2f5d2388922f h1:QM2QVxvDoW9PFSPp/zy9FgxJLfaWTZlS61KEPtBwacM= golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -136,25 +99,9 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/indexer.go b/indexer.go index 9760bd6..1a8856d 100644 --- a/indexer.go +++ b/indexer.go @@ -3,6 +3,7 @@ package indexer import ( "bytes" "database/sql" + _ "embed" "encoding/json" "fmt" "io/ioutil" @@ -383,7 +384,7 @@ func MapIndexAlias(elasticURL string, alias string, newIndex string) error { const contactQuery = ` SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( SELECT - id, org_id, uuid, name, language, status, status = 'S' AS is_stopped, status = 'B' AS is_blocked, is_active, created_on, modified_on, last_seen_on, + id, org_id, uuid, name, language, status, is_active, created_on, modified_on, last_seen_on, EXTRACT(EPOCH FROM modified_on) * 1000000 as modified_on_mu, ( SELECT array_to_json(array_agg(row_to_json(u))) @@ -443,189 +444,8 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( ` // settings and mappings for our index -const indexSettings = ` -{ - "settings": { - "index": { - "number_of_shards": 2, - "number_of_replicas": 1, - "routing_partition_size": 1 - }, - "analysis": { - "analyzer": { - "trigrams": { - "type": "custom", - "tokenizer": "trigram", - "filter": [ - "lowercase" - ] - }, - "locations": { - "tokenizer": "location_tokenizer", - "filter": [ - "lowercase", - "word_delimiter" - ] - }, - "prefix": { - "type": "custom", - "tokenizer": "standard", - "filter": [ - "lowercase", - "prefix_filter" - ] - }, - "name_search": { - "type": "custom", - "tokenizer": "standard", - "filter": [ - "lowercase", - "max_length" - ] - } - }, - "tokenizer": { - "location_tokenizer": { - "type": "pattern", - "pattern": "(.* > )?([^>]+)", - "group": 2 - }, - "trigram": { - "type" : "ngram", - "min_gram" : 3, - "max_gram" : 3 - } - }, - "normalizer": { - "lowercase": { - "type": "custom", - "char_filter": [], - "filter": ["lowercase", "trim"] - } - }, - "filter": { - "prefix_filter": { - "type": "edge_ngram", - "min_gram": 2, - "max_gram": 8 - }, - "max_length":{ - "type": "truncate", - "length": 8 - } - } - } - }, - - "mappings": { - "_doc": { - "_routing": { - "required": true - }, - "properties": { - "fields": { - "type": "nested", - "properties": { - "field": { - "type": "keyword" - }, - "text": { - "type": "keyword", - "normalizer": "lowercase" - }, - "number": { - "type": "scaled_float", - "scaling_factor": 10000 - }, - "datetime": { - "type": "date" - }, - "state": { - "type": "text", - "analyzer": "locations" - }, - "state_keyword": { - "type": "keyword", - "normalizer": "lowercase" - }, - "district": { - "type": "text", - "analyzer": "locations" - }, - "district_keyword": { - "type": "keyword", - "normalizer": "lowercase" - }, - "ward": { - "type": "text", - "analyzer": "locations" - }, - "ward_keyword": { - "type": "keyword", - "normalizer": "lowercase" - } - } - }, - "urns": { - "type": "nested", - "properties": { - "path": { - "type": "text", - "analyzer": "trigrams", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase" - } - } - }, - "scheme": { - "type": "keyword", - "normalizer": "lowercase" - } - } - }, - "groups": { - "type": "keyword" - }, - "uuid": { - "type": "keyword" - }, - "status": { - "type": "keyword" - }, - "language": { - "type": "keyword", - "normalizer": "lowercase" - }, - "modified_on": { - "type": "date" - }, - "created_on": { - "type": "date" - }, - "modified_on_mu": { - "type": "long" - }, - "last_seen_on": { - "type": "date" - }, - "name": { - "type": "text", - "analyzer": "prefix", - "search_analyzer": "name_search", - "fields": { - "keyword": { - "type": "keyword", - "normalizer": "lowercase" - } - } - } - } - } - } -} -` +//go:embed contacts/index_settings.json +var indexSettings string // gets our last modified contact const lastModifiedQuery = `{ "sort": [{ "modified_on_mu": "desc" }]}` diff --git a/indexer_test.go b/indexer_test.go index 9ce9818..14545b9 100644 --- a/indexer_test.go +++ b/indexer_test.go @@ -92,9 +92,6 @@ func TestIndexing(t *testing.T) { // test contact, not indexed assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "fra"), []int64{}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_blocked", "true"), []int64{3}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_stopped", "true"), []int64{2}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("status", "B"), []int64{3}) assertQuery(t, client, physicalName, elastic.NewMatchQuery("status", "S"), []int64{2}) @@ -337,8 +334,6 @@ func TestRetryServer(t *testing.T) { "uuid": "c7a2dd87-a80e-420b-8431-ca48d422e924", "name": null, "language": "eng", - "is_stopped": false, - "is_blocked": false, "is_active": true, "created_on": "2017-11-10T16:11:59.890662-05:00", "modified_on": "2017-11-10T16:11:59.890662-05:00", From 455077550b02f1291f29917297a6a5fc14100404 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 29 Jul 2021 17:32:54 -0500 Subject: [PATCH 7/9] Include contact.ticket_count as tickets in index --- contacts/index_settings.json | 3 +++ indexer.go | 2 +- indexer_test.go | 4 ++++ testdb.sql | 25 +++++++++++++------------ 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/contacts/index_settings.json b/contacts/index_settings.json index 8abbe5e..9706745 100644 --- a/contacts/index_settings.json +++ b/contacts/index_settings.json @@ -150,6 +150,9 @@ "status": { "type": "keyword" }, + "tickets": { + "type": "integer" + }, "language": { "type": "keyword", "normalizer": "lowercase" diff --git a/indexer.go b/indexer.go index 1a8856d..be30942 100644 --- a/indexer.go +++ b/indexer.go @@ -384,7 +384,7 @@ func MapIndexAlias(elasticURL string, alias string, newIndex string) error { const contactQuery = ` SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( SELECT - id, org_id, uuid, name, language, status, is_active, created_on, modified_on, last_seen_on, + id, org_id, uuid, name, language, status, ticket_count AS tickets, is_active, created_on, modified_on, last_seen_on, EXTRACT(EPOCH FROM modified_on) * 1000000 as modified_on_mu, ( SELECT array_to_json(array_agg(row_to_json(u))) diff --git a/indexer_test.go b/indexer_test.go index 14545b9..40b6296 100644 --- a/indexer_test.go +++ b/indexer_test.go @@ -97,6 +97,10 @@ func TestIndexing(t *testing.T) { assertQuery(t, client, physicalName, elastic.NewMatchQuery("org_id", "1"), []int64{1, 2, 3, 4}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("tickets", 2), []int64{1}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("tickets", 1), []int64{2, 3}) + assertQuery(t, client, physicalName, elastic.NewRangeQuery("tickets").Gt(0), []int64{1, 2, 3}) + // created_on range query assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8}) diff --git a/testdb.sql b/testdb.sql index 5914f8b..99f379b 100644 --- a/testdb.sql +++ b/testdb.sql @@ -12,7 +12,8 @@ CREATE TABLE contacts_contact ( name character varying(128), language character varying(3), uuid character varying(36) NOT NULL, - fields jsonb + fields jsonb, + ticket_count integer NOT NULL ); CREATE SEQUENCE contacts_contact_id_seq @@ -87,42 +88,42 @@ ALTER SEQUENCE contacts_contactgroup_contacts_id_seq OWNED BY contacts_contactgr -- fcab2439-861c-4832-aa54-0c97f38f24ab - home_district (district) -- a551ade4-e5a0-4d83-b185-53b515ad2f2a - home_ward (ward) -INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, last_seen_on, org_id, status, name, language, uuid, fields) VALUES +INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, last_seen_on, org_id, status, name, language, uuid, fields, ticket_count) VALUES ( 1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:11', 1, 'A', NULL, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', - '{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}' + '{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}', 2 ), ( 2, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', '2020-08-03 13:11', 1, 'S', NULL, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', - '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}' + '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}', 1 ), ( 3, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', '2018-05-04 21:11', 1, 'B', NULL, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', - '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}' + '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}', 1 ), ( 4, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', '2015-12-31 23:59', 1, 'A', 'John Doe', NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', - '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}' - ), + '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}', 0 +), ( 5, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', '2020-08-04 21:11', 2, 'A', 'Ajodinabiff Dane', NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', - '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }' + '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }', 0 ), ( 6, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', '2020-08-04 21:00', 2, 'A', 'Joanne Stone', NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', - '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }' + '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }', 0 ), ( 7, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', NULL, 2, 'A', NULL, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', - '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King-Côunty", "district": "USA > Washington > King-Côunty"} }' + '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King-Côunty", "district": "USA > Washington > King-Côunty"} }', 0 ), ( 8, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', NULL, 2, 'A', NULL, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', - '{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King-Côunty > Central District", "ward": "USA > Washington > King-Côunty > Central District"} }' + '{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King-Côunty > Central District", "ward": "USA > Washington > King-Côunty > Central District"} }', 0 ), ( 9, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', NULL, 2, 'A', NULL, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', - '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }' + '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }', 0 ); INSERT INTO contacts_contacturn(id, contact_id, scheme, org_id, priority, path, display, identity) VALUES From 0b2a44c7c97f8f49906181342ed14279db9e8469 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 2 Aug 2021 14:35:48 -0500 Subject: [PATCH 8/9] Update CHANGELOG.md for v6.5.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 626ad79..9b7822a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +v6.5.0 +---------- + * Include contact.ticket_count as tickets in index + * Update to go 1.16 + * Use embedded file for index settings + * Remove no longer used is_blocked and and is_stopped fields + v6.4.0 ---------- * 6.4.0 candidate From 9892c9c75cc99e125d060da33fb5e9f763bf8539 Mon Sep 17 00:00:00 2001 From: Joao Date: Tue, 3 Aug 2021 01:05:35 -0300 Subject: [PATCH 9/9] update golang to 1.16 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 56ee219..c61f1eb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.13.14-alpine3.12 +FROM golang:1.16.6-alpine3.14 WORKDIR /app