From 8c838bd358ea7a50d8464e794c3a09f58ba838ff Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 15 Feb 2019 09:48:39 -0500 Subject: [PATCH 1/2] Don't filter on is_test Should only be deployed once https://github.com/nyaruka/rapidpro/pull/2319 is deployed --- indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer.go b/indexer.go index 4cb1cd7..9e0c0b1 100644 --- a/indexer.go +++ b/indexer.go @@ -397,7 +397,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( ) g ) as groups FROM contacts_contact - WHERE is_test = FALSE AND modified_on >= $1 + WHERE modified_on >= $1 ORDER BY modified_on ASC LIMIT 500000 ) t; From dc5af4556f511b3653a9228a5696822ab33505a3 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 15 Feb 2019 14:49:20 -0500 Subject: [PATCH 2/2] Remove test contact from test data --- indexer_test.go | 52 +++++++++++++++++++++++------------------------ testdb.sql | 41 +++++++++++++++++-------------------- testdb_update.sql | 6 +++--- 3 files changed, 48 insertions(+), 51 deletions(-) diff --git a/indexer_test.go b/indexer_test.go index e682b23..276355b 100644 --- a/indexer_test.go +++ b/indexer_test.go @@ -70,45 +70,45 @@ func TestIndexing(t *testing.T) { time.Sleep(2 * time.Second) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JOHn"), []int64{5}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JOHn"), []int64{4}) // prefix on name matches both john and joanne, but no ajodi - assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JO"), []int64{5, 7}) - assertQuery(t, client, physicalName, elastic.NewTermQuery("name.keyword", "JOHN DOE"), []int64{5}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JO"), []int64{4, 6}) + assertQuery(t, client, physicalName, elastic.NewTermQuery("name.keyword", "JOHN DOE"), []int64{4}) // can search on both first and last name boolQuery := elastic.NewBoolQuery().Must( elastic.NewMatchQuery("name", "john"), elastic.NewMatchQuery("name", "doe")) - assertQuery(t, client, physicalName, boolQuery, []int64{5}) + assertQuery(t, client, physicalName, boolQuery, []int64{4}) // can search on a long name - assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "Ajodinabiff"), []int64{6}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "Ajodinabiff"), []int64{5}) assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "eng"), []int64{1}) // test contact, not indexed assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "fra"), []int64{}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_blocked", "true"), []int64{4}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_stopped", "true"), []int64{3}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_blocked", "true"), []int64{3}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_stopped", "true"), []int64{2}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("org_id", "1"), []int64{1, 3, 4, 5}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("org_id", "1"), []int64{1, 2, 3, 4}) // created_on range query - assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 7, 9}) + assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8}) // urn query query := elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("urns.scheme", "facebook"), elastic.NewMatchQuery("urns.path.keyword", "1000001"))) - assertQuery(t, client, physicalName, query, []int64{9}) + assertQuery(t, client, physicalName, query, []int64{8}) // urn substring query query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("urns.scheme", "tel"), elastic.NewMatchPhraseQuery("urns.path", "779"))) - assertQuery(t, client, physicalName, query, []int64{1, 3, 4, 7}) + assertQuery(t, client, physicalName, query, []int64{1, 2, 3, 6}) // urn substring query with more characters (77911) query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must( @@ -120,7 +120,7 @@ func TestIndexing(t *testing.T) { query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("urns.scheme", "tel"), elastic.NewMatchPhraseQuery("urns.path", "600055"))) - assertQuery(t, client, physicalName, query, []int64{6}) + assertQuery(t, client, physicalName, query, []int64{5}) // match a contact with multiple tel urns query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must( @@ -139,7 +139,7 @@ func TestIndexing(t *testing.T) { elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"), elastic.NewExistsQuery("fields.text")))) - assertQuery(t, client, physicalName, notQuery, []int64{3, 4, 5, 6, 7, 8, 9, 10}) + assertQuery(t, client, physicalName, notQuery, []int64{2, 3, 4, 5, 6, 7, 8, 9}) // no tokenizing of field text query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( @@ -151,24 +151,24 @@ func TestIndexing(t *testing.T) { query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "05bca1cd-e322-4837-9595-86d0d85e5adb"), elastic.NewRangeQuery("fields.number").Gt(10))) - assertQuery(t, client, physicalName, query, []int64{3}) + assertQuery(t, client, physicalName, query, []int64{2}) // datetime field range query query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "e0eac267-463a-4c00-9732-cab62df07b16"), elastic.NewRangeQuery("fields.datetime").Lt(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)))) - assertQuery(t, client, physicalName, query, []int64{4}) + assertQuery(t, client, physicalName, query, []int64{3}) // state query query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"), elastic.NewMatchPhraseQuery("fields.state", "washington"))) - assertQuery(t, client, physicalName, query, []int64{6}) + assertQuery(t, client, physicalName, query, []int64{5}) query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"), elastic.NewMatchQuery("fields.state_keyword", " washington"))) - assertQuery(t, client, physicalName, query, []int64{6}) + assertQuery(t, client, physicalName, query, []int64{5}) // doesn't include country query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( @@ -185,29 +185,29 @@ func TestIndexing(t *testing.T) { query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"), elastic.NewMatchPhraseQuery("fields.district", "king"))) - assertQuery(t, client, physicalName, query, []int64{8, 10}) + assertQuery(t, client, physicalName, query, []int64{7, 9}) // phrase matches all query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"), elastic.NewMatchPhraseQuery("fields.district", "King Côunty"))) - assertQuery(t, client, physicalName, query, []int64{8}) + assertQuery(t, client, physicalName, query, []int64{7}) query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"), elastic.NewMatchQuery("fields.district_keyword", "King Côunty"))) - assertQuery(t, client, physicalName, query, []int64{8}) + assertQuery(t, client, physicalName, query, []int64{7}) // ward query query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"), elastic.NewMatchPhraseQuery("fields.ward", "district"))) - assertQuery(t, client, physicalName, query, []int64{9}) + assertQuery(t, client, physicalName, query, []int64{8}) query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"), elastic.NewMatchQuery("fields.ward_keyword", "central district"))) - assertQuery(t, client, physicalName, query, []int64{9}) + assertQuery(t, client, physicalName, query, []int64{8}) // no substring though on keyword query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must( @@ -217,7 +217,7 @@ func TestIndexing(t *testing.T) { // group query assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4ea0f313-2f62-4e57-bdf0-232b5191dd57"), []int64{1}) - assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 3}) + assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2}) assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{}) lastModified, err := GetLastModified(elasticURL, physicalName) @@ -230,7 +230,7 @@ func TestIndexing(t *testing.T) { time.Sleep(5 * time.Second) // try a test query to check it worked - assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{5}) + assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{4}) // look up our mapping physical := FindPhysicalIndexes(elasticURL, indexName) @@ -265,7 +265,7 @@ func TestIndexing(t *testing.T) { assert.Equal(t, resp.StatusCode, http.StatusNotFound) // new index still works - assertQuery(t, client, newIndex, elastic.NewMatchQuery("name", "john"), []int64{5}) + assertQuery(t, client, newIndex, elastic.NewMatchQuery("name", "john"), []int64{4}) // update our database, removing one contact, updating another dbUpdate, err := ioutil.ReadFile("testdb_update.sql") @@ -281,7 +281,7 @@ func TestIndexing(t *testing.T) { time.Sleep(5 * time.Second) // should only match new john, old john is gone - assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{3}) + assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{2}) // 3 is no longer in our group assertQuery(t, client, indexName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1}) diff --git a/testdb.sql b/testdb.sql index fa6c89c..7abd916 100644 --- a/testdb.sql +++ b/testdb.sql @@ -91,37 +91,35 @@ ALTER SEQUENCE contacts_contactgroup_contacts_id_seq OWNED BY contacts_contactgr INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, org_id, is_blocked, name, is_test, language, uuid, is_stopped, fields) VALUES (1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 1, FALSE, NULL, FALSE, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', FALSE, '{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}'), -(2, TRUE, -1, '2015-03-25 17:15:12.982168+00', -1, '2015-03-25 17:15:12.982168+00', 1, FALSE, NULL, TRUE, 'fra', '1ad43adc-c4fc-4244-8b3d-a938b8eba57a', FALSE, NULL), -(3, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', 1, FALSE, NULL, FALSE, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', TRUE, +(2, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', 1, FALSE, NULL, FALSE, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', TRUE, '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}'), -(4, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', 1, TRUE, NULL, FALSE, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', FALSE, +(3, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', 1, TRUE, NULL, FALSE, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', FALSE, '{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}'), -(5, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', 1, FALSE, 'John Doe', FALSE, NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', FALSE, +(4, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', 1, FALSE, 'John Doe', FALSE, NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', FALSE, '{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}'), -(6, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', 2, FALSE, 'Ajodinabiff Dane', FALSE, NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', FALSE, +(5, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', 2, FALSE, 'Ajodinabiff Dane', FALSE, NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', FALSE, '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }'), -(7, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, 'Joanne Stone', FALSE, NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', FALSE, +(6, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, 'Joanne Stone', FALSE, NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', FALSE, '{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }'), -(8, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', 2, FALSE, NULL, FALSE, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', FALSE, +(7, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', 2, FALSE, NULL, FALSE, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', FALSE, '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King Côunty", "district": "USA > Washington > King Côunty"} }'), -(9, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, NULL, FALSE, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', FALSE, +(8, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, NULL, FALSE, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', FALSE, '{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King Côunty > Central District", "ward": "USA > Washington > King Côunty > Central District"} }'), -(10, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', 2, FALSE, NULL, FALSE, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', FALSE, +(9, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', 2, FALSE, NULL, FALSE, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', FALSE, '{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }'); INSERT INTO contacts_contacturn(id, contact_id, scheme, org_id, priority, path, display, identity) VALUES (1, 1, 'tel', 1, 50, '+12067791111', NULL, 'tel:+12067791111'), (2, 1, 'tel', 1, 50, '+12067792222', NULL, 'tel:+12067792222'), -(3, 2, 'tel', 1, 50, '+12067793333', NULL, 'tel:+12067793333'), -(4, 3, 'tel', 1, 50, '+12067794444', NULL, 'tel:+12067794444'), -(5, 4, 'tel', 1, 50, '+12067795555', NULL, 'tel:+12067795555'), -(6, 5, 'tel', 1, 50, '+12060000556', NULL, 'tel:+12067796666'), -(7, 6, 'tel', 2, 50, '+12060005577', NULL, 'tel:+12067797777'), -(8, 7, 'tel', 2, 50, '+12067798888', NULL, 'tel:+12067798888'), -(9, 8, 'viber', 2, 90, 'viberpath==', NULL, 'viber:viberpath=='), -(10, 9, 'facebook', 2, 90, 1000001, 'funguy', 'facebook:1000001'), -(11, 10, 'twitterid', 2, 90, 1000001, 'fungal', 'twitterid:1000001'), -(12, 11, 'whatsapp', 2, 90, 1000003, NULL, 'whatsapp:1000003'); +(3, 2, 'tel', 1, 50, '+12067794444', NULL, 'tel:+12067794444'), +(4, 3, 'tel', 1, 50, '+12067795555', NULL, 'tel:+12067795555'), +(5, 4, 'tel', 1, 50, '+12060000556', NULL, 'tel:+12067796666'), +(6, 5, 'tel', 2, 50, '+12060005577', NULL, 'tel:+12067797777'), +(7, 6, 'tel', 2, 50, '+12067798888', NULL, 'tel:+12067798888'), +(8, 7, 'viber', 2, 90, 'viberpath==', NULL, 'viber:viberpath=='), +(9, 8, 'facebook', 2, 90, 1000001, 'funguy', 'facebook:1000001'), +(10, 9, 'twitterid', 2, 90, 1000001, 'fungal', 'twitterid:1000001'), +(11, 10, 'whatsapp', 2, 90, 1000003, NULL, 'whatsapp:1000003'); INSERT INTO contacts_contactgroup(id, uuid, name) VALUES (1, '4ea0f313-2f62-4e57-bdf0-232b5191dd57', 'Group 1'), @@ -131,6 +129,5 @@ INSERT INTO contacts_contactgroup(id, uuid, name) VALUES INSERT INTO contacts_contactgroup_contacts(id, contact_id, contactgroup_id) VALUES (1, 1, 1), -(2, 2, 1), -(3, 1, 4), -(4, 3, 4); +(2, 1, 4), +(3, 2, 4); diff --git a/testdb_update.sql b/testdb_update.sql index af2cbf0..e551846 100644 --- a/testdb_update.sql +++ b/testdb_update.sql @@ -1,8 +1,8 @@ -- update one of our contacts -DELETE FROM contacts_contactgroup_contacts WHERE id = 4; -UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 3; +DELETE FROM contacts_contactgroup_contacts WHERE id = 3; +UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 2; -- delete one of our others -UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 5; +UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;