Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

🧽 Don't filter on is_test #16

Merged
merged 2 commits into from
Jun 13, 2019
Merged

🧽 Don't filter on is_test #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8c838bd
Don't filter on is_test
rowanseymour Feb 15, 2019
dc5af45
Remove test contact from test data
rowanseymour Feb 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion indexer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,7 +397,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM (
) g
) as groups
FROM contacts_contact
WHERE is_test = FALSE AND modified_on >= $1
WHERE modified_on >= $1
ORDER BY modified_on ASC
LIMIT 500000
) t;
Expand Down
52 changes: 26 additions & 26 deletions indexer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,45 +70,45 @@ func TestIndexing(t *testing.T) {

time.Sleep(2 * time.Second)

assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JOHn"), []int64{5})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JOHn"), []int64{4})

// prefix on name matches both john and joanne, but no ajodi
assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JO"), []int64{5, 7})
assertQuery(t, client, physicalName, elastic.NewTermQuery("name.keyword", "JOHN DOE"), []int64{5})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "JO"), []int64{4, 6})
assertQuery(t, client, physicalName, elastic.NewTermQuery("name.keyword", "JOHN DOE"), []int64{4})

// can search on both first and last name
boolQuery := elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("name", "john"),
elastic.NewMatchQuery("name", "doe"))
assertQuery(t, client, physicalName, boolQuery, []int64{5})
assertQuery(t, client, physicalName, boolQuery, []int64{4})

// can search on a long name
assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "Ajodinabiff"), []int64{6})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("name", "Ajodinabiff"), []int64{5})

assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "eng"), []int64{1})

// test contact, not indexed
assertQuery(t, client, physicalName, elastic.NewMatchQuery("language", "fra"), []int64{})

assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_blocked", "true"), []int64{4})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_stopped", "true"), []int64{3})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_blocked", "true"), []int64{3})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("is_stopped", "true"), []int64{2})

assertQuery(t, client, physicalName, elastic.NewMatchQuery("org_id", "1"), []int64{1, 3, 4, 5})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("org_id", "1"), []int64{1, 2, 3, 4})

// created_on range query
assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 7, 9})
assertQuery(t, client, physicalName, elastic.NewRangeQuery("created_on").Gt("2017-01-01"), []int64{1, 6, 8})

// urn query
query := elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("urns.scheme", "facebook"),
elastic.NewMatchQuery("urns.path.keyword", "1000001")))
assertQuery(t, client, physicalName, query, []int64{9})
assertQuery(t, client, physicalName, query, []int64{8})

// urn substring query
query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("urns.scheme", "tel"),
elastic.NewMatchPhraseQuery("urns.path", "779")))
assertQuery(t, client, physicalName, query, []int64{1, 3, 4, 7})
assertQuery(t, client, physicalName, query, []int64{1, 2, 3, 6})

// urn substring query with more characters (77911)
query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
Expand All @@ -120,7 +120,7 @@ func TestIndexing(t *testing.T) {
query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("urns.scheme", "tel"),
elastic.NewMatchPhraseQuery("urns.path", "600055")))
assertQuery(t, client, physicalName, query, []int64{6})
assertQuery(t, client, physicalName, query, []int64{5})

// match a contact with multiple tel urns
query = elastic.NewNestedQuery("urns", elastic.NewBoolQuery().Must(
Expand All @@ -139,7 +139,7 @@ func TestIndexing(t *testing.T) {
elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "17103bb1-1b48-4b70-92f7-1f6b73bd3488"),
elastic.NewExistsQuery("fields.text"))))
assertQuery(t, client, physicalName, notQuery, []int64{3, 4, 5, 6, 7, 8, 9, 10})
assertQuery(t, client, physicalName, notQuery, []int64{2, 3, 4, 5, 6, 7, 8, 9})

// no tokenizing of field text
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
Expand All @@ -151,24 +151,24 @@ func TestIndexing(t *testing.T) {
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "05bca1cd-e322-4837-9595-86d0d85e5adb"),
elastic.NewRangeQuery("fields.number").Gt(10)))
assertQuery(t, client, physicalName, query, []int64{3})
assertQuery(t, client, physicalName, query, []int64{2})

// datetime field range query
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "e0eac267-463a-4c00-9732-cab62df07b16"),
elastic.NewRangeQuery("fields.datetime").Lt(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC))))
assertQuery(t, client, physicalName, query, []int64{4})
assertQuery(t, client, physicalName, query, []int64{3})

// state query
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
elastic.NewMatchPhraseQuery("fields.state", "washington")))
assertQuery(t, client, physicalName, query, []int64{6})
assertQuery(t, client, physicalName, query, []int64{5})

query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "22d11697-edba-4186-b084-793e3b876379"),
elastic.NewMatchQuery("fields.state_keyword", " washington")))
assertQuery(t, client, physicalName, query, []int64{6})
assertQuery(t, client, physicalName, query, []int64{5})

// doesn't include country
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
Expand All @@ -185,29 +185,29 @@ func TestIndexing(t *testing.T) {
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
elastic.NewMatchPhraseQuery("fields.district", "king")))
assertQuery(t, client, physicalName, query, []int64{8, 10})
assertQuery(t, client, physicalName, query, []int64{7, 9})

// phrase matches all
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
elastic.NewMatchPhraseQuery("fields.district", "King Côunty")))
assertQuery(t, client, physicalName, query, []int64{8})
assertQuery(t, client, physicalName, query, []int64{7})

query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "fcab2439-861c-4832-aa54-0c97f38f24ab"),
elastic.NewMatchQuery("fields.district_keyword", "King Côunty")))
assertQuery(t, client, physicalName, query, []int64{8})
assertQuery(t, client, physicalName, query, []int64{7})

// ward query
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
elastic.NewMatchPhraseQuery("fields.ward", "district")))
assertQuery(t, client, physicalName, query, []int64{9})
assertQuery(t, client, physicalName, query, []int64{8})

query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
elastic.NewMatchQuery("fields.field", "a551ade4-e5a0-4d83-b185-53b515ad2f2a"),
elastic.NewMatchQuery("fields.ward_keyword", "central district")))
assertQuery(t, client, physicalName, query, []int64{9})
assertQuery(t, client, physicalName, query, []int64{8})

// no substring though on keyword
query = elastic.NewNestedQuery("fields", elastic.NewBoolQuery().Must(
Expand All @@ -217,7 +217,7 @@ func TestIndexing(t *testing.T) {

// group query
assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4ea0f313-2f62-4e57-bdf0-232b5191dd57"), []int64{1})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 3})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1, 2})
assertQuery(t, client, physicalName, elastic.NewMatchQuery("groups", "4c016340-468d-4675-a974-15cb7a45a5ab"), []int64{})

lastModified, err := GetLastModified(elasticURL, physicalName)
Expand All @@ -230,7 +230,7 @@ func TestIndexing(t *testing.T) {
time.Sleep(5 * time.Second)

// try a test query to check it worked
assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{5})
assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{4})

// look up our mapping
physical := FindPhysicalIndexes(elasticURL, indexName)
Expand Down Expand Up @@ -265,7 +265,7 @@ func TestIndexing(t *testing.T) {
assert.Equal(t, resp.StatusCode, http.StatusNotFound)

// new index still works
assertQuery(t, client, newIndex, elastic.NewMatchQuery("name", "john"), []int64{5})
assertQuery(t, client, newIndex, elastic.NewMatchQuery("name", "john"), []int64{4})

// update our database, removing one contact, updating another
dbUpdate, err := ioutil.ReadFile("testdb_update.sql")
Expand All @@ -281,7 +281,7 @@ func TestIndexing(t *testing.T) {
time.Sleep(5 * time.Second)

// should only match new john, old john is gone
assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{3})
assertQuery(t, client, indexName, elastic.NewMatchQuery("name", "john"), []int64{2})

// 3 is no longer in our group
assertQuery(t, client, indexName, elastic.NewMatchQuery("groups", "529bac39-550a-4d6f-817c-1833f3449007"), []int64{1})
Expand Down
41 changes: 19 additions & 22 deletions testdb.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,37 +91,35 @@ ALTER SEQUENCE contacts_contactgroup_contacts_id_seq OWNED BY contacts_contactgr
INSERT INTO contacts_contact(id, is_active, created_by_id, created_on, modified_by_id, modified_on, org_id, is_blocked, name, is_test, language, uuid, is_stopped, fields) VALUES
(1, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 1, FALSE, NULL, FALSE, 'eng', 'c7a2dd87-a80e-420b-8431-ca48d422e924', FALSE,
'{ "17103bb1-1b48-4b70-92f7-1f6b73bd3488": {"text": "the rock"}}'),
(2, TRUE, -1, '2015-03-25 17:15:12.982168+00', -1, '2015-03-25 17:15:12.982168+00', 1, FALSE, NULL, TRUE, 'fra', '1ad43adc-c4fc-4244-8b3d-a938b8eba57a', FALSE, NULL),
(3, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', 1, FALSE, NULL, FALSE, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', TRUE,
(2, TRUE, -1, '2015-03-26 10:07:14.054521+00', -1, '2015-03-26 10:07:14.054521+00', 1, FALSE, NULL, FALSE, NULL, '7a6606c7-ff41-4203-aa98-454a10d37209', TRUE,
'{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "11", "number": 11 }}'),
(4, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', 1, TRUE, NULL, FALSE, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', FALSE,
(3, TRUE, -1, '2015-03-26 13:04:58.699648+00', -1, '2015-03-26 13:04:58.699648+00', 1, TRUE, NULL, FALSE, NULL, '29b45297-15ad-4061-a7d4-e0b33d121541', FALSE,
'{ "05bca1cd-e322-4837-9595-86d0d85e5adb": {"text": "9", "number": 9 }, "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2018-04-06T18:37:59+00:00", "datetime": "2018-04-06T18:37:59+00:00"}}'),
(5, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', 1, FALSE, 'John Doe', FALSE, NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', FALSE,
(4, TRUE, -1, '2015-03-27 07:39:28.955051+00', -1, '2015-03-27 07:39:28.955051+00', 1, FALSE, 'John Doe', FALSE, NULL, '51762bba-01a2-4c4e-b5cd-b182d0405cd4', FALSE,
'{ "e0eac267-463a-4c00-9732-cab62df07b16": { "text": "2030-04-06T18:37:59+00:00", "datetime": "2030-04-06T18:37:59+00:00"}}'),
(6, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', 2, FALSE, 'Ajodinabiff Dane', FALSE, NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', FALSE,
(5, TRUE, -1, '2015-10-30 19:42:27.001837+00', -1, '2015-10-30 19:42:27.001837+00', 2, FALSE, 'Ajodinabiff Dane', FALSE, NULL, '3e814add-e614-41f7-8b5d-a07f670a698f', FALSE,
'{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Washington", "state": "USA > Washington"} }'),
(7, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, 'Joanne Stone', FALSE, NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', FALSE,
(6, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, 'Joanne Stone', FALSE, NULL, '7051dff0-0a27-49d7-af1f-4494239139e6', FALSE,
'{ "22d11697-edba-4186-b084-793e3b876379": { "text": "USA > Colorado", "state": "USA > Colorado"} }'),
(8, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', 2, FALSE, NULL, FALSE, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', FALSE,
(7, TRUE, -1, '2015-03-27 13:39:43.995812+00', -1, '2015-03-27 13:39:43.995812+00', 2, FALSE, NULL, FALSE, NULL, 'b46f6e18-95b4-4984-9926-dded047f4eb3', FALSE,
'{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Washington > King Côunty", "district": "USA > Washington > King Côunty"} }'),
(9, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, NULL, FALSE, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', FALSE,
(8, TRUE, -1, '2017-11-10 21:11:59.890662+00', -1, '2017-11-10 21:11:59.890662+00', 2, FALSE, NULL, FALSE, NULL, '9195c8b7-6138-4d84-ac56-5192cc3d8ceb', FALSE,
'{ "a551ade4-e5a0-4d83-b185-53b515ad2f2a": { "text": "USA > Washington > King Côunty > Central District", "ward": "USA > Washington > King Côunty > Central District"} }'),
(10, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', 2, FALSE, NULL, FALSE, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', FALSE,
(9, TRUE, -1, '2016-08-22 14:20:05.690311+00', -1, '2016-08-22 14:20:05.690311+00', 2, FALSE, NULL, FALSE, NULL, '2b8bd28d-43e0-4c34-a4bb-0f10b11fdb8a', FALSE,
'{ "fcab2439-861c-4832-aa54-0c97f38f24ab": { "text": "USA > Colorado > King", "district": "USA > Colorado > King"} }');

INSERT INTO contacts_contacturn(id, contact_id, scheme, org_id, priority, path, display, identity) VALUES
(1, 1, 'tel', 1, 50, '+12067791111', NULL, 'tel:+12067791111'),
(2, 1, 'tel', 1, 50, '+12067792222', NULL, 'tel:+12067792222'),
(3, 2, 'tel', 1, 50, '+12067793333', NULL, 'tel:+12067793333'),
(4, 3, 'tel', 1, 50, '+12067794444', NULL, 'tel:+12067794444'),
(5, 4, 'tel', 1, 50, '+12067795555', NULL, 'tel:+12067795555'),
(6, 5, 'tel', 1, 50, '+12060000556', NULL, 'tel:+12067796666'),
(7, 6, 'tel', 2, 50, '+12060005577', NULL, 'tel:+12067797777'),
(8, 7, 'tel', 2, 50, '+12067798888', NULL, 'tel:+12067798888'),
(9, 8, 'viber', 2, 90, 'viberpath==', NULL, 'viber:viberpath=='),
(10, 9, 'facebook', 2, 90, 1000001, 'funguy', 'facebook:1000001'),
(11, 10, 'twitterid', 2, 90, 1000001, 'fungal', 'twitterid:1000001'),
(12, 11, 'whatsapp', 2, 90, 1000003, NULL, 'whatsapp:1000003');
(3, 2, 'tel', 1, 50, '+12067794444', NULL, 'tel:+12067794444'),
(4, 3, 'tel', 1, 50, '+12067795555', NULL, 'tel:+12067795555'),
(5, 4, 'tel', 1, 50, '+12060000556', NULL, 'tel:+12067796666'),
(6, 5, 'tel', 2, 50, '+12060005577', NULL, 'tel:+12067797777'),
(7, 6, 'tel', 2, 50, '+12067798888', NULL, 'tel:+12067798888'),
(8, 7, 'viber', 2, 90, 'viberpath==', NULL, 'viber:viberpath=='),
(9, 8, 'facebook', 2, 90, 1000001, 'funguy', 'facebook:1000001'),
(10, 9, 'twitterid', 2, 90, 1000001, 'fungal', 'twitterid:1000001'),
(11, 10, 'whatsapp', 2, 90, 1000003, NULL, 'whatsapp:1000003');

INSERT INTO contacts_contactgroup(id, uuid, name) VALUES
(1, '4ea0f313-2f62-4e57-bdf0-232b5191dd57', 'Group 1'),
Expand All @@ -131,6 +129,5 @@ INSERT INTO contacts_contactgroup(id, uuid, name) VALUES

INSERT INTO contacts_contactgroup_contacts(id, contact_id, contactgroup_id) VALUES
(1, 1, 1),
(2, 2, 1),
(3, 1, 4),
(4, 3, 4);
(2, 1, 4),
(3, 2, 4);
6 changes: 3 additions & 3 deletions testdb_update.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
-- update one of our contacts
DELETE FROM contacts_contactgroup_contacts WHERE id = 4;
UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 3;
DELETE FROM contacts_contactgroup_contacts WHERE id = 3;
UPDATE contacts_contact SET name = 'John Deer', modified_on = '2020-08-20 14:00:00+00' where id = 2;

-- delete one of our others
UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 5;
UPDATE contacts_contact SET is_active = FALSE, modified_on = '2020-08-22 15:00:00+00' where id = 4;