Skip to content

Commit fc2236a

Browse files
paulobernardoafpaulobernardoaf
authored
Merge pull request #19 from Ilhasoft/update/v7.5.0
Update/v7.5.0
2 parents bd8f651 + 41750c7 commit fc2236a

File tree

13 files changed

+802
-89
lines changed

13 files changed

+802
-89
lines changed

‎.github/workflows/ci.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: CI
22
on: [push, pull_request]
33
env:
4-
go-version: "1.18.x"
4+
go-version: "1.19.x"
55
es-version: "7.17.2"
66
jobs:
77
test:
@@ -40,6 +40,7 @@ jobs:
4040
uses: codecov/codecov-action@v1
4141
with:
4242
token: ${{ secrets.CODECOV_TOKEN }}
43+
fail_ci_if_error: true
4344

4445
release:
4546
name: Release
@@ -62,4 +63,3 @@ jobs:
6263
args: release --rm-dist
6364
env:
6465
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
65-
fail_ci_if_error: true

‎.github/workflows/cla.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: "CLA Assistant"
2+
on:
3+
issue_comment:
4+
types: [created]
5+
pull_request_target:
6+
types: [opened,closed,synchronize]
7+
8+
jobs:
9+
CLAssistant:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: "CLA Assistant"
13+
if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
14+
# Beta Release
15+
uses: contributor-assistant/[email protected]
16+
env:
17+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
18+
PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_TOKEN }}
19+
with:
20+
path-to-signatures: 'signatures/version1/cla.json'
21+
path-to-document: 'https://github.com/nyaruka/license/blob/main/TextIt_CLA.md'
22+
branch: 'main'
23+
allowlist: bot*
24+
remote-organization-name: 'nyaruka'
25+
remote-repository-name: 'legal'

‎CHANGELOG.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
v7.5.0
2+
----------
3+
* Change index setting to contacts_index
4+
* Use go 1.19
5+
* Allow configurable number of shards/replicas
6+
17
v7.4.0
28
----------
39
* Update README

‎LICENSE

+677-29
Large diffs are not rendered by default.

‎cmd/rp-indexer/main.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func main() {
5555
}
5656

5757
idxrs := []indexers.Indexer{
58-
indexers.NewContactIndexer(cfg.ElasticURL, cfg.Index, 500),
58+
indexers.NewContactIndexer(cfg.ElasticURL, cfg.ContactsIndex, cfg.ContactsShards, cfg.ContactsReplicas, 500),
5959
}
6060

6161
if cfg.Rebuild {

‎config.go

+15-10
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,20 @@ package indexer
33
import "os"
44

55
type Config struct {
6-
ElasticURL string `help:"the url for our elastic search instance"`
7-
DB string `help:"the connection string for our database"`
8-
Index string `help:"the alias for our contact index"`
9-
Poll int `help:"the number of seconds to wait between checking for updated contacts"`
10-
Rebuild bool `help:"whether to rebuild the index, swapping it when complete, then exiting (default false)"`
11-
Cleanup bool `help:"whether to remove old indexes after a rebuild"`
12-
LogLevel string `help:"the log level, one of error, warn, info, debug"`
13-
SentryDSN string `help:"the sentry configuration to log errors to, if any"`
14-
6+
ElasticURL string `help:"the url for our elastic search instance"`
7+
DB string `help:"the connection string for our database"`
8+
Poll int `help:"the number of seconds to wait between checking for database updates"`
9+
Rebuild bool `help:"whether to rebuild the index, swapping it when complete, then exiting (default false)"`
10+
Cleanup bool `help:"whether to remove old indexes after a rebuild"`
11+
LogLevel string `help:"the log level, one of error, warn, info, debug"`
12+
SentryDSN string `help:"the sentry configuration to log errors to, if any"`
1513
LibratoUsername string `help:"the username that will be used to authenticate to Librato"`
1614
LibratoToken string `help:"the token that will be used to authenticate to Librato"`
1715
InstanceName string `help:"the unique name of this instance used for analytics"`
16+
17+
ContactsIndex string `help:"the alias to use for the contact index"`
18+
ContactsShards int `help:"the number of shards to use for the contacts index"`
19+
ContactsReplicas int `help:"the number of replicas to use for the contacts index"`
1820
}
1921

2022
func NewDefaultConfig() *Config {
@@ -23,11 +25,14 @@ func NewDefaultConfig() *Config {
2325
return &Config{
2426
ElasticURL: "http://localhost:9200",
2527
DB: "postgres://localhost/temba?sslmode=disable",
26-
Index: "contacts",
2728
Poll: 5,
2829
Rebuild: false,
2930
Cleanup: false,
3031
LogLevel: "info",
3132
InstanceName: hostname,
33+
34+
ContactsIndex: "contacts",
35+
ContactsShards: 2,
36+
ContactsReplicas: 1,
3237
}
3338
}

‎go.mod

+9-9
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
11
module github.com/nyaruka/rp-indexer
22

3-
go 1.18
3+
go 1.19
44

55
require (
66
github.com/evalphobia/logrus_sentry v0.8.2
7-
github.com/lib/pq v1.10.5
7+
github.com/lib/pq v1.10.7
88
github.com/nyaruka/ezconf v0.2.1
9-
github.com/nyaruka/gocommon v1.21.0
9+
github.com/nyaruka/gocommon v1.31.0
1010
github.com/olivere/elastic/v7 v7.0.32
1111
github.com/pkg/errors v0.9.1
12-
github.com/sirupsen/logrus v1.8.1
13-
github.com/stretchr/testify v1.7.1
12+
github.com/sirupsen/logrus v1.9.0
13+
github.com/stretchr/testify v1.8.0
1414
)
1515

1616
require (
1717
github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect
1818
github.com/davecgh/go-spew v1.1.1 // indirect
1919
github.com/fatih/structs v1.1.0 // indirect
20-
github.com/gabriel-vasile/mimetype v1.4.0 // indirect
20+
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
2121
github.com/getsentry/raven-go v0.2.0 // indirect
2222
github.com/go-chi/chi v4.1.2+incompatible // indirect
2323
github.com/josharian/intern v1.0.0 // indirect
@@ -28,7 +28,7 @@ require (
2828
github.com/nyaruka/librato v1.0.0 // indirect
2929
github.com/pmezard/go-difflib v1.0.0 // indirect
3030
github.com/shopspring/decimal v1.3.1 // indirect
31-
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
32-
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
33-
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
31+
golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect
32+
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect
33+
gopkg.in/yaml.v3 v3.0.1 // indirect
3434
)

‎go.sum

+25-22
Original file line numberDiff line numberDiff line change
@@ -9,21 +9,21 @@ github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga
99
github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
1010
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
1111
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
12-
github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro=
13-
github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
12+
github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
13+
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
1414
github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs=
1515
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
1616
github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec=
1717
github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
1818
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
19-
github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w=
19+
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
2020
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
2121
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
2222
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
2323
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
2424
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
25-
github.com/lib/pq v1.10.5 h1:J+gdV2cUmX7ZqL2B0lFcW0m+egaHC2V3lpO8nWxyYiQ=
26-
github.com/lib/pq v1.10.5/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
25+
github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
26+
github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
2727
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
2828
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
2929
github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
@@ -32,8 +32,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
3232
github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
3333
github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
3434
github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
35-
github.com/nyaruka/gocommon v1.21.0 h1:nu7M2cdSPrkqUPdGsEeWX047+neo69H4x+4g/OKpoLM=
36-
github.com/nyaruka/gocommon v1.21.0/go.mod h1:cv9r6amof1gSktfPZROClZhLFzdSIH/N9KbW6Nny4g8=
35+
github.com/nyaruka/gocommon v1.31.0 h1:eVRxmyTZxRQ4mBs3JoYaPe33LlNuQD63pxq2M+eHQA8=
36+
github.com/nyaruka/gocommon v1.31.0/go.mod h1:PApT/06fP5Tzs4/kbkJ+rVoyOc9Lbqm1lR0ow8Vqzp0=
3737
github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
3838
github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
3939
github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E=
@@ -45,30 +45,33 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
4545
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
4646
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
4747
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
48-
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
49-
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
48+
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
49+
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
5050
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
5151
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
52+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
5253
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
5354
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
54-
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
55+
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
5556
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
56-
golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
57-
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
58-
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
57+
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
58+
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
59+
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
60+
golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4=
61+
golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
5962
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
60-
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
61-
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
62-
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
63-
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60=
64-
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
65-
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
66-
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
63+
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
64+
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
65+
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
66+
golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc=
67+
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
68+
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
6769
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
70+
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
6871
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
6972
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
7073
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
7174
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
7275
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
73-
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
74-
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
76+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
77+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎indexers/base.go

+29-3
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,39 @@ type Indexer interface {
3333
Stats() Stats
3434
}
3535

36+
// IndexDefinition is what we pass to elastic to create an index,
37+
// see https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html
38+
type IndexDefinition struct {
39+
Settings struct {
40+
Index struct {
41+
NumberOfShards int `json:"number_of_shards"`
42+
NumberOfReplicas int `json:"number_of_replicas"`
43+
RoutingPartitionSize int `json:"routing_partition_size"`
44+
} `json:"index"`
45+
Analysis json.RawMessage `json:"analysis"`
46+
} `json:"settings"`
47+
Mappings json.RawMessage `json:"mappings"`
48+
}
49+
50+
func newIndexDefinition(base []byte, shards, replicas int) *IndexDefinition {
51+
d := &IndexDefinition{}
52+
jsonx.MustUnmarshal(contactsIndexDef, d)
53+
54+
d.Settings.Index.NumberOfShards = shards
55+
d.Settings.Index.NumberOfReplicas = replicas
56+
return d
57+
}
58+
3659
type baseIndexer struct {
3760
elasticURL string
3861
name string // e.g. contacts, used as the alias
62+
definition *IndexDefinition
3963

4064
stats Stats
4165
}
4266

43-
func newBaseIndexer(elasticURL, name string) baseIndexer {
44-
return baseIndexer{elasticURL: elasticURL, name: name}
67+
func newBaseIndexer(elasticURL, name string, def *IndexDefinition) baseIndexer {
68+
return baseIndexer{elasticURL: elasticURL, name: name, definition: def}
4569
}
4670

4771
func (i *baseIndexer) Name() string {
@@ -99,7 +123,7 @@ func (i *baseIndexer) FindIndexes() []string {
99123
// that index to `contacts`.
100124
//
101125
// If the day-specific name already exists, we append a .1 or .2 to the name.
102-
func (i *baseIndexer) createNewIndex(settings json.RawMessage) (string, error) {
126+
func (i *baseIndexer) createNewIndex(def *IndexDefinition) (string, error) {
103127
// create our day-specific name
104128
index := fmt.Sprintf("%s_%s", i.name, time.Now().Format("2006_01_02"))
105129
idx := 0
@@ -121,6 +145,8 @@ func (i *baseIndexer) createNewIndex(settings json.RawMessage) (string, error) {
121145
}
122146

123147
// create the new index
148+
settings := jsonx.MustMarshal(def)
149+
124150
_, err := utils.MakeJSONRequest(http.MethodPut, fmt.Sprintf("%s/%s?include_type_name=true", i.elasticURL, index), settings, nil)
125151
if err != nil {
126152
return "", err

‎indexers/base_test.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package indexers_test
33
import (
44
"context"
55
"database/sql"
6-
"io/ioutil"
76
"log"
87
"os"
98
"sort"
@@ -22,7 +21,7 @@ const elasticURL = "http://localhost:9200"
2221
const aliasName = "indexer_test"
2322

2423
func setup(t *testing.T) (*sql.DB, *elastic.Client) {
25-
testDB, err := ioutil.ReadFile("../testdb.sql")
24+
testDB, err := os.ReadFile("../testdb.sql")
2625
require.NoError(t, err)
2726

2827
db, err := sql.Open("postgres", "postgres://nyaruka:nyaruka@localhost:5432/elastic_test?sslmode=disable")

‎indexers/contacts.go

+7-6
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,15 @@ import (
44
"bytes"
55
"database/sql"
66
_ "embed"
7-
"encoding/json"
87
"fmt"
98
"time"
109

1110
"github.com/pkg/errors"
1211
"github.com/sirupsen/logrus"
1312
)
1413

15-
//go:embed contacts.settings.json
16-
var contactsSettings json.RawMessage
14+
//go:embed contacts.index.json
15+
var contactsIndexDef []byte
1716

1817
// ContactIndexer is an indexer for contacts
1918
type ContactIndexer struct {
@@ -23,9 +22,11 @@ type ContactIndexer struct {
2322
}
2423

2524
// NewContactIndexer creates a new contact indexer
26-
func NewContactIndexer(elasticURL, name string, batchSize int) *ContactIndexer {
25+
func NewContactIndexer(elasticURL, name string, shards, replicas, batchSize int) *ContactIndexer {
26+
def := newIndexDefinition(contactsIndexDef, shards, replicas)
27+
2728
return &ContactIndexer{
28-
baseIndexer: newBaseIndexer(elasticURL, name),
29+
baseIndexer: newBaseIndexer(elasticURL, name, def),
2930
batchSize: batchSize,
3031
}
3132
}
@@ -47,7 +48,7 @@ func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error
4748

4849
// doesn't exist or we are rebuilding, create it
4950
if physicalIndex == "" || rebuild {
50-
physicalIndex, err = i.createNewIndex(contactsSettings)
51+
physicalIndex, err = i.createNewIndex(i.definition)
5152
if err != nil {
5253
return "", errors.Wrap(err, "error creating new index")
5354
}

‎indexers/contacts.settings.json renamed to ‎indexers/contacts.index.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
22
"settings": {
33
"index": {
4-
"number_of_shards": 2,
5-
"number_of_replicas": 1,
4+
"number_of_shards": -1,
5+
"number_of_replicas": -1,
66
"routing_partition_size": 1
77
},
88
"analysis": {

‎indexers/contacts_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ var contactQueryTests = []struct {
186186
func TestContacts(t *testing.T) {
187187
db, es := setup(t)
188188

189-
ix1 := indexers.NewContactIndexer(elasticURL, aliasName, 4)
189+
ix1 := indexers.NewContactIndexer(elasticURL, aliasName, 2, 1, 4)
190190
assert.Equal(t, "indexer_test", ix1.Name())
191191

192192
expectedIndexName := fmt.Sprintf("indexer_test_%s", time.Now().Format("2006_01_02"))
@@ -238,7 +238,7 @@ func TestContacts(t *testing.T) {
238238
require.NoError(t, err)
239239

240240
// and simulate another indexer doing a parallel rebuild
241-
ix2 := indexers.NewContactIndexer(elasticURL, aliasName, 4)
241+
ix2 := indexers.NewContactIndexer(elasticURL, aliasName, 2, 1, 4)
242242

243243
indexName2, err := ix2.Index(db, true, false)
244244
assert.NoError(t, err)
@@ -254,7 +254,7 @@ func TestContacts(t *testing.T) {
254254
assertQuery(t, es, elastic.NewMatchQuery("name", "eric"), []int64{2})
255255

256256
// simulate another indexer doing a parallel rebuild with cleanup
257-
ix3 := indexers.NewContactIndexer(elasticURL, aliasName, 4)
257+
ix3 := indexers.NewContactIndexer(elasticURL, aliasName, 2, 1, 4)
258258
indexName3, err := ix3.Index(db, true, true)
259259
assert.NoError(t, err)
260260
assert.Equal(t, expectedIndexName+"_2", indexName3) // new index used

0 commit comments

Comments
 (0)