From 0a5a2c933debb3e4207d24fd7ed35f485d9cbbda Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 8 Oct 2021 18:10:52 +0200 Subject: [PATCH 01/62] Throttle courier queues when the channel has rate limit redis key not expired --- handlers/whatsapp/whatsapp.go | 18 ++++++++++-- handlers/whatsapp/whatsapp_test.go | 6 ++++ queue/queue.go | 14 +++++++++ queue/queue_test.go | 46 ++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index e1fd4116d..651b960cd 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -12,6 +12,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" @@ -495,6 +496,9 @@ const maxMsgLength = 4096 // SendMsg sends the passed in message, returning any error func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { start := time.Now() + conn := h.Backend().RedisPool().Get() + defer conn.Close() + // get our token token := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") if token == "" { @@ -525,7 +529,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat for i, payload := range payloads { externalID := "" - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(conn, msg, sendPath, payload) // add logs to our status for _, log := range logs { status.AddLog(log) @@ -827,7 +831,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return mediaID, logs, nil } -func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { +func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) @@ -839,6 +843,16 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (s req, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) req.Header = buildWhatsAppHeaders(msg.Channel()) rr, err := utils.MakeHTTPRequest(req) + + if rr.StatusCode == 429 || rr.StatusCode == 503 { + rateLimitKey := fmt.Sprintf("rate_limit:%s", msg.Channel().UUID().String()) + rc.Do("set", rateLimitKey, "engaged") + rc.Do("expire", rateLimitKey, 2) + + log := courier.NewChannelLogFromRR("rate limit engaged", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + return "", "", []*courier.ChannelLog{log}, err + } + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) errPayload := &mtErrorPayload{} err = json.Unmarshal(rr.Body, errPayload) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 40d940445..653a62051 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -388,6 +388,12 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "errors": [{ "title": "Error Sending" }] }`, ResponseStatus: 403, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, SendPrep: setSendURL}, + {Label: "Rate Limit Engaged", + Text: "Error", URN: "whatsapp:250788123123", + Status: "E", + ResponseBody: `{ "errors": [{ "title": "Too many requests" }] }`, ResponseStatus: 429, + RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, + SendPrep: setSendURL}, {Label: "No Message ID", Text: "Error", URN: "whatsapp:250788123123", Status: "E", diff --git a/queue/queue.go b/queue/queue.go index 7aa4bebaa..da6b96e83 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -83,10 +83,24 @@ var luaPop = redis.NewScript(2, `-- KEYS: [EpochMS QueueType] local delim = string.find(queue, "|") local tps = 0 local tpsKey = "" + + local queueName = "" + if delim then + queueName = string.sub(queue, string.len(KEYS[2])+2, delim-1) tps = tonumber(string.sub(queue, delim+1)) end + if queueName then + local rateLimitKey = "rate_limit:" .. queueName + local rateLimitEngaged = redis.call("get", rateLimitKey) + if rateLimitEngaged then + redis.call("zincrby", KEYS[2] .. ":throttled", workers, queue) + redis.call("zrem", KEYS[2] .. ":active", queue) + return {"retry", ""} + end + end + -- if we have a tps, then check whether we exceed it if tps > 0 then tpsKey = queue .. ":tps:" .. math.floor(KEYS[1]) diff --git a/queue/queue_test.go b/queue/queue_test.go index 157793a0e..81cd415a3 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -50,6 +50,7 @@ func TestLua(t *testing.T) { defer close(quitter) rate := 10 + for i := 0; i < 20; i++ { err := PushOntoQueue(conn, "msgs", "chan1", rate, fmt.Sprintf(`[{"id":%d}]`, i), LowPriority) assert.NoError(err) @@ -166,6 +167,51 @@ func TestLua(t *testing.T) { assert.NoError(err) assert.Equal(EmptyQueue, queue) assert.Empty(value) + + err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":34}]`, HighPriority) + assert.NoError(err) + + conn.Do("set", "rate_limit:chan1", "engaged") + conn.Do("EXPIRE", "rate_limit:chan1", 5) + + // we have the rate limit set, + queue, value, err = PopFromQueue(conn, "msgs") + if value != "" && queue != EmptyQueue { + t.Fatal("Should be throttled") + } + + time.Sleep(2 * time.Second) + queue, value, err = PopFromQueue(conn, "msgs") + if value != "" && queue != EmptyQueue { + t.Fatal("Should be throttled") + } + + count, err = redis.Int(conn.Do("zcard", "msgs:throttled")) + assert.NoError(err) + assert.Equal(1, count, "Expected chan1 to be throttled") + + count, err = redis.Int(conn.Do("zcard", "msgs:active")) + assert.NoError(err) + assert.Equal(0, count, "Expected chan1 to not be active") + + // but if we wait for the rate limit to expire + time.Sleep(3 * time.Second) + + // next should be 34 + queue, value, err = PopFromQueue(conn, "msgs") + assert.NotEqual(queue, EmptyQueue) + assert.Equal(`{"id":34}`, value) + assert.NoError(err) + + // nothing should be left + queue = Retry + for queue == Retry { + queue, value, err = PopFromQueue(conn, "msgs") + } + assert.NoError(err) + assert.Equal(EmptyQueue, queue) + assert.Empty(value) + } func nTestThrottle(t *testing.T) { From d63ccf354fa1b5ffd92ce9d53df1d76ad695ff84 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Thu, 28 Oct 2021 18:42:42 -0300 Subject: [PATCH 02/62] refactor external channel handler to use headers config on send --- channel.go | 3 +++ handlers/external/external.go | 9 +++++--- handlers/external/external_test.go | 34 +++++++++++++++--------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/channel.go b/channel.go index d64394cca..7a71a40d1 100644 --- a/channel.go +++ b/channel.go @@ -52,6 +52,9 @@ const ( // ConfigUseNational is a constant key for channel configs ConfigUseNational = "use_national" + + // ConfigSendHeaders is a constant key for channel configs + ConfigSendHeaders = "headers" ) // ChannelType is our typing of the two char channel types diff --git a/handlers/external/external.go b/handlers/external/external.go index c447228d2..97f3f6c2b 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,9 +353,12 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) - authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") - if authorization != "" { - req.Header.Set("Authorization", authorization) + headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) + + if len(headers) > 0 { + for hKey, hValue := range headers { + req.Header.Set(hKey, fmt.Sprint(hValue)) + } } rr, err := utils.MakeHTTPRequest(req) diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index 1c4b84685..bbed3d214 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -423,11 +423,11 @@ func TestSending(t *testing.T) { var jsonChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) var xmlChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", @@ -472,22 +472,22 @@ func TestSending(t *testing.T) { var jsonChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) var xmlChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, - courier.ConfigContentType: contentXML, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, + courier.ConfigContentType: contentXML, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) RunChannelSendTestCases(t, getChannel30IntLength, newHandler(), longSendTestCases, nil) From 2c33fc2dbae4de5681d7651ae5353fc43be0b092 Mon Sep 17 00:00:00 2001 From: Jason Rogena Date: Wed, 10 Nov 2021 16:25:56 +0300 Subject: [PATCH 03/62] Add support for 'Expired' status in the AT handler Add support for the 'Expired' status from Africa's Talking which seems to be thrown but isn't handled. Courier currently throws the following error when it encounters this status: time="2021-11-10T10:29:10+03:00" level=info msg="request errored" channel_uuid= elapsed_ms=0.109525 error="unknown status 'Expired', must be one of 'Success','Sent', 'Buffered','Rejected' or 'Failed'" url=/c/at//status Mark messages that have this status as failed as it doesn't appear messages that have this status get delivered to users. Signed-off-by: Jason Rogena --- handlers/africastalking/africastalking.go | 3 ++- handlers/africastalking/africastalking_test.go | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/handlers/africastalking/africastalking.go b/handlers/africastalking/africastalking.go index b64553d59..e196212fb 100644 --- a/handlers/africastalking/africastalking.go +++ b/handlers/africastalking/africastalking.go @@ -91,6 +91,7 @@ var statusMapping = map[string]courier.MsgStatusValue{ "Buffered": courier.MsgSent, "Rejected": courier.MsgFailed, "Failed": courier.MsgFailed, + "Expired": courier.MsgFailed, } // receiveStatus is our HTTP handler function for status updates @@ -105,7 +106,7 @@ func (h *handler) receiveStatus(ctx context.Context, channel courier.Channel, w msgStatus, found := statusMapping[form.Status] if !found { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, - fmt.Errorf("unknown status '%s', must be one of 'Success','Sent','Buffered','Rejected' or 'Failed'", form.Status)) + fmt.Errorf("unknown status '%s', must be one of 'Success','Sent','Buffered','Rejected', 'Failed', or 'Expired'", form.Status)) } // write our status diff --git a/handlers/africastalking/africastalking_test.go b/handlers/africastalking/africastalking_test.go index cc36711e3..c3941a58a 100644 --- a/handlers/africastalking/africastalking_test.go +++ b/handlers/africastalking/africastalking_test.go @@ -26,7 +26,8 @@ var ( missingStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7" invalidStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Borked" - validStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Success" + successStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Success" + expiredStatus = "id=ATXid_dda018a640edfcc5d2ce455de3e4a6e7&status=Expired" ) var testCases = []ChannelHandleTestCase{ @@ -42,7 +43,8 @@ var testCases = []ChannelHandleTestCase{ {Label: "Invalid Date", URL: receiveURL, Data: invalidDate, Status: 400, Response: "invalid date format"}, {Label: "Status Invalid", URL: statusURL, Status: 400, Data: invalidStatus, Response: "unknown status"}, {Label: "Status Missing", URL: statusURL, Status: 400, Data: missingStatus, Response: "field 'status' required"}, - {Label: "Status Valid", URL: statusURL, Status: 200, Data: validStatus, Response: `"status":"D"`}, + {Label: "Status Success", URL: statusURL, Status: 200, Data: successStatus, Response: `"status":"D"`}, + {Label: "Status Expired", URL: statusURL, Status: 200, Data: expiredStatus, Response: `"status":"F"`}, } func TestHandler(t *testing.T) { From 0ae35b6b3d6fd40eb01318e376ce8db92c05a876 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 19 Nov 2021 09:42:45 -0500 Subject: [PATCH 04/62] Test with Redis 3.2.4 --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e4b9d207..5b6f2486b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: CI on: [push, pull_request] env: go-version: '1.17.x' + redis-version: '3.2.4' jobs: test: name: Test @@ -16,7 +17,7 @@ jobs: - name: Install Redis uses: zhulik/redis-action@v1.0.0 with: - redis version: '5' + redis version: ${{ env.redis-version }} - name: Install PostgreSQL uses: harmon758/postgresql-action@v1 From 7d499982c40b555875af2e0e7806d284b1821ccd Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 22 Nov 2021 16:15:41 -0500 Subject: [PATCH 05/62] Remove chatbase support --- backends/rapidpro/backend.go | 16 --------- chatbase/chatbase.go | 50 ---------------------------- chatbase/chatbase_test.go | 64 ------------------------------------ 3 files changed, 130 deletions(-) delete mode 100644 chatbase/chatbase.go delete mode 100644 chatbase/chatbase_test.go diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 3125bf374..da6df33c6 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -17,7 +17,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" "github.com/nyaruka/courier/batch" - "github.com/nyaruka/courier/chatbase" "github.com/nyaruka/courier/queue" "github.com/nyaruka/courier/utils" "github.com/nyaruka/gocommon/storage" @@ -33,11 +32,6 @@ const msgQueueName = "msgs" // the name of our set for tracking sends const sentSetName = "msgs_sent_%s" -// constants used in org configs for chatbase -const chatbaseAPIKey = "CHATBASE_API_KEY" -const chatbaseVersion = "CHATBASE_VERSION" -const chatbaseMessageType = "agent" - // our timeout for backend operations const backendTimeout = time.Second * 20 @@ -292,16 +286,6 @@ func (b *backend) MarkOutgoingMsgComplete(ctx context.Context, msg courier.Msg, } } } - - // if this org has chatbase connected, notify chatbase - chatKey, _ := msg.Channel().OrgConfigForKey(chatbaseAPIKey, "").(string) - if chatKey != "" { - chatVersion, _ := msg.Channel().OrgConfigForKey(chatbaseVersion, "").(string) - err := chatbase.SendChatbaseMessage(chatKey, chatVersion, chatbaseMessageType, dbMsg.ContactID_.String(), msg.Channel().Name(), msg.Text(), time.Now().UTC()) - if err != nil { - logrus.WithError(err).WithField("chatbase_api_key", chatKey).WithField("chatbase_version", chatVersion).WithField("msg_id", dbMsg.ID().String()).Error("unable to write chatbase message") - } - } } // WriteMsg writes the passed in message to our store diff --git a/chatbase/chatbase.go b/chatbase/chatbase.go deleted file mode 100644 index e5bca39c9..000000000 --- a/chatbase/chatbase.go +++ /dev/null @@ -1,50 +0,0 @@ -package chatbase - -import ( - "bytes" - "encoding/json" - "net/http" - "time" - - "github.com/nyaruka/courier/utils" -) - -// ChatbaseAPIURL is the URL chatbase API messages will be sent to -var chatbaseAPIURL = "https://chatbase.com/api/message" - -// chatbaseLog is the payload for a chatbase request -type chatbaseLog struct { - Type string `json:"type"` - UserID string `json:"user_id"` - Platform string `json:"platform"` - Message string `json:"message"` - TimeStamp int64 `json:"time_stamp"` - - APIKey string `json:"api_key"` - APIVersion string `json:"version,omitempty"` -} - -// SendChatbaseMessage sends a chatbase message with the passed in api key and message details -func SendChatbaseMessage(apiKey string, apiVersion string, messageType string, userID string, platform string, message string, timestamp time.Time) error { - body := chatbaseLog{ - Type: messageType, - UserID: userID, - Platform: platform, - Message: message, - TimeStamp: timestamp.UnixNano() / int64(time.Millisecond), - - APIKey: apiKey, - APIVersion: apiVersion, - } - - jsonBody, err := json.Marshal(body) - if err != nil { - return err - } - - req, _ := http.NewRequest(http.MethodPost, chatbaseAPIURL, bytes.NewReader(jsonBody)) - req.Header.Set("Content-Type", "application/json") - - _, err = utils.MakeHTTPRequest(req) - return err -} diff --git a/chatbase/chatbase_test.go b/chatbase/chatbase_test.go deleted file mode 100644 index 12ee9dbe7..000000000 --- a/chatbase/chatbase_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package chatbase - -import ( - "bytes" - "io/ioutil" - "net/http" - "net/http/httptest" - "testing" - "time" - - "github.com/buger/jsonparser" - "github.com/stretchr/testify/assert" -) - -func TestChatbase(t *testing.T) { - var testRequest *http.Request - var statusCode = 200 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, _ := ioutil.ReadAll(r.Body) - testRequest = httptest.NewRequest(r.Method, r.URL.String(), bytes.NewBuffer(body)) - testRequest.Header = r.Header - w.WriteHeader(statusCode) - w.Write([]byte("ok")) - })) - defer server.Close() - - chatbaseAPIURL = server.URL - - now := time.Now() - err := SendChatbaseMessage("apiKey", "apiVersion", "messageType", "userID", "platform", "message", now) - assert.NoError(t, err) - - // parse our body - bytes, err := ioutil.ReadAll(testRequest.Body) - assert.NoError(t, err) - - // check our request body - str, err := jsonparser.GetString(bytes, "type") - assert.NoError(t, err) - assert.Equal(t, "messageType", str) - - str, err = jsonparser.GetString(bytes, "version") - assert.NoError(t, err) - assert.Equal(t, "apiVersion", str) - - ts, err := jsonparser.GetInt(bytes, "time_stamp") - assert.NoError(t, err) - assert.Equal(t, now.UnixNano()/int64(time.Millisecond), ts) - - // simulate an error - statusCode = 500 - err = SendChatbaseMessage("apiKey", "apiVersion", "messageType", "userID", "platform", "message", now) - assert.Error(t, err) - - // simulate error when messageType is invalid - statusCode = 400 - err = SendChatbaseMessage("apiKey", "apiVersion", "msg", "userID", "platform", "message", now) - assert.Error(t, err) - - bytes, err = ioutil.ReadAll(testRequest.Body) - str, err = jsonparser.GetString(bytes, "type") - assert.NoError(t, err) - assert.Equal(t, "msg", str) -} From 9fef1baa5f46618ecd3fbf49ce9b9b8346c5996d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 22 Nov 2021 19:31:47 -0500 Subject: [PATCH 06/62] Update CHANGELOG.md for v7.1.0 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8af3263b..bf834618c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v7.1.0 +---------- + * Remove chatbase support + * Test with Redis 3.2.4 + * Add support for 'Expired' status in the AT handler + v7.0.0 ---------- * Tweak README From 71f1763092fa3c468d499a753ebbc110e6153e73 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 08:10:47 -0500 Subject: [PATCH 07/62] Pin to go 1.17.2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b6f2486b..d43f9d7f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: '1.17.x' + go-version: '1.17.2' # https://github.com/golang/go/issues/49366 redis-version: '3.2.4' jobs: test: From 7466834b6755d72f67ac39d4e8cf6f922dbf578d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 09:32:36 -0500 Subject: [PATCH 08/62] Update CHANGELOG.md for v7.1.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf834618c..57302032d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.1 +---------- + * Pin to go 1.17.2 + v7.1.0 ---------- * Remove chatbase support From a1f5a5d2387e06c4f3ef9f17b383655ae3f17303 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Tue, 23 Nov 2021 11:49:28 -0300 Subject: [PATCH 09/62] add support to old way to set authorization token header --- handlers/external/external.go | 5 +++++ handlers/external/external_test.go | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/handlers/external/external.go b/handlers/external/external.go index 97f3f6c2b..62ee86876 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,6 +353,11 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) + authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) if len(headers) > 0 { diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index bbed3d214..8a9343c68 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -503,4 +503,14 @@ func TestSending(t *testing.T) { RunChannelSendTestCases(t, nationalChannel, newHandler(), nationalGetSendTestCases, nil) + var jsonChannelWithSendAuthorization = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendAuthorization: "Token ABCDEF", + }) + RunChannelSendTestCases(t, jsonChannelWithSendAuthorization, newHandler(), jsonSendTestCases, nil) + } From 538fdcf7f8f7ddd79e04c3027648b77fd5834180 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 10:36:24 -0500 Subject: [PATCH 10/62] Add comment --- handlers/external/external.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/handlers/external/external.go b/handlers/external/external.go index 62ee86876..da01ecce2 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,17 +353,15 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) + // TODO can drop this when channels have been migrated to use ConfigSendHeaders authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") if authorization != "" { req.Header.Set("Authorization", authorization) } headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) - - if len(headers) > 0 { - for hKey, hValue := range headers { - req.Header.Set(hKey, fmt.Sprint(hValue)) - } + for hKey, hValue := range headers { + req.Header.Set(hKey, fmt.Sprint(hValue)) } rr, err := utils.MakeHTTPRequest(req) From 34471015ea0db7aab0e30a490059c1e5154db5bf Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 10:37:22 -0500 Subject: [PATCH 11/62] Update CHANGELOG.md for v7.1.2 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57302032d..d7838a92a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.2 +---------- + * External channel handler should use headers config setting if provided + v7.1.1 ---------- * Pin to go 1.17.2 From 42f1110764d9403c5ff9d54ee305c89c046944f8 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 30 Nov 2021 16:38:14 -0500 Subject: [PATCH 12/62] Use response_to_external_id instead of response_to_id --- backends/rapidpro/backend.go | 2 +- backends/rapidpro/backend_test.go | 8 ++------ backends/rapidpro/msg.go | 2 -- handlers/chikka/chikka_test.go | 2 -- handlers/facebook/facebook.go | 2 +- handlers/facebook/facebook_test.go | 2 +- handlers/facebookapp/facebookapp.go | 2 +- handlers/facebookapp/facebookapp_test.go | 2 +- handlers/test.go | 3 +-- msg.go | 1 - test.go | 11 ++--------- 11 files changed, 10 insertions(+), 27 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index da6df33c6..0d362e471 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -238,7 +238,7 @@ func (b *backend) IsMsgLoop(ctx context.Context, msg courier.Msg) (bool, error) m := msg.(*DBMsg) // things that aren't replies can't be loops, neither do we count retries - if m.ResponseToID_ == courier.NilMsgID || m.ErrorCount_ > 0 { + if m.ResponseToExternalID_ == "" || m.ErrorCount_ > 0 { return false, nil } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 14982c8b1..b803f8e7b 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -120,7 +120,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "sent_on": null, "high_priority": true, "channel_id": 11, - "response_to_id": 15, "response_to_external_id": "external-id", "external_id": null, "is_resend": true, @@ -137,7 +136,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.Equal(msg.ExternalID(), "") ts.Equal([]string{"Yes", "No"}, msg.QuickReplies()) ts.Equal("event", msg.Topic()) - ts.Equal(courier.NewMsgID(15), msg.ResponseToID()) ts.Equal("external-id", msg.ResponseToExternalID()) ts.True(msg.HighPriority()) ts.True(msg.IsResend()) @@ -162,8 +160,7 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "sent_on": null, "high_priority": true, "channel_id": 11, - "response_to_id": null, - "response_to_external_id": "", + "response_to_external_id": null, "external_id": null, "metadata": null }` @@ -173,7 +170,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.NoError(err) ts.Equal([]string{}, msg.QuickReplies()) ts.Equal("", msg.Topic()) - ts.Equal(courier.NilMsgID, msg.ResponseToID()) ts.Equal("", msg.ResponseToExternalID()) ts.False(msg.IsResend()) } @@ -756,7 +752,7 @@ func (ts *BackendTestSuite) TestLoop() { ctx := context.Background() dbMsg := readMsgFromDB(ts.b, courier.NewMsgID(10000)) - dbMsg.ResponseToID_ = courier.MsgID(5) + dbMsg.ResponseToExternalID_ = "65474" loop, err := ts.b.IsMsgLoop(ctx, dbMsg) ts.NoError(err) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 6f7ef5dfd..020071f84 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -489,7 +489,6 @@ type DBMsg struct { Text_ string `json:"text" db:"text"` Attachments_ pq.StringArray `json:"attachments" db:"attachments"` ExternalID_ null.String `json:"external_id" db:"external_id"` - ResponseToID_ courier.MsgID `json:"response_to_id" db:"response_to_id"` ResponseToExternalID_ string `json:"response_to_external_id"` IsResend_ bool `json:"is_resend,omitempty"` Metadata_ json.RawMessage `json:"metadata" db:"metadata"` @@ -534,7 +533,6 @@ func (m *DBMsg) ContactName() string { return m.ContactName_ } func (m *DBMsg) HighPriority() bool { return m.HighPriority_ } func (m *DBMsg) ReceivedOn() *time.Time { return m.SentOn_ } func (m *DBMsg) SentOn() *time.Time { return m.SentOn_ } -func (m *DBMsg) ResponseToID() courier.MsgID { return m.ResponseToID_ } func (m *DBMsg) ResponseToExternalID() string { return m.ResponseToExternalID_ } func (m *DBMsg) IsResend() bool { return m.IsResend_ } diff --git a/handlers/chikka/chikka_test.go b/handlers/chikka/chikka_test.go index dd6fbc38a..b31a9533e 100644 --- a/handlers/chikka/chikka_test.go +++ b/handlers/chikka/chikka_test.go @@ -71,7 +71,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Plain Reply", Text: "Simple Message", URN: "tel:+63911231234", Status: "W", - ResponseToID: 5, ResponseToExternalID: "external-id", ResponseBody: "Success", ResponseStatus: 200, PostParams: map[string]string{ @@ -88,7 +87,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Failed Reply use Send", Text: "Simple Message", URN: "tel:+63911231234", - ResponseToID: 5, ResponseToExternalID: "external-id", ResponseBody: `{"status":400,"message":"BAD REQUEST","description":"Invalid\\/Used Request ID"}`, ResponseStatus: 400, diff --git a/handlers/facebook/facebook.go b/handlers/facebook/facebook.go index 62fd688bc..75aff3de4 100644 --- a/handlers/facebook/facebook.go +++ b/handlers/facebook/facebook.go @@ -481,7 +481,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/facebook/facebook_test.go b/handlers/facebook/facebook_test.go index 6a22ebbc9..42f2ad648 100644 --- a/handlers/facebook/facebook_test.go +++ b/handlers/facebook/facebook_test.go @@ -569,7 +569,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 09686c217..77982f4cb 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -488,7 +488,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 79e27f7e8..5514c7114 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -570,7 +570,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, diff --git a/handlers/test.go b/handlers/test.go index 6f139d194..9f5a88e3a 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -87,7 +87,6 @@ type ChannelSendTestCase struct { QuickReplies []string Topic string HighPriority bool - ResponseToID int64 ResponseToExternalID string Metadata json.RawMessage @@ -221,7 +220,7 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour t.Run(testCase.Label, func(t *testing.T) { require := require.New(t) - msg := mb.NewOutgoingMsg(channel, courier.NewMsgID(10), urns.URN(testCase.URN), testCase.Text, testCase.HighPriority, testCase.QuickReplies, testCase.Topic, testCase.ResponseToID, testCase.ResponseToExternalID) + msg := mb.NewOutgoingMsg(channel, courier.NewMsgID(10), urns.URN(testCase.URN), testCase.Text, testCase.HighPriority, testCase.QuickReplies, testCase.Topic, testCase.ResponseToExternalID) for _, a := range testCase.Attachments { msg.WithAttachment(a) diff --git a/msg.go b/msg.go index 7ee4044e9..809370f20 100644 --- a/msg.go +++ b/msg.go @@ -95,7 +95,6 @@ type Msg interface { QuickReplies() []string Topic() string Metadata() json.RawMessage - ResponseToID() MsgID ResponseToExternalID() string IsResend() bool diff --git a/test.go b/test.go index 527853648..09762d438 100644 --- a/test.go +++ b/test.go @@ -119,13 +119,8 @@ func (mb *MockBackend) NewIncomingMsg(channel Channel, urn urns.URN, text string } // NewOutgoingMsg creates a new outgoing message from the given params -func (mb *MockBackend) NewOutgoingMsg(channel Channel, id MsgID, urn urns.URN, text string, highPriority bool, quickReplies []string, topic string, responseToID int64, responseToExternalID string) Msg { - msgResponseToID := NilMsgID - if responseToID != 0 { - msgResponseToID = NewMsgID(responseToID) - } - - return &mockMsg{channel: channel, id: id, urn: urn, text: text, highPriority: highPriority, quickReplies: quickReplies, topic: topic, responseToID: msgResponseToID, responseToExternalID: responseToExternalID} +func (mb *MockBackend) NewOutgoingMsg(channel Channel, id MsgID, urn urns.URN, text string, highPriority bool, quickReplies []string, topic string, responseToExternalID string) Msg { + return &mockMsg{channel: channel, id: id, urn: urn, text: text, highPriority: highPriority, quickReplies: quickReplies, topic: topic, responseToExternalID: responseToExternalID} } // PushOutgoingMsg is a test method to add a message to our queue of messages to send @@ -568,7 +563,6 @@ type mockMsg struct { highPriority bool quickReplies []string topic string - responseToID MsgID responseToExternalID string metadata json.RawMessage alreadyWritten bool @@ -594,7 +588,6 @@ func (m *mockMsg) ContactName() string { return m.contactName } func (m *mockMsg) HighPriority() bool { return m.highPriority } func (m *mockMsg) QuickReplies() []string { return m.quickReplies } func (m *mockMsg) Topic() string { return m.topic } -func (m *mockMsg) ResponseToID() MsgID { return m.responseToID } func (m *mockMsg) ResponseToExternalID() string { return m.responseToExternalID } func (m *mockMsg) Metadata() json.RawMessage { return m.metadata } func (m *mockMsg) IsResend() bool { return m.isResend } From b609d29ab99b70079e2626dbe5122f1930a37cca Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 1 Dec 2021 11:52:45 -0500 Subject: [PATCH 13/62] Update CHANGELOG.md for v7.1.3 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7838a92a..a94dd65fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.3 +---------- + * Use response_to_external_id instead of response_to_id + v7.1.2 ---------- * External channel handler should use headers config setting if provided From 30f9a91d87b4b50cf6fc950cc167d97489209192 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 1 Dec 2021 13:41:49 -0500 Subject: [PATCH 14/62] Remove loop detection now that mailroom does this --- backend.go | 4 --- backends/rapidpro/backend.go | 60 ------------------------------- backends/rapidpro/backend_test.go | 30 ---------------- sender.go | 13 ------- test.go | 5 --- 5 files changed, 112 deletions(-) diff --git a/backend.go b/backend.go index 196d02d4f..9516a0cdc 100644 --- a/backend.go +++ b/backend.go @@ -74,10 +74,6 @@ type Backend interface { // a message is being forced in being resent by a user ClearMsgSent(context.Context, MsgID) error - // IsMsgLoop returns whether the passed in message is part of a message loop, possibly with another bot. Backends should - // implement their own logic to implement this. - IsMsgLoop(ctx context.Context, msg Msg) (bool, error) - // MarkOutgoingMsgComplete marks the passed in message as having been processed. Note this should be called even in the case // of errors during sending as it will manage the number of active workers per channel. The optional status parameter can be // used to determine any sort of deduping of msg sends diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 0d362e471..364d0d3fe 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -35,9 +35,6 @@ const sentSetName = "msgs_sent_%s" // our timeout for backend operations const backendTimeout = time.Second * 20 -// number of messages for loop detection -const msgLoopThreshold = 20 - func init() { courier.RegisterBackend("rapidpro", newBackend) } @@ -202,63 +199,6 @@ func (b *backend) ClearMsgSent(ctx context.Context, id courier.MsgID) error { return err } -var luaMsgLoop = redis.NewScript(3, `-- KEYS: [key, contact_id, text] - local key = KEYS[1] - local contact_id = KEYS[2] - local text = KEYS[3] - local count = 1 - - -- try to look up in window - local record = redis.call("hget", key, contact_id) - if record then - local record_count = tonumber(string.sub(record, 1, 2)) - local record_text = string.sub(record, 4, -1) - - if record_text == text then - count = math.min(record_count + 1, 99) - else - count = 1 - end - end - - -- create our new record with our updated count - record = string.format("%02d:%s", count, text) - - -- write our new record with updated count - redis.call("hset", key, contact_id, record) - - -- sets its expiration - redis.call("expire", key, 300) - - return count -`) - -// IsMsgLoop checks whether the passed in message is part of a loop -func (b *backend) IsMsgLoop(ctx context.Context, msg courier.Msg) (bool, error) { - m := msg.(*DBMsg) - - // things that aren't replies can't be loops, neither do we count retries - if m.ResponseToExternalID_ == "" || m.ErrorCount_ > 0 { - return false, nil - } - - // otherwise run our script to check whether this is a loop in the past 5 minutes - rc := b.redisPool.Get() - defer rc.Close() - - keyTime := time.Now().UTC().Round(time.Minute * 5) - key := fmt.Sprintf(sentSetName, fmt.Sprintf("loop_msgs:%s", keyTime.Format("2006-01-02-15:04"))) - count, err := redis.Int(luaMsgLoop.Do(rc, key, m.ContactID_, m.Text_)) - if err != nil { - return false, errors.Wrapf(err, "error while checking for msg loop") - } - - if count >= msgLoopThreshold { - return true, nil - } - return false, nil -} - // MarkOutgoingMsgComplete marks the passed in message as having completed processing, freeing up a worker for that channel func (b *backend) MarkOutgoingMsgComplete(ctx context.Context, msg courier.Msg, status courier.MsgStatus) { rc := b.redisPool.Get() diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index b803f8e7b..f58bb3186 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -748,36 +748,6 @@ func (ts *BackendTestSuite) TestExternalIDDupes() { ts.True(m2.alreadyWritten) } -func (ts *BackendTestSuite) TestLoop() { - ctx := context.Background() - dbMsg := readMsgFromDB(ts.b, courier.NewMsgID(10000)) - - dbMsg.ResponseToExternalID_ = "65474" - - loop, err := ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.False(loop) - - // call it 18 times more, no loop still - for i := 0; i < 18; i++ { - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.False(loop) - } - - // last one should make us a loop - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.True(loop) - - // make sure this keeps working even in hundreds of loops - for i := 0; i < 100; i++ { - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.True(loop) - } -} - func (ts *BackendTestSuite) TestStatus() { // our health should just contain the header ts.True(strings.Contains(ts.b.Status(), "Channel"), ts.b.Status()) diff --git a/sender.go b/sender.go index 63d966822..dbed21aab 100644 --- a/sender.go +++ b/sender.go @@ -189,23 +189,10 @@ func (w *Sender) sendMessage(msg Msg) { log.WithError(err).Error("error looking up msg was sent") } - // is this msg in a loop? - loop, err := backend.IsMsgLoop(sendCTX, msg) - - // failing on loop lookup isn't permanent, but log - if err != nil { - log.WithError(err).Error("error looking up msg loop") - } - if sent { // if this message was already sent, create a wired status for it status = backend.NewMsgStatusForID(msg.Channel(), msg.ID(), MsgWired) log.Warning("duplicate send, marking as wired") - } else if loop { - // if this contact is in a loop, fail the message immediately without sending - status = backend.NewMsgStatusForID(msg.Channel(), msg.ID(), MsgFailed) - status.AddLog(NewChannelLogFromError("Message Loop", msg.Channel(), msg.ID(), 0, fmt.Errorf("message loop detected, failing message without send"))) - log.Error("message loop detected, failing message") } else { // send our message status, err = server.SendMsg(sendCTX, msg) diff --git a/test.go b/test.go index 09762d438..012c67a1b 100644 --- a/test.go +++ b/test.go @@ -161,11 +161,6 @@ func (mb *MockBackend) ClearMsgSent(ctx context.Context, id MsgID) error { return nil } -// IsMsgLoop returns whether the passed in msg is a loop -func (mb *MockBackend) IsMsgLoop(ctx context.Context, msg Msg) (bool, error) { - return false, nil -} - // MarkOutgoingMsgComplete marks the passed msg as having been dealt with func (mb *MockBackend) MarkOutgoingMsgComplete(ctx context.Context, msg Msg, s MsgStatus) { mb.mutex.Lock() From 99c9b6aaa2cb43171f07ee07e5cfaf0f02b3783d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 2 Dec 2021 15:20:03 -0500 Subject: [PATCH 15/62] Update CHANGELOG.md for v7.1.4 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a94dd65fc..ec33438ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.4 +---------- + * Remove loop detection now that mailroom does this + * Smarter organization of quick replies for viber keyboards + v7.1.3 ---------- * Use response_to_external_id instead of response_to_id From 756b32f6f22382e320c1ac462fa43b4ef8afcdaf Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 6 Dec 2021 13:45:36 -0500 Subject: [PATCH 16/62] Add Msg.failed_reason --- backends/rapidpro/backend_test.go | 4 ++++ backends/rapidpro/msg.go | 6 ++++-- backends/rapidpro/schema.sql | 4 +++- backends/rapidpro/status.go | 16 ++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index f58bb3186..74de96d55 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -481,6 +481,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { ts.Equal(null.String("ext0"), m.ExternalID_) ts.True(m.ModifiedOn_.After(now)) ts.True(m.SentOn_.After(now)) + ts.Equal(null.NullString, m.FailedReason_) sentOn := *m.SentOn_ @@ -584,6 +585,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { ts.Equal(m.ErrorCount_, 1) ts.True(m.ModifiedOn_.After(now)) ts.True(m.NextAttempt_.After(now)) + ts.Equal(null.NullString, m.FailedReason_) // second go status = ts.b.NewMsgStatusForExternalID(channel, "ext1", courier.MsgErrored) @@ -594,6 +596,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { m = readMsgFromDB(ts.b, courier.NewMsgID(10000)) ts.Equal(m.Status_, courier.MsgErrored) ts.Equal(m.ErrorCount_, 2) + ts.Equal(null.NullString, m.FailedReason_) // third go status = ts.b.NewMsgStatusForExternalID(channel, "ext1", courier.MsgErrored) @@ -604,6 +607,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { m = readMsgFromDB(ts.b, courier.NewMsgID(10000)) ts.Equal(m.Status_, courier.MsgFailed) ts.Equal(m.ErrorCount_, 3) + ts.Equal(null.String("E"), m.FailedReason_) // update URN when the new doesn't exist tx, _ := ts.b.db.BeginTxx(ctx, nil) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 020071f84..a194a9363 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -175,6 +175,7 @@ SELECT attachments, msg_count, error_count, + failed_reason, high_priority, status, visibility, @@ -497,8 +498,9 @@ type DBMsg struct { ContactID_ ContactID `json:"contact_id" db:"contact_id"` ContactURNID_ ContactURNID `json:"contact_urn_id" db:"contact_urn_id"` - MessageCount_ int `json:"msg_count" db:"msg_count"` - ErrorCount_ int `json:"error_count" db:"error_count"` + MessageCount_ int `json:"msg_count" db:"msg_count"` + ErrorCount_ int `json:"error_count" db:"error_count"` + FailedReason_ null.String `json:"failed_reason" db:"failed_reason"` ChannelUUID_ courier.ChannelUUID `json:"channel_uuid"` ContactName_ string `json:"contact_name"` diff --git a/backends/rapidpro/schema.sql b/backends/rapidpro/schema.sql index 191169480..e21cbb7c0 100644 --- a/backends/rapidpro/schema.sql +++ b/backends/rapidpro/schema.sql @@ -72,6 +72,7 @@ CREATE TABLE msgs_msg ( msg_count integer NOT NULL, error_count integer NOT NULL, next_attempt timestamp with time zone NOT NULL, + failed_reason character varying(1), external_id character varying(255), attachments character varying(255)[], channel_id integer references channels_channel(id) on delete cascade, @@ -79,7 +80,8 @@ CREATE TABLE msgs_msg ( contact_urn_id integer NOT NULL references contacts_contacturn(id) on delete cascade, org_id integer NOT NULL references orgs_org(id) on delete cascade, metadata text, - topup_id integer + topup_id integer, + delete_from_counts boolean ); DROP TABLE IF EXISTS channels_channellog CASCADE; diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index 6a81d6335..a707bb7a4 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -107,6 +107,14 @@ UPDATE msgs_msg SET ELSE next_attempt END, + failed_reason = CASE + WHEN + error_count >= 2 + THEN + 'E' + ELSE + failed_reason + END, sent_on = CASE WHEN :status = 'W' @@ -164,6 +172,14 @@ UPDATE msgs_msg SET ELSE next_attempt END, + failed_reason = CASE + WHEN + error_count >= 2 + THEN + 'E' + ELSE + failed_reason + END, sent_on = CASE WHEN :status IN ('W', 'S', 'D') From ca40349d6822883ce64fc54067ddcf2bca439303 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 6 Dec 2021 14:12:40 -0500 Subject: [PATCH 17/62] Update CHANGELOG.md for v7.1.5 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec33438ce..5a14e5798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.5 +---------- + * Add Msg.failed_reason and set when msg fails due to reaching error limit + v7.1.4 ---------- * Remove loop detection now that mailroom does this From 9560dc96c3d6feb0c89ebe2d87101cb728c6bc7f Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 8 Dec 2021 16:11:43 +0200 Subject: [PATCH 18/62] Add comment about the 2 seconds pause choice --- handlers/whatsapp/whatsapp.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 651b960cd..ee9f30022 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -847,6 +847,10 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload if rr.StatusCode == 429 || rr.StatusCode == 503 { rateLimitKey := fmt.Sprintf("rate_limit:%s", msg.Channel().UUID().String()) rc.Do("set", rateLimitKey, "engaged") + + // The rate limit is 50 requests per second + // We pause sending 2 seconds so the limit count is reset + // TODO: In the future we should the header value when available rc.Do("expire", rateLimitKey, 2) log := courier.NewChannelLogFromRR("rate limit engaged", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) From cb453c1451772e3c1e153291398625d46c73233b Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 8 Dec 2021 09:27:06 -0500 Subject: [PATCH 19/62] Update CHANGELOG.md for v7.1.6 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a14e5798..a481824da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.6 +---------- + * Throttle WA queues when we get 429 responses + v7.1.5 ---------- * Add Msg.failed_reason and set when msg fails due to reaching error limit From 718aaa137c2885b8d951ef810272b66763d5cf32 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 10 Dec 2021 15:34:46 -0500 Subject: [PATCH 20/62] Update .gitignore to include deploy/ --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6b5579d0b..86bba308f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.a *.so *~ +deploy fabric fabfile.py fabfile.pyc @@ -34,4 +35,4 @@ _testmain.go dist/ .envrc courier -_storage \ No newline at end of file +_storage From 3233f3d7b0b229e8ae7ec700d4f7d64a1b3b9bbe Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 29 Nov 2021 10:51:22 -0300 Subject: [PATCH 21/62] Add instagram handler --- handlers/instagram/instagram.go | 600 +++++++++++++++++++++++++++ handlers/instagram/instagram_test.go | 461 ++++++++++++++++++++ 2 files changed, 1061 insertions(+) create mode 100644 handlers/instagram/instagram.go create mode 100644 handlers/instagram/instagram_test.go diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go new file mode 100644 index 000000000..6c4c9f35f --- /dev/null +++ b/handlers/instagram/instagram.go @@ -0,0 +1,600 @@ +package instagram + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/buger/jsonparser" + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/urns" + "github.com/pkg/errors" +) + +// Endpoints we hit +var ( + sendURL = "https://graph.facebook.com/v12.0/me/messages" + graphURL = "https://graph.facebook.com/v12.0/" + + signatureHeader = "X-Hub-Signature" + + // max for the body + maxMsgLength = 1000 + + //Only Human_Agent tag available for instagram + tagByTopic = map[string]string{ + "agent": "HUMAN_AGENT", + } +) + +// keys for extra in channel events +const ( + titleKey = "title" + payloadKey = "payload" +) + +func init() { + courier.RegisterHandler(newHandler()) +} + +type handler struct { + handlers.BaseHandler +} + +func newHandler() courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("IG"), "Instagram", false)} +} + +// Initialize is called by the engine once everything is loaded +func (h *handler) Initialize(s courier.Server) error { + h.SetServer(s) + s.AddHandlerRoute(h, http.MethodGet, "receive", h.receiveVerify) + s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveEvent) + return nil +} + +type igSender struct { + ID string `json:"id"` +} + +type igUser struct { + ID string `json:"id"` +} + +// { +// "object":"instagram", +// "entry":[{ +// "id":"180005062406476", +// "time":1514924367082, +// "messaging":[{ +// "sender": {"id":"1630934236957797"}, +// "recipient":{"id":"180005062406476"}, +// "timestamp":1514924366807, +// "message":{ +// "mid":"mid.$cAAD5QiNHkz1m6cyj11guxokwkhi2", +// "text":"65863634" +// } +// }] +// }] +// } + +type moPayload struct { + Object string `json:"object"` + Entry []struct { + ID string `json:"id"` + Time int64 `json:"time"` + Messaging []struct { + Sender igSender `json:"sender"` + Recipient igUser `json:"recipient"` + Timestamp int64 `json:"timestamp"` + + Postback *struct { + MID string `json:"mid"` + Title string `json:"title"` + Payload string `json:"payload"` + } `json:"postback,omitempty"` + + Message *struct { + IsEcho bool `json:"is_echo,omitempty"` + MID string `json:"mid"` + Text string `json:"text,omitempty"` + QuickReply struct { + Payload string `json:"payload"` + } `json:"quick_replies,omitempty"` + Attachments []struct { + Type string `json:"type"` + Payload *struct { + URL string `json:"url"` + } `json:"payload"` + } `json:"attachments,omitempty"` + } `json:"message,omitempty"` + } `json:"messaging"` + } `json:"entry"` +} + +/*type moPayload struct { + Object string `json:"object"` + Entry []struct { + ID string `json:"id"` + Time int64 `json:"time"` + Changes []struct { + Field string `json:"field"` + Value struct { + Sender struct { + ID string `json:"id"` + } `json:"sender"` + + Recipient struct { + ID string `json:"id"` + } `json:"recipient"` + Timestamp int64 `json:"timestamp"` + + Postback *struct { + MID string `json:"mid"` + Title string `json:"title"` + Payload string `json:"payload"` + } `json:"postback,omitempty"` + + Message *struct { + IsEcho bool `json:"is_echo,omitempty"` + MID string `json:"mid"` + Text string `json:"text,omitempty"` + QuickReply struct { + Payload string `json:"payload"` + } `json:"quick_replies,omitempty"` + Attachments []struct { + Type string `json:"type"` + Payload *struct { + URL string `json:"url"` + } `json:"payload"` + } `json:"attachments,omitempty"` + } `json:"message,omitempty"` + } `json:"value"` + } `json:"changes"` + } `json:"entry"` +}*/ + +// GetChannel returns the channel +func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Channel, error) { + + if r.Method == http.MethodGet { + + return nil, nil + } + + payload := &moPayload{} + + err := handlers.DecodeAndValidateJSON(payload, r) + + if err != nil { + + return nil, err + } + + // not a instagram object? ignore + if payload.Object != "instagram" { + + return nil, fmt.Errorf("object expected 'instagram', found %s", payload.Object) + } + + // no entries? ignore this request + if len(payload.Entry) == 0 { + + return nil, fmt.Errorf("no entries found") + } + + igID := payload.Entry[0].ID + + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(igID)) +} + +// receiveVerify handles Instagram's webhook verification callback +func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + mode := r.URL.Query().Get("hub.mode") + + // this isn't a subscribe verification, that's an error + if mode != "subscribe" { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown request")) + } + + // verify the token against our server facebook webhook secret, if the same return the challenge IG sent us + secret := r.URL.Query().Get("hub.verify_token") + + if secret != h.Server().Config().FacebookWebhookSecret { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("token does not match secret")) + } + // and respond with the challenge token + _, err := fmt.Fprint(w, r.URL.Query().Get("hub.challenge")) + return nil, err +} + +// receiveEvent is our HTTP handler function for incoming messages and status updates +func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + err := h.validateSignature(r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + payload := &moPayload{} + err = handlers.DecodeAndValidateJSON(payload, r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // not a instagram object? ignore + if payload.Object != "instagram" { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") + } + + // no entries? ignore this request + if len(payload.Entry) == 0 { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request, no entries") + } + + // the list of events we deal with + events := make([]courier.Event, 0, 2) + + // the list of data we will return in our response + data := make([]interface{}, 0, 2) + + // for each entry + for _, entry := range payload.Entry { + // no entry, ignore + if len(entry.Messaging) == 0 { + continue + } + + // grab our message, there is always a single one + msg := entry.Messaging[0] + + //msg.Value.Recipient.ID = "218041941572367" + + // ignore this entry if it is to another page + if channel.Address() != msg.Recipient.ID { + continue + } + + // create our date from the timestamp (they give us millis, arg is nanos) + date := time.Unix(0, msg.Timestamp*1000000).UTC() + + sender := msg.Sender.ID + if sender == "" { + sender = msg.Sender.ID + } + + // create our URN + urn, err := urns.NewInstagramURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + if msg.Postback != nil { + // by default postbacks are treated as new conversations + eventType := courier.NewConversation + event := h.Backend().NewChannelEvent(channel, eventType, urn).WithOccurredOn(date) + + // build our extra + extra := map[string]interface{}{ + titleKey: msg.Postback.Title, + payloadKey: msg.Postback.Payload, + } + + event = event.WithExtra(extra) + + err := h.Backend().WriteChannelEvent(ctx, event) + if err != nil { + return nil, err + } + + events = append(events, event) + data = append(data, courier.NewEventReceiveData(event)) + } else if msg.Message != nil { + // this is an incoming message + // ignore echos + if msg.Message.IsEcho { + data = append(data, courier.NewInfoData("ignoring echo")) + continue + } + + text := msg.Message.Text + + attachmentURLs := make([]string, 0, 2) + + for _, att := range msg.Message.Attachments { + if att.Payload != nil && att.Payload.URL != "" { + attachmentURLs = append(attachmentURLs, att.Payload.URL) + } + } + + // create our message + ev := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(msg.Message.MID).WithReceivedOn(date) + event := h.Backend().CheckExternalIDSeen(ev) + + // add any attachment URL found + for _, attURL := range attachmentURLs { + event.WithAttachment(attURL) + } + + err := h.Backend().WriteMsg(ctx, event) + if err != nil { + return nil, err + } + + h.Backend().WriteExternalIDSeen(event) + + events = append(events, event) + data = append(data, courier.NewMsgReceiveData(event)) + + } else { + data = append(data, courier.NewInfoData("ignoring unknown entry type")) + } + } + return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) +} + +// { +// "messaging_type": "" +// "recipient":{ +// "id":"" +// }, +// "message":{ +// "text":"hello, world!" +// "attachment":{ +// "type":"image", +// "payload":{ +// "url":"http://www.messenger-rocks.com/image.jpg", +// "is_reusable":true +// } +// } +// } +// } +type mtPayload struct { + MessagingType string `json:"messaging_type"` + Tag string `json:"tag,omitempty"` + Recipient struct { + //UserRef string `json:"user_ref,omitempty"` + ID string `json:"id,omitempty"` + } `json:"recipient"` + Message struct { + Text string `json:"text,omitempty"` + QuickReplies []mtQuickReply `json:"quick_replies,omitempty"` + Attachment *mtAttachment `json:"attachment,omitempty"` + } `json:"message"` +} + +type mtAttachment struct { + Type string `json:"type"` + Payload struct { + URL string `json:"url,omitempty"` + IsReusable bool `json:"is_reusable,omitempty"` + } `json:"payload"` +} +type mtQuickReply struct { + Title string `json:"title"` + Payload string `json:"payload"` + ContentType string `json:"content_type"` +} + +func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { + // can't do anything without an access token + accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") + if accessToken == "" { + return nil, fmt.Errorf("missing access token") + } + + topic := msg.Topic() + payload := mtPayload{} + + // set our message type + if msg.ResponseToID() != courier.NilMsgID { + payload.MessagingType = "RESPONSE" + } else if topic != "" { + payload.MessagingType = "MESSAGE_TAG" + payload.Tag = tagByTopic[topic] + } else { + payload.MessagingType = "UPDATE" + } + + payload.Recipient.ID = msg.URN().Path() + + msgURL, _ := url.Parse(sendURL) + query := url.Values{} + query.Set("access_token", accessToken) + msgURL.RawQuery = query.Encode() + + status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) + + msgParts := make([]string, 0) + if msg.Text() != "" { + msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + } + + // send each part and each attachment separately. we send attachments first as otherwise quick replies + // attached to text messages get hidden when images get delivered + for i := 0; i < len(msgParts)+len(msg.Attachments()); i++ { + if i < len(msg.Attachments()) { + // this is an attachment + payload.Message.Attachment = &mtAttachment{} + attType, attURL := handlers.SplitAttachment(msg.Attachments()[i]) + attType = strings.Split(attType, "/")[0] + payload.Message.Attachment.Type = attType + payload.Message.Attachment.Payload.URL = attURL + payload.Message.Attachment.Payload.IsReusable = true + payload.Message.Text = "" + } else { + // this is still a msg part + payload.Message.Text = msgParts[i-len(msg.Attachments())] + payload.Message.Attachment = nil + } + + // include any quick replies on the last piece we send + if i == (len(msgParts)+len(msg.Attachments()))-1 { + for _, qr := range msg.QuickReplies() { + payload.Message.QuickReplies = append(payload.Message.QuickReplies, mtQuickReply{qr, qr, "text"}) + } + } else { + payload.Message.QuickReplies = nil + } + + jsonBody, err := json.Marshal(payload) + if err != nil { + return status, err + } + + req, err := http.NewRequest(http.MethodPost, msgURL.String(), bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + rr, err := utils.MakeHTTPRequest(req) + + // record our status and log + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + status.AddLog(log) + if err != nil { + return status, nil + } + + externalID, err := jsonparser.GetString(rr.Body, "message_id") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to get message_id from body")) + return status, nil + } + + // if this is our first message, record the external id + if i == 0 { + status.SetExternalID(externalID) + if msg.URN().IsInstagramRef() { + recipientID, err := jsonparser.GetString(rr.Body, "recipient_id") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to get recipient_id from body")) + return status, nil + } + + referralID := msg.URN().InstagramRef() + + realIDURN, err := urns.NewInstagramURN(recipientID) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to make Instagram urn from %s", recipientID)) + } + + contact, err := h.Backend().GetContact(ctx, msg.Channel(), msg.URN(), "", "") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to get contact for %s", msg.URN().String())) + } + realURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, realIDURN) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to add real Instagram URN %s to contact with uuid %s", realURN.String(), contact.UUID())) + } + referralIDExtURN, err := urns.NewURNFromParts(urns.ExternalScheme, referralID, "", "") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to make ext urn from %s", referralID)) + } + extURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, referralIDExtURN) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to add URN %s to contact with uuid %s", extURN.String(), contact.UUID())) + } + + referralInstagramURN, err := h.Backend().RemoveURNfromContact(ctx, msg.Channel(), contact, msg.URN()) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to remove referral Instagram URN %s from contact with uuid %s", referralInstagramURN.String(), contact.UUID())) + } + + } + + } + + // this was wired successfully + status.SetStatus(courier.MsgWired) + } + + return status, nil +} + +// DescribeURN looks up URN metadata for new contacts +func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { + // can't do anything with Instagram refs, ignore them + if urn.IsInstagramRef() { + return map[string]string{}, nil + } + + accessToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") + if accessToken == "" { + return nil, fmt.Errorf("missing access token") + } + + // build a request to lookup the stats for this contact + base, _ := url.Parse(graphURL) + path, _ := url.Parse(fmt.Sprintf("/%s", urn.Path())) + u := base.ResolveReference(path) + + query := url.Values{} + query.Set("fields", "first_name,last_name") + query.Set("access_token", accessToken) + u.RawQuery = query.Encode() + req, _ := http.NewRequest(http.MethodGet, u.String(), nil) + rr, err := utils.MakeHTTPRequest(req) + if err != nil { + return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) + } + + // read our first and last name + firstName, _ := jsonparser.GetString(rr.Body, "first_name") + lastName, _ := jsonparser.GetString(rr.Body, "last_name") + + return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil +} + +// see https://developers.facebook.com/docs/messenger-platform/webhook#security +func (h *handler) validateSignature(r *http.Request) error { + headerSignature := r.Header.Get(signatureHeader) + if headerSignature == "" { + return fmt.Errorf("missing request signature") + } + appSecret := h.Server().Config().FacebookApplicationSecret + + body, err := handlers.ReadBody(r, 100000) + if err != nil { + return fmt.Errorf("unable to read request body: %s", err) + } + + expectedSignature, err := fbCalculateSignature(appSecret, body) + if err != nil { + return err + } + + signature := "" + if len(headerSignature) == 45 && strings.HasPrefix(headerSignature, "sha1=") { + signature = strings.TrimPrefix(headerSignature, "sha1=") + } + + // compare signatures in way that isn't sensitive to a timing attack + if !hmac.Equal([]byte(expectedSignature), []byte(signature)) { + return fmt.Errorf("invalid request signature, expected: %s got: %s for body: '%s'", expectedSignature, signature, string(body)) + } + + return nil +} + +func fbCalculateSignature(appSecret string, body []byte) (string, error) { + var buffer bytes.Buffer + buffer.Write(body) + + // hash with SHA1 + mac := hmac.New(sha1.New, []byte(appSecret)) + mac.Write(buffer.Bytes()) + + return hex.EncodeToString(mac.Sum(nil)), nil +} diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go new file mode 100644 index 000000000..b18ab10f3 --- /dev/null +++ b/handlers/instagram/instagram_test.go @@ -0,0 +1,461 @@ +package instagram + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + . "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/gocommon/urns" + "github.com/stretchr/testify/assert" +) + +var testChannels = []courier.Channel{ + courier.NewMockChannel("8ab23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +} + +var helloMsg = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var duplicateMsg = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }, + { + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var invalidURN = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var attachment = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"image", + "payload":{ + "url":"https://image-url/foo.png" + } + }] + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var like_heart = `{ + "object":"instagram", + "entry":[{ + "id":"1234", + "messaging":[{ + "sender":{"id":"5678"}, + "recipient":{"id":"1234"}, + "timestamp":1459991487970, + "message":{ + "mid":"external_id", + "attachments":[{ + "type":"like_heart" + }] + } + }], + "time":1459991487970 + }] +}` + +var differentPage = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var echo = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + }] + }] +}` + +var icebreakerGetStarted = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "postback": { + "title": "icebreaker question", + "payload": "get_started" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var notInstagram = `{ + "object":"notinstagram", + "entry": [{}] +}` + +var noEntries = `{ + "object":"instagram", + "entry": [] +}` + +var noMessagingEntries = `{ + "object":"instagram", + "entry": [{ + "id": "1234" + }] +}` + +var unkownMessagingEntry = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }] + }] +}` + +var notJSON = `blargh` + +var testCases = []ChannelHandleTestCase{ + {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + + {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsg, Status: 200, Response: "Handled", + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachment, Status: 200, Response: "Handled", + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", + Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", + URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, + PrepRequest: addValidSignature}, + + {Label: "Different Page", URL: "/c/ig/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/ig/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "expected 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/ig/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURN, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, +} + +func addValidSignature(r *http.Request) { + body, _ := handlers.ReadBody(r, 100000) + sig, _ := fbCalculateSignature("fb_app_secret", body) + r.Header.Set(signatureHeader, fmt.Sprintf("sha1=%s", string(sig))) +} + +func addInvalidSignature(r *http.Request) { + r.Header.Set(signatureHeader, "invalidsig") +} + +// mocks the call to the Facebook graph API +func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + accessToken := r.URL.Query().Get("access_token") + defer r.Body.Close() + + // invalid auth token + if accessToken != "a123" { + http.Error(w, "invalid auth token", 403) + } + + // user has a name + if strings.HasSuffix(r.URL.Path, "1337") { + w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + return + } + + // no name + w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + })) + graphURL = server.URL + + return server +} + +func TestDescribe(t *testing.T) { + fbGraph := buildMockFBGraph(testCases) + defer fbGraph.Close() + + handler := newHandler().(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}, + {"instagram:ref:1337", map[string]string{}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } +} + +func TestHandler(t *testing.T) { + RunChannelTestCases(t, testChannels, newHandler(), testCases) +} + +func BenchmarkHandler(b *testing.B) { + fbService := buildMockFBGraph(testCases) + defer fbService.Close() + + RunChannelBenchmarks(b, testChannels, newHandler(), testCases) +} + +func TestVerify(t *testing.T) { + + RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, + {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, + {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, + {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + }) + +} + +// setSendURL takes care of setting the send_url to our test server host +func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { + sendURL = s.URL +} + +var defaultSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + + {Label: "Plain Response", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + + {Label: "Tag Human Agent", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", Topic: "agent", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + + {Label: "Plain Send using ref URN", + Text: "Simple Message", URN: "instagram:ref:67890", + ContactURNs: map[string]bool{"instagram:12345": true, "ext:67890": true, "instagram:ref:67890": false}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Long Message", + Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", + URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + + {Label: "Send caption and photo with Quick Reply", + Text: "This is some text.", + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + + {Label: "ID Error", + Text: "ID Error", URN: "instagram12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, + SendPrep: setSendURL}, + + {Label: "Error", + Text: "Error", URN: "instagram12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, + SendPrep: setSendURL}, + + {Label: "Quick Reply", + URN: "instagram:12345", Text: "Are you happy?", QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + + {Label: "Send Photo", + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + SendPrep: setSendURL}, +} + +func TestSending(t *testing.T) { + // shorter max msg length for testing + maxMsgLength = 100 + var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) + RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) +} + +func TestSigning(t *testing.T) { + tcs := []struct { + Body string + Signature string + }{ + { + "hello world", + "308de7627fe19e92294c4572a7f831bc1002809d", + }, + { + "hello world2", + "ab6f902b58b9944032d4a960f470d7a8ebfd12b7", + }, + } + + for i, tc := range tcs { + sig, err := fbCalculateSignature("sesame", []byte(tc.Body)) + assert.NoError(t, err) + assert.Equal(t, tc.Signature, sig, "%d: mismatched signature", i) + } +} From ba9732855b18521bafc4847defcd2a55d09226e8 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 29 Nov 2021 11:05:06 -0300 Subject: [PATCH 22/62] refactor instagram.go --- handlers/instagram/instagram.go | 44 --------------------------------- 1 file changed, 44 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index 6c4c9f35f..bb803f818 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -122,48 +122,6 @@ type moPayload struct { } `json:"entry"` } -/*type moPayload struct { - Object string `json:"object"` - Entry []struct { - ID string `json:"id"` - Time int64 `json:"time"` - Changes []struct { - Field string `json:"field"` - Value struct { - Sender struct { - ID string `json:"id"` - } `json:"sender"` - - Recipient struct { - ID string `json:"id"` - } `json:"recipient"` - Timestamp int64 `json:"timestamp"` - - Postback *struct { - MID string `json:"mid"` - Title string `json:"title"` - Payload string `json:"payload"` - } `json:"postback,omitempty"` - - Message *struct { - IsEcho bool `json:"is_echo,omitempty"` - MID string `json:"mid"` - Text string `json:"text,omitempty"` - QuickReply struct { - Payload string `json:"payload"` - } `json:"quick_replies,omitempty"` - Attachments []struct { - Type string `json:"type"` - Payload *struct { - URL string `json:"url"` - } `json:"payload"` - } `json:"attachments,omitempty"` - } `json:"message,omitempty"` - } `json:"value"` - } `json:"changes"` - } `json:"entry"` -}*/ - // GetChannel returns the channel func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Channel, error) { @@ -257,8 +215,6 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h // grab our message, there is always a single one msg := entry.Messaging[0] - //msg.Value.Recipient.ID = "218041941572367" - // ignore this entry if it is to another page if channel.Address() != msg.Recipient.ID { continue From 91885b23c94da08b796ef6c5708eaf73c5f2e0ff Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 30 Nov 2021 15:34:26 -0300 Subject: [PATCH 23/62] Refactor instagram handler --- handlers/instagram/instagram.go | 46 ---------------------------- handlers/instagram/instagram_test.go | 12 ++------ 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index bb803f818..9c7da6198 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -318,7 +318,6 @@ type mtPayload struct { MessagingType string `json:"messaging_type"` Tag string `json:"tag,omitempty"` Recipient struct { - //UserRef string `json:"user_ref,omitempty"` ID string `json:"id,omitempty"` } `json:"recipient"` Message struct { @@ -422,54 +421,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat if err != nil { return status, nil } - externalID, err := jsonparser.GetString(rr.Body, "message_id") if err != nil { log.WithError("Message Send Error", errors.Errorf("unable to get message_id from body")) return status, nil } - - // if this is our first message, record the external id if i == 0 { status.SetExternalID(externalID) - if msg.URN().IsInstagramRef() { - recipientID, err := jsonparser.GetString(rr.Body, "recipient_id") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to get recipient_id from body")) - return status, nil - } - - referralID := msg.URN().InstagramRef() - - realIDURN, err := urns.NewInstagramURN(recipientID) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to make Instagram urn from %s", recipientID)) - } - - contact, err := h.Backend().GetContact(ctx, msg.Channel(), msg.URN(), "", "") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to get contact for %s", msg.URN().String())) - } - realURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, realIDURN) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to add real Instagram URN %s to contact with uuid %s", realURN.String(), contact.UUID())) - } - referralIDExtURN, err := urns.NewURNFromParts(urns.ExternalScheme, referralID, "", "") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to make ext urn from %s", referralID)) - } - extURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, referralIDExtURN) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to add URN %s to contact with uuid %s", extURN.String(), contact.UUID())) - } - - referralInstagramURN, err := h.Backend().RemoveURNfromContact(ctx, msg.Channel(), contact, msg.URN()) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to remove referral Instagram URN %s from contact with uuid %s", referralInstagramURN.String(), contact.UUID())) - } - - } - } // this was wired successfully @@ -481,10 +439,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat // DescribeURN looks up URN metadata for new contacts func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - // can't do anything with Instagram refs, ignore them - if urn.IsInstagramRef() { - return map[string]string{}, nil - } accessToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") if accessToken == "" { diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index b18ab10f3..217ccb2ac 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -110,7 +110,7 @@ var attachment = `{ "attachments":[{ "type":"image", "payload":{ - "url":"https://image-url/foo.png" + "url":"https://image-url/foo.png" } }] }, @@ -320,8 +320,7 @@ func TestDescribe(t *testing.T) { urn urns.URN metadata map[string]string }{{"instagram:1337", map[string]string{"name": "John Doe"}}, - {"instagram:4567", map[string]string{"name": ""}}, - {"instagram:ref:1337", map[string]string{}}} + {"instagram:4567", map[string]string{"name": ""}}} for _, tc := range tcs { metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) @@ -380,13 +379,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, - {Label: "Plain Send using ref URN", - Text: "Simple Message", URN: "instagram:ref:67890", - ContactURNs: map[string]bool{"instagram:12345": true, "ext:67890": true, "instagram:ref:67890": false}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", From 86f48fef11117f44abfb10d7e8751b39c79ab0eb Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 3 Dec 2021 16:55:04 -0300 Subject: [PATCH 24/62] Add environment variables to instagram --- config.go | 94 +++++++++++++++------------- handlers/instagram/instagram.go | 6 +- handlers/instagram/instagram_test.go | 6 +- handlers/test.go | 2 + 4 files changed, 57 insertions(+), 51 deletions(-) diff --git a/config.go b/config.go index 5e6447690..336664e96 100644 --- a/config.go +++ b/config.go @@ -4,31 +4,33 @@ import "github.com/nyaruka/ezconf" // Config is our top level configuration object type Config struct { - Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` - SentryDSN string `help:"the DSN used for logging errors to Sentry"` - Domain string `help:"the domain courier is exposed on"` - Address string `help:"the network interface address courier will bind to"` - Port int `help:"the port courier will listen on"` - DB string `help:"URL describing how to connect to the RapidPro database"` - Redis string `help:"URL describing how to connect to Redis"` - SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` - S3Endpoint string `help:"the S3 endpoint we will write attachments to"` - S3Region string `help:"the S3 region we will write attachments to"` - S3MediaBucket string `help:"the S3 bucket we will write attachments to"` - S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` - S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` - S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` - AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` - AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - FacebookApplicationSecret string `help:"the Facebook app secret"` - FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` - MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` - LibratoUsername string `help:"the username that will be used to authenticate to Librato"` - LibratoToken string `help:"the token that will be used to authenticate to Librato"` - StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` - StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` - LogLevel string `help:"the logging level courier should use"` - Version string `help:"the version that will be used in request and response headers"` + Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` + SentryDSN string `help:"the DSN used for logging errors to Sentry"` + Domain string `help:"the domain courier is exposed on"` + Address string `help:"the network interface address courier will bind to"` + Port int `help:"the port courier will listen on"` + DB string `help:"URL describing how to connect to the RapidPro database"` + Redis string `help:"URL describing how to connect to Redis"` + SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` + S3Endpoint string `help:"the S3 endpoint we will write attachments to"` + S3Region string `help:"the S3 region we will write attachments to"` + S3MediaBucket string `help:"the S3 bucket we will write attachments to"` + S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` + S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` + S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` + AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` + AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` + FacebookApplicationSecret string `help:"the Facebook app secret"` + FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` + InstagramApplicationSecret string `help:"the Instagram app secret"` + InstagramWebhookSecret string `help:"the secret for Instagram webhook URL verification"` + MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` + LibratoUsername string `help:"the username that will be used to authenticate to Librato"` + LibratoToken string `help:"the token that will be used to authenticate to Librato"` + StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` + StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` + LogLevel string `help:"the logging level courier should use"` + Version string `help:"the version that will be used in request and response headers"` // IncludeChannels is the list of channels to enable, empty means include all IncludeChannels []string @@ -40,26 +42,28 @@ type Config struct { // NewConfig returns a new default configuration object func NewConfig() *Config { return &Config{ - Backend: "rapidpro", - Domain: "localhost", - Address: "", - Port: 8080, - DB: "postgres://temba:temba@localhost/temba?sslmode=disable", - Redis: "redis://localhost:6379/15", - SpoolDir: "/var/spool/courier", - S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", - S3MediaBucket: "courier-media", - S3MediaPrefix: "/media/", - S3DisableSSL: false, - S3ForcePathStyle: false, - AWSAccessKeyID: "", - AWSSecretAccessKey: "", - FacebookApplicationSecret: "missing_facebook_app_secret", - FacebookWebhookSecret: "missing_facebook_webhook_secret", - MaxWorkers: 32, - LogLevel: "error", - Version: "Dev", + Backend: "rapidpro", + Domain: "localhost", + Address: "", + Port: 8080, + DB: "postgres://temba:temba@localhost/temba?sslmode=disable", + Redis: "redis://localhost:6379/15", + SpoolDir: "/var/spool/courier", + S3Endpoint: "https://s3.amazonaws.com", + S3Region: "us-east-1", + S3MediaBucket: "courier-media", + S3MediaPrefix: "/media/", + S3DisableSSL: false, + S3ForcePathStyle: false, + AWSAccessKeyID: "", + AWSSecretAccessKey: "", + FacebookApplicationSecret: "missing_facebook_app_secret", + FacebookWebhookSecret: "missing_facebook_webhook_secret", + InstagramApplicationSecret: "missing_instagram_app_secret", + InstagramWebhookSecret: "missing_instagram_webhook_secret", + MaxWorkers: 32, + LogLevel: "error", + Version: "Dev", } } diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index 9c7da6198..f51ec1cef 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -165,10 +165,10 @@ func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown request")) } - // verify the token against our server facebook webhook secret, if the same return the challenge IG sent us + // verify the token against our server instagram webhook secret, if the same return the challenge IG sent us secret := r.URL.Query().Get("hub.verify_token") - if secret != h.Server().Config().FacebookWebhookSecret { + if secret != h.Server().Config().InstagramWebhookSecret { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("token does not match secret")) } // and respond with the challenge token @@ -473,7 +473,7 @@ func (h *handler) validateSignature(r *http.Request) error { if headerSignature == "" { return fmt.Errorf("missing request signature") } - appSecret := h.Server().Config().FacebookApplicationSecret + appSecret := h.Server().Config().InstagramApplicationSecret body, err := handlers.ReadBody(r, 100000) if err != nil { diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index 217ccb2ac..249f0e0b3 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -278,7 +278,7 @@ var testCases = []ChannelHandleTestCase{ func addValidSignature(r *http.Request) { body, _ := handlers.ReadBody(r, 100000) - sig, _ := fbCalculateSignature("fb_app_secret", body) + sig, _ := fbCalculateSignature("ig_app_secret", body) r.Header.Set(signatureHeader, fmt.Sprintf("sha1=%s", string(sig))) } @@ -342,12 +342,12 @@ func BenchmarkHandler(b *testing.B) { func TestVerify(t *testing.T) { RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, }) } diff --git a/handlers/test.go b/handlers/test.go index 9f5a88e3a..bd1354628 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -200,6 +200,8 @@ func newServer(backend courier.Backend) courier.Server { config := courier.NewConfig() config.FacebookWebhookSecret = "fb_webhook_secret" config.FacebookApplicationSecret = "fb_app_secret" + config.InstagramWebhookSecret = "ig_webhook_secret" + config.InstagramApplicationSecret = "ig_app_secret" return courier.NewServerWithLogger(config, backend, logger) From 2cf769958fc55a8868ba36e256a4de1f5a55fd68 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 16 Dec 2021 16:39:10 -0300 Subject: [PATCH 25/62] fix: Metadata search for a new contact --- handlers/instagram/instagram.go | 8 +++----- handlers/instagram/instagram_test.go | 10 ++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index f51ec1cef..fab5ab6be 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -451,7 +451,6 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn u := base.ResolveReference(path) query := url.Values{} - query.Set("fields", "first_name,last_name") query.Set("access_token", accessToken) u.RawQuery = query.Encode() req, _ := http.NewRequest(http.MethodGet, u.String(), nil) @@ -460,11 +459,10 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) } - // read our first and last name - firstName, _ := jsonparser.GetString(rr.Body, "first_name") - lastName, _ := jsonparser.GetString(rr.Body, "last_name") + // read our name + name, _ := jsonparser.GetString(rr.Body, "name") - return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil + return map[string]string{"name": name}, nil } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index 249f0e0b3..0d891554d 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -299,12 +299,12 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { // user has a name if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + w.Write([]byte(`{ "name": "John Doe"}`)) return } // no name - w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + w.Write([]byte(`{ "name": ""}`)) })) graphURL = server.URL @@ -319,8 +319,10 @@ func TestDescribe(t *testing.T) { tcs := []struct { urn urns.URN metadata map[string]string - }{{"instagram:1337", map[string]string{"name": "John Doe"}}, - {"instagram:4567", map[string]string{"name": ""}}} + }{ + {"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}, + } for _, tc := range tcs { metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) From 3cfcb5a3238756f5d2c50d7d541efe29391c694f Mon Sep 17 00:00:00 2001 From: Matheus Soares Date: Fri, 17 Dec 2021 11:23:50 -0300 Subject: [PATCH 26/62] add import for instagram handler --- cmd/courier/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/courier/main.go b/cmd/courier/main.go index 5c44ef92f..62f0afee5 100644 --- a/cmd/courier/main.go +++ b/cmd/courier/main.go @@ -34,6 +34,7 @@ import ( _ "github.com/nyaruka/courier/handlers/hub9" _ "github.com/nyaruka/courier/handlers/i2sms" _ "github.com/nyaruka/courier/handlers/infobip" + _ "github.com/nyaruka/courier/handlers/instagram" _ "github.com/nyaruka/courier/handlers/jasmin" _ "github.com/nyaruka/courier/handlers/jiochat" _ "github.com/nyaruka/courier/handlers/junebug" From 7965bd8f85a645777e432a9cb5b65f417f506794 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 17 Dec 2021 12:40:55 -0300 Subject: [PATCH 27/62] Refactor response field to external ID --- handlers/instagram/instagram.go | 35 +--------------------------- handlers/instagram/instagram_test.go | 2 +- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index fab5ab6be..1cdea4a2e 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -71,23 +71,6 @@ type igUser struct { ID string `json:"id"` } -// { -// "object":"instagram", -// "entry":[{ -// "id":"180005062406476", -// "time":1514924367082, -// "messaging":[{ -// "sender": {"id":"1630934236957797"}, -// "recipient":{"id":"180005062406476"}, -// "timestamp":1514924366807, -// "message":{ -// "mid":"mid.$cAAD5QiNHkz1m6cyj11guxokwkhi2", -// "text":"65863634" -// } -// }] -// }] -// } - type moPayload struct { Object string `json:"object"` Entry []struct { @@ -298,22 +281,6 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) } -// { -// "messaging_type": "" -// "recipient":{ -// "id":"" -// }, -// "message":{ -// "text":"hello, world!" -// "attachment":{ -// "type":"image", -// "payload":{ -// "url":"http://www.messenger-rocks.com/image.jpg", -// "is_reusable":true -// } -// } -// } -// } type mtPayload struct { MessagingType string `json:"messaging_type"` Tag string `json:"tag,omitempty"` @@ -351,7 +318,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index 0d891554d..648a3ed68 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -369,7 +369,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Plain Response", Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, From d27ee1aa6638e005e026288da81e2207a9194977 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 23 Dec 2021 15:09:24 -0300 Subject: [PATCH 28/62] feat: Add Instagram channel support to Facebook handler --- cmd/courier/main.go | 1 - handlers/facebookapp/facebookapp.go | 64 +-- handlers/facebookapp/facebookapp_test.go | 533 +++++++++++++++++++---- 3 files changed, 483 insertions(+), 115 deletions(-) diff --git a/cmd/courier/main.go b/cmd/courier/main.go index 62f0afee5..5c44ef92f 100644 --- a/cmd/courier/main.go +++ b/cmd/courier/main.go @@ -34,7 +34,6 @@ import ( _ "github.com/nyaruka/courier/handlers/hub9" _ "github.com/nyaruka/courier/handlers/i2sms" _ "github.com/nyaruka/courier/handlers/infobip" - _ "github.com/nyaruka/courier/handlers/instagram" _ "github.com/nyaruka/courier/handlers/jasmin" _ "github.com/nyaruka/courier/handlers/jiochat" _ "github.com/nyaruka/courier/handlers/junebug" diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 77982f4cb..3781adce8 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -23,13 +23,13 @@ import ( // Endpoints we hit var ( - sendURL = "https://graph.facebook.com/v7.0/me/messages" - graphURL = "https://graph.facebook.com/v7.0/" + sendURL = "https://graph.facebook.com/v12.0/me/messages" + graphURL = "https://graph.facebook.com/v12.0/" signatureHeader = "X-Hub-Signature" - // Facebook API says 640 is max for the body - maxMsgLength = 640 + // max for the body + maxMsgLength = 1000 // Sticker ID substitutions stickerIDToEmoji = map[int64]string{ @@ -56,18 +56,20 @@ const ( payloadKey = "payload" ) +func newHandler(channelType courier.ChannelType, name string, validateSignatures bool) courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(channelType, name, validateSignatures)} +} + func init() { - courier.RegisterHandler(newHandler()) + courier.RegisterHandler(newHandler("IG", "Instagram", false)) + courier.RegisterHandler(newHandler("FBA", "Facebook", false)) + } type handler struct { handlers.BaseHandler } -func newHandler() courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("FBA"), "Facebook", false)} -} - // Initialize is called by the engine once everything is loaded func (h *handler) Initialize(s courier.Server) error { h.SetServer(s) @@ -76,12 +78,12 @@ func (h *handler) Initialize(s courier.Server) error { return nil } -type fbSender struct { +type Sender struct { ID string `json:"id"` - UserRef string `json:"user_ref"` + UserRef string `json:"user_ref,omitempty"` } -type fbUser struct { +type User struct { ID string `json:"id"` } @@ -108,9 +110,9 @@ type moPayload struct { ID string `json:"id"` Time int64 `json:"time"` Messaging []struct { - Sender fbSender `json:"sender"` - Recipient fbUser `json:"recipient"` - Timestamp int64 `json:"timestamp"` + Sender Sender `json:"sender"` + Recipient User `json:"recipient"` + Timestamp int64 `json:"timestamp"` OptIn *struct { Ref string `json:"ref"` @@ -125,6 +127,7 @@ type moPayload struct { } `json:"referral"` Postback *struct { + MID string `json:"mid"` Title string `json:"title"` Payload string `json:"payload"` Referral struct { @@ -172,9 +175,9 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, err } - // not a page object? ignore - if payload.Object != "page" { - return nil, fmt.Errorf("object expected 'page', found %s", payload.Object) + // is not a 'page' and 'instagram' object? ignore it + if payload.Object != "page" && payload.Object != "instagram" { + return nil, fmt.Errorf("object expected 'page' or 'instagram', found %s", payload.Object) } // no entries? ignore this request @@ -182,9 +185,14 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no entries found") } - pageID := payload.Entry[0].ID + EntryID := payload.Entry[0].ID - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(pageID)) + //if object is 'page' returns type FBA, if object is 'instagram' returns type IG + if payload.Object == "page" { + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(EntryID)) + } else { + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(EntryID)) + } } // receiveVerify handles Facebook's webhook verification callback @@ -219,9 +227,9 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } - // not a page object? ignore - if payload.Object != "page" { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring non-page request") + // // is not a 'page' and 'instagram' object? ignore it + if payload.Object != "page" && payload.Object != "instagram" { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") } // no entries? ignore this request @@ -494,7 +502,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload.MessagingType = "MESSAGE_TAG" payload.Tag = tagByTopic[topic] } else { - payload.MessagingType = "NON_PROMOTIONAL_SUBSCRIPTION" // only allowed until Jan 15, 2020 + payload.MessagingType = "UPDATE" } // build our recipient @@ -641,7 +649,6 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn u := base.ResolveReference(path) query := url.Values{} - query.Set("fields", "first_name,last_name") query.Set("access_token", accessToken) u.RawQuery = query.Encode() req, _ := http.NewRequest(http.MethodGet, u.String(), nil) @@ -650,11 +657,10 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) } - // read our first and last name - firstName, _ := jsonparser.GetString(rr.Body, "first_name") - lastName, _ := jsonparser.GetString(rr.Body, "last_name") + // read our name + name, _ := jsonparser.GetString(rr.Body, "name") - return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil + return map[string]string{"name": name}, nil } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 5514c7114..008a2d840 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -17,20 +17,45 @@ import ( ) var testChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } -var helloMsg = `{ +var testChannelsIG = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +} + +var helloMsgFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var helloMsgIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -41,17 +66,55 @@ var helloMsg = `{ }] }` -var duplicateMsg = `{ +var duplicateMsgFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }, + { + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var duplicateMsgIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -61,14 +124,14 @@ var duplicateMsg = `{ "time": 1459991487970 }, { - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -79,17 +142,38 @@ var duplicateMsg = `{ }] }` -var invalidURN = `{ +var invalidURNFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var invalidURNIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "abc5678" @@ -100,10 +184,10 @@ var invalidURN = `{ }] }` -var attachment = `{ +var attachmentFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "mid": "external_id", @@ -115,7 +199,33 @@ var attachment = `{ }] }, "recipient": { - "id": "1234" + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var attachmentIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"image", + "payload":{ + "url":"https://image-url/foo.png" + } + }] + }, + "recipient": { + "id": "12345" }, "sender": { "id": "5678" @@ -129,7 +239,7 @@ var attachment = `{ var locationAttachment = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "mid": "external_id", @@ -144,7 +254,7 @@ var locationAttachment = `{ }] }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -158,11 +268,11 @@ var locationAttachment = `{ var thumbsUp = `{ "object":"page", "entry":[{ - "id":"1234", + "id":"12345", "time":1459991487970, "messaging":[{ "sender":{"id":"5678"}, - "recipient":{"id":"1234"}, + "recipient":{"id":"12345"}, "timestamp":1459991487970, "message":{ "mid":"external_id", @@ -178,10 +288,50 @@ var thumbsUp = `{ }] }` -var differentPage = `{ +var like_heart = `{ + "object":"instagram", + "entry":[{ + "id":"12345", + "messaging":[{ + "sender":{"id":"5678"}, + "recipient":{"id":"12345"}, + "timestamp":1459991487970, + "message":{ + "mid":"external_id", + "attachments":[{ + "type":"like_heart" + }] + } + }], + "time":1459991487970 + }] +}` + +var differentPageIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var differentPageFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", @@ -199,13 +349,33 @@ var differentPage = `{ }] }` -var echo = `{ +var echoFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + }] + }] +}` + +var echoIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -219,17 +389,38 @@ var echo = `{ }] }` +var icebreakerGetStarted = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "postback": { + "title": "icebreaker question", + "payload": "get_started" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + var optInUserRef = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "optin": { "ref": "optin_ref", "user_ref": "optin_user_ref" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -243,13 +434,13 @@ var optInUserRef = `{ var optIn = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "optin": { "ref": "optin_ref" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -263,7 +454,7 @@ var optIn = `{ var postback = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", @@ -275,7 +466,7 @@ var postback = `{ } }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -289,7 +480,7 @@ var postback = `{ var postbackReferral = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", @@ -302,7 +493,7 @@ var postbackReferral = `{ } }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -316,14 +507,14 @@ var postbackReferral = `{ var postbackGetStarted = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", "payload": "get_started" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -337,7 +528,7 @@ var postbackGetStarted = `{ var referral = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "referral": { "ref": "referral id", @@ -346,7 +537,7 @@ var referral = `{ "type": "referral type" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678", @@ -361,7 +552,7 @@ var referral = `{ var dlr = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "delivery":{ "mids":[ @@ -371,7 +562,7 @@ var dlr = `{ "seq":37 }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -387,25 +578,58 @@ var notPage = `{ "entry": [{}] }` -var noEntries = `{ +var notInstagram = `{ + "object":"notinstagram", + "entry": [{}] +}` + +var noEntriesFBA = `{ "object":"page", "entry": [] }` -var noMessagingEntries = `{ +var noEntriesIG = `{ + "object":"instagram", + "entry": [] +}` + +var noMessagingEntriesFBA = `{ "object":"page", "entry": [{ - "id": "1234" + "id": "12345" + }] +}` + +var noMessagingEntriesIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345" }] }` -var unkownMessagingEntry = `{ +var unkownMessagingEntryFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }] + }] +}` + +var unkownMessagingEntryIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -417,17 +641,16 @@ var unkownMessagingEntry = `{ var notJSON = `blargh` -var testCases = []ChannelHandleTestCase{ - {Label: "Receive Message", URL: "/c/fba/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, +var testCasesFBA = []ChannelHandleTestCase{ + {Label: "Receive Message FBA", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, + {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - - {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsg, Status: 200, Response: "Handled", + {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsgFBA, Status: 200, Response: "Handled", Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachment, Status: 200, Response: "Handled", + {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachmentFBA, Status: 200, Response: "Handled", Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, @@ -469,14 +692,47 @@ var testCases = []ChannelHandleTestCase{ Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), MsgStatus: Sp(courier.MsgDelivered), ExternalID: Sp("mid.1458668856218:ed81099e15d3f4f233"), PrepRequest: addValidSignature}, - {Label: "Different Page", URL: "/c/fba/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/fba/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "expected 'page', found notpage", PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/fba/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Different Page", URL: "/c/fba/receive", Data: differentPageFBA, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/fba/receive", Data: echoFBA, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/fba/receive", Data: noEntriesFBA, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURN, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, +} +var testCasesIG = []ChannelHandleTestCase{ + {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsgIG, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsgIG, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + + {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsgIG, Status: 200, Response: "Handled", + Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachmentIG, Status: 200, Response: "Handled", + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", + Text: Sp(""), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", + URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, + PrepRequest: addValidSignature}, + + {Label: "Different Page", URL: "/c/ig/receive", Data: differentPageIG, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/ig/receive", Data: echoIG, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/ig/receive", Data: noEntriesIG, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { @@ -502,12 +758,12 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { // user has a name if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + w.Write([]byte(`{ "name": "John Doe"}`)) return } // no name - w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + w.Write([]byte(`{ "name": ""}`)) })) graphURL = server.URL @@ -515,37 +771,69 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { } func TestDescribe(t *testing.T) { - fbGraph := buildMockFBGraph(testCases) - defer fbGraph.Close() + var testCases [][]ChannelHandleTestCase + testCases = append(testCases, testCasesFBA) + testCases = append(testCases, testCasesIG) + + for i, tc := range testCases { + fbGraph := buildMockFBGraph(tc) + defer fbGraph.Close() + + if i == 0 { + handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{ + {"facebook:1337", map[string]string{"name": "John Doe"}}, + {"facebook:4567", map[string]string{"name": ""}}, + } + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } + } else { + handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{ + {"facebook:1337", map[string]string{"name": "John Doe"}}, + {"facebook:4567", map[string]string{"name": ""}}, + } + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } + } - handler := newHandler().(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{{"facebook:1337", map[string]string{"name": "John Doe"}}, - {"facebook:4567", map[string]string{"name": ""}}, - {"facebook:ref:1337", map[string]string{}}} - - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) } + } func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), testCases) + RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) + } func BenchmarkHandler(b *testing.B) { - fbService := buildMockFBGraph(testCases) - defer fbService.Close() + fbService := buildMockFBGraph(testCasesFBA) - RunChannelBenchmarks(b, testChannels, newHandler(), testCases) + RunChannelBenchmarks(b, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + fbService.Close() + + fbServiceIG := buildMockFBGraph(testCasesIG) + + RunChannelBenchmarks(b, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) + fbServiceIG.Close() } func TestVerify(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ + RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/fba/receive", Status: 400, Response: "unknown request"}, @@ -554,6 +842,15 @@ func TestVerify(t *testing.T) { {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, }) + RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), []ChannelHandleTestCase{ + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, + {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, + {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, + {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + }) + } // setSendURL takes care of setting the send_url to our test server host @@ -561,12 +858,12 @@ func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, sendURL = s.URL } -var defaultSendTestCases = []ChannelSendTestCase{ +var SendTestCasesFBA = []ChannelSendTestCase{ {Label: "Plain Send", Text: "Simple Message", URN: "facebook:12345", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", @@ -579,13 +876,13 @@ var defaultSendTestCases = []ChannelSendTestCase{ ContactURNs: map[string]bool{"facebook:12345": true, "ext:67890": true, "facebook:ref:67890": false}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Quick Reply", Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", @@ -598,7 +895,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "Send caption and photo with Quick Reply", Text: "This is some text.", @@ -612,7 +909,71 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, + SendPrep: setSendURL}, + {Label: "ID Error", + Text: "ID Error", URN: "facebook:12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, + SendPrep: setSendURL}, + {Label: "Error", + Text: "Error", URN: "facebook:12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, + SendPrep: setSendURL}, +} + +var SendTestCasesIG = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "facebook:12345", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Plain Response", + Text: "Simple Message", URN: "facebook:12345", + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Quick Reply", + Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Long Message", + Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", + URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Send Photo", + URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + SendPrep: setSendURL}, + {Label: "Send caption and photo with Quick Reply", + Text: "This is some text.", + URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Tag Human Agent", + Text: "Simple Message", URN: "facebook:12345", + Status: "W", ExternalID: "mid.133", Topic: "agent", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Send Document", + URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "ID Error", Text: "ID Error", URN: "facebook:12345", @@ -629,8 +990,10 @@ var defaultSendTestCases = []ChannelSendTestCase{ func TestSending(t *testing.T) { // shorter max msg length for testing maxMsgLength = 100 - var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) - RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) + var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + RunChannelSendTestCases(t, ChannelFBA, newHandler("FBA", "Facebook", false), SendTestCasesFBA, nil) + RunChannelSendTestCases(t, ChannelIG, newHandler("IG", "Instagram", false), SendTestCasesIG, nil) } func TestSigning(t *testing.T) { From b914672dd4e227d3000899be332849987bdd04d1 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 23 Dec 2021 17:56:29 -0300 Subject: [PATCH 29/62] Remove unused instagram handler files --- handlers/instagram/instagram.go | 475 --------------------------- handlers/instagram/instagram_test.go | 455 ------------------------- 2 files changed, 930 deletions(-) delete mode 100644 handlers/instagram/instagram.go delete mode 100644 handlers/instagram/instagram_test.go diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go deleted file mode 100644 index 1cdea4a2e..000000000 --- a/handlers/instagram/instagram.go +++ /dev/null @@ -1,475 +0,0 @@ -package instagram - -import ( - "bytes" - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "fmt" - "net/http" - "net/url" - "strings" - "time" - - "github.com/buger/jsonparser" - "github.com/nyaruka/courier" - "github.com/nyaruka/courier/handlers" - "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/urns" - "github.com/pkg/errors" -) - -// Endpoints we hit -var ( - sendURL = "https://graph.facebook.com/v12.0/me/messages" - graphURL = "https://graph.facebook.com/v12.0/" - - signatureHeader = "X-Hub-Signature" - - // max for the body - maxMsgLength = 1000 - - //Only Human_Agent tag available for instagram - tagByTopic = map[string]string{ - "agent": "HUMAN_AGENT", - } -) - -// keys for extra in channel events -const ( - titleKey = "title" - payloadKey = "payload" -) - -func init() { - courier.RegisterHandler(newHandler()) -} - -type handler struct { - handlers.BaseHandler -} - -func newHandler() courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("IG"), "Instagram", false)} -} - -// Initialize is called by the engine once everything is loaded -func (h *handler) Initialize(s courier.Server) error { - h.SetServer(s) - s.AddHandlerRoute(h, http.MethodGet, "receive", h.receiveVerify) - s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveEvent) - return nil -} - -type igSender struct { - ID string `json:"id"` -} - -type igUser struct { - ID string `json:"id"` -} - -type moPayload struct { - Object string `json:"object"` - Entry []struct { - ID string `json:"id"` - Time int64 `json:"time"` - Messaging []struct { - Sender igSender `json:"sender"` - Recipient igUser `json:"recipient"` - Timestamp int64 `json:"timestamp"` - - Postback *struct { - MID string `json:"mid"` - Title string `json:"title"` - Payload string `json:"payload"` - } `json:"postback,omitempty"` - - Message *struct { - IsEcho bool `json:"is_echo,omitempty"` - MID string `json:"mid"` - Text string `json:"text,omitempty"` - QuickReply struct { - Payload string `json:"payload"` - } `json:"quick_replies,omitempty"` - Attachments []struct { - Type string `json:"type"` - Payload *struct { - URL string `json:"url"` - } `json:"payload"` - } `json:"attachments,omitempty"` - } `json:"message,omitempty"` - } `json:"messaging"` - } `json:"entry"` -} - -// GetChannel returns the channel -func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Channel, error) { - - if r.Method == http.MethodGet { - - return nil, nil - } - - payload := &moPayload{} - - err := handlers.DecodeAndValidateJSON(payload, r) - - if err != nil { - - return nil, err - } - - // not a instagram object? ignore - if payload.Object != "instagram" { - - return nil, fmt.Errorf("object expected 'instagram', found %s", payload.Object) - } - - // no entries? ignore this request - if len(payload.Entry) == 0 { - - return nil, fmt.Errorf("no entries found") - } - - igID := payload.Entry[0].ID - - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(igID)) -} - -// receiveVerify handles Instagram's webhook verification callback -func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { - mode := r.URL.Query().Get("hub.mode") - - // this isn't a subscribe verification, that's an error - if mode != "subscribe" { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown request")) - } - - // verify the token against our server instagram webhook secret, if the same return the challenge IG sent us - secret := r.URL.Query().Get("hub.verify_token") - - if secret != h.Server().Config().InstagramWebhookSecret { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("token does not match secret")) - } - // and respond with the challenge token - _, err := fmt.Fprint(w, r.URL.Query().Get("hub.challenge")) - return nil, err -} - -// receiveEvent is our HTTP handler function for incoming messages and status updates -func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { - err := h.validateSignature(r) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - payload := &moPayload{} - err = handlers.DecodeAndValidateJSON(payload, r) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - // not a instagram object? ignore - if payload.Object != "instagram" { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") - } - - // no entries? ignore this request - if len(payload.Entry) == 0 { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request, no entries") - } - - // the list of events we deal with - events := make([]courier.Event, 0, 2) - - // the list of data we will return in our response - data := make([]interface{}, 0, 2) - - // for each entry - for _, entry := range payload.Entry { - // no entry, ignore - if len(entry.Messaging) == 0 { - continue - } - - // grab our message, there is always a single one - msg := entry.Messaging[0] - - // ignore this entry if it is to another page - if channel.Address() != msg.Recipient.ID { - continue - } - - // create our date from the timestamp (they give us millis, arg is nanos) - date := time.Unix(0, msg.Timestamp*1000000).UTC() - - sender := msg.Sender.ID - if sender == "" { - sender = msg.Sender.ID - } - - // create our URN - urn, err := urns.NewInstagramURN(sender) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - if msg.Postback != nil { - // by default postbacks are treated as new conversations - eventType := courier.NewConversation - event := h.Backend().NewChannelEvent(channel, eventType, urn).WithOccurredOn(date) - - // build our extra - extra := map[string]interface{}{ - titleKey: msg.Postback.Title, - payloadKey: msg.Postback.Payload, - } - - event = event.WithExtra(extra) - - err := h.Backend().WriteChannelEvent(ctx, event) - if err != nil { - return nil, err - } - - events = append(events, event) - data = append(data, courier.NewEventReceiveData(event)) - } else if msg.Message != nil { - // this is an incoming message - // ignore echos - if msg.Message.IsEcho { - data = append(data, courier.NewInfoData("ignoring echo")) - continue - } - - text := msg.Message.Text - - attachmentURLs := make([]string, 0, 2) - - for _, att := range msg.Message.Attachments { - if att.Payload != nil && att.Payload.URL != "" { - attachmentURLs = append(attachmentURLs, att.Payload.URL) - } - } - - // create our message - ev := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(msg.Message.MID).WithReceivedOn(date) - event := h.Backend().CheckExternalIDSeen(ev) - - // add any attachment URL found - for _, attURL := range attachmentURLs { - event.WithAttachment(attURL) - } - - err := h.Backend().WriteMsg(ctx, event) - if err != nil { - return nil, err - } - - h.Backend().WriteExternalIDSeen(event) - - events = append(events, event) - data = append(data, courier.NewMsgReceiveData(event)) - - } else { - data = append(data, courier.NewInfoData("ignoring unknown entry type")) - } - } - return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) -} - -type mtPayload struct { - MessagingType string `json:"messaging_type"` - Tag string `json:"tag,omitempty"` - Recipient struct { - ID string `json:"id,omitempty"` - } `json:"recipient"` - Message struct { - Text string `json:"text,omitempty"` - QuickReplies []mtQuickReply `json:"quick_replies,omitempty"` - Attachment *mtAttachment `json:"attachment,omitempty"` - } `json:"message"` -} - -type mtAttachment struct { - Type string `json:"type"` - Payload struct { - URL string `json:"url,omitempty"` - IsReusable bool `json:"is_reusable,omitempty"` - } `json:"payload"` -} -type mtQuickReply struct { - Title string `json:"title"` - Payload string `json:"payload"` - ContentType string `json:"content_type"` -} - -func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { - // can't do anything without an access token - accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") - if accessToken == "" { - return nil, fmt.Errorf("missing access token") - } - - topic := msg.Topic() - payload := mtPayload{} - - // set our message type - if msg.ResponseToExternalID() != "" { - payload.MessagingType = "RESPONSE" - } else if topic != "" { - payload.MessagingType = "MESSAGE_TAG" - payload.Tag = tagByTopic[topic] - } else { - payload.MessagingType = "UPDATE" - } - - payload.Recipient.ID = msg.URN().Path() - - msgURL, _ := url.Parse(sendURL) - query := url.Values{} - query.Set("access_token", accessToken) - msgURL.RawQuery = query.Encode() - - status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - - msgParts := make([]string, 0) - if msg.Text() != "" { - msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) - } - - // send each part and each attachment separately. we send attachments first as otherwise quick replies - // attached to text messages get hidden when images get delivered - for i := 0; i < len(msgParts)+len(msg.Attachments()); i++ { - if i < len(msg.Attachments()) { - // this is an attachment - payload.Message.Attachment = &mtAttachment{} - attType, attURL := handlers.SplitAttachment(msg.Attachments()[i]) - attType = strings.Split(attType, "/")[0] - payload.Message.Attachment.Type = attType - payload.Message.Attachment.Payload.URL = attURL - payload.Message.Attachment.Payload.IsReusable = true - payload.Message.Text = "" - } else { - // this is still a msg part - payload.Message.Text = msgParts[i-len(msg.Attachments())] - payload.Message.Attachment = nil - } - - // include any quick replies on the last piece we send - if i == (len(msgParts)+len(msg.Attachments()))-1 { - for _, qr := range msg.QuickReplies() { - payload.Message.QuickReplies = append(payload.Message.QuickReplies, mtQuickReply{qr, qr, "text"}) - } - } else { - payload.Message.QuickReplies = nil - } - - jsonBody, err := json.Marshal(payload) - if err != nil { - return status, err - } - - req, err := http.NewRequest(http.MethodPost, msgURL.String(), bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - rr, err := utils.MakeHTTPRequest(req) - - // record our status and log - log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) - status.AddLog(log) - if err != nil { - return status, nil - } - externalID, err := jsonparser.GetString(rr.Body, "message_id") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to get message_id from body")) - return status, nil - } - if i == 0 { - status.SetExternalID(externalID) - } - - // this was wired successfully - status.SetStatus(courier.MsgWired) - } - - return status, nil -} - -// DescribeURN looks up URN metadata for new contacts -func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - - accessToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") - if accessToken == "" { - return nil, fmt.Errorf("missing access token") - } - - // build a request to lookup the stats for this contact - base, _ := url.Parse(graphURL) - path, _ := url.Parse(fmt.Sprintf("/%s", urn.Path())) - u := base.ResolveReference(path) - - query := url.Values{} - query.Set("access_token", accessToken) - u.RawQuery = query.Encode() - req, _ := http.NewRequest(http.MethodGet, u.String(), nil) - rr, err := utils.MakeHTTPRequest(req) - if err != nil { - return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) - } - - // read our name - name, _ := jsonparser.GetString(rr.Body, "name") - - return map[string]string{"name": name}, nil -} - -// see https://developers.facebook.com/docs/messenger-platform/webhook#security -func (h *handler) validateSignature(r *http.Request) error { - headerSignature := r.Header.Get(signatureHeader) - if headerSignature == "" { - return fmt.Errorf("missing request signature") - } - appSecret := h.Server().Config().InstagramApplicationSecret - - body, err := handlers.ReadBody(r, 100000) - if err != nil { - return fmt.Errorf("unable to read request body: %s", err) - } - - expectedSignature, err := fbCalculateSignature(appSecret, body) - if err != nil { - return err - } - - signature := "" - if len(headerSignature) == 45 && strings.HasPrefix(headerSignature, "sha1=") { - signature = strings.TrimPrefix(headerSignature, "sha1=") - } - - // compare signatures in way that isn't sensitive to a timing attack - if !hmac.Equal([]byte(expectedSignature), []byte(signature)) { - return fmt.Errorf("invalid request signature, expected: %s got: %s for body: '%s'", expectedSignature, signature, string(body)) - } - - return nil -} - -func fbCalculateSignature(appSecret string, body []byte) (string, error) { - var buffer bytes.Buffer - buffer.Write(body) - - // hash with SHA1 - mac := hmac.New(sha1.New, []byte(appSecret)) - mac.Write(buffer.Bytes()) - - return hex.EncodeToString(mac.Sum(nil)), nil -} diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go deleted file mode 100644 index 648a3ed68..000000000 --- a/handlers/instagram/instagram_test.go +++ /dev/null @@ -1,455 +0,0 @@ -package instagram - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/nyaruka/courier" - "github.com/nyaruka/courier/handlers" - . "github.com/nyaruka/courier/handlers" - "github.com/nyaruka/gocommon/urns" - "github.com/stretchr/testify/assert" -) - -var testChannels = []courier.Channel{ - courier.NewMockChannel("8ab23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), -} - -var helloMsg = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var duplicateMsg = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }, - { - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var invalidURN = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "abc5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var attachment = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "mid": "external_id", - "attachments":[{ - "type":"image", - "payload":{ - "url":"https://image-url/foo.png" - } - }] - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var like_heart = `{ - "object":"instagram", - "entry":[{ - "id":"1234", - "messaging":[{ - "sender":{"id":"5678"}, - "recipient":{"id":"1234"}, - "timestamp":1459991487970, - "message":{ - "mid":"external_id", - "attachments":[{ - "type":"like_heart" - }] - } - }], - "time":1459991487970 - }] -}` - -var differentPage = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1235" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var echo = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970, - "message": { - "is_echo": true, - "mid": "qT7ywaK" - } - }] - }] -}` - -var icebreakerGetStarted = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "postback": { - "title": "icebreaker question", - "payload": "get_started" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var notInstagram = `{ - "object":"notinstagram", - "entry": [{}] -}` - -var noEntries = `{ - "object":"instagram", - "entry": [] -}` - -var noMessagingEntries = `{ - "object":"instagram", - "entry": [{ - "id": "1234" - }] -}` - -var unkownMessagingEntry = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }] - }] -}` - -var notJSON = `blargh` - -var testCases = []ChannelHandleTestCase{ - {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, - Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - - {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsg, Status: 200, Response: "Handled", - Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachment, Status: 200, Response: "Handled", - Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", - Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", - URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), - ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, - PrepRequest: addValidSignature}, - - {Label: "Different Page", URL: "/c/ig/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/ig/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "expected 'instagram', found notinstagram", PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/ig/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURN, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, -} - -func addValidSignature(r *http.Request) { - body, _ := handlers.ReadBody(r, 100000) - sig, _ := fbCalculateSignature("ig_app_secret", body) - r.Header.Set(signatureHeader, fmt.Sprintf("sha1=%s", string(sig))) -} - -func addInvalidSignature(r *http.Request) { - r.Header.Set(signatureHeader, "invalidsig") -} - -// mocks the call to the Facebook graph API -func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - accessToken := r.URL.Query().Get("access_token") - defer r.Body.Close() - - // invalid auth token - if accessToken != "a123" { - http.Error(w, "invalid auth token", 403) - } - - // user has a name - if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "name": "John Doe"}`)) - return - } - - // no name - w.Write([]byte(`{ "name": ""}`)) - })) - graphURL = server.URL - - return server -} - -func TestDescribe(t *testing.T) { - fbGraph := buildMockFBGraph(testCases) - defer fbGraph.Close() - - handler := newHandler().(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{ - {"instagram:1337", map[string]string{"name": "John Doe"}}, - {"instagram:4567", map[string]string{"name": ""}}, - } - - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) - } -} - -func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), testCases) -} - -func BenchmarkHandler(b *testing.B) { - fbService := buildMockFBGraph(testCases) - defer fbService.Close() - - RunChannelBenchmarks(b, testChannels, newHandler(), testCases) -} - -func TestVerify(t *testing.T) { - - RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, - Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, - {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, - {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, - {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, - }) - -} - -// setSendURL takes care of setting the send_url to our test server host -func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { - sendURL = s.URL -} - -var defaultSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, - - {Label: "Plain Response", - Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, - - {Label: "Tag Human Agent", - Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", Topic: "agent", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, - - {Label: "Long Message", - Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", - URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, - SendPrep: setSendURL}, - - {Label: "Send caption and photo with Quick Reply", - Text: "This is some text.", - URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - QuickReplies: []string{"Yes", "No"}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, - SendPrep: setSendURL}, - - {Label: "ID Error", - Text: "ID Error", URN: "instagram12345", - Status: "E", - ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, - SendPrep: setSendURL}, - - {Label: "Error", - Text: "Error", URN: "instagram12345", - Status: "E", - ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, - SendPrep: setSendURL}, - - {Label: "Quick Reply", - URN: "instagram:12345", Text: "Are you happy?", QuickReplies: []string{"Yes", "No"}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, - SendPrep: setSendURL}, - - {Label: "Send Photo", - URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, - SendPrep: setSendURL}, -} - -func TestSending(t *testing.T) { - // shorter max msg length for testing - maxMsgLength = 100 - var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) - RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) -} - -func TestSigning(t *testing.T) { - tcs := []struct { - Body string - Signature string - }{ - { - "hello world", - "308de7627fe19e92294c4572a7f831bc1002809d", - }, - { - "hello world2", - "ab6f902b58b9944032d4a960f470d7a8ebfd12b7", - }, - } - - for i, tc := range tcs { - sig, err := fbCalculateSignature("sesame", []byte(tc.Body)) - assert.NoError(t, err) - assert.Equal(t, tc.Signature, sig, "%d: mismatched signature", i) - } -} From fe834f7c1ac4a790b1d5498629bac0a842aea20b Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 28 Dec 2021 16:35:34 -0300 Subject: [PATCH 30/62] Ignore story mention callback --- handlers/facebookapp/facebookapp.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 3781adce8..aa4c5b4eb 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -142,6 +142,7 @@ type moPayload struct { IsEcho bool `json:"is_echo"` MID string `json:"mid"` Text string `json:"text"` + IsDeleted bool `json:"is_deleted"` Attachments []struct { Type string `json:"type"` Payload *struct { @@ -388,6 +389,11 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h attachmentURLs = append(attachmentURLs, fmt.Sprintf("geo:%f,%f", att.Payload.Coordinates.Lat, att.Payload.Coordinates.Long)) } + if att.Type == "story_mention" { + data = append(data, courier.NewInfoData("ignoring story_mention")) + continue + } + if att.Payload != nil && att.Payload.URL != "" { attachmentURLs = append(attachmentURLs, att.Payload.URL) } From 20553306b419251a97c4d6cfce9cc7b04b525c44 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 30 Dec 2021 12:17:16 -0300 Subject: [PATCH 31/62] Add quick replies for vk --- handlers/vk/vk.go | 88 ++++++++++++++++++++++++++++++++++++++++-- handlers/vk/vk_test.go | 50 +++++++++++++++++++++++- 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index e82ceeaa3..8fc2bbc43 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -56,6 +56,7 @@ var ( paramMessage = "message" paramAttachments = "attachment" paramRandomId = "random_id" + paramKeyboard = "keyboard" // base upload media values paramServerId = "server" @@ -113,6 +114,7 @@ type moNewMessagePayload struct { Lng float64 `json:"longitude"` } `json:"coordinates"` } `json:"geo"` + Payload string `json:"payload"` } `json:"message" validate:"required"` } `json:"object" validate:"required"` } @@ -189,6 +191,23 @@ type mediaUploadInfoPayload struct { OwnerId int64 `json:"owner_id"` } +type Keyboard struct { + One_Time bool `json:"one_time"` + Buttons [][]buttonPayload `json:"buttons"` + Inline bool `json:"inline"` +} + +type buttonPayload struct { + Action buttonAction `json:"action"` + Color string `json:"color"` +} + +type buttonAction struct { + Type string `json:"type"` + Label string `json:"label"` + Payload string `json:"payload"` +} + // receiveEvent handles request event type func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { // read request body @@ -384,11 +403,7 @@ func takeFirstAttachmentUrl(payload moNewMessagePayload) string { func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) - if err != nil { - return status, errors.New("Cannot create send message request") - } params := buildApiBaseParams(msg.Channel()) params.Set(paramUserId, msg.URN().Path()) params.Set(paramRandomId, msg.ID().String()) @@ -397,6 +412,19 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat params.Set(paramMessage, text) params.Set(paramAttachments, attachments) + if len(msg.QuickReplies()) != 0 { + keyboard := buildQuickRepliesParams(msg) + keyboardString := ToJSON(keyboard) + + params.Set(paramKeyboard, keyboardString) + } + + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) + + if err != nil { + return status, errors.New("Cannot create send message request") + } + req.URL.RawQuery = params.Encode() res, err := utils.MakeHTTPRequest(req) @@ -417,6 +445,58 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } +// ToJSON returns the JSON encoding of Keyboard. +func ToJSON(keyboard Keyboard) string { + b, _ := json.Marshal(keyboard) + return string(b) +} + +//keyboard +func buildQuickRepliesParams(msg courier.Msg) Keyboard { + + var keyboard Keyboard + keyboard = AddRowKeyboard(keyboard) + + for _, qr := range msg.QuickReplies() { + keyboard = AddTextButton(qr, qr, "secondary", keyboard) + } + return keyboard +} + +func AddRowKeyboard(keyboard Keyboard) Keyboard { + if len(keyboard.Buttons) == 0 { + keyboard.Buttons = make([][]buttonPayload, 1) + } else { + row := make([]buttonPayload, 0) + keyboard.Buttons = append(keyboard.Buttons, row) + + } + keyboard.One_Time = true + + return keyboard +} + +func AddTextButton(label string, payload interface{}, color string, keyboard Keyboard) Keyboard { + b, err := json.Marshal(payload) + if err != nil { + panic(err) + } + + button := buttonPayload{ + Action: buttonAction{ + Type: "text", + Label: label, + Payload: string(b), + }, + Color: color, + } + + lastRow := len(keyboard.Buttons) - 1 + keyboard.Buttons[lastRow] = append(keyboard.Buttons[lastRow], button) + + return keyboard +} + // buildTextAndAttachmentParams builds msg text with attachment links (if needed) and attachments list param, also returns the errors that occurred func buildTextAndAttachmentParams(msg courier.Msg, status courier.MsgStatus) (string, string) { var msgAttachments []string diff --git a/handlers/vk/vk_test.go b/handlers/vk/vk_test.go index 01cef3e58..dbca85cfb 100644 --- a/handlers/vk/vk_test.go +++ b/handlers/vk/vk_test.go @@ -2,8 +2,6 @@ package vk import ( "context" - "github.com/nyaruka/gocommon/urns" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "net/url" @@ -11,6 +9,9 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/urns" + "github.com/stretchr/testify/assert" + "github.com/nyaruka/courier" . "github.com/nyaruka/courier/handlers" ) @@ -210,6 +211,22 @@ const eventServerVerification = `{ "secret": "abc123xyz" }` +const msgKeyboard = `{ + "type": "message_new", + "object": { + "message": { + "id": 1, + "date": 1580125800, + "from_id": 123456, + "text": "Yes", + "payload": "\"Yes\"" + } + }, + "secret": "abc123xyz" + }` + +const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"Yes","payload":"\"Yes\""},"color":"secondary"}]],"inline":false}` + var testCases = []ChannelHandleTestCase{ { Label: "Receive Message", @@ -281,6 +298,16 @@ var testCases = []ChannelHandleTestCase{ ExternalID: Sp("1"), Date: Tp(time.Date(2020, 1, 27, 11, 50, 0, 0, time.UTC)), Attachments: []string{"https://foo.bar/doc.pdf"}, }, + { + Label: "Receive Message Keyboard", + URL: receiveURL, + Data: msgKeyboard, + Status: 200, + Response: "ok", + URN: Sp("vk:123456"), + ExternalID: Sp("1"), + Date: Tp(time.Date(2020, 1, 27, 11, 50, 0, 0, time.UTC)), + }, { Label: "Receive Geolocation Attachment", URL: receiveURL, @@ -445,6 +472,25 @@ var sendTestCases = []ChannelSendTestCase{ }, }, }, + { + Label: "Send keyboard", + Text: "Send keyboard", + URN: "vk:123456789", + QuickReplies: []string{"Yes"}, + Status: "S", + SendPrep: setSendURL, + ExternalID: "1", + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: actionSendMessage, + RawQuery: "access_token=token123xyz&attachment=&keyboard=" + url.QueryEscape(keyboardJson) + "&message=Send+keyboard&random_id=10&user_id=123456789&v=5.103", + }: { + Status: 200, + Body: `{"response": 1}`, + }, + }, + }, } func mockAttachmentURLs(mediaServer *httptest.Server, testCases []ChannelSendTestCase) []ChannelSendTestCase { From 220295b235129966e9d8cdc10e4e6a14da5f564e Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 4 Jan 2022 13:04:56 -0300 Subject: [PATCH 32/62] Add support for new keyboard rows --- handlers/vk/keyboard.go | 41 +++++++++++++++ handlers/vk/keyboard_test.go | 98 ++++++++++++++++++++++++++++++++++++ handlers/vk/vk.go | 76 ++-------------------------- handlers/vk/vk_test.go | 4 +- 4 files changed, 145 insertions(+), 74 deletions(-) create mode 100644 handlers/vk/keyboard.go create mode 100644 handlers/vk/keyboard_test.go diff --git a/handlers/vk/keyboard.go b/handlers/vk/keyboard.go new file mode 100644 index 000000000..a8b435254 --- /dev/null +++ b/handlers/vk/keyboard.go @@ -0,0 +1,41 @@ +package vk + +import ( + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/jsonx" +) + +type Keyboard struct { + One_Time bool `json:"one_time"` + Buttons [][]ButtonPayload `json:"buttons"` + Inline bool `json:"inline"` +} + +type ButtonPayload struct { + Action ButtonAction `json:"action"` + Color string `json:"color"` +} + +type ButtonAction struct { + Type string `json:"type"` + Label string `json:"label"` + Payload string `json:"payload"` +} + +// NewKeyboardFromReplies creates a keyboard from the given quick replies +func NewKeyboardFromReplies(replies []string) *Keyboard { + rows := utils.StringsToRows(replies, 10, 30, 2) + buttons := make([][]ButtonPayload, len(rows)) + + for i := range rows { + buttons[i] = make([]ButtonPayload, len(rows[i])) + for j := range rows[i] { + buttons[i][j].Action.Label = rows[i][j] + buttons[i][j].Action.Type = "text" + buttons[i][j].Action.Payload = string(jsonx.MustMarshal(rows[i][j])) + buttons[i][j].Color = "primary" + } + } + + return &Keyboard{One_Time: true, Buttons: buttons, Inline: false} +} diff --git a/handlers/vk/keyboard_test.go b/handlers/vk/keyboard_test.go new file mode 100644 index 000000000..6a2491b68 --- /dev/null +++ b/handlers/vk/keyboard_test.go @@ -0,0 +1,98 @@ +package vk_test + +import ( + "testing" + + "github.com/nyaruka/courier/handlers/vk" + "github.com/stretchr/testify/assert" +) + +func TestKeyboardFromReplies(t *testing.T) { + tcs := []struct { + replies []string + expected *vk.Keyboard + }{ + { + + []string{"OK"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + { + {vk.ButtonAction{Type: "text", Label: "OK", Payload: "\"OK\""}, "primary"}, + }, + }, + false, + }, + }, + { + []string{"Yes", "No", "Maybe"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + { + {vk.ButtonAction{Type: "text", Label: "Yes", Payload: "\"Yes\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "No", Payload: "\"No\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "Maybe", Payload: "\"Maybe\""}, "primary"}, + }, + }, + false, + }, + }, + { + []string{"Vanilla", "Chocolate", "Mint", "Lemon Sorbet", "Papaya", "Strawberry"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + {{vk.ButtonAction{Type: "text", Label: "Vanilla", Payload: "\"Vanilla\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Chocolate", Payload: "\"Chocolate\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Mint", Payload: "\"Mint\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Lemon Sorbet", Payload: "\"Lemon Sorbet\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Papaya", Payload: "\"Papaya\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Strawberry", Payload: "\"Strawberry\""}, "primary"}}, + }, + false, + }, + }, + { + []string{"A", "B", "C", "D", "Chicken", "Fish", "Peanut Butter Pickle"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + {{vk.ButtonAction{Type: "text", Label: "A", Payload: "\"A\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "B", Payload: "\"B\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "C", Payload: "\"C\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "D", Payload: "\"D\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Chicken", Payload: "\"Chicken\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Fish", Payload: "\"Fish\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Peanut Butter Pickle", Payload: "\"Peanut Butter Pickle\""}, "primary"}}, + }, + false, + }, + }, + { + []string{"A", "B", "C", "D", "E"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + { + {vk.ButtonAction{Type: "text", Label: "A", Payload: "\"A\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "B", Payload: "\"B\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "C", Payload: "\"C\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "D", Payload: "\"D\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "E", Payload: "\"E\""}, "primary"}, + }, + }, + false, + }, + }, + } + + for _, tc := range tcs { + kb := vk.NewKeyboardFromReplies(tc.replies) + assert.Equal(t, tc.expected, kb, "keyboard mismatch for replies %v", tc.replies) + } +} diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index 8fc2bbc43..1f39c9a85 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -19,6 +19,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/gocommon/urns" ) @@ -191,23 +192,6 @@ type mediaUploadInfoPayload struct { OwnerId int64 `json:"owner_id"` } -type Keyboard struct { - One_Time bool `json:"one_time"` - Buttons [][]buttonPayload `json:"buttons"` - Inline bool `json:"inline"` -} - -type buttonPayload struct { - Action buttonAction `json:"action"` - Color string `json:"color"` -} - -type buttonAction struct { - Type string `json:"type"` - Label string `json:"label"` - Payload string `json:"payload"` -} - // receiveEvent handles request event type func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { // read request body @@ -413,10 +397,10 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat params.Set(paramAttachments, attachments) if len(msg.QuickReplies()) != 0 { - keyboard := buildQuickRepliesParams(msg) - keyboardString := ToJSON(keyboard) + qrs := msg.QuickReplies() + keyboard := NewKeyboardFromReplies(qrs) - params.Set(paramKeyboard, keyboardString) + params.Set(paramKeyboard, string(jsonx.MustMarshal(keyboard))) } req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) @@ -445,58 +429,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } -// ToJSON returns the JSON encoding of Keyboard. -func ToJSON(keyboard Keyboard) string { - b, _ := json.Marshal(keyboard) - return string(b) -} - -//keyboard -func buildQuickRepliesParams(msg courier.Msg) Keyboard { - - var keyboard Keyboard - keyboard = AddRowKeyboard(keyboard) - - for _, qr := range msg.QuickReplies() { - keyboard = AddTextButton(qr, qr, "secondary", keyboard) - } - return keyboard -} - -func AddRowKeyboard(keyboard Keyboard) Keyboard { - if len(keyboard.Buttons) == 0 { - keyboard.Buttons = make([][]buttonPayload, 1) - } else { - row := make([]buttonPayload, 0) - keyboard.Buttons = append(keyboard.Buttons, row) - - } - keyboard.One_Time = true - - return keyboard -} - -func AddTextButton(label string, payload interface{}, color string, keyboard Keyboard) Keyboard { - b, err := json.Marshal(payload) - if err != nil { - panic(err) - } - - button := buttonPayload{ - Action: buttonAction{ - Type: "text", - Label: label, - Payload: string(b), - }, - Color: color, - } - - lastRow := len(keyboard.Buttons) - 1 - keyboard.Buttons[lastRow] = append(keyboard.Buttons[lastRow], button) - - return keyboard -} - // buildTextAndAttachmentParams builds msg text with attachment links (if needed) and attachments list param, also returns the errors that occurred func buildTextAndAttachmentParams(msg courier.Msg, status courier.MsgStatus) (string, string) { var msgAttachments []string diff --git a/handlers/vk/vk_test.go b/handlers/vk/vk_test.go index dbca85cfb..980755132 100644 --- a/handlers/vk/vk_test.go +++ b/handlers/vk/vk_test.go @@ -225,7 +225,7 @@ const msgKeyboard = `{ "secret": "abc123xyz" }` -const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"Yes","payload":"\"Yes\""},"color":"secondary"}]],"inline":false}` +const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"A","payload":"\"A\""},"color":"primary"},{"action":{"type":"text","label":"B","payload":"\"B\""},"color":"primary"},{"action":{"type":"text","label":"C","payload":"\"C\""},"color":"primary"},{"action":{"type":"text","label":"D","payload":"\"D\""},"color":"primary"},{"action":{"type":"text","label":"E","payload":"\"E\""},"color":"primary"}]],"inline":false}` var testCases = []ChannelHandleTestCase{ { @@ -476,7 +476,7 @@ var sendTestCases = []ChannelSendTestCase{ Label: "Send keyboard", Text: "Send keyboard", URN: "vk:123456789", - QuickReplies: []string{"Yes"}, + QuickReplies: []string{"A", "B", "C", "D", "E"}, Status: "S", SendPrep: setSendURL, ExternalID: "1", From 238050c36e57d55e32f7a994d4cac1aae90cbc68 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 4 Jan 2022 16:32:09 -0300 Subject: [PATCH 33/62] Remove unused Instagram-type configuration variables --- config.go | 94 +++++++++++++++++++++++------------------------- handlers/test.go | 2 -- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/config.go b/config.go index 336664e96..5e6447690 100644 --- a/config.go +++ b/config.go @@ -4,33 +4,31 @@ import "github.com/nyaruka/ezconf" // Config is our top level configuration object type Config struct { - Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` - SentryDSN string `help:"the DSN used for logging errors to Sentry"` - Domain string `help:"the domain courier is exposed on"` - Address string `help:"the network interface address courier will bind to"` - Port int `help:"the port courier will listen on"` - DB string `help:"URL describing how to connect to the RapidPro database"` - Redis string `help:"URL describing how to connect to Redis"` - SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` - S3Endpoint string `help:"the S3 endpoint we will write attachments to"` - S3Region string `help:"the S3 region we will write attachments to"` - S3MediaBucket string `help:"the S3 bucket we will write attachments to"` - S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` - S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` - S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` - AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` - AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - FacebookApplicationSecret string `help:"the Facebook app secret"` - FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` - InstagramApplicationSecret string `help:"the Instagram app secret"` - InstagramWebhookSecret string `help:"the secret for Instagram webhook URL verification"` - MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` - LibratoUsername string `help:"the username that will be used to authenticate to Librato"` - LibratoToken string `help:"the token that will be used to authenticate to Librato"` - StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` - StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` - LogLevel string `help:"the logging level courier should use"` - Version string `help:"the version that will be used in request and response headers"` + Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` + SentryDSN string `help:"the DSN used for logging errors to Sentry"` + Domain string `help:"the domain courier is exposed on"` + Address string `help:"the network interface address courier will bind to"` + Port int `help:"the port courier will listen on"` + DB string `help:"URL describing how to connect to the RapidPro database"` + Redis string `help:"URL describing how to connect to Redis"` + SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` + S3Endpoint string `help:"the S3 endpoint we will write attachments to"` + S3Region string `help:"the S3 region we will write attachments to"` + S3MediaBucket string `help:"the S3 bucket we will write attachments to"` + S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` + S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` + S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` + AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` + AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` + FacebookApplicationSecret string `help:"the Facebook app secret"` + FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` + MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` + LibratoUsername string `help:"the username that will be used to authenticate to Librato"` + LibratoToken string `help:"the token that will be used to authenticate to Librato"` + StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` + StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` + LogLevel string `help:"the logging level courier should use"` + Version string `help:"the version that will be used in request and response headers"` // IncludeChannels is the list of channels to enable, empty means include all IncludeChannels []string @@ -42,28 +40,26 @@ type Config struct { // NewConfig returns a new default configuration object func NewConfig() *Config { return &Config{ - Backend: "rapidpro", - Domain: "localhost", - Address: "", - Port: 8080, - DB: "postgres://temba:temba@localhost/temba?sslmode=disable", - Redis: "redis://localhost:6379/15", - SpoolDir: "/var/spool/courier", - S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", - S3MediaBucket: "courier-media", - S3MediaPrefix: "/media/", - S3DisableSSL: false, - S3ForcePathStyle: false, - AWSAccessKeyID: "", - AWSSecretAccessKey: "", - FacebookApplicationSecret: "missing_facebook_app_secret", - FacebookWebhookSecret: "missing_facebook_webhook_secret", - InstagramApplicationSecret: "missing_instagram_app_secret", - InstagramWebhookSecret: "missing_instagram_webhook_secret", - MaxWorkers: 32, - LogLevel: "error", - Version: "Dev", + Backend: "rapidpro", + Domain: "localhost", + Address: "", + Port: 8080, + DB: "postgres://temba:temba@localhost/temba?sslmode=disable", + Redis: "redis://localhost:6379/15", + SpoolDir: "/var/spool/courier", + S3Endpoint: "https://s3.amazonaws.com", + S3Region: "us-east-1", + S3MediaBucket: "courier-media", + S3MediaPrefix: "/media/", + S3DisableSSL: false, + S3ForcePathStyle: false, + AWSAccessKeyID: "", + AWSSecretAccessKey: "", + FacebookApplicationSecret: "missing_facebook_app_secret", + FacebookWebhookSecret: "missing_facebook_webhook_secret", + MaxWorkers: 32, + LogLevel: "error", + Version: "Dev", } } diff --git a/handlers/test.go b/handlers/test.go index bd1354628..9f5a88e3a 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -200,8 +200,6 @@ func newServer(backend courier.Backend) courier.Server { config := courier.NewConfig() config.FacebookWebhookSecret = "fb_webhook_secret" config.FacebookApplicationSecret = "fb_app_secret" - config.InstagramWebhookSecret = "ig_webhook_secret" - config.InstagramApplicationSecret = "ig_app_secret" return courier.NewServerWithLogger(config, backend, logger) From 42951fd54406b511d15f464b2ffa7420a7573eb2 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 4 Jan 2022 16:58:21 -0300 Subject: [PATCH 34/62] Add story mention skip test coverage --- handlers/facebookapp/facebookapp.go | 3 +-- handlers/facebookapp/facebookapp_test.go | 27 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index aa4c5b4eb..f9c1705b8 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -142,7 +142,6 @@ type moPayload struct { IsEcho bool `json:"is_echo"` MID string `json:"mid"` Text string `json:"text"` - IsDeleted bool `json:"is_deleted"` Attachments []struct { Type string `json:"type"` Payload *struct { @@ -228,7 +227,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } - // // is not a 'page' and 'instagram' object? ignore it + // is not a 'page' and 'instagram' object? ignore it if payload.Object != "page" && payload.Object != "instagram" { return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") } diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 008a2d840..95b06189a 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -639,6 +639,32 @@ var unkownMessagingEntryIG = `{ }] }` +var storyMentionIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"story_mention", + "payload":{ + "url":"https://story-url" + } + }] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + var notJSON = `blargh` var testCasesFBA = []ChannelHandleTestCase{ @@ -733,6 +759,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { From df30df8ad24bfa8192da1da4ebde82718797e740 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 11:56:38 -0300 Subject: [PATCH 35/62] Update to gocommon v1.15.1 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ccb11b67c..324f55a31 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.14.1 + github.com/nyaruka/gocommon v1.15.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 606fbc4d4..3826c538a 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.14.1 h1:/ScvLmg4zzVAuZ78TaENrvSEvW3WnUdqRd/t9hX7z7E= github.com/nyaruka/gocommon v1.14.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.15.1 h1:iMbI/CtCBNKSTl7ez+3tg+TGqQ1KqtIY4i4O5+dl1Tc= +github.com/nyaruka/gocommon v1.15.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 140b70c4be1bf2e85d95ff0f3117644bfa26cb2c Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 12:07:59 -0300 Subject: [PATCH 36/62] Change of urn in instagram tests --- handlers/facebookapp/facebookapp.go | 16 +++++++++--- handlers/facebookapp/facebookapp_test.go | 32 ++++++++++++------------ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index f9c1705b8..cbb6769b9 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -266,11 +266,21 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h sender = msg.Sender.ID } + var urn urns.URN + // create our URN - urn, err := urns.NewFacebookURN(sender) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + if payload.Object == "instagram" { + urn, err = urns.NewInstagramURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + } else { + urn, err = urns.NewFacebookURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } } + if msg.OptIn != nil { // this is an opt in, if we have a user_ref, use that as our URN (this is a checkbox plugin) // TODO: diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 95b06189a..7b75f97af 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -729,25 +729,25 @@ var testCasesFBA = []ChannelHandleTestCase{ } var testCasesIG = []ChannelHandleTestCase{ {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsgIG, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, - Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsgIG, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsgIG, Status: 200, Response: "Handled", - Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachmentIG, Status: 200, Response: "Handled", - Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", - Text: Sp(""), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", - URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, PrepRequest: addValidSignature}, @@ -758,7 +758,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, } @@ -952,63 +952,63 @@ var SendTestCasesFBA = []ChannelSendTestCase{ var SendTestCasesIG = []ChannelSendTestCase{ {Label: "Plain Send", - Text: "Simple Message", URN: "facebook:12345", + Text: "Simple Message", URN: "instagram:12345", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Plain Response", - Text: "Simple Message", URN: "facebook:12345", + Text: "Simple Message", URN: "instagram:12345", Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Quick Reply", - Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, + Text: "Are you happy?", URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", - URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Send Photo", - URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "Send caption and photo with Quick Reply", Text: "This is some text.", - URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Tag Human Agent", - Text: "Simple Message", URN: "facebook:12345", + Text: "Simple Message", URN: "instagram:12345", Status: "W", ExternalID: "mid.133", Topic: "agent", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Send Document", - URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + URN: "instagram:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "ID Error", - Text: "ID Error", URN: "facebook:12345", + Text: "ID Error", URN: "instagram:12345", Status: "E", ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, SendPrep: setSendURL}, {Label: "Error", - Text: "Error", URN: "facebook:12345", + Text: "Error", URN: "instagram:12345", Status: "E", ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, SendPrep: setSendURL}, From 4a7fcc92021fafebf8e64226baac9f564ed12f02 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 15:07:00 -0300 Subject: [PATCH 37/62] Rename validateSignatures parameter to useUUIDRoutes --- handlers/facebookapp/facebookapp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index cbb6769b9..d39a122a3 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -56,8 +56,8 @@ const ( payloadKey = "payload" ) -func newHandler(channelType courier.ChannelType, name string, validateSignatures bool) courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(channelType, name, validateSignatures)} +func newHandler(channelType courier.ChannelType, name string, useUUIDRoutes bool) courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(channelType, name, useUUIDRoutes)} } func init() { From eb6d0aa1ac91618d66fd212c84454a9cd0e9c154 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 15:18:58 -0300 Subject: [PATCH 38/62] Separate TestDescribe for type IG and FBA --- handlers/facebookapp/facebookapp_test.go | 74 +++++++++++------------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 7b75f97af..3123df7fe 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/assert" ) -var testChannels = []courier.Channel{ +var testChannelsFBA = []courier.Channel{ courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } @@ -797,51 +797,43 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { return server } -func TestDescribe(t *testing.T) { - var testCases [][]ChannelHandleTestCase - testCases = append(testCases, testCasesFBA) - testCases = append(testCases, testCasesIG) - - for i, tc := range testCases { - fbGraph := buildMockFBGraph(tc) - defer fbGraph.Close() - - if i == 0 { - handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{ - {"facebook:1337", map[string]string{"name": "John Doe"}}, - {"facebook:4567", map[string]string{"name": ""}}, - } +func TestDescribeFBA(t *testing.T) { + fbGraph := buildMockFBGraph(testCasesFBA) + defer fbGraph.Close() - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) - } - } else { - handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{ - {"facebook:1337", map[string]string{"name": "John Doe"}}, - {"facebook:4567", map[string]string{"name": ""}}, - } + handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"facebook:1337", map[string]string{"name": "John Doe"}}, + {"facebook:4567", map[string]string{"name": ""}}, + {"facebook:ref:1337", map[string]string{}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsFBA[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } +} - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) - } - } +func TestDescribeIG(t *testing.T) { + fbGraph := buildMockFBGraph(testCasesIG) + defer fbGraph.Close() + handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) } - } func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) } @@ -849,7 +841,7 @@ func TestHandler(t *testing.T) { func BenchmarkHandler(b *testing.B) { fbService := buildMockFBGraph(testCasesFBA) - RunChannelBenchmarks(b, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelBenchmarks(b, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) fbService.Close() fbServiceIG := buildMockFBGraph(testCasesIG) @@ -860,7 +852,7 @@ func BenchmarkHandler(b *testing.B) { func TestVerify(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ + RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/fba/receive", Status: 400, Response: "unknown request"}, From 4ade345ff956b19a1ae1cff147afab2cb0f63226 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 7 Jan 2022 13:26:42 -0500 Subject: [PATCH 39/62] Use dbutil package from gocommon --- backends/rapidpro/backend.go | 20 +++++++- backends/rapidpro/backend_test.go | 5 +- batch/batch.go | 82 ++----------------------------- batch/batch_test.go | 27 +++------- go.mod | 16 +++--- go.sum | 31 +++++++----- utils/misc.go | 10 ---- utils/misc_test.go | 5 -- 8 files changed, 57 insertions(+), 139 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 364d0d3fe..32e56c061 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -19,6 +19,7 @@ import ( "github.com/nyaruka/courier/batch" "github.com/nyaruka/courier/queue" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/librato" @@ -651,7 +652,15 @@ func (b *backend) Start() error { // create our status committer and start it b.statusCommitter = batch.NewCommitter("status committer", b.db, bulkUpdateMsgStatusSQL, time.Millisecond*500, b.committerWG, func(err error, value batch.Value) { - logrus.WithField("comp", "status committer").WithError(err).Error("error writing status") + log := logrus.WithField("comp", "status committer") + + if qerr := dbutil.AsQueryError(err); qerr != nil { + query, params := qerr.Query() + log = log.WithFields(logrus.Fields{"sql": query, "sql_params": params}) + } + + log.WithError(err).Error("error writing status") + err = courier.WriteToSpool(b.config.SpoolDir, "statuses", value) if err != nil { logrus.WithField("comp", "status committer").WithError(err).Error("error writing status to spool") @@ -662,7 +671,14 @@ func (b *backend) Start() error { // create our log committer and start it b.logCommitter = batch.NewCommitter("log committer", b.db, insertLogSQL, time.Millisecond*500, b.committerWG, func(err error, value batch.Value) { - logrus.WithField("comp", "log committer").WithError(err).Error("error writing channel log") + log := logrus.WithField("comp", "log committer") + + if qerr := dbutil.AsQueryError(err); qerr != nil { + query, params := qerr.Query() + log = log.WithFields(logrus.Fields{"sql": query, "sql_params": params}) + } + + log.WithError(err).Error("error writing channel log") }) b.logCommitter.Start() diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 74de96d55..df34f0af5 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -17,6 +17,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/queue" + "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/null" @@ -1172,9 +1173,7 @@ func (ts *BackendTestSuite) TestSessionTimeout() { ts.NoError(err) // make sure that took - count := 0 - ts.b.db.Get(&count, "SELECT count(*) from flows_flowsession WHERE timeout_on > NOW()") - ts.Equal(1, count) + assertdb.Query(ts.T(), ts.b.db, `SELECT count(*) from flows_flowsession WHERE timeout_on > NOW()`).Returns(1) } func (ts *BackendTestSuite) TestMailroomEvents() { diff --git a/batch/batch.go b/batch/batch.go index bfec78c46..1848ce911 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -2,11 +2,11 @@ package batch import ( "context" - "strings" "sync" "time" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/dbutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -147,7 +147,7 @@ func (c *committer) flush(size int) bool { err = batchSQL(ctx, c.label, c.db, c.sql, []interface{}{v}) if err != nil { if c.callback != nil { - c.callback(errors.Wrapf(err, "%s: error comitting value", c.label), v.(Value)) + c.callback(errors.Wrapf(err, "%s: error committing value", c.label), v.(Value)) } } } @@ -180,86 +180,12 @@ func batchSQL(ctx context.Context, label string, db *sqlx.DB, sql string, vs []i start := time.Now() - // this will be our SQL placeholders ($1, $2,..) for values in our final query, built dynamically - values := strings.Builder{} - values.Grow(7 * len(vs)) - - // this will be each of the arguments to match the positional values above - args := make([]interface{}, 0, len(vs)*5) - - // for each value we build a bound SQL statement, then extract the values clause - for i, value := range vs { - valueSQL, valueArgs, err := sqlx.Named(sql, value) - if err != nil { - return errors.Wrapf(err, "error converting bulk insert args") - } - - args = append(args, valueArgs...) - argValues, err := extractValues(valueSQL) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", valueSQL) - } - - // append to our global values, adding comma if necessary - values.WriteString(argValues) - if i+1 < len(vs) { - values.WriteString(",") - } - } - - valuesSQL, err := extractValues(sql) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", sql) - } - - bulkInsert := db.Rebind(strings.Replace(sql, valuesSQL, values.String(), -1)) - - // insert them all at once - rows, err := db.QueryxContext(ctx, bulkInsert, args...) + err := dbutil.BulkQuery(ctx, db, sql, vs) if err != nil { - return errors.Wrapf(err, "error during bulk insert") - } - defer rows.Close() - - // iterate our remaining rows - for rows.Next() { - } - - // check for any error - if rows.Err() != nil { - return errors.Wrapf(rows.Err(), "error in row cursor") + return err } logrus.WithField("elapsed", time.Since(start)).WithField("rows", len(vs)).Infof("%s bulk sql complete", label) return nil } - -// extractValues extracts the portion between `VALUE(` and `)` in the passed in string. (leaving VALUE but not the parentheses) -func extractValues(sql string) (string, error) { - startValues := strings.Index(sql, "VALUES(") - if startValues <= 0 { - return "", errors.Errorf("unable to find VALUES( in bulk insert SQL: %s", sql) - } - - // find the matching end parentheses, we need to count balanced parentheses here - openCount := 1 - endValues := -1 - for i, r := range sql[startValues+7:] { - if r == '(' { - openCount++ - } else if r == ')' { - openCount-- - if openCount == 0 { - endValues = i + startValues + 7 - break - } - } - } - - if endValues <= 0 { - return "", errors.Errorf("unable to find end of VALUES() in bulk insert sql: %s", sql) - } - - return sql[startValues+6 : endValues+1], nil -} diff --git a/batch/batch_test.go b/batch/batch_test.go index 73947201d..0c2de2d2d 100644 --- a/batch/batch_test.go +++ b/batch/batch_test.go @@ -8,6 +8,7 @@ import ( "github.com/jmoiron/sqlx" _ "github.com/lib/pq" + "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/stretchr/testify/assert" ) @@ -44,9 +45,7 @@ func TestBatchInsert(t *testing.T) { time.Sleep(time.Second) assert.NoError(t, callbackErr) - count := 0 - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 3, count) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(3) committer.Queue(&Label{0, "label4"}) committer.Queue(&Label{0, "label3"}) @@ -54,9 +53,8 @@ func TestBatchInsert(t *testing.T) { time.Sleep(time.Second) assert.Error(t, callbackErr) - assert.Equal(t, `labels: error comitting value: error during bulk insert: pq: duplicate key value violates unique constraint "labels_label_key"`, callbackErr.Error()) - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 4, count) + assert.Equal(t, `labels: error committing value: error making bulk query: pq: duplicate key value violates unique constraint "labels_label_key"`, callbackErr.Error()) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(4) } func TestBatchUpdate(t *testing.T) { @@ -94,17 +92,8 @@ func TestBatchUpdate(t *testing.T) { time.Sleep(time.Second) assert.NoError(t, callbackErr) - count := 0 - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 3, count) - - label := "" - db.Get(&label, "SELECT label FROM labels WHERE id = 1;") - assert.Equal(t, "label001", label) - - db.Get(&label, "SELECT label FROM labels WHERE id = 2;") - assert.Equal(t, "label02", label) - - db.Get(&label, "SELECT label FROM labels WHERE id = 3;") - assert.Equal(t, "label03", label) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(3) + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 1`).Returns("label001") + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 2`).Returns("label02") + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 3`).Returns("label03") } diff --git a/go.mod b/go.mod index ccb11b67c..6f95063a3 100644 --- a/go.mod +++ b/go.mod @@ -11,19 +11,17 @@ require ( github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 - github.com/go-playground/locales v0.11.2 // indirect - github.com/go-playground/universal-translator v0.16.0 // indirect - github.com/go-sql-driver/mysql v1.5.0 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/schema v1.0.2 - github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 + github.com/jmoiron/sqlx v1.3.4 github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect - github.com/lib/pq v1.0.0 - github.com/mattn/go-sqlite3 v1.10.0 // indirect + github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.14.1 + github.com/nyaruka/gocommon v1.16.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -32,8 +30,7 @@ require ( github.com/stretchr/testify v1.7.0 golang.org/x/mod v0.4.2 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/go-playground/assert.v1 v1.2.1 // indirect - gopkg.in/go-playground/validator.v9 v9.11.0 + gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/h2non/filetype.v1 v1.0.5 ) @@ -43,6 +40,7 @@ require ( github.com/golang/protobuf v1.3.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/phonenumbers v1.0.71 // indirect diff --git a/go.sum b/go.sum index 606fbc4d4..23c139a6c 100644 --- a/go.sum +++ b/go.sum @@ -23,10 +23,10 @@ github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyN github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-playground/locales v0.11.2 h1:wH6Ksuvzk0SU9M6wUeGz/EaRWnavAHCOsFre1njzgi8= -github.com/go-playground/locales v0.11.2/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= @@ -41,8 +41,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -52,18 +52,22 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.14.1 h1:/ScvLmg4zzVAuZ78TaENrvSEvW3WnUdqRd/t9hX7z7E= -github.com/nyaruka/gocommon v1.14.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.16.0 h1:F2DXo8075ErYm3pIJ5209HXRIyGWvwH8mtyxjwRYN0w= +github.com/nyaruka/gocommon v1.16.0/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= @@ -85,6 +89,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -115,8 +120,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.11.0 h1:ER548TqE6ZknRRDDo0/tP8I12UHYxNlIfss8tMd4iCo= -gopkg.in/go-playground/validator.v9 v9.11.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/utils/misc.go b/utils/misc.go index 222b00251..8f73b9a15 100644 --- a/utils/misc.go +++ b/utils/misc.go @@ -5,7 +5,6 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "encoding/json" "net/url" "path" "regexp" @@ -22,15 +21,6 @@ func SignHMAC256(privateKey string, value string) string { return signedParams } -// MapAsJSON serializes the given map as a JSON string -func MapAsJSON(m map[string]string) []byte { - bytes, err := json.Marshal(m) - if err != nil { - panic(err) - } - return bytes -} - // JoinNonEmpty takes a vararg of strings and return the join of all the non-empty strings with a delimiter between them func JoinNonEmpty(delim string, strings ...string) string { var buf bytes.Buffer diff --git a/utils/misc_test.go b/utils/misc_test.go index 23c30c26d..6e760b825 100644 --- a/utils/misc_test.go +++ b/utils/misc_test.go @@ -14,11 +14,6 @@ func TestSignHMAC256(t *testing.T) { assert.Len(t, utils.SignHMAC256("ZXwAumfRSejDxJGa", "newValueToEncrypt"), 64) } -func TestMapAsJSON(t *testing.T) { - assert.Equal(t, "{}", string(utils.MapAsJSON(map[string]string{}))) - assert.Equal(t, "{\"foo\":\"bar\"}", string(utils.MapAsJSON(map[string]string{"foo": "bar"}))) -} - func TestJoinNonEmpty(t *testing.T) { assert.Equal(t, "", utils.JoinNonEmpty(" ")) assert.Equal(t, "hello world", utils.JoinNonEmpty(" ", "", "hello", "", "world")) From 9ce5e88d648c5bb8eab265f00f702f1a13d8810f Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 7 Jan 2022 15:46:24 -0300 Subject: [PATCH 40/62] Change 'EntryID' to 'entryID' --- handlers/facebookapp/facebookapp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index d39a122a3..54fd3099e 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -185,13 +185,13 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no entries found") } - EntryID := payload.Entry[0].ID + entryID := payload.Entry[0].ID //if object is 'page' returns type FBA, if object is 'instagram' returns type IG if payload.Object == "page" { - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(EntryID)) + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(entryID)) } else { - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(EntryID)) + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(entryID)) } } From eef3716d879b9fc3d06570f24114e5220bcec7fa Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 7 Jan 2022 15:48:07 -0300 Subject: [PATCH 41/62] Fix variable names --- handlers/facebookapp/facebookapp_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 3123df7fe..72ce44003 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -607,7 +607,7 @@ var noMessagingEntriesIG = `{ }] }` -var unkownMessagingEntryFBA = `{ +var unknownMessagingEntryFBA = `{ "object":"page", "entry": [{ "id": "12345", @@ -623,7 +623,7 @@ var unkownMessagingEntryFBA = `{ }] }` -var unkownMessagingEntryIG = `{ +var unknownMessagingEntryIG = `{ "object":"instagram", "entry": [{ "id": "12345", From c97ea0b1bb1d0585aaf409fb2fab2ccd53902188 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 7 Jan 2022 16:03:10 -0300 Subject: [PATCH 42/62] Rename variable in test cases --- handlers/facebookapp/facebookapp_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 72ce44003..625664971 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -723,7 +723,7 @@ var testCasesFBA = []ChannelHandleTestCase{ {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, {Label: "No Entries", URL: "/c/fba/receive", Data: noEntriesFBA, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unknownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, } @@ -756,7 +756,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "No Entries", URL: "/c/ig/receive", Data: noEntriesIG, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unknownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, From 3163cff466705fae27c580ad5a137e436e6ac209 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 7 Jan 2022 14:20:43 -0500 Subject: [PATCH 43/62] Update CHANGELOG.md for v7.1.7 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a481824da..55cc8365e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.7 +---------- + * Use dbutil package from gocommon + * Add quick replies for vk + v7.1.6 ---------- * Throttle WA queues when we get 429 responses From db8727b3e9f678109e9697ca1c8a9bef073d8891 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 7 Jan 2022 15:09:09 -0500 Subject: [PATCH 44/62] Do more error wrapping when creating contacts and URNs --- backends/rapidpro/contact.go | 24 ++++++++++++------------ backends/rapidpro/msg.go | 7 ++++--- backends/rapidpro/urn.go | 9 +++++---- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/backends/rapidpro/contact.go b/backends/rapidpro/contact.go index 4384436af..722ab4c76 100644 --- a/backends/rapidpro/contact.go +++ b/backends/rapidpro/contact.go @@ -9,13 +9,14 @@ import ( "unicode/utf8" "github.com/nyaruka/courier" + "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/librato" "github.com/nyaruka/null" + "github.com/pkg/errors" "github.com/jmoiron/sqlx" - "github.com/lib/pq" "github.com/sirupsen/logrus" ) @@ -102,7 +103,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne err := b.db.GetContext(ctx, contact, lookupContactFromURNSQL, urn.Identity(), org) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") - return nil, err + return nil, errors.Wrap(err, "error looking up contact by URN") } // we found it, return it @@ -111,14 +112,14 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne tx, err := b.db.BeginTxx(ctx, nil) if err != nil { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") - return nil, err + return nil, errors.Wrap(err, "error beginning transaction") } err = setDefaultURN(tx, channel, contact, urn, auth) if err != nil { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") tx.Rollback() - return nil, err + return nil, errors.Wrap(err, "error setting default URN for contact") } return contact, tx.Commit() } @@ -166,13 +167,13 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne // insert it tx, err := b.db.BeginTxx(ctx, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error beginning transaction") } err = insertContact(tx, contact) if err != nil { tx.Rollback() - return nil, err + return nil, errors.Wrap(err, "error inserting contact") } // used for unit testing contact races @@ -186,13 +187,12 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne contactURN, err := contactURNForURN(tx, channel, contact.ID_, urn, auth) if err != nil { tx.Rollback() - if pqErr, ok := err.(*pq.Error); ok { + + if dbutil.IsUniqueViolation(err) { // if this was a duplicate URN, start over with a contact lookup - if pqErr.Code.Name() == "unique_violation" { - return contactForURN(ctx, b, org, channel, urn, auth, name) - } + return contactForURN(ctx, b, org, channel, urn, auth, name) } - return nil, err + return nil, errors.Wrap(err, "error getting URN for contact") } // we stole the URN from another contact, roll back and start over @@ -204,7 +204,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne // all is well, we created the new contact, commit and move forward err = tx.Commit() if err != nil { - return nil, err + return nil, errors.Wrap(err, "error commiting transaction") } // store this URN on our contact diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index a194a9363..cd96c5b4e 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -15,6 +15,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/pkg/errors" "mime" @@ -134,7 +135,7 @@ func writeMsgToDB(ctx context.Context, b *backend, m *DBMsg) error { // our db is down, write to the spool, we will write/queue this later if err != nil { - return err + return errors.Wrap(err, "error getting contact for message") } // set our contact and urn ids from our contact @@ -143,14 +144,14 @@ func writeMsgToDB(ctx context.Context, b *backend, m *DBMsg) error { rows, err := b.db.NamedQueryContext(ctx, insertMsgSQL, m) if err != nil { - return err + return errors.Wrap(err, "error inserting message") } defer rows.Close() rows.Next() err = rows.Scan(&m.ID_) if err != nil { - return err + return errors.Wrap(err, "error scanning for inserted message id") } // queue this up to be handled by RapidPro diff --git a/backends/rapidpro/urn.go b/backends/rapidpro/urn.go index 4e25cddd1..6e71e9402 100644 --- a/backends/rapidpro/urn.go +++ b/backends/rapidpro/urn.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/nyaruka/null" + "github.com/pkg/errors" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" @@ -209,14 +210,14 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn } err := db.Get(contactURN, selectOrgURN, channel.OrgID(), urn.Identity()) if err != nil && err != sql.ErrNoRows { - return nil, err + return nil, errors.Wrap(err, "error looking up URN by identity") } // we didn't find it, let's insert it if err == sql.ErrNoRows { err = insertContactURN(db, contactURN) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error inserting URN") } } @@ -232,7 +233,7 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn contactURN.Display = display err = updateContactURN(db, contactURN) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error updating URN") } } @@ -242,7 +243,7 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn err = updateContactURN(db, contactURN) } - return contactURN, err + return contactURN, errors.Wrap(err, "error updating URN auth") } const insertURN = ` From 207d7bf93b1c699fadf644defb8f6be7bbf163bd Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 09:24:59 -0500 Subject: [PATCH 45/62] Update CHANGELOG.md for v7.1.8 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55cc8365e..3148d5de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.8 +---------- + * Do more error wrapping when creating contacts and URNs + v7.1.7 ---------- * Use dbutil package from gocommon From 50064d1cbb3494b807a505a8a3783c12b91e29ba Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 10:26:19 -0500 Subject: [PATCH 46/62] Fix bulk status updates --- backends/rapidpro/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index a707bb7a4..998101042 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -305,7 +305,7 @@ WHERE msgs_msg.channel_id = s.channel_id::int AND msgs_msg.direction = 'O' RETURNING - msgs_msg.id + msgs_msg.id AS msg_id ` //----------------------------------------------------------------------------- From 2fd9ac0773b1e47bf5702978daa246e0fed59cde Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 11:01:17 -0500 Subject: [PATCH 47/62] Update CHANGELOG.md for v7.1.9 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3148d5de0..0668c1f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.9 +---------- + * Fix bulk status updates + v7.1.8 ---------- * Do more error wrapping when creating contacts and URNs From 63e6361dd4c3bd801d67f9e51b4193bcf1945c45 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 13:26:48 -0500 Subject: [PATCH 48/62] Update to latest gocommon --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6f95063a3..469cef140 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.16.0 + github.com/nyaruka/gocommon v1.16.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 23c139a6c..1126f1e6d 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.16.0 h1:F2DXo8075ErYm3pIJ5209HXRIyGWvwH8mtyxjwRYN0w= -github.com/nyaruka/gocommon v1.16.0/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.16.1 h1:RvIEx/w6a2c0T3eO+SaOGZjiajK4+t8S8Utqd3PnoGg= +github.com/nyaruka/gocommon v1.16.1/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 4a8a9e6919d6c2057b0add50a1a4763c8efe29c5 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 13:33:32 -0500 Subject: [PATCH 49/62] Update CHANGELOG.md for v7.1.10 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0668c1f39..b62e7a96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.10 +---------- + * Update to latest gocommon + v7.1.9 ---------- * Fix bulk status updates From 6f444e72c3940eab8fbc635ab64193a8a7b32514 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 15:40:02 -0500 Subject: [PATCH 50/62] More bulk sql tweaks --- backends/rapidpro/status.go | 2 -- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index 998101042..56759fcf2 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -304,8 +304,6 @@ WHERE msgs_msg.id = s.msg_id::bigint AND msgs_msg.channel_id = s.channel_id::int AND msgs_msg.direction = 'O' -RETURNING - msgs_msg.id AS msg_id ` //----------------------------------------------------------------------------- diff --git a/go.mod b/go.mod index 469cef140..34edb620f 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.16.1 + github.com/nyaruka/gocommon v1.16.2 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 1126f1e6d..a4af1d246 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.16.1 h1:RvIEx/w6a2c0T3eO+SaOGZjiajK4+t8S8Utqd3PnoGg= -github.com/nyaruka/gocommon v1.16.1/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.16.2 h1:fvDKKRVsaES76SyvWujDttwplVWVZwHL3AJNbk+rDno= +github.com/nyaruka/gocommon v1.16.2/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 71d5ddf7cb05115a8e67a72b548848e29e116b46 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 15:47:14 -0500 Subject: [PATCH 51/62] Update CHANGELOG.md for v7.1.11 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b62e7a96d..a102cf7fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.11 +---------- + * More bulk sql tweaks + v7.1.10 ---------- * Update to latest gocommon From 7e8a19845aef536f38b2fffec6f51fa2f026bdd6 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 12 Jan 2022 16:53:54 -0500 Subject: [PATCH 52/62] Update to latest gocommon --- go.mod | 9 +++++---- go.sum | 18 ++++++++++++++---- handlers/whatsapp/whatsapp.go | 7 ++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 34edb620f..bb6f37861 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/nyaruka/courier +go 1.17 + require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect @@ -14,16 +16,17 @@ require ( github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible - github.com/gomodule/redigo v2.0.0+incompatible + github.com/gomodule/redigo v1.8.8 github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v1.3.4 github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.16.2 + github.com/nyaruka/gocommon v1.17.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 + github.com/nyaruka/redisx v0.2.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 @@ -51,5 +54,3 @@ require ( golang.org/x/text v0.3.6 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) - -go 1.17 diff --git a/go.sum b/go.sum index a4af1d246..5db31a84a 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/aws/aws-sdk-go v1.34.31/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= @@ -33,8 +34,8 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= +github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -66,14 +67,18 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.16.2 h1:fvDKKRVsaES76SyvWujDttwplVWVZwHL3AJNbk+rDno= -github.com/nyaruka/gocommon v1.16.2/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= +github.com/nyaruka/gocommon v1.17.0 h1:cTiDLSUgmYJ9OZw752jva0P2rz0utRtv5WGuKFc9kxw= +github.com/nyaruka/gocommon v1.17.0/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= +github.com/nyaruka/phonenumbers v1.0.58/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/nyaruka/phonenumbers v1.0.71 h1:itkCGhxkQkHrJ6OyZSApdjQVlPmrWs88MF283pPvbFU= github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/redisx v0.2.1 h1:BavpQRCsK5xV2uxPdJJ26yVmjSo+q6bdjWqeNNf0s5w= +github.com/nyaruka/redisx v0.2.1/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -94,21 +99,26 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index ee9f30022..a05061655 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -16,8 +16,8 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/rcache" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/redisx" "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -765,7 +765,8 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri defer rc.Close() cacheKey := fmt.Sprintf(mediaCacheKeyPattern, msg.Channel().UUID().String()) - mediaID, err := rcache.Get(rc, cacheKey, mediaURL) + mediaCache := redisx.NewIntervalHash(cacheKey, time.Hour*24, 2) + mediaID, err := mediaCache.Get(rc, mediaURL) if err != nil { return "", logs, errors.Wrapf(err, "error reading media id from redis: %s : %s", cacheKey, mediaURL) } else if mediaID != "" { @@ -823,7 +824,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri } // put in cache - err = rcache.Set(rc, cacheKey, mediaURL, mediaID) + err = mediaCache.Set(rc, mediaURL, mediaID) if err != nil { return "", logs, errors.Wrapf(err, "error setting media id in cache") } From 966c851c1885a7478f3a52ca761f9649f0760e12 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 13 Jan 2022 09:07:33 -0500 Subject: [PATCH 53/62] Update CHANGELOG.md for v7.1.12 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a102cf7fb..97b05ce13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.12 +---------- + * Update to latest gocommon + * Add instagram handler + v7.1.11 ---------- * More bulk sql tweaks From 93f74a0c454b356ac2217ce41e2e4dea78ed9bfe Mon Sep 17 00:00:00 2001 From: alviriseup Date: Tue, 18 Jan 2022 18:57:46 +0600 Subject: [PATCH 54/62] Added Session Status and modified test case --- handlers/firebase/firebase.go | 12 +++++++----- handlers/firebase/firebase_test.go | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/handlers/firebase/firebase.go b/handlers/firebase/firebase.go index 70e6ced82..5e9d414e8 100644 --- a/handlers/firebase/firebase.go +++ b/handlers/firebase/firebase.go @@ -117,11 +117,12 @@ func (h *handler) registerContact(ctx context.Context, channel courier.Channel, type mtPayload struct { Data struct { - Type string `json:"type"` - Title string `json:"title"` - Message string `json:"message"` - MessageID int64 `json:"message_id"` - QuickReplies []string `json:"quick_replies,omitempty"` + Type string `json:"type"` + Title string `json:"title"` + Message string `json:"message"` + MessageID int64 `json:"message_id"` + SessionStatus string `json:"session_status"` + QuickReplies []string `json:"quick_replies,omitempty"` } `json:"data"` Notification *mtNotification `json:"notification,omitempty"` ContentAvailable bool `json:"content_available"` @@ -162,6 +163,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload.Data.Title = title payload.Data.Message = part payload.Data.MessageID = int64(msg.ID()) + payload.Data.SessionStatus = msg.SessionStatus() // include any quick replies on the last piece we send if i == len(msgParts)-1 { diff --git a/handlers/firebase/firebase_test.go b/handlers/firebase/firebase_test.go index b245946d7..3d4498602 100644 --- a/handlers/firebase/firebase_test.go +++ b/handlers/firebase/firebase_test.go @@ -77,7 +77,7 @@ var notificationSendTestCases = []ChannelSendTestCase{ Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10},"notification":{"title":"FCMTitle","body":"Simple Message"},"content_available":true,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10,"session_status":""},"notification":{"title":"FCMTitle","body":"Simple Message"},"content_available":true,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, } @@ -87,22 +87,22 @@ var sendTestCases = []ChannelSendTestCase{ Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10,"session_status":""},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Long Message", Text: longMsg, - URN: "fcm:250788123123", URNAuth: "auth1", + URN: "fcm:250788123123", URNAuth: "auth1", Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"ate ac.","message_id":10},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"ate ac.","message_id":10,"session_status":""},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Quick Reply", Text: "Simple Message", URN: "fcm:250788123123", URNAuth: "auth1", QuickReplies: []string{"yes", "no"}, Attachments: []string{"image/jpeg:https://foo.bar"}, Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message\nhttps://foo.bar","message_id":10,"quick_replies":["yes","no"]},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message\nhttps://foo.bar","message_id":10,"session_status":"","quick_replies":["yes","no"]},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Error", Text: "Error", URN: "fcm:250788123123", URNAuth: "auth1", From a23b58d0d223b1ada1a487c95b1fc7f0cf137923 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 18 Jan 2022 10:41:51 -0500 Subject: [PATCH 55/62] Send db and redis stats to librato in backed heartbeat --- backends/rapidpro/backend.go | 43 ++++++++++++++++++++++++++----- backends/rapidpro/backend_test.go | 5 ++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 32e56c061..6aa335f32 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" @@ -438,10 +437,39 @@ func (b *backend) Heartbeat() error { bulkSize += count } - // log our total + // get our DB and redis stats + dbStats := b.db.Stats() + redisStats := b.redisPool.Stats() + + dbWaitDurationInPeriod := dbStats.WaitDuration - b.dbWaitDuration + dbWaitCountInPeriod := dbStats.WaitCount - b.dbWaitCount + redisWaitDurationInPeriod := redisStats.WaitDuration - b.redisWaitDuration + redisWaitCountInPeriod := redisStats.WaitCount - b.redisWaitCount + + b.dbWaitDuration = dbStats.WaitDuration + b.dbWaitCount = dbStats.WaitCount + b.redisWaitDuration = redisStats.WaitDuration + b.redisWaitCount = redisStats.WaitCount + + librato.Gauge("courier.db_busy", float64(dbStats.InUse)) + librato.Gauge("courier.db_idle", float64(dbStats.Idle)) + librato.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) + librato.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod)) + librato.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) + librato.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod)) librato.Gauge("courier.bulk_queue", float64(bulkSize)) librato.Gauge("courier.priority_queue", float64(prioritySize)) - logrus.WithField("bulk_queue", bulkSize).WithField("priority_queue", prioritySize).Info("heartbeat queue sizes calculated") + + logrus.WithFields(logrus.Fields{ + "db_busy": dbStats.InUse, + "db_idle": dbStats.Idle, + "db_wait_time": dbWaitDurationInPeriod, + "db_wait_count": dbWaitCountInPeriod, + "redis_wait_time": dbWaitDurationInPeriod, + "redis_wait_count": dbWaitCountInPeriod, + "priority_size": prioritySize, + "bulk_size": bulkSize, + }).Info("current analytics") return nil } @@ -753,10 +781,13 @@ type backend struct { db *sqlx.DB redisPool *redis.Pool storage storage.Storage - awsCreds *credentials.Credentials - - popScript *redis.Script stopChan chan bool waitGroup *sync.WaitGroup + + // both sqlx and redis provide wait stats which are cummulative that we need to convert into increments + dbWaitDuration time.Duration + dbWaitCount int64 + redisWaitDuration time.Duration + redisWaitCount int64 } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index df34f0af5..07298585e 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -680,6 +680,11 @@ func (ts *BackendTestSuite) TestHealth() { ts.Equal(ts.b.Health(), "") } +func (ts *BackendTestSuite) TestHeartbeat() { + // TODO make analytics abstraction layer so we can test what we report + ts.NoError(ts.b.Heartbeat()) +} + func (ts *BackendTestSuite) TestDupes() { r := ts.b.redisPool.Get() defer r.Close() From fd95b68b39c1b1d0d66afec230fe24aef7db0d5c Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 18 Jan 2022 12:02:12 -0500 Subject: [PATCH 56/62] Update CHANGELOG.md for v7.1.13 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b05ce13..e2074147a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.13 +---------- + * Send db and redis stats to librato in backed heartbeat + * Include session_status in FCM payloads + v7.1.12 ---------- * Update to latest gocommon From 54ca176b9462a43590b4af7a02c2f2e514fe81b4 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 19 Jan 2022 12:17:35 +0200 Subject: [PATCH 57/62] Add support to receive button text from Twilio WhatsApp --- handlers/twiml/twiml.go | 8 +++++++- handlers/twiml/twiml_test.go | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/handlers/twiml/twiml.go b/handlers/twiml/twiml.go index c3fde453f..220a1cac5 100644 --- a/handlers/twiml/twiml.go +++ b/handlers/twiml/twiml.go @@ -79,6 +79,7 @@ type moForm struct { To string `validate:"required"` ToCountry string Body string + ButtonText string NumMedia int } @@ -138,8 +139,13 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w form.Body = handlers.DecodePossibleBase64(form.Body) } + text := form.Body + if channel.IsScheme(urns.WhatsAppScheme) && form.ButtonText != "" { + text = form.ButtonText + } + // build our msg - msg := h.Backend().NewIncomingMsg(channel, urn, form.Body).WithExternalID(form.MessageSID) + msg := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(form.MessageSID) // process any attached media for i := 0; i < form.NumMedia; i++ { diff --git a/handlers/twiml/twiml_test.go b/handlers/twiml/twiml_test.go index 19ff7d98b..9dd223e64 100644 --- a/handlers/twiml/twiml_test.go +++ b/handlers/twiml/twiml_test.go @@ -54,10 +54,11 @@ var ( twaStatusIDURL = "/c/twa/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status?id=12345" twaStatusInvalidIDURL = "/c/twa/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status?id=asdf" - receiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" - receiveMedia = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" - receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" - receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveButtonIgnored = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&ButtonText=Confirm&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveMedia = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" + receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" + receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" statusInvalid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=huh" statusValid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=delivered" @@ -67,6 +68,7 @@ var ( tmsReceiveExtra = "ToCountry=US&ToState=&SmsMessageSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&NumMedia=0&ToCity=&FromZip=27609&SmsSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&FromState=NC&SmsStatus=received&FromCity=RALEIGH&Body=John+Cruz&FromCountry=US&To=384387&ToZip=&NumSegments=1&MessageSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" waReceiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=whatsapp:%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=whatsapp:%2B14133881111&ApiVersion=2010-04-01" + waReceiveButtonValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&ButtonText=Confirm&FromCountry=US&To=whatsapp:%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=whatsapp:%2B14133881111&ApiVersion=2010-04-01" waReceivePrefixlessURN = "ToCountry=US&ToState=CA&SmsMessageSid=SM681a1f26d9ec591431ce406e8f399525&NumMedia=0&ToCity=&FromZip=60625&SmsSid=SM681a1f26d9ec591431ce406e8f399525&FromState=IL&SmsStatus=received&FromCity=CHICAGO&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SM681a1f26d9ec591431ce406e8f399525&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" ) @@ -74,6 +76,9 @@ var testCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: receiveURL, Data: receiveValid, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Receive Button Ignored", URL: receiveURL, Data: receiveButtonIgnored, Status: 200, Response: "", + Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), + PrepRequest: addValidSignature}, {Label: "Receive Invalid Signature", URL: receiveURL, Data: receiveValid, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, {Label: "Receive Missing Signature", URL: receiveURL, Data: receiveValid, Status: 400, Response: "missing request signature"}, @@ -199,6 +204,9 @@ var twaTestCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: twaReceiveURL, Data: waReceiveValid, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Receive Valid", URL: twaReceiveURL, Data: waReceiveButtonValid, Status: 200, Response: "", + Text: Sp("Confirm"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), + PrepRequest: addValidSignature}, {Label: "Receive Prefixless URN", URL: twaReceiveURL, Data: waReceivePrefixlessURN, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SM681a1f26d9ec591431ce406e8f399525"), PrepRequest: addValidSignature}, From e3f960077c4f0f6cc9703994aaf17d75d5444a9d Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 19 Jan 2022 14:51:11 +0200 Subject: [PATCH 58/62] Support sending WA quick replies when we have attachments too --- handlers/whatsapp/whatsapp.go | 72 ++++++++++++++++++++++++++---- handlers/whatsapp/whatsapp_test.go | 46 +++++++++++++++++++ 2 files changed, 109 insertions(+), 9 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index a05061655..542ddbc45 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -568,6 +568,13 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann var logs []*courier.ChannelLog var err error + parts := handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + + qrs := msg.QuickReplies() + wppVersion := msg.Channel().ConfigForKey("version", "0").(string) + isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) + isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) + if len(msg.Attachments()) > 0 { for attachmentCount, attachment := range msg.Attachments() { @@ -595,7 +602,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "document", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } mediaPayload.Filename, err = utils.BasePathForURL(mediaURL) @@ -611,7 +618,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "image", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } payload.Image = mediaPayload @@ -621,7 +628,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "video", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } payload.Video = mediaPayload @@ -634,6 +641,59 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann break } } + + if isInteractiveMsg { + for i, part := range parts { + if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive + payload := mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + payload.Text.Body = part + payloads = append(payloads, payload) + + } else { + payload := mtInteractivePayload{ + To: msg.URN().Path(), + Type: "interactive", + } + + // up to 3 qrs the interactive message will be button type, otherwise it will be list + if len(qrs) <= 3 { + payload.Interactive.Type = "button" + payload.Interactive.Body.Text = part + btns := make([]mtButton, len(qrs)) + for i, qr := range qrs { + btns[i] = mtButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + payload.Interactive.Action.Buttons = btns + payloads = append(payloads, payload) + } else { + payload.Interactive.Type = "list" + payload.Interactive.Body.Text = part + payload.Interactive.Action.Button = "Menu" + section := mtSection{ + Rows: make([]mtSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = mtSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + payload.Interactive.Action.Sections = []mtSection{ + section, + } + payloads = append(payloads, payload) + } + } + } + } + } else { // do we have a template? var templating *MsgTemplating @@ -684,12 +744,6 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann payloads = append(payloads, payload) } } else { - parts := handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) - - qrs := msg.QuickReplies() - wppVersion := msg.Channel().ConfigForKey("version", "0").(string) - isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) - isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) if isInteractiveMsg { for i, part := range parts { diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 653a62051..edb54c7bb 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -605,6 +605,52 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, SendPrep: setSendURL}, + {Label: "Interactive Button Message Send with attachment", + Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + {Label: "Interactive List Message Send with attachment", + Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, } var mediaCacheSendTestCases = []ChannelSendTestCase{ From 8b0131886c730175831bd6b0b12a69c3b7a9f155 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 19 Jan 2022 09:10:22 -0500 Subject: [PATCH 59/62] Allow more active redis connections --- backends/rapidpro/backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 6aa335f32..a36e87888 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -592,11 +592,11 @@ func (b *backend) Start() error { // create our pool redisPool := &redis.Pool{ Wait: true, // makes callers wait for a connection - MaxActive: 8, // only open this many concurrent connections at once + MaxActive: 36, // only open this many concurrent connections at once MaxIdle: 4, // only keep up to this many idle IdleTimeout: 240 * time.Second, // how long to wait before reaping a connection Dial: func() (redis.Conn, error) { - conn, err := redis.Dial("tcp", fmt.Sprintf("%s", redisURL.Host)) + conn, err := redis.Dial("tcp", redisURL.Host) if err != nil { return nil, err } From cbfd24d843f2ab48584a2ea833426692c23cac57 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 19 Jan 2022 09:59:06 -0500 Subject: [PATCH 60/62] Update CHANGELOG.md for v7.1.14 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2074147a..d0ee8624b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v7.1.14 +---------- + * Allow more active redis connections + * Support sending WA quick replies when we have attachments too + * Add support to receive button text from Twilio WhatsApp + v7.1.13 ---------- * Send db and redis stats to librato in backed heartbeat From 4be66532ff64bc2e0ba03dede04cb4914ac91cc4 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 25 Jan 2022 18:28:51 -0300 Subject: [PATCH 61/62] Fix payload build --- go.sum | 8 ++++ handlers/whatsapp/whatsapp.go | 69 +++++++++++++++++++++++++++++++---- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/go.sum b/go.sum index 5db31a84a..b8190da2f 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -68,6 +70,12 @@ github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4 github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= +github.com/nyaruka/gocommon v1.14.1 h1:/ScvLmg4zzVAuZ78TaENrvSEvW3WnUdqRd/t9hX7z7E= +github.com/nyaruka/gocommon v1.14.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.15.0 h1:n0jdOEkorVIxyD+53XHRi49d/C3hD1q70o9HILnfz9U= +github.com/nyaruka/gocommon v1.15.0/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.16.0 h1:F2DXo8075ErYm3pIJ5209HXRIyGWvwH8mtyxjwRYN0w= +github.com/nyaruka/gocommon v1.16.0/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/gocommon v1.17.0 h1:cTiDLSUgmYJ9OZw752jva0P2rz0utRtv5WGuKFc9kxw= github.com/nyaruka/gocommon v1.17.0/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 630ff49fa..14d9b02a7 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -620,7 +620,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetStatus(courier.MsgWired) } - return status, nil } @@ -630,14 +629,14 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann var logs []*courier.ChannelLog var err error - // do we have a template? + //do we have a template? templating, err := h.getTemplate(msg) - if templating != nil { - - if err != nil { + if err != nil { return nil, nil, errors.Wrapf(err, "unable to decode template: %s for channel: %s", string(msg.Metadata()), msg.Channel().UUID()) - } - + } + + if templating != nil { + namespace := templating.Namespace if namespace == "" { namespace = msg.Channel().StringConfigForKey(configNamespace, "") @@ -799,6 +798,58 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } } + if isInteractiveMsg { + for i, part := range parts { + if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive + payload := mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + payload.Text.Body = part + payloads = append(payloads, payload) + + } else { + payload := mtInteractivePayload{ + To: msg.URN().Path(), + Type: "interactive", + } + + // up to 3 qrs the interactive message will be button type, otherwise it will be list + if len(qrs) <= 3 { + payload.Interactive.Type = "button" + payload.Interactive.Body.Text = part + btns := make([]mtButton, len(qrs)) + for i, qr := range qrs { + btns[i] = mtButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + payload.Interactive.Action.Buttons = btns + payloads = append(payloads, payload) + } else { + payload.Interactive.Type = "list" + payload.Interactive.Body.Text = part + payload.Interactive.Action.Button = "Menu" + section := mtSection{ + Rows: make([]mtSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = mtSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + payload.Interactive.Action.Sections = []mtSection{ + section, + } + payloads = append(payloads, payload) + } + } + } + } + } else { if isInteractiveMsg { for i, part := range parts { if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive @@ -859,8 +910,10 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann payloads = append(payloads, payload) } } + } - } + + } return payloads, logs, err } From 2666dcf5e920339b1a65a9a6fb6839e965b46d95 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 26 Jan 2022 10:02:57 -0300 Subject: [PATCH 62/62] Fix facebookapp handler tests --- handlers/facebookapp/facebookapp_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index a4e0e0324..1a402116d 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -643,12 +643,12 @@ var customerFeedbackResponse = `{ "object": "page", "entry": [ { - "id": "1234", + "id": "12345", "time": 1459991487970, "messaging": [ { "recipient": { - "id": "1234" + "id": "12345" }, "timestamp": 1459991487970, "sender": { @@ -759,7 +759,7 @@ var testCasesFBA = []ChannelHandleTestCase{ {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unknownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURN, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, {Label: "Receive Customer Feedback Message", URL: "/c/fba/receive", Data: customerFeedbackResponse, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("4"), URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature},