From 1e419907d2cd316d32a59ad3fcd7f831c8160d63 Mon Sep 17 00:00:00 2001 From: milan-elastic Date: Wed, 20 Sep 2023 18:50:30 +0530 Subject: [PATCH 1/4] add request timeout parameter in configuration --- packages/salesforce/_dev/build/docs/README.md | 8 ++++++++ packages/salesforce/changelog.yml | 5 +++++ .../data_stream/apex/agent/stream/httpjson.yml.hbs | 3 +++ packages/salesforce/data_stream/apex/manifest.yml | 8 ++++++++ .../data_stream/login_rest/agent/stream/httpjson.yml.hbs | 3 +++ packages/salesforce/data_stream/login_rest/manifest.yml | 8 ++++++++ .../data_stream/logout_rest/agent/stream/httpjson.yml.hbs | 3 +++ packages/salesforce/data_stream/logout_rest/manifest.yml | 8 ++++++++ .../setupaudittrail/agent/stream/httpjson.yml.hbs | 3 +++ .../salesforce/data_stream/setupaudittrail/manifest.yml | 8 ++++++++ packages/salesforce/docs/README.md | 8 ++++++++ packages/salesforce/manifest.yml | 2 +- 12 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/salesforce/_dev/build/docs/README.md b/packages/salesforce/_dev/build/docs/README.md index 64f9d1f99fa..d5747f3e815 100644 --- a/packages/salesforce/_dev/build/docs/README.md +++ b/packages/salesforce/_dev/build/docs/README.md @@ -151,6 +151,14 @@ After the integration is successfully configured, clicking on the Assets tab of ## Troubleshooting +### Request timeout + +In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the response is getting delayed from the Salesforce server side due to any reason then the following error might occur. +``` +Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: failed to execute http client.Do: failed to read http.response.body +``` +In this case, consider increasing `Request Timeout` configuration from `Advanced options` section of that data stream. + ### Data ingestion error In case of data ingestion if the user finds the following type of error logs: diff --git a/packages/salesforce/changelog.yml b/packages/salesforce/changelog.yml index 0e1a4955ba7..27d493f587c 100644 --- a/packages/salesforce/changelog.yml +++ b/packages/salesforce/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.10.1" + changes: + - description: Add 'Request Timeout' configuration. + type: enhancement + link: https://github.com/elastic/integrations/pull/1 #FIX ME - version: "0.10.0" changes: - description: Rename ownership from obs-service-integrations to obs-infraobs-integrations diff --git a/packages/salesforce/data_stream/apex/agent/stream/httpjson.yml.hbs b/packages/salesforce/data_stream/apex/agent/stream/httpjson.yml.hbs index 87b94b8a800..655d199cda7 100644 --- a/packages/salesforce/data_stream/apex/agent/stream/httpjson.yml.hbs +++ b/packages/salesforce/data_stream/apex/agent/stream/httpjson.yml.hbs @@ -11,6 +11,9 @@ auth.oauth2: token_url: {{token_url}} user: {{username}} password: {{password}} +{{#if request_timeout}} +request.timeout: {{request_timeout}} +{{/if}} request.url: {{instance_url}}/services/data/v54.0/query?q=apex request.transforms: - set: diff --git a/packages/salesforce/data_stream/apex/manifest.yml b/packages/salesforce/data_stream/apex/manifest.yml index d9ca8a8e379..089459def87 100644 --- a/packages/salesforce/data_stream/apex/manifest.yml +++ b/packages/salesforce/data_stream/apex/manifest.yml @@ -28,6 +28,14 @@ streams: type: bool multi: false default: false + - name: request_timeout + type: text + title: Request Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + show_user: false + multi: false + required: false + default: 5m - name: processors type: yaml title: Processors diff --git a/packages/salesforce/data_stream/login_rest/agent/stream/httpjson.yml.hbs b/packages/salesforce/data_stream/login_rest/agent/stream/httpjson.yml.hbs index d1d1a8547c5..1b5d7f215b5 100644 --- a/packages/salesforce/data_stream/login_rest/agent/stream/httpjson.yml.hbs +++ b/packages/salesforce/data_stream/login_rest/agent/stream/httpjson.yml.hbs @@ -11,6 +11,9 @@ auth.oauth2: token_url: {{token_url}} user: {{username}} password: {{password}} +{{#if request_timeout}} +request.timeout: {{request_timeout}} +{{/if}} request.url: {{instance_url}}/services/data/v54.0/query?q=login+rest request.transforms: - set: diff --git a/packages/salesforce/data_stream/login_rest/manifest.yml b/packages/salesforce/data_stream/login_rest/manifest.yml index d80a3b7c85f..595e820ebf3 100644 --- a/packages/salesforce/data_stream/login_rest/manifest.yml +++ b/packages/salesforce/data_stream/login_rest/manifest.yml @@ -28,6 +28,14 @@ streams: type: bool multi: false default: false + - name: request_timeout + type: text + title: Request Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + show_user: false + multi: false + required: false + default: 5m - name: processors type: yaml title: Processors diff --git a/packages/salesforce/data_stream/logout_rest/agent/stream/httpjson.yml.hbs b/packages/salesforce/data_stream/logout_rest/agent/stream/httpjson.yml.hbs index 6b172fc59e8..281f068bdf1 100644 --- a/packages/salesforce/data_stream/logout_rest/agent/stream/httpjson.yml.hbs +++ b/packages/salesforce/data_stream/logout_rest/agent/stream/httpjson.yml.hbs @@ -11,6 +11,9 @@ auth.oauth2: token_url: {{token_url}} user: {{username}} password: {{password}} +{{#if request_timeout}} +request.timeout: {{request_timeout}} +{{/if}} request.url: {{instance_url}}/services/data/v54.0/query?q=logout+rest request.transforms: - set: diff --git a/packages/salesforce/data_stream/logout_rest/manifest.yml b/packages/salesforce/data_stream/logout_rest/manifest.yml index bbaa28038d8..8ac53f8dd88 100644 --- a/packages/salesforce/data_stream/logout_rest/manifest.yml +++ b/packages/salesforce/data_stream/logout_rest/manifest.yml @@ -28,6 +28,14 @@ streams: type: bool multi: false default: false + - name: request_timeout + type: text + title: Request Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + show_user: false + multi: false + required: false + default: 5m - name: processors type: yaml title: Processors diff --git a/packages/salesforce/data_stream/setupaudittrail/agent/stream/httpjson.yml.hbs b/packages/salesforce/data_stream/setupaudittrail/agent/stream/httpjson.yml.hbs index 0b1c74333f4..cd53a5ae532 100644 --- a/packages/salesforce/data_stream/setupaudittrail/agent/stream/httpjson.yml.hbs +++ b/packages/salesforce/data_stream/setupaudittrail/agent/stream/httpjson.yml.hbs @@ -11,6 +11,9 @@ auth.oauth2: token_url: {{token_url}} user: {{username}} password: {{password}} +{{#if request_timeout}} +request.timeout: {{request_timeout}} +{{/if}} request.url: {{instance_url}}/services/data/v54.0/query?q=setupaudittrail+rest request.transforms: - set: diff --git a/packages/salesforce/data_stream/setupaudittrail/manifest.yml b/packages/salesforce/data_stream/setupaudittrail/manifest.yml index 949cc70ca7c..f65a6d32a22 100644 --- a/packages/salesforce/data_stream/setupaudittrail/manifest.yml +++ b/packages/salesforce/data_stream/setupaudittrail/manifest.yml @@ -28,6 +28,14 @@ streams: type: bool multi: false default: false + - name: request_timeout + type: text + title: Request Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + show_user: false + multi: false + required: false + default: 5m - name: processors type: yaml title: Processors diff --git a/packages/salesforce/docs/README.md b/packages/salesforce/docs/README.md index e30ac2dffb1..174fa1b3de3 100644 --- a/packages/salesforce/docs/README.md +++ b/packages/salesforce/docs/README.md @@ -151,6 +151,14 @@ After the integration is successfully configured, clicking on the Assets tab of ## Troubleshooting +### Request timeout + +In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the response is getting delayed from the Salesforce server side due to any reason then the following error might occur. +``` +Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: failed to execute http client.Do: failed to read http.response.body +``` +In this case, consider increasing `Request Timeout` configuration from `Advanced options` section of that data stream. + ### Data ingestion error In case of data ingestion if the user finds the following type of error logs: diff --git a/packages/salesforce/manifest.yml b/packages/salesforce/manifest.yml index 9e37347ff8b..02e55c441e3 100644 --- a/packages/salesforce/manifest.yml +++ b/packages/salesforce/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: salesforce title: Salesforce -version: "0.10.0" +version: "0.10.1" license: basic description: Collect logs from Salesforce with Elastic Agent. type: integration From 051d326ab4da7e8546e3ff4a3e5083154251d494 Mon Sep 17 00:00:00 2001 From: milan-elastic Date: Wed, 20 Sep 2023 19:42:33 +0530 Subject: [PATCH 2/4] update changelog.yml --- packages/salesforce/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/salesforce/changelog.yml b/packages/salesforce/changelog.yml index 27d493f587c..f1010255ed0 100644 --- a/packages/salesforce/changelog.yml +++ b/packages/salesforce/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add 'Request Timeout' configuration. type: enhancement - link: https://github.com/elastic/integrations/pull/1 #FIX ME + link: https://github.com/elastic/integrations/pull/7891 - version: "0.10.0" changes: - description: Rename ownership from obs-service-integrations to obs-infraobs-integrations From a2b52808e3b377c94c2f27f7a43968fa93e61a1f Mon Sep 17 00:00:00 2001 From: milan-elastic Date: Thu, 21 Sep 2023 15:11:58 +0530 Subject: [PATCH 3/4] address review comments --- packages/salesforce/_dev/build/docs/README.md | 2 +- packages/salesforce/data_stream/apex/manifest.yml | 4 ++-- packages/salesforce/data_stream/login_rest/manifest.yml | 4 ++-- packages/salesforce/data_stream/logout_rest/manifest.yml | 4 ++-- packages/salesforce/data_stream/setupaudittrail/manifest.yml | 4 ++-- packages/salesforce/docs/README.md | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/salesforce/_dev/build/docs/README.md b/packages/salesforce/_dev/build/docs/README.md index d5747f3e815..d35c2dee8ee 100644 --- a/packages/salesforce/_dev/build/docs/README.md +++ b/packages/salesforce/_dev/build/docs/README.md @@ -153,7 +153,7 @@ After the integration is successfully configured, clicking on the Assets tab of ### Request timeout -In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the response is getting delayed from the Salesforce server side due to any reason then the following error might occur. +In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the response is getting delayed from the Salesforce server side due to any reason then the following error might occur: ``` Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: failed to execute http client.Do: failed to read http.response.body ``` diff --git a/packages/salesforce/data_stream/apex/manifest.yml b/packages/salesforce/data_stream/apex/manifest.yml index 089459def87..499224e507c 100644 --- a/packages/salesforce/data_stream/apex/manifest.yml +++ b/packages/salesforce/data_stream/apex/manifest.yml @@ -30,8 +30,8 @@ streams: default: false - name: request_timeout type: text - title: Request Timeout - description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + title: Request timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are `s` (seconds) and `m` (minutes). Default is 30s if left empty. show_user: false multi: false required: false diff --git a/packages/salesforce/data_stream/login_rest/manifest.yml b/packages/salesforce/data_stream/login_rest/manifest.yml index 595e820ebf3..f15fb899c00 100644 --- a/packages/salesforce/data_stream/login_rest/manifest.yml +++ b/packages/salesforce/data_stream/login_rest/manifest.yml @@ -30,8 +30,8 @@ streams: default: false - name: request_timeout type: text - title: Request Timeout - description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + title: Request timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are `s` (seconds) and `m` (minutes). Default is 30s if left empty. show_user: false multi: false required: false diff --git a/packages/salesforce/data_stream/logout_rest/manifest.yml b/packages/salesforce/data_stream/logout_rest/manifest.yml index 8ac53f8dd88..00f27352802 100644 --- a/packages/salesforce/data_stream/logout_rest/manifest.yml +++ b/packages/salesforce/data_stream/logout_rest/manifest.yml @@ -30,8 +30,8 @@ streams: default: false - name: request_timeout type: text - title: Request Timeout - description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + title: Request timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are `s` (seconds) and `m` (minutes). Default is 30s if left empty. show_user: false multi: false required: false diff --git a/packages/salesforce/data_stream/setupaudittrail/manifest.yml b/packages/salesforce/data_stream/setupaudittrail/manifest.yml index f65a6d32a22..9ac116c79ad 100644 --- a/packages/salesforce/data_stream/setupaudittrail/manifest.yml +++ b/packages/salesforce/data_stream/setupaudittrail/manifest.yml @@ -30,8 +30,8 @@ streams: default: false - name: request_timeout type: text - title: Request Timeout - description: Duration before declaring that the HTTP client connection has timed out. Valid time units are s and m. Default is 30s. + title: Request timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are `s` (seconds) and `m` (minutes). Default is 30s if left empty. show_user: false multi: false required: false diff --git a/packages/salesforce/docs/README.md b/packages/salesforce/docs/README.md index 174fa1b3de3..cc30913f1cf 100644 --- a/packages/salesforce/docs/README.md +++ b/packages/salesforce/docs/README.md @@ -153,7 +153,7 @@ After the integration is successfully configured, clicking on the Assets tab of ### Request timeout -In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the response is getting delayed from the Salesforce server side due to any reason then the following error might occur. +In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the response is getting delayed from the Salesforce server side due to any reason then the following error might occur: ``` Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: failed to execute http client.Do: failed to read http.response.body ``` From ed063da7652a3dbfa65a0a3aabd664ad1f4277ba Mon Sep 17 00:00:00 2001 From: milan-elastic Date: Thu, 21 Sep 2023 15:30:35 +0530 Subject: [PATCH 4/4] resolve review comments --- packages/salesforce/_dev/build/docs/README.md | 2 +- packages/salesforce/changelog.yml | 2 +- packages/salesforce/docs/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/salesforce/_dev/build/docs/README.md b/packages/salesforce/_dev/build/docs/README.md index d35c2dee8ee..4caa956ba46 100644 --- a/packages/salesforce/_dev/build/docs/README.md +++ b/packages/salesforce/_dev/build/docs/README.md @@ -157,7 +157,7 @@ In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the ``` Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: failed to execute http client.Do: failed to read http.response.body ``` -In this case, consider increasing `Request Timeout` configuration from `Advanced options` section of that data stream. +In this case, consider increasing `Request timeout` configuration from `Advanced options` section of that data stream. ### Data ingestion error diff --git a/packages/salesforce/changelog.yml b/packages/salesforce/changelog.yml index f1010255ed0..eda10f3b5ed 100644 --- a/packages/salesforce/changelog.yml +++ b/packages/salesforce/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "0.10.1" changes: - - description: Add 'Request Timeout' configuration. + - description: Add 'Request timeout' configuration. type: enhancement link: https://github.com/elastic/integrations/pull/7891 - version: "0.10.0" diff --git a/packages/salesforce/docs/README.md b/packages/salesforce/docs/README.md index cc30913f1cf..e6180062042 100644 --- a/packages/salesforce/docs/README.md +++ b/packages/salesforce/docs/README.md @@ -157,7 +157,7 @@ In `Apex`, `Login Rest`, `Logout Rest`, or `SetupAuditTrail` datastreams, if the ``` Error while processing http request: failed to execute rf.collectResponse: failed to execute http client.Do: failed to execute http client.Do: failed to read http.response.body ``` -In this case, consider increasing `Request Timeout` configuration from `Advanced options` section of that data stream. +In this case, consider increasing `Request timeout` configuration from `Advanced options` section of that data stream. ### Data ingestion error