Skip to content

Commit 1c303d6

Browse files
BonapartePCBonapartePC
authored
Merge branch 'master' into feat.adaptiveRateLimiting
2 parents 64feab0 + a61ab5f commit 1c303d6

14 files changed

+529
-65
lines changed

‎.github/workflows/builds.yml

+9-9
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ jobs:
1919
-
2020
name: Docker meta
2121
id: meta
22-
uses: docker/metadata-action@v4
22+
uses: docker/metadata-action@v5
2323
with:
2424
images: |
2525
name=rudderlabs/develop-rudder-server
@@ -34,13 +34,13 @@ jobs:
3434
type=semver,pattern={{major}}
3535
-
3636
name: Login to DockerHub
37-
uses: docker/login-action@v2
37+
uses: docker/login-action@v3
3838
with:
3939
username: rudderlabs
4040
password: ${{ secrets.DOCKERHUB_TOKEN }}
4141
-
4242
name: Build and push
43-
uses: docker/build-push-action@v3
43+
uses: docker/build-push-action@v5
4444
with:
4545
context: .
4646
# platforms: linux/amd64,linux/arm64
@@ -61,7 +61,7 @@ jobs:
6161
-
6262
name: Docker meta
6363
id: meta
64-
uses: docker/metadata-action@v4
64+
uses: docker/metadata-action@v5
6565
with:
6666
images: |
6767
name=rudderstack/develop-rudder-server-enterprise
@@ -75,13 +75,13 @@ jobs:
7575
type=semver,pattern={{major}}
7676
-
7777
name: Login to DockerHub
78-
uses: docker/login-action@v2
78+
uses: docker/login-action@v3
7979
with:
8080
username: rudderlabs
8181
password: ${{ secrets.DOCKERHUB_TOKEN }}
8282
-
8383
name: Build and push
84-
uses: docker/build-push-action@v3
84+
uses: docker/build-push-action@v5
8585
with:
8686
context: .
8787
# platforms: linux/amd64,linux/arm64
@@ -103,7 +103,7 @@ jobs:
103103
-
104104
name: Docker meta
105105
id: meta
106-
uses: docker/metadata-action@v4
106+
uses: docker/metadata-action@v5
107107
with:
108108
images: |
109109
name=rudderstack/develop-suppression-backup-service
@@ -117,13 +117,13 @@ jobs:
117117
type=semver,pattern={{major}}
118118
-
119119
name: Login to DockerHub
120-
uses: docker/login-action@v2
120+
uses: docker/login-action@v3
121121
with:
122122
username: rudderlabs
123123
password: ${{ secrets.DOCKERHUB_TOKEN }}
124124
-
125125
name: Build and push
126-
uses: docker/build-push-action@v3
126+
uses: docker/build-push-action@v5
127127
with:
128128
context: .
129129
file: ./suppression-backup-service/Dockerfile

‎.github/workflows/housekeeping.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
pull-requests: write
1414

1515
steps:
16-
- uses: actions/stale@v5
16+
- uses: actions/stale@v8
1717
with:
1818
repo-token: ${{ secrets.GITHUB_TOKEN }}
1919
operations-per-run: 200
@@ -29,7 +29,7 @@ jobs:
2929
- name: Checkout repository
3030
uses: actions/checkout@v4
3131
- name: Run delete-old-branches-action
32-
uses: beatlabs/[email protected].9
32+
uses: beatlabs/[email protected].10
3333
with:
3434
repo_token: ${{ github.token }}
3535
date: '2 months ago'

‎.github/workflows/semantic-pr.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ jobs:
1717
runs-on: ubuntu-latest
1818
steps:
1919
-
20-
uses: amannn/action-semantic-pull-request@v4
20+
uses: amannn/action-semantic-pull-request@v5
2121
env:
2222
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2323
with:

‎.github/workflows/tests.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
- run: go version
7272
- run: go mod download # Not required, used to segregate module download vs test times
7373
- name: Login to DockerHub
74-
uses: docker/login-action@v2
74+
uses: docker/login-action@v3
7575
with:
7676
username: rudderlabs
7777
password: ${{ secrets.DOCKERHUB_TOKEN }}

‎CHANGELOG.md

+7
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
# Changelog
22

3+
## [1.17.2](https://github.com/rudderlabs/rudder-server/compare/v1.17.1...v1.17.2) (2023-11-20)
4+
5+
6+
### Bug Fixes
7+
8+
* gateway responds with http status 500 and body pq: invalid byte sequence for encoding UTF8: 0x00 ([#4161](https://github.com/rudderlabs/rudder-server/issues/4161)) ([2c168ef](https://github.com/rudderlabs/rudder-server/commit/2c168ef96308443206ec93e3527f401de1431eb4))
9+
310
## [1.17.1](https://github.com/rudderlabs/rudder-server/compare/v1.17.0...v1.17.1) (2023-11-17)
411

512

‎gateway/handle.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -324,8 +324,8 @@ func (gw *Handle) getJobDataFromRequest(req *webRequestT) (jobData *jobFromReq,
324324
return
325325
}
326326

327-
anonIDFromReq := strings.TrimSpace(misc.GetStringifiedData(toSet["anonymousId"]))
328-
userIDFromReq := strings.TrimSpace(misc.GetStringifiedData(toSet["userId"]))
327+
anonIDFromReq := strings.TrimSpace(misc.SanitizeUnicode(misc.GetStringifiedData(toSet["anonymousId"])))
328+
userIDFromReq := strings.TrimSpace(misc.SanitizeUnicode(misc.GetStringifiedData(toSet["userId"])))
329329
eventTypeFromReq, _ := misc.MapLookup(
330330
toSet,
331331
"type",

‎go.mod

+10-10
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ require (
4545
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0
4646
github.com/denisenkom/go-mssqldb v0.12.3
4747
github.com/dgraph-io/badger/v4 v4.2.0
48-
github.com/docker/docker v24.0.6+incompatible
48+
github.com/docker/docker v24.0.7+incompatible
4949
github.com/go-chi/chi/v5 v5.0.10
5050
github.com/go-redis/redis v6.15.9+incompatible
5151
github.com/go-redis/redis/v8 v8.11.5
@@ -81,7 +81,7 @@ require (
8181
github.com/rs/cors v1.10.1
8282
github.com/rudderlabs/analytics-go v3.3.3+incompatible
8383
github.com/rudderlabs/compose-test v0.1.3
84-
github.com/rudderlabs/rudder-go-kit v0.16.3
84+
github.com/rudderlabs/rudder-go-kit v0.17.0
8585
github.com/rudderlabs/sql-tunnels v0.1.5
8686
github.com/samber/lo v1.38.1
8787
github.com/segmentio/kafka-go v0.4.44
@@ -103,19 +103,17 @@ require (
103103
go.uber.org/atomic v1.11.0
104104
go.uber.org/automaxprocs v1.5.3
105105
go.uber.org/goleak v1.3.0
106-
golang.org/x/crypto v0.14.0
106+
golang.org/x/crypto v0.15.0
107107
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
108-
golang.org/x/oauth2 v0.13.0
108+
golang.org/x/oauth2 v0.14.0
109109
golang.org/x/sync v0.5.0
110110
golang.org/x/text v0.14.0
111-
google.golang.org/api v0.149.0
111+
google.golang.org/api v0.150.0
112112
google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405
113113
google.golang.org/grpc v1.59.0
114114
google.golang.org/protobuf v1.31.0
115115
)
116116

117-
require github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
118-
119117
require (
120118
cloud.google.com/go v0.110.9 // indirect
121119
cloud.google.com/go/compute v1.23.2 // indirect
@@ -222,6 +220,7 @@ require (
222220
github.com/mattn/go-ieproxy v0.0.1 // indirect
223221
github.com/mattn/go-isatty v0.0.19 // indirect
224222
github.com/mattn/go-runewidth v0.0.12 // indirect
223+
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
225224
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
226225
github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
227226
github.com/minio/md5-simd v1.1.2 // indirect
@@ -251,7 +250,7 @@ require (
251250
github.com/rudderlabs/bing-ads-go-sdk v0.2.0
252251
github.com/russross/blackfriday/v2 v2.1.0 // indirect
253252
github.com/segmentio/backo-go v1.0.1 // indirect
254-
github.com/shirou/gopsutil/v3 v3.23.9 // indirect
253+
github.com/shirou/gopsutil/v3 v3.23.10 // indirect
255254
github.com/sirupsen/logrus v1.9.3 // indirect
256255
github.com/spf13/afero v1.10.0 // indirect
257256
github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -288,9 +287,10 @@ require (
288287
go.uber.org/multierr v1.11.0 // indirect
289288
go.uber.org/zap v1.26.0 // indirect
290289
golang.org/x/mod v0.12.0 // indirect
291-
golang.org/x/net v0.17.0 // indirect
292-
golang.org/x/sys v0.13.0 // indirect
290+
golang.org/x/net v0.18.0 // indirect
291+
golang.org/x/sys v0.14.0 // indirect
293292
golang.org/x/term v0.13.0 // indirect
293+
golang.org/x/time v0.3.0 // indirect
294294
golang.org/x/tools v0.13.0 // indirect
295295
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
296296
google.golang.org/appengine v1.6.7 // indirect

‎go.sum

+12-9
Original file line numberDiff line numberDiff line change
@@ -950,8 +950,8 @@ github.com/rudderlabs/compose-test v0.1.3 h1:uyep6jDCIF737sfv4zIaMsKRQKX95IDz5Xb
950950
github.com/rudderlabs/compose-test v0.1.3/go.mod h1:tuvS1eQdSfwOYv1qwyVAcpdJxPLQXJgy5xGDd/9XmMg=
951951
github.com/rudderlabs/parquet-go v0.0.2 h1:ZXRdZdimB0PdJtmxeSSxfI0fDQ3kZjwzBxRi6Ut1J8k=
952952
github.com/rudderlabs/parquet-go v0.0.2/go.mod h1:g6guum7o8uhj/uNhunnt7bw5Vabu/goI5i21/3fnxWQ=
953-
github.com/rudderlabs/rudder-go-kit v0.16.3 h1:IZIg7RjwbQN0GAHpiZgNLW388AwBmgVnh3bYPXP7SKQ=
954-
github.com/rudderlabs/rudder-go-kit v0.16.3/go.mod h1:vRRTcYmAtYg87R4liGy24wO3452WlGHkFwtEopgme3k=
953+
github.com/rudderlabs/rudder-go-kit v0.17.0 h1:bpgBSJU0wgB/JLqE4wjaNxP3efChJskDNcpcJ1Q1pfM=
954+
github.com/rudderlabs/rudder-go-kit v0.17.0/go.mod h1:+hOAdqg3AHdVWIKB4QGFcWczQWabRZ7s8uKLQvFiXSQ=
955955
github.com/rudderlabs/sql-tunnels v0.1.5 h1:L/e9GQtqJlTVMauAE+ym/XUqhg+Va6RZQiOvBgbhspY=
956956
github.com/rudderlabs/sql-tunnels v0.1.5/go.mod h1:ZwQkCLb/5hHm5U90juAj9idkkFGv2R2dzDHJoPbKIto=
957957
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -969,8 +969,8 @@ github.com/segmentio/kafka-go v0.4.44 h1:Vjjksniy0WSTZ7CuVJrz1k04UoZeTc77UV6Yyk6
969969
github.com/segmentio/kafka-go v0.4.44/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
970970
github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c=
971971
github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE=
972-
github.com/shirou/gopsutil/v3 v3.23.9 h1:ZI5bWVeu2ep4/DIxB4U9okeYJ7zp/QLTO4auRb/ty/E=
973-
github.com/shirou/gopsutil/v3 v3.23.9/go.mod h1:x/NWSb71eMcjFIO0vhyGW5nZ7oSIgVjrCnADckb85GA=
972+
github.com/shirou/gopsutil/v3 v3.23.10 h1:/N42opWlYzegYaVkWejXWJpbzKv2JDy3mrgGzKsh9hM=
973+
github.com/shirou/gopsutil/v3 v3.23.10/go.mod h1:JIE26kpucQi+innVlAUnIEOSBhBUkirr5b44yr55+WE=
974974
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
975975
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
976976
github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
@@ -1223,8 +1223,8 @@ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ
12231223
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
12241224
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
12251225
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc=
1226-
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
1227-
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
1226+
golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0=
1227+
golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM=
12281228
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
12291229
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
12301230
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1329,8 +1329,9 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13291329
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13301330
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
13311331
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1332-
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
13331332
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1333+
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
1334+
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
13341335
golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
13351336
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
13361337
golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
@@ -1342,6 +1343,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb
13421343
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
13431344
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
13441345
golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
1346+
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
1347+
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
13451348
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
13461349
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
13471350
golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1468,8 +1471,8 @@ google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7
14681471
google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
14691472
google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
14701473
google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
1471-
google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY=
1472-
google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI=
1474+
google.golang.org/api v0.150.0 h1:Z9k22qD289SZ8gCJrk4DrWXkNjtfvKAUo/l1ma8eBYE=
1475+
google.golang.org/api v0.150.0/go.mod h1:ccy+MJ6nrYFgE3WgRx/AMXOxOmU8Q4hSa+jjibzhxcg=
14731476
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
14741477
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
14751478
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

‎processor/delayed/stats.go

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package delayed
2+
3+
import (
4+
"strings"
5+
"time"
6+
7+
"github.com/rudderlabs/rudder-go-kit/config"
8+
"github.com/rudderlabs/rudder-go-kit/stats"
9+
backendconfig "github.com/rudderlabs/rudder-server/backend-config"
10+
"github.com/rudderlabs/rudder-server/processor/transformer"
11+
"github.com/rudderlabs/rudder-server/utils/misc"
12+
)
13+
14+
type eventStats struct {
15+
stats stats.Stats
16+
threshold time.Duration
17+
}
18+
19+
func NewEventStats(stats stats.Stats, config *config.Config) *eventStats {
20+
threshold := config.GetDuration("processor.delayed_events.threshold", 10*24, time.Hour)
21+
22+
return &eventStats{
23+
stats: stats,
24+
threshold: threshold,
25+
}
26+
}
27+
28+
func (s *eventStats) ObserveSourceEvents(source *backendconfig.SourceT, events []transformer.TransformerEvent) {
29+
statusCount := map[string]map[string]int{
30+
"missing_original_timestamp": {},
31+
"missing_sent_at": {},
32+
"late": {},
33+
"on-time": {},
34+
}
35+
36+
for _, event := range events {
37+
sdkVersion := "unknown"
38+
39+
sdkContext, err := misc.NestedMapLookup(event.Message, "context", "library")
40+
if err == nil {
41+
m, ok := sdkContext.(map[string]interface{})
42+
if ok {
43+
sdkLibVersion, _ := m["version"].(string)
44+
sdkLibName, _ := m["name"].(string)
45+
46+
if sdkLibName != "" || sdkLibVersion != "" {
47+
sdkVersion = strings.Join([]string{sdkLibName, sdkLibVersion}, "/")
48+
}
49+
}
50+
}
51+
52+
originalTimestamp, ok := misc.GetParsedTimestamp(event.Message["originalTimestamp"])
53+
if !ok {
54+
statusCount["missing_original_timestamp"][sdkVersion]++
55+
continue
56+
}
57+
58+
sentAt, ok := misc.GetParsedTimestamp(event.Message["sentAt"])
59+
if !ok {
60+
statusCount["missing_sent_at"][sdkVersion]++
61+
continue
62+
}
63+
64+
if sentAt.Sub(originalTimestamp) > s.threshold {
65+
statusCount["late"][sdkVersion]++
66+
} else {
67+
statusCount["on-time"][sdkVersion]++
68+
}
69+
}
70+
71+
for status, versions := range statusCount {
72+
for version, count := range versions {
73+
s.stats.NewTaggedStat("processor.delayed_events", stats.CountType, stats.Tags{
74+
"sourceId": source.ID,
75+
"sourceType": source.SourceDefinition.Category,
76+
"workspaceId": source.WorkspaceID,
77+
"status": status,
78+
"sdkVersion": version,
79+
}).Count(count)
80+
}
81+
}
82+
}

0 commit comments

Comments
 (0)