From cdadf352412cffc66bb0206573be0cef737a45b4 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Thu, 17 Nov 2022 12:41:05 -0500 Subject: [PATCH 01/13] feat: change log format to json --- src/services/logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/logger.ts b/src/services/logger.ts index 7bf8d6e2..63715366 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -88,7 +88,7 @@ const options = { level: 'debug', handleExceptions: true, colorize: true, - format: logFormat, + format: format.json(), silent, }, aws: { From 34c0ccdd8c218bfd26c77c60ad8eef0d5320d913 Mon Sep 17 00:00:00 2001 From: Wesley Charles Blake Date: Thu, 17 Nov 2022 20:16:54 +0200 Subject: [PATCH 02/13] build docker images --- .github/workflows/tag_and_release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tag_and_release.yml b/.github/workflows/tag_and_release.yml index ad646eb6..ebcefa66 100644 --- a/.github/workflows/tag_and_release.yml +++ b/.github/workflows/tag_and_release.yml @@ -3,6 +3,7 @@ on: push: branches: - develop + - develop-k8s - master jobs: From a770220949896de762287e2787bf89829e94b087 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Thu, 17 Nov 2022 13:40:50 -0500 Subject: [PATCH 03/13] feat: log raw altruist url --- src/services/pocket-relayer.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/pocket-relayer.ts b/src/services/pocket-relayer.ts index cdb6bf98..2f927dc7 100644 --- a/src/services/pocket-relayer.ts +++ b/src/services/pocket-relayer.ts @@ -528,6 +528,7 @@ export class PocketRelayer { relayType: 'FALLBACK', typeID: application.id, serviceNode: 'fallback:' + redactedAltruistURL, + rawAltruist: JSON.stringify(redactedAltruistURL), blockchainID, origin: this.origin, forcedFallback: !notForceFallback, From 86d02feadbe3275f08022d4f18941251963a4352 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Thu, 17 Nov 2022 13:57:11 -0500 Subject: [PATCH 04/13] feat: update redacted altruist --- src/services/pocket-relayer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/pocket-relayer.ts b/src/services/pocket-relayer.ts index 2f927dc7..dcf96f94 100644 --- a/src/services/pocket-relayer.ts +++ b/src/services/pocket-relayer.ts @@ -528,7 +528,7 @@ export class PocketRelayer { relayType: 'FALLBACK', typeID: application.id, serviceNode: 'fallback:' + redactedAltruistURL, - rawAltruist: JSON.stringify(redactedAltruistURL), + rawAltruist: redactedAltruistURL, blockchainID, origin: this.origin, forcedFallback: !notForceFallback, From df1c27939c64d3771a8bc287d80428fbf6688b13 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Thu, 17 Nov 2022 14:54:42 -0500 Subject: [PATCH 05/13] revert: rawAltruist log --- src/services/pocket-relayer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/pocket-relayer.ts b/src/services/pocket-relayer.ts index dcf96f94..cdb6bf98 100644 --- a/src/services/pocket-relayer.ts +++ b/src/services/pocket-relayer.ts @@ -528,7 +528,6 @@ export class PocketRelayer { relayType: 'FALLBACK', typeID: application.id, serviceNode: 'fallback:' + redactedAltruistURL, - rawAltruist: redactedAltruistURL, blockchainID, origin: this.origin, forcedFallback: !notForceFallback, From 47688e47b24938189c5f3c574a4f78942ea88f8b Mon Sep 17 00:00:00 2001 From: Wesley Charles Blake Date: Fri, 25 Nov 2022 15:56:17 +0200 Subject: [PATCH 06/13] enable multi arch in gh action --- .github/workflows/dockerhub.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml index 15d6c7dd..799435a8 100644 --- a/.github/workflows/dockerhub.yml +++ b/.github/workflows/dockerhub.yml @@ -4,6 +4,7 @@ on: push: branches: - 'develop' + - 'feat/arm-support' tags: - 'v*' pull_request: @@ -46,6 +47,7 @@ jobs: uses: docker/build-push-action@v3 with: context: . + platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 22f9c3a5ac17745b87e87cb43745a3a244e07d60 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Tue, 29 Nov 2022 12:53:16 -0400 Subject: [PATCH 07/13] feat: update error db script --- scripts/error.init.sql | 14 ++++++++++++++ stacks/local.init.sql | 34 ---------------------------------- 2 files changed, 14 insertions(+), 34 deletions(-) create mode 100644 scripts/error.init.sql delete mode 100644 stacks/local.init.sql diff --git a/scripts/error.init.sql b/scripts/error.init.sql new file mode 100644 index 00000000..e4d5f9d4 --- /dev/null +++ b/scripts/error.init.sql @@ -0,0 +1,14 @@ +CREATE TABLE error ( + timestamp TIMESTAMPTZ NOT NULL, + applicationpublickey TEXT NOT NULL, + blockchain TEXT NOT NULL, + nodepublickey TEXT, + elapsedtime DOUBLE PRECISION NOT NULL, + bytes NUMERIC NOT NULL, + method TEXT, + message TEXT, + code TEXT +); +CREATE INDEX error_node_public_key_timestamp_idx ON error(nodepublickey, timestamp DESC); +CREATE INDEX error_timestamp_application_public_key_idx ON error(timestamp DESC, applicationpublickey); +CREATE INDEX error_timestamp_idx ON error(timestamp DESC); \ No newline at end of file diff --git a/stacks/local.init.sql b/stacks/local.init.sql deleted file mode 100644 index d8854da9..00000000 --- a/stacks/local.init.sql +++ /dev/null @@ -1,34 +0,0 @@ --- Local setup of timescaledb relations - -CREATE TABLE relay ( - timestamp TIMESTAMPTZ NOT NULL, - app_pub_key TEXT NOT NULL, - blockchain TEXT NOT NULL, - service_node TEXT, - elapsed_time DOUBLE PRECISION NOT NULL, - result NUMERIC, - bytes NUMERIC NOT NULL, - method TEXT -); - -CREATE INDEX relay_app_pub_key_method_timestamp_idx ON relay(app_pub_key, method, timestamp); -CREATE INDEX relay_app_pub_key_result_timestamp_idx ON relay(app_pub_key, result, timestamp); -CREATE INDEX relay_app_pub_key_timestamp_idx ON relay(app_pub_key, timestamp DESC); -CREATE INDEX relay_service_node_timestamp_idx ON relay(service_node, timestamp DESC); -CREATE INDEX relay_timestamp_app_pub_key_idx ON relay(timestamp DESC, app_pub_key); -CREATE INDEX relay_timestamp_idx ON relay(timestamp DESC); - -CREATE TABLE error ( - timestamp TIMESTAMPTZ NOT NULL, - app_pub_key TEXT NOT NULL, - blockchain TEXT NOT NULL, - service_node TEXT, - elapsed_time DOUBLE PRECISION NOT NULL, - bytes NUMERIC NOT NULL, - method TEXT, - message TEXT -); - -CREATE INDEX error_service_node_timestamp_idx ON error(service_node, timestamp DESC); -CREATE INDEX error_timestamp_app_pub_key_idx ON error(timestamp DESC, app_pub_key); -CREATE INDEX error_timestamp_idx ON error(timestamp DESC); \ No newline at end of file From 360c1242229f4af02b1d26588c2e33cf79bcaa7e Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Tue, 29 Nov 2022 12:53:29 -0400 Subject: [PATCH 08/13] infra: update local stack for error db --- stacks/local.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stacks/local.yml b/stacks/local.yml index 023440c6..9964a94a 100644 --- a/stacks/local.yml +++ b/stacks/local.yml @@ -18,7 +18,7 @@ services: depends_on: - db - cache - - metricsdb + - errorsdb networks: - pocket @@ -57,9 +57,9 @@ services: - pocket # DB to store errors - metricsdb: - container_name: metricsdb - image: timescale/timescaledb:2.0.2-pg12 + errorsdb: + image: postgres:15.1 + container_name: errorsdb environment: - POSTGRES_USER=pguser - POSTGRES_PASSWORD=pgpassword @@ -69,7 +69,7 @@ services: networks: - pocket volumes: - - ./local.init.sql:/docker-entrypoint-initdb.d/init.sql + - ../scripts/error.init.sql:/docker-entrypoint-initdb.d/init.sql influxdb: image: quay.io/influxdb/influxdb:v2.0.3 From bb141a393fc538165e4840f075bf2c26910d2702 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Tue, 29 Nov 2022 13:30:48 -0400 Subject: [PATCH 09/13] chore: remove unused branch --- .github/workflows/production-eu-central-1.yml | 4 ++-- .../tasks/production/eu-central-1/production-eu-central-1.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-eu-central-1.yml b/.github/workflows/production-eu-central-1.yml index ae68a8e8..42a1bc11 100644 --- a/.github/workflows/production-eu-central-1.yml +++ b/.github/workflows/production-eu-central-1.yml @@ -4,7 +4,7 @@ name: Terraform Production Deployment eu-central-1 on: push: - branches: [master, eth-altruist] + branches: [master] jobs: deploy: @@ -119,4 +119,4 @@ jobs: task-definition: ${{ steps.task-def-eu-central-1-datadog-agent.outputs.task-definition }} service: gateway-prod cluster: gateway-prod - wait-for-service-stability: true + wait-for-service-stability: true \ No newline at end of file diff --git a/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml b/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml index 07f9ab0f..b5863895 100644 --- a/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml +++ b/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml @@ -6,7 +6,7 @@ name: Terraform Production Deployment eu-central-1 on: push: - branches: [master, eth-altruist] + branches: [master] jobs: deploy: From 6e96437168e559d1d3cf0f4824a60e618c9d29a7 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Tue, 29 Nov 2022 13:31:01 -0400 Subject: [PATCH 10/13] chore: lint --- .github/workflows/production-eu-central-1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/production-eu-central-1.yml b/.github/workflows/production-eu-central-1.yml index 42a1bc11..e2247acd 100644 --- a/.github/workflows/production-eu-central-1.yml +++ b/.github/workflows/production-eu-central-1.yml @@ -119,4 +119,4 @@ jobs: task-definition: ${{ steps.task-def-eu-central-1-datadog-agent.outputs.task-definition }} service: gateway-prod cluster: gateway-prod - wait-for-service-stability: true \ No newline at end of file + wait-for-service-stability: true From 3fed3d1b5b85481da464433a58aa32ff93906750 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Tue, 29 Nov 2022 13:36:49 -0400 Subject: [PATCH 11/13] chore: remove unused branch --- .github/workflows/production-eu-central-1.yml | 4 ++-- .../tasks/production/eu-central-1/production-eu-central-1.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/production-eu-central-1.yml b/.github/workflows/production-eu-central-1.yml index ae68a8e8..42a1bc11 100644 --- a/.github/workflows/production-eu-central-1.yml +++ b/.github/workflows/production-eu-central-1.yml @@ -4,7 +4,7 @@ name: Terraform Production Deployment eu-central-1 on: push: - branches: [master, eth-altruist] + branches: [master] jobs: deploy: @@ -119,4 +119,4 @@ jobs: task-definition: ${{ steps.task-def-eu-central-1-datadog-agent.outputs.task-definition }} service: gateway-prod cluster: gateway-prod - wait-for-service-stability: true + wait-for-service-stability: true \ No newline at end of file diff --git a/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml b/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml index 07f9ab0f..b5863895 100644 --- a/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml +++ b/pocket-gateway/tasks/production/eu-central-1/production-eu-central-1.yml @@ -6,7 +6,7 @@ name: Terraform Production Deployment eu-central-1 on: push: - branches: [master, eth-altruist] + branches: [master] jobs: deploy: From 59b0aee50817d120e6d6eb2f5f86488c48f81df3 Mon Sep 17 00:00:00 2001 From: Roniel Valdez Date: Tue, 29 Nov 2022 13:42:07 -0400 Subject: [PATCH 12/13] lint: remove unused variable --- src/services/logger.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/services/logger.ts b/src/services/logger.ts index 98953245..f185f0da 100644 --- a/src/services/logger.ts +++ b/src/services/logger.ts @@ -72,15 +72,6 @@ const consoleFormat = printf( const startTime = new Date().toISOString() -const logFormat = format.combine( - format.colorize(), - format.simple(), - format.timestamp({ - format: 'YYYY-MM-DD HH:mm:ss.SSS', - }), - consoleFormat -) - const logName = (process.env.REGION_NAME || '') + '/ecs/gateway' const options = { From 17b35ed6d7d2f3b7c2abf8f89321887cd8769daa Mon Sep 17 00:00:00 2001 From: crisog Date: Wed, 30 Nov 2022 11:29:08 -0400 Subject: [PATCH 13/13] feat: add method to log --- src/services/metrics-recorder.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/services/metrics-recorder.ts b/src/services/metrics-recorder.ts index 480d819b..6783e242 100644 --- a/src/services/metrics-recorder.ts +++ b/src/services/metrics-recorder.ts @@ -123,6 +123,7 @@ export class MetricsRecorder { relayType: 'APP', typeID: applicationID, gigastakeAppID, + method, serviceNode, serviceURL, serviceDomain, @@ -142,6 +143,7 @@ export class MetricsRecorder { relayType: 'APP', typeID: applicationID, gigastakeAppID, + method, serviceNode, serviceURL, serviceDomain, @@ -162,6 +164,7 @@ export class MetricsRecorder { relayType: 'APP', typeID: applicationID, gigastakeAppID, + method, serviceNode, serviceURL, serviceDomain,