From 78c8f642afed6e7c603fdaf794f03dbfab03ce53 Mon Sep 17 00:00:00 2001 From: codyborn Date: Mon, 29 Mar 2021 09:48:34 -0700 Subject: [PATCH 01/16] Add PR reminder to add/update docs --- .github/pull_request_template.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 0c22012583a..43b60853b36 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -17,4 +17,8 @@ _An explanation of how the changes were tested or an explanation as to why they ### Backwards compatibility -_Brief explanation of why these changes are/are not backwards compatible._ \ No newline at end of file +_Brief explanation of why these changes are/are not backwards compatible._ + +### Documentation + +_The set of community facing docs that have been added/modified because of this change_ \ No newline at end of file From 28b7a715f69134e0da5b1b8356a7b2c16ba64b43 Mon Sep 17 00:00:00 2001 From: codyborn Date: Thu, 1 Apr 2021 19:42:50 -0700 Subject: [PATCH 02/16] WIP --- .env.alfajores | 8 ++ .../src/cmds/deploy/initial/komenci.ts | 19 ++- .../src/cmds/deploy/upgrade/komenci.ts | 19 ++- packages/celotool/src/lib/env-utils.ts | 3 + packages/celotool/src/lib/komenci.ts | 108 +++++++++++++++--- .../komenci/templates/_helpers.tpl | 7 ++ 6 files changed, 134 insertions(+), 30 deletions(-) diff --git a/.env.alfajores b/.env.alfajores index 76f111e454f..a406fc4dfc9 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -186,6 +186,14 @@ AZURE_KOMENCI_WESTEU_KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED=true #
:: AZURE_KOMENCI_EASTUS_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x00454cac6dae53f8800f71395b9a174f07a784b1:staging-komenci-eus,0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e:staging-komenci-eus AZURE_KOMENCI_WESTEU_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x0f812be74511b90ea6b2f80e77bea047e69a0b2a:staging-komenci-weu,0xb354d3d2908ba6a2b791683b0f454a38f69cb282:staging-komenci-weu +AZURE_KOMENCI_EASTUS_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards +AZURE_KOMENCI_WESTEU_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards +AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards +AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards + +# Celo Rewards +AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 +AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 # Network AZURE_KOMENCI_EASTUS_KOMENCI_NETWORK=alfajores diff --git a/packages/celotool/src/cmds/deploy/initial/komenci.ts b/packages/celotool/src/cmds/deploy/initial/komenci.ts index a008b1fb55d..32c99cb3571 100644 --- a/packages/celotool/src/cmds/deploy/initial/komenci.ts +++ b/packages/celotool/src/cmds/deploy/initial/komenci.ts @@ -11,19 +11,26 @@ export const describe = 'deploy the komenci for the specified network' type KomenciInitialArgv = InitialArgv & ContextArgv & { useForno: boolean + deployRewards: boolean } export const builder = (argv: yargs.Argv) => { - return addContextMiddleware(argv).option('useForno', { - description: 'Uses forno for RPCs from the komenci clients', - default: false, - type: 'boolean', - }) + return addContextMiddleware(argv) + .option('useForno', { + description: 'Uses forno for RPCs from the komenci clients', + default: false, + type: 'boolean', + }) + .option('deployRewards', { + description: 'Deploy Rewards Service alongside Komenci', + default: true, + type: 'boolean', + }) } export const handler = async (argv: KomenciInitialArgv) => { // Do not allow --helmdryrun because komenciIdentityHelmParameters function. It could be refactored to allow exitIfCelotoolHelmDryRun() await switchToContextCluster(argv.celoEnv, argv.context) - await installHelmChart(argv.celoEnv, argv.context, argv.useForno) + await installHelmChart(argv.celoEnv, argv.context, argv.useForno, argv.deployRewards) } diff --git a/packages/celotool/src/cmds/deploy/upgrade/komenci.ts b/packages/celotool/src/cmds/deploy/upgrade/komenci.ts index 153505c414d..ae339f0196c 100644 --- a/packages/celotool/src/cmds/deploy/upgrade/komenci.ts +++ b/packages/celotool/src/cmds/deploy/upgrade/komenci.ts @@ -11,19 +11,26 @@ export const describe = 'upgrade komenci on an AKS cluster' type OracleUpgradeArgv = UpgradeArgv & ContextArgv & { useForno: boolean + deployRewards: boolean } export const builder = (argv: yargs.Argv) => { - return addContextMiddleware(argv).option('useForno', { - description: 'Uses forno for RPCs from the komenci clients', - default: false, - type: 'boolean', - }) + return addContextMiddleware(argv) + .option('useForno', { + description: 'Uses forno for RPCs from the komenci clients', + default: false, + type: 'boolean', + }) + .option('deployRewards', { + description: 'Deploy Rewards Service alongside Komenci', + default: true, + type: 'boolean', + }) } export const handler = async (argv: OracleUpgradeArgv) => { // Do not allow --helmdryrun because komenciIdentityHelmParameters function. It could be refactored to allow exitIfCelotoolHelmDryRun() await switchToContextCluster(argv.celoEnv, argv.context) - await upgradeKomenciChart(argv.celoEnv, argv.context, argv.useForno) + await upgradeKomenciChart(argv.celoEnv, argv.context, argv.useForno, argv.deployRewards) } diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts index d6dab3b1c66..089d2b8db71 100644 --- a/packages/celotool/src/lib/env-utils.ts +++ b/packages/celotool/src/lib/env-utils.ts @@ -184,6 +184,9 @@ export enum DynamicEnvVar { KUBERNETES_CLUSTER_NAME = '{{ context }}_KUBERNETES_CLUSTER_NAME', KOMENCI_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_ADDRESS_AZURE_KEY_VAULTS', KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT = '{{ context }}_KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT', + KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT = '{{ context }}_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT', + KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT = '{{ context }}_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT', + KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = '{{ context }}_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT', KOMENCI_DB_HOST = '{{ context }}_KOMENCI_DB_HOST', KOMENCI_DB_PORT = '{{ context }}_KOMENCI_DB_PORT', KOMENCI_DB_USERNAME = '{{ context }}_KOMENCI_DB_USERNAME', diff --git a/packages/celotool/src/lib/komenci.ts b/packages/celotool/src/lib/komenci.ts index 0b8d378c04d..ed7a56dd7ac 100644 --- a/packages/celotool/src/lib/komenci.ts +++ b/packages/celotool/src/lib/komenci.ts @@ -43,7 +43,10 @@ interface KomenciIdentity { * Configuration of multiple relayers */ interface KomenciConfig { - identities: KomenciIdentity[] + relayerIdentities: KomenciIdentity[] + foundationRewardsIdentities: KomenciIdentity[] + cLabsRewardsIdentities: KomenciIdentity[] + rewardServiceInstanceCount: number } interface KomenciKeyVaultIdentityConfig { @@ -61,6 +64,11 @@ interface KomenciDatabaseConfig { passwordVaultName: string } +enum RewardType { + Foundation, + CeloLabs, +} + /** * Env vars corresponding to each value for the KomenciKeyVaultIdentityConfig for a particular context */ @@ -79,6 +87,33 @@ const contextKomenciMnemonicIdentityConfigDynamicEnvVars: { addressesFromMnemonicCount: DynamicEnvVar.KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT, } +/** + * Env vars corresponding to each value for the KomenciFoundationRewardsKeyVaultIdentityConfig for a particular context + */ +const contextKomenciFoundationRewardsKeyVaultIdentityConfigDynamicEnvVars: { + [k in keyof KomenciKeyVaultIdentityConfig]: DynamicEnvVar +} = { + addressAzureKeyVaults: DynamicEnvVar.KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT, +} + +/** + * Env vars corresponding to each value for the KomenciCeloLabsRewardsKeyVaultIdentityConfig for a particular context + */ +const contextKomenciCeloLabsRewardsKeyVaultIdentityConfigDynamicEnvVars: { + [k in keyof KomenciKeyVaultIdentityConfig]: DynamicEnvVar +} = { + addressAzureKeyVaults: DynamicEnvVar.KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT, +} + +/** + * Env vars corresponding to each value for the KomenciCeloLabsRewardsKeyVaultIdentityConfig for a particular context + */ +const contextKomenciRewardsServiceInstanceCountConfigDynamicEnvVars: { + [k in keyof KomenciKeyVaultIdentityConfig]: DynamicEnvVar +} = { + addressAzureKeyVaults: DynamicEnvVar.KOMENCI_REWARD_SERVICE_INSTANCE_COUNT, +} + const contextDatabaseConfigDynamicEnvVars: { [k in keyof KomenciDatabaseConfig]: DynamicEnvVar } = { host: DynamicEnvVar.KOMENCI_DB_HOST, port: DynamicEnvVar.KOMENCI_DB_PORT, @@ -90,7 +125,12 @@ function releaseName(celoEnv: string) { return `${celoEnv}-komenci` } -export async function installHelmChart(celoEnv: string, context: string, useForno: boolean) { +export async function installHelmChart( + celoEnv: string, + context: string, + useForno: boolean, + deployRewards: boolean +) { // First install the komenci-rbac helm chart. // This must be deployed before so we can use a resulting auth token so that // komenci pods can reach the K8s API server to change their aad labels @@ -100,17 +140,22 @@ export async function installHelmChart(celoEnv: string, context: string, useForn celoEnv, releaseName(celoEnv), helmChartPath, - await helmParameters(celoEnv, context, useForno) + await helmParameters(celoEnv, context, useForno, deployRewards) ) } -export async function upgradeKomenciChart(celoEnv: string, context: string, useFullNodes: boolean) { +export async function upgradeKomenciChart( + celoEnv: string, + context: string, + useFullNodes: boolean, + deployRewards: boolean +) { await upgradeKomenciRBACHelmChart(celoEnv, context) return upgradeGenericHelmChart( celoEnv, releaseName(celoEnv), helmChartPath, - await helmParameters(celoEnv, context, useFullNodes) + await helmParameters(celoEnv, context, useFullNodes, deployRewards) ) } @@ -118,7 +163,7 @@ export async function removeHelmRelease(celoEnv: string, context: string) { await removeGenericHelmChart(releaseName(celoEnv), celoEnv) await removeKomenciRBACHelmRelease(celoEnv) const komenciConfig = getKomenciConfig(context) - for (const identity of komenciConfig.identities) { + for (const identity of komenciConfig.relayerIdentities) { // If the identity is using Azure HSM signing, clean it up too if (identity.azureHsmIdentity) { await deleteAzureKeyVaultIdentity( @@ -142,11 +187,16 @@ async function getPasswordFromKeyVaultSecret(vaultName: string, secretName: stri return password.replace(/\n|"/g, '') } -async function helmParameters(celoEnv: string, context: string, useForno: boolean) { +async function helmParameters( + celoEnv: string, + context: string, + useForno: boolean, + deployRewards: boolean +) { const komenciConfig = getKomenciConfig(context) - const replicas = komenciConfig.identities.length - const kubeServiceAccountSecretNames = await rbacServiceAccountSecretNames(celoEnv, replicas) + const relayerCount = komenciConfig.relayerIdentities.length + const kubeServiceAccountSecretNames = await rbacServiceAccountSecretNames(celoEnv, relayerCount) const databaseConfig = getContextDynamicEnvVarValues(contextDatabaseConfigDynamicEnvVars, context) const vars = getContextDynamicEnvVarValues( @@ -189,21 +239,20 @@ async function helmParameters(celoEnv: string, context: string, useForno: boolea `--set komenci.azureHsm.initTryCount=5`, `--set komenci.azureHsm.initMaxRetryBackoffMs=30000`, `--set onboarding.recaptchaToken=${recaptchaToken}`, - `--set onboarding.replicas=${replicas}`, + `--set onboarding.replicas=${relayerCount}`, `--set onboarding.relayer.host=${celoEnv + '-relayer'}`, `--set onboarding.db.host=${databaseConfig.host}`, `--set onboarding.db.port=${databaseConfig.port}`, `--set onboarding.db.username=${databaseConfig.username}`, `--set onboarding.db.password=${databasePassword}`, `--set onboarding.publicHostname=${getPublicHostname(clusterConfig.regionName, celoEnv)}`, - `--set onboarding.publicUrl=${ - 'https://' + getPublicHostname(clusterConfig.regionName, celoEnv) - }`, + `--set onboarding.publicUrl=${'https://' + + getPublicHostname(clusterConfig.regionName, celoEnv)}`, `--set onboarding.ruleConfig.captcha.bypassEnabled=${vars.captchaBypassEnabled}`, `--set onboarding.ruleConfig.captcha.bypassToken=${fetchEnv( envVar.KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_TOKEN )}`, - `--set relayer.replicas=${replicas}`, + `--set relayer.replicas=${relayerCount}`, `--set relayer.rpcProviderUrls.http=${httpRpcProviderUrl}`, `--set relayer.rpcProviderUrls.ws=${wsRpcProviderUrl}`, `--set relayer.metrics.enabled=true`, @@ -226,10 +275,10 @@ function getPublicHostname(regionName: string, celoEnv: string): string { * Supports both private key and Azure HSM signing. */ async function komenciIdentityHelmParameters(context: string, komenciConfig: KomenciConfig) { - const replicas = komenciConfig.identities.length + const replicas = komenciConfig.relayerIdentities.length let params: string[] = [] for (let i = 0; i < replicas; i++) { - const komenciIdentity = komenciConfig.identities[i] + const komenciIdentity = komenciConfig.relayerIdentities[i] const prefix = `--set relayer.identities[${i}]` params.push(`${prefix}.address=${komenciIdentity.address}`) // An komenci identity can specify either a private key or some information @@ -263,7 +312,9 @@ async function komenciIdentityHelmParameters(context: string, komenciConfig: Kom */ function getKomenciConfig(context: string): KomenciConfig { return { - identities: getKomenciIdentities(context), + relayerIdentities: getKomenciRelayerIdentities(context), + cLabsRewardsIdentities: getKomenciRewardIdentities(context, RewardType.Foundation), + foundationRewardsIdentities: getKomenciRewardIdentities(context, RewardType.CeloLabs), } } @@ -272,7 +323,7 @@ function getKomenciConfig(context: string): KomenciConfig { * the identities are created from that. Otherwise, the identities are created * with private keys generated by the mnemonic. */ -function getKomenciIdentities(context: string): KomenciIdentity[] { +function getKomenciRelayerIdentities(context: string): KomenciIdentity[] { const { addressAzureKeyVaults } = getContextDynamicEnvVarValues( contextKomenciKeyVaultIdentityConfigDynamicEnvVars, context, @@ -301,6 +352,27 @@ function getKomenciIdentities(context: string): KomenciIdentity[] { throw Error('No komenci identity env vars specified') } +/** + * Returns an array of komenci reward identities. The identities are created from the Azure Key Vault env var. + */ +function getKomenciRewardIdentities(context: string, rewardType: RewardType): KomenciIdentity[] { + const envVars = + rewardType === RewardType.Foundation + ? contextKomenciFoundationRewardsKeyVaultIdentityConfigDynamicEnvVars + : contextKomenciCeloLabsRewardsKeyVaultIdentityConfigDynamicEnvVars + const { addressAzureKeyVaults } = getContextDynamicEnvVarValues(envVars, context, { + addressAzureKeyVaults: '', + }) + + const rewardServiceCount = getContextDynamicEnvVarValues() + + if (addressAzureKeyVaults) { + return getAzureHsmKomenciIdentities(addressAzureKeyVaults) + } + + throw Error('No komenci identity env vars specified') +} + /** * Given a string addressAzureKeyVaults of the form: *
:,
: diff --git a/packages/helm-charts/komenci/templates/_helpers.tpl b/packages/helm-charts/komenci/templates/_helpers.tpl index d92b63640d2..9cf6a682f49 100644 --- a/packages/helm-charts/komenci/templates/_helpers.tpl +++ b/packages/helm-charts/komenci/templates/_helpers.tpl @@ -41,6 +41,13 @@ Label specific to the komenci onboarding component app.kubernetes.io/component: komenci-onboarding {{- end -}} +{{/* +Label specific to the komenci rewards component +*/}} +{{- define "komenci-rewards-component-label" -}} +app.kubernetes.io/component: komenci-rewards +{{- end -}} + {{/* The name of the azure identity binding for all relayers */}} From cae5f9ba9939a59b71901ce0833b3b8556973f76 Mon Sep 17 00:00:00 2001 From: codyborn Date: Mon, 3 May 2021 15:40:11 -0700 Subject: [PATCH 03/16] Add helm charts --- .env.alfajores | 11 +- .../src/cmds/deploy/initial/komenci.ts | 19 +-- .../src/cmds/deploy/upgrade/komenci.ts | 19 +-- packages/celotool/src/lib/env-utils.ts | 7 +- packages/celotool/src/lib/komenci.ts | 123 ++++++++++----- .../komenci/templates/rewards-deployment.yaml | 41 +++++ .../rewards-relayer-statefulset.yaml | 141 ++++++++++++++++++ 7 files changed, 288 insertions(+), 73 deletions(-) create mode 100644 packages/helm-charts/komenci/templates/rewards-deployment.yaml create mode 100644 packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml diff --git a/.env.alfajores b/.env.alfajores index a406fc4dfc9..d6961fc7a6c 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -173,6 +173,11 @@ AZURE_KOMENCI_WESTEU_KOMENCI_DB_PORT=5432 AZURE_KOMENCI_WESTEU_KOMENCI_DB_USERNAME=postgres@staging-komenci-weu AZURE_KOMENCI_WESTEU_KOMENCI_DB_PASSWORD_VAULT_NAME=staging-komenci-weu +AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_HOST=staging-komenci-weu.postgres.database.azure.com +AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_PORT=5432 +AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_USERNAME=postgres@staging-komenci-weu +AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME=staging-komenci-weu + # Secrets AZURE_KOMENCI_EASTUS_KOMENCI_APP_SECRETS_VAULT_NAME=staging-komenci-eus AZURE_KOMENCI_WESTEU_KOMENCI_APP_SECRETS_VAULT_NAME=staging-komenci-weu @@ -186,10 +191,8 @@ AZURE_KOMENCI_WESTEU_KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED=true #
:: AZURE_KOMENCI_EASTUS_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x00454cac6dae53f8800f71395b9a174f07a784b1:staging-komenci-eus,0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e:staging-komenci-eus AZURE_KOMENCI_WESTEU_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x0f812be74511b90ea6b2f80e77bea047e69a0b2a:staging-komenci-weu,0xb354d3d2908ba6a2b791683b0f454a38f69cb282:staging-komenci-weu -AZURE_KOMENCI_EASTUS_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards -AZURE_KOMENCI_WESTEU_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards -AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards -AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards +AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards +AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards # Celo Rewards AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 diff --git a/packages/celotool/src/cmds/deploy/initial/komenci.ts b/packages/celotool/src/cmds/deploy/initial/komenci.ts index 32c99cb3571..a008b1fb55d 100644 --- a/packages/celotool/src/cmds/deploy/initial/komenci.ts +++ b/packages/celotool/src/cmds/deploy/initial/komenci.ts @@ -11,26 +11,19 @@ export const describe = 'deploy the komenci for the specified network' type KomenciInitialArgv = InitialArgv & ContextArgv & { useForno: boolean - deployRewards: boolean } export const builder = (argv: yargs.Argv) => { - return addContextMiddleware(argv) - .option('useForno', { - description: 'Uses forno for RPCs from the komenci clients', - default: false, - type: 'boolean', - }) - .option('deployRewards', { - description: 'Deploy Rewards Service alongside Komenci', - default: true, - type: 'boolean', - }) + return addContextMiddleware(argv).option('useForno', { + description: 'Uses forno for RPCs from the komenci clients', + default: false, + type: 'boolean', + }) } export const handler = async (argv: KomenciInitialArgv) => { // Do not allow --helmdryrun because komenciIdentityHelmParameters function. It could be refactored to allow exitIfCelotoolHelmDryRun() await switchToContextCluster(argv.celoEnv, argv.context) - await installHelmChart(argv.celoEnv, argv.context, argv.useForno, argv.deployRewards) + await installHelmChart(argv.celoEnv, argv.context, argv.useForno) } diff --git a/packages/celotool/src/cmds/deploy/upgrade/komenci.ts b/packages/celotool/src/cmds/deploy/upgrade/komenci.ts index ae339f0196c..153505c414d 100644 --- a/packages/celotool/src/cmds/deploy/upgrade/komenci.ts +++ b/packages/celotool/src/cmds/deploy/upgrade/komenci.ts @@ -11,26 +11,19 @@ export const describe = 'upgrade komenci on an AKS cluster' type OracleUpgradeArgv = UpgradeArgv & ContextArgv & { useForno: boolean - deployRewards: boolean } export const builder = (argv: yargs.Argv) => { - return addContextMiddleware(argv) - .option('useForno', { - description: 'Uses forno for RPCs from the komenci clients', - default: false, - type: 'boolean', - }) - .option('deployRewards', { - description: 'Deploy Rewards Service alongside Komenci', - default: true, - type: 'boolean', - }) + return addContextMiddleware(argv).option('useForno', { + description: 'Uses forno for RPCs from the komenci clients', + default: false, + type: 'boolean', + }) } export const handler = async (argv: OracleUpgradeArgv) => { // Do not allow --helmdryrun because komenciIdentityHelmParameters function. It could be refactored to allow exitIfCelotoolHelmDryRun() await switchToContextCluster(argv.celoEnv, argv.context) - await upgradeKomenciChart(argv.celoEnv, argv.context, argv.useForno, argv.deployRewards) + await upgradeKomenciChart(argv.celoEnv, argv.context, argv.useForno) } diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts index 089d2b8db71..c8a50de1f67 100644 --- a/packages/celotool/src/lib/env-utils.ts +++ b/packages/celotool/src/lib/env-utils.ts @@ -184,13 +184,16 @@ export enum DynamicEnvVar { KUBERNETES_CLUSTER_NAME = '{{ context }}_KUBERNETES_CLUSTER_NAME', KOMENCI_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_ADDRESS_AZURE_KEY_VAULTS', KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT = '{{ context }}_KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT', - KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT = '{{ context }}_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT', - KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT = '{{ context }}_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT', + KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT', KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = '{{ context }}_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT', KOMENCI_DB_HOST = '{{ context }}_KOMENCI_DB_HOST', KOMENCI_DB_PORT = '{{ context }}_KOMENCI_DB_PORT', KOMENCI_DB_USERNAME = '{{ context }}_KOMENCI_DB_USERNAME', KOMENCI_DB_PASSWORD_VAULT_NAME = '{{ context }}_KOMENCI_DB_PASSWORD_VAULT_NAME', + KOMENCI_REWARD_SERVICE_DB_HOST = '{{ context }}_KOMENCI_REWARD_SERVICE_DB_HOST', + KOMENCI_REWARD_SERVICE_DB_PORT = '{{ context }}_KOMENCI_REWARD_SERVICE_DB_PORT', + KOMENCI_REWARD_SERVICE_DB_USERNAME = '{{ context }}_KOMENCI_REWARD_SERVICE_DB_USERNAME', + KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME = '{{ context }}_KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME', KOMENCI_NETWORK = '{{ context }}_KOMENCI_NETWORK', KOMENCI_APP_SECRETS_VAULT_NAME = '{{ context }}_KOMENCI_APP_SECRETS_VAULT_NAME', KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED = '{{ context }}_KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED', diff --git a/packages/celotool/src/lib/komenci.ts b/packages/celotool/src/lib/komenci.ts index ed7a56dd7ac..2be71ca42c4 100644 --- a/packages/celotool/src/lib/komenci.ts +++ b/packages/celotool/src/lib/komenci.ts @@ -44,9 +44,9 @@ interface KomenciIdentity { */ interface KomenciConfig { relayerIdentities: KomenciIdentity[] - foundationRewardsIdentities: KomenciIdentity[] + // TODO: For Signup rewards + // foundationRewardsIdentities: KomenciIdentity[] cLabsRewardsIdentities: KomenciIdentity[] - rewardServiceInstanceCount: number } interface KomenciKeyVaultIdentityConfig { @@ -57,6 +57,10 @@ interface KomenciMnemonicIdentityConfig { addressesFromMnemonicCount: string } +interface KomenciRewardServiceConfig { + instanceCount: number +} + interface KomenciDatabaseConfig { host: string port: string @@ -108,10 +112,10 @@ const contextKomenciCeloLabsRewardsKeyVaultIdentityConfigDynamicEnvVars: { /** * Env vars corresponding to each value for the KomenciCeloLabsRewardsKeyVaultIdentityConfig for a particular context */ -const contextKomenciRewardsServiceInstanceCountConfigDynamicEnvVars: { - [k in keyof KomenciKeyVaultIdentityConfig]: DynamicEnvVar +const contextKomenciRewardsServiceConfigDynamicEnvVars: { + [k in keyof KomenciRewardServiceConfig]: DynamicEnvVar } = { - addressAzureKeyVaults: DynamicEnvVar.KOMENCI_REWARD_SERVICE_INSTANCE_COUNT, + instanceCount: DynamicEnvVar.KOMENCI_REWARD_SERVICE_INSTANCE_COUNT, } const contextDatabaseConfigDynamicEnvVars: { [k in keyof KomenciDatabaseConfig]: DynamicEnvVar } = { @@ -121,16 +125,20 @@ const contextDatabaseConfigDynamicEnvVars: { [k in keyof KomenciDatabaseConfig]: passwordVaultName: DynamicEnvVar.KOMENCI_DB_PASSWORD_VAULT_NAME, } +const contextRewardServiceDatabaseConfigDynamicEnvVars: { + [k in keyof KomenciDatabaseConfig]: DynamicEnvVar +} = { + host: DynamicEnvVar.KOMENCI_REWARD_SERVICE_DB_HOST, + port: DynamicEnvVar.KOMENCI_REWARD_SERVICE_DB_PORT, + username: DynamicEnvVar.KOMENCI_REWARD_SERVICE_DB_USERNAME, + passwordVaultName: DynamicEnvVar.KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME, +} + function releaseName(celoEnv: string) { return `${celoEnv}-komenci` } -export async function installHelmChart( - celoEnv: string, - context: string, - useForno: boolean, - deployRewards: boolean -) { +export async function installHelmChart(celoEnv: string, context: string, useForno: boolean) { // First install the komenci-rbac helm chart. // This must be deployed before so we can use a resulting auth token so that // komenci pods can reach the K8s API server to change their aad labels @@ -140,22 +148,17 @@ export async function installHelmChart( celoEnv, releaseName(celoEnv), helmChartPath, - await helmParameters(celoEnv, context, useForno, deployRewards) + await helmParameters(celoEnv, context, useForno) ) } -export async function upgradeKomenciChart( - celoEnv: string, - context: string, - useFullNodes: boolean, - deployRewards: boolean -) { +export async function upgradeKomenciChart(celoEnv: string, context: string, useFullNodes: boolean) { await upgradeKomenciRBACHelmChart(celoEnv, context) return upgradeGenericHelmChart( celoEnv, releaseName(celoEnv), helmChartPath, - await helmParameters(celoEnv, context, useFullNodes, deployRewards) + await helmParameters(celoEnv, context, useFullNodes) ) } @@ -187,18 +190,21 @@ async function getPasswordFromKeyVaultSecret(vaultName: string, secretName: stri return password.replace(/\n|"/g, '') } -async function helmParameters( - celoEnv: string, - context: string, - useForno: boolean, - deployRewards: boolean -) { +async function helmParameters(celoEnv: string, context: string, useForno: boolean) { const komenciConfig = getKomenciConfig(context) - const relayerCount = komenciConfig.relayerIdentities.length - const kubeServiceAccountSecretNames = await rbacServiceAccountSecretNames(celoEnv, relayerCount) + const onboardingRelayerCount = komenciConfig.relayerIdentities.length + const rewardsRelayerCount = komenciConfig.cLabsRewardsIdentities.length + const kubeServiceAccountSecretNames = await rbacServiceAccountSecretNames( + celoEnv, + onboardingRelayerCount + ) const databaseConfig = getContextDynamicEnvVarValues(contextDatabaseConfigDynamicEnvVars, context) + const rewardDatabaseConfig = getContextDynamicEnvVarValues( + contextRewardServiceDatabaseConfigDynamicEnvVars, + context + ) const vars = getContextDynamicEnvVarValues( { network: DynamicEnvVar.KOMENCI_NETWORK, @@ -216,6 +222,10 @@ async function helmParameters( databaseConfig.passwordVaultName, 'DB-PASSWORD' ) + const rewardDatabasePassword = await getPasswordFromKeyVaultSecret( + rewardDatabaseConfig.passwordVaultName, + 'DB-PASSWORD' + ) const recaptchaToken = await getPasswordFromKeyVaultSecret( vars.appSecretsKeyVault, 'RECAPTCHA-SECRET-KEY' @@ -224,6 +234,10 @@ async function helmParameters( vars.appSecretsKeyVault, 'LOGGER-SERVICE-ACCOUNT' ) + const rewardServiceConfig = getContextDynamicEnvVarValues( + contextKomenciRewardsServiceConfigDynamicEnvVars, + context + ) const clusterConfig = getAksClusterConfig(context) return [ @@ -239,20 +253,21 @@ async function helmParameters( `--set komenci.azureHsm.initTryCount=5`, `--set komenci.azureHsm.initMaxRetryBackoffMs=30000`, `--set onboarding.recaptchaToken=${recaptchaToken}`, - `--set onboarding.replicas=${relayerCount}`, + `--set onboarding.replicas=${onboardingRelayerCount}`, `--set onboarding.relayer.host=${celoEnv + '-relayer'}`, `--set onboarding.db.host=${databaseConfig.host}`, `--set onboarding.db.port=${databaseConfig.port}`, `--set onboarding.db.username=${databaseConfig.username}`, `--set onboarding.db.password=${databasePassword}`, `--set onboarding.publicHostname=${getPublicHostname(clusterConfig.regionName, celoEnv)}`, - `--set onboarding.publicUrl=${'https://' + - getPublicHostname(clusterConfig.regionName, celoEnv)}`, + `--set onboarding.publicUrl=${ + 'https://' + getPublicHostname(clusterConfig.regionName, celoEnv) + }`, `--set onboarding.ruleConfig.captcha.bypassEnabled=${vars.captchaBypassEnabled}`, `--set onboarding.ruleConfig.captcha.bypassToken=${fetchEnv( envVar.KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_TOKEN )}`, - `--set relayer.replicas=${relayerCount}`, + `--set relayer.replicas=${onboardingRelayerCount}`, `--set relayer.rpcProviderUrls.http=${httpRpcProviderUrl}`, `--set relayer.rpcProviderUrls.ws=${wsRpcProviderUrl}`, `--set relayer.metrics.enabled=true`, @@ -263,7 +278,30 @@ async function helmParameters( ) .split(',') .join('\\,')}'`, - ].concat(await komenciIdentityHelmParameters(context, komenciConfig)) + `--set rewards.replicas=${rewardServiceConfig.instanceCount}`, + `--set rewards.db.host=${rewardDatabaseConfig.host}`, + `--set rewards.db.port=${rewardDatabaseConfig.port}`, + `--set rewards.db.username=${rewardDatabaseConfig.username}`, + `--set rewards.db.password=${rewardDatabasePassword}`, + `--set rewards.metrics.enabled=true`, + `--set rewards.metrics.prometheusPort=9090`, + `--set rewards.relayer.replicas=${rewardsRelayerCount}`, + `--set rewards.relayer.rpcProviderUrls.http=${httpRpcProviderUrl}`, + `--set rewards.relayer.rpcProviderUrls.ws=${wsRpcProviderUrl}`, + `--set rewards.relayer.metrics.enabled=true`, + `--set rewards.relayer.metrics.prometheusPort=9090`, + `--set rewards.relayer.host=${celoEnv + '-rewards-relayer'}`, + ] + .concat( + await komenciIdentityHelmParameters(context, komenciConfig.relayerIdentities, 'relayer') + ) + .concat( + await komenciIdentityHelmParameters( + context, + komenciConfig.cLabsRewardsIdentities, + 'rewards.relayer' + ) + ) } function getPublicHostname(regionName: string, celoEnv: string): string { @@ -274,12 +312,16 @@ function getPublicHostname(regionName: string, celoEnv: string): string { * Returns an array of helm command line parameters for the komenci relayer identities. * Supports both private key and Azure HSM signing. */ -async function komenciIdentityHelmParameters(context: string, komenciConfig: KomenciConfig) { - const replicas = komenciConfig.relayerIdentities.length +async function komenciIdentityHelmParameters( + context: string, + relayerIdentities: KomenciIdentity[], + envVarPrefix: string +) { + const replicas = relayerIdentities.length let params: string[] = [] for (let i = 0; i < replicas; i++) { - const komenciIdentity = komenciConfig.relayerIdentities[i] - const prefix = `--set relayer.identities[${i}]` + const komenciIdentity = relayerIdentities[i] + const prefix = `--set ${envVarPrefix}.identities[${i}]` params.push(`${prefix}.address=${komenciIdentity.address}`) // An komenci identity can specify either a private key or some information // about an Azure Key Vault that houses an HSM with the address provided. @@ -308,13 +350,14 @@ async function komenciIdentityHelmParameters(context: string, komenciConfig: Kom } /** - * Gives a config for all komencis for a particular context + * Gives a config for all komenci services for a particular context */ function getKomenciConfig(context: string): KomenciConfig { return { relayerIdentities: getKomenciRelayerIdentities(context), cLabsRewardsIdentities: getKomenciRewardIdentities(context, RewardType.Foundation), - foundationRewardsIdentities: getKomenciRewardIdentities(context, RewardType.CeloLabs), + // TODO: For Signup rewards + // foundationRewardsIdentities: getKomenciRewardIdentities(context, RewardType.CeloLabs), } } @@ -364,8 +407,6 @@ function getKomenciRewardIdentities(context: string, rewardType: RewardType): Ko addressAzureKeyVaults: '', }) - const rewardServiceCount = getContextDynamicEnvVarValues() - if (addressAzureKeyVaults) { return getAzureHsmKomenciIdentities(addressAzureKeyVaults) } @@ -453,7 +494,7 @@ function removeKomenciRBACHelmRelease(celoEnv: string) { function rbacHelmParameters(celoEnv: string, context: string) { const komenciConfig = getKomenciConfig(context) console.info(komenciConfig) - const replicas = komenciConfig.identities.length + const replicas = komenciConfig.relayerIdentities.length return [`--set environment.name=${celoEnv}`, `--set relayer.replicas=${replicas}`] } diff --git a/packages/helm-charts/komenci/templates/rewards-deployment.yaml b/packages/helm-charts/komenci/templates/rewards-deployment.yaml new file mode 100644 index 00000000000..1039ae80ac3 --- /dev/null +++ b/packages/helm-charts/komenci/templates/rewards-deployment.yaml @@ -0,0 +1,41 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "komenci-rewards-fullname" . }} + labels: +{{- include "komenci-rewards-component-label" . | nindent 4 }} +spec: + replicas: {{ .Values.rewards.replicaCount }} + selector: + matchLabels: + {{- include "komenci-rewards-component-label" . | nindent 6 }} + template: + metadata: + labels: +{{- include "komenci-rewards-component-label" . | nindent 8 }} + spec: + containers: + - name: komenci-rewards + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: Always + command: + - bash + - "-c" + - | + node dist/apps/rewards/main.js + env: + - name: REPLICA_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +{{ include "common.env-var" (dict "name" "DB_HOST" "dict" .Values.rewards.db "value_name" "host" "optional" true) | indent 10 }} +{{ include "common.env-var" (dict "name" "DB_PORT" "dict" .Values.rewards.db "value_name" "port" "optional" true) | indent 10 }} +{{ include "common.env-var" (dict "name" "DB_USERNAME" "dict" .Values.rewards.db "value_name" "username") | indent 10 }} +{{ include "common.env-var" (dict "name" "DB_PASSWORD" "dict" .Values.rewards.db "value_name" "password") | indent 10 }} +{{ include "common.env-var" (dict "name" "DB_DATABASE" "dict" .Values.rewards.db "value_name" "database") | indent 10 }} +{{ include "common.env-var" (dict "name" "DB_SYNCHRONIZE" "dict" .Values.rewards.db "value_name" "synchronize") | indent 10 }} +{{ include "common.env-var" (dict "name" "DB_SSL" "dict" .Values.rewards.db "value_name" "ssl") | indent 10 }} +{{ include "common.env-var" (dict "name" "RELAYER_HOST" "dict" .Values.rewards.relayer "value_name" "host") | indent 10 }} +{{ include "common.env-var" (dict "name" "RELAYER_PORT" "dict" .Values.rewards.relayer "value_name" "port") | indent 10 }} diff --git a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml new file mode 100644 index 00000000000..c53e87568df --- /dev/null +++ b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml @@ -0,0 +1,141 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "name" . }} + labels: +{{ include "labels" . | indent 4 }} +{{ include "komenci-rewards-relayer-component-label" . | indent 4 }} +spec: + ports: + - name: http + port: 3000 + clusterIP: None + selector: +{{ include "komenci-rewards-relayer-component-label" . | indent 4 }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ template "name" . }} + labels: +{{ include "labels" . | indent 4 }} +{{ include "komenci-rewards-relayer-component-label" . | indent 4 }} +spec: + podManagementPolicy: Parallel + updateStrategy: + type: RollingUpdate + replicas: {{ .Values.rewards.relayer.replicas }} + serviceName: relayer + selector: + matchLabels: +{{ include "labels" . | indent 6 }} +{{ include "komenci-rewards-relayer-component-label" . | indent 6 }} + template: + metadata: + labels: +{{ include "labels" . | indent 8 }} +{{ include "komenci-rewards-relayer-component-label" . | indent 8 }} + annotations: +{{ if .Values.rewards.relayer.metrics.enabled }} +{{ include "metric-annotations" . | indent 8 }} +{{ end }} + spec: + initContainers: + - name: set-label + image: {{ .Values.kubectl.image.repository }}:{{ .Values.kubectl.image.tag }} + command: + - /bin/bash + - -c + args: + - | + RID=${POD_NAME##*-} + TOKEN_ENV_VAR_NAME="TOKEN_$RID" + kubectl \ + --namespace "$POD_NAMESPACE" \ + --server="https://kubernetes.default.svc" \ + --token="${!TOKEN_ENV_VAR_NAME}" \ + --certificate-authority="/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" \ + label pod "$POD_NAME" \ + --overwrite \ + "aadpodidbinding=$POD_NAME-identity-binding" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + {{ range $index, $e := .Values.kube.serviceAccountSecretNames }} + - name: TOKEN_{{ $index }} + valueFrom: + secretKeyRef: + key: token + name: {{ $e }} + {{ end }} + containers: + - name: komenci-rewards-relayer + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: Always + ports: + - name: prometheus + containerPort: {{ .Values.rewards.relayer.metrics.prometheusPort }} + - name: relayer + containerPort: 3000 + command: + - bash + - "-c" + - | + [[ $REPLICA_NAME =~ -([0-9]+)$ ]] || exit 1 + RID=${BASH_REMATCH[1]} + + # Set the private key path. If Azure HSM signing is specified, + # it will take precedence. + export PRIVATE_KEY_PATH="/private-keys/private-key-$RID" + + # Get the correct key vault name. If this relayer's identity is not + # using Azure HSM signing, the key vault name will be empty and ignored + AZURE_KEY_VAULT_NAMES={{- range $index, $identity := .Values.rewards.relayer.identities -}}{{- if (hasKey $identity "azure" ) -}}{{ $identity.azure.keyVaultName | default "" }}{{- end }},{{- end }} + export AZURE_KEY_NAME=`echo -n $AZURE_KEY_VAULT_NAMES | cut -d ',' -f $((RID + 1))` + export AZURE_VAULT_NAME=`echo -n $AZURE_KEY_VAULT_NAMES | cut -d ',' -f $((RID + 1))` + + # Get the correct relayer account address + ADDRESSES={{- range $index, $identity := .Values.rewards.relayer.identities -}}{{ $identity.address }},{{- end }} + export WALLET_ADDRESS=`echo -n $ADDRESSES | cut -d ',' -f $((RID + 1))` + + node dist/apps/relayer/main.js + env: + - name: REPLICA_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name +{{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_TRY_COUNT" "dict" .Values.rewards.relayer.azureHsm "value_name" "initTryCount") | indent 8 }} +{{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_MAX_RETRY_BACKOFF_MS" "dict" .Values.rewards.relayer.azureHsm "value_name" "initMaxRetryBackoffMs") | indent 8 }} +{{ include "common.env-var" (dict "name" "METRICS" "dict" .Values.rewards.relayer.metrics "value_name" "enabled") | indent 8 }} +{{ include "common.env-var" (dict "name" "OVERRIDE_INDEX" "dict" .Values.rewards.relayer "value_name" "overrideIndex" "optional" true) | indent 8 }} +{{ include "common.env-var" (dict "name" "PRIVATE_KEY_PATH" "dict" .Values.rewards.relayer "value_name" "privateKeyPath" "optional" true) | indent 8 }} +{{ include "common.env-var" (dict "name" "PROMETHEUS_PORT" "dict" .Values.rewards.relayer.metrics "value_name" "prometheusPort") | indent 8 }} +{{ include "common.env-var" (dict "name" "NODE_ENV" "dict" .Values.rewards.relayer "value_name" "node_env") | indent 8 }} +{{ include "common.env-var" (dict "name" "RELAYER_PORT" "dict" .Values.rewards.relayer "value_name" "port") | indent 8 }} +{{ include "common.env-var" (dict "name" "NETWORK" "dict" .Values.environment "value_name" "network") | indent 8 }} +{{ include "common.env-var" (dict "name" "WALLET_TYPE" "dict" .Values.rewards.relayer "value_name" "walletType") | indent 8 }} +{{ include "common.env-var" (dict "name" "GAS_PRICE_UPDATE_INTERVAL_MS" "dict" .Values.rewards.relayer "value_name" "gasPriceUpdateIntervalMs") | indent 8 }} + readinessProbe: + tcpSocket: + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + tcpSocket: + port: 3000 + initialDelaySeconds: 15 + periodSeconds: 20 + volumeMounts: + - name: private-key-volume + readOnly: true + mountPath: "/private-keys" + volumes: + - name: private-key-volume + secret: + secretName: pkey-secret From f963cbd5c63442c2dd182ba0b8c7955e0fe07a8a Mon Sep 17 00:00:00 2001 From: codyborn Date: Mon, 3 May 2021 21:04:02 -0700 Subject: [PATCH 04/16] WIP --- .env.alfajores | 5 +++++ packages/celotool/src/lib/env-utils.ts | 3 ++- packages/celotool/src/lib/komenci.ts | 11 +++++------ packages/helm-charts/komenci/templates/_helpers.tpl | 11 +++++++++++ .../templates/rewards-relayer-statefulset.yaml | 6 +++--- 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/.env.alfajores b/.env.alfajores index d6961fc7a6c..13095364764 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -173,6 +173,11 @@ AZURE_KOMENCI_WESTEU_KOMENCI_DB_PORT=5432 AZURE_KOMENCI_WESTEU_KOMENCI_DB_USERNAME=postgres@staging-komenci-weu AZURE_KOMENCI_WESTEU_KOMENCI_DB_PASSWORD_VAULT_NAME=staging-komenci-weu +AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_DB_HOST=staging-komenci-weu.postgres.database.azure.com +AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_DB_PORT=5432 +AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_DB_USERNAME=postgres@staging-komenci-weu +AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME=staging-komenci-weu + AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_HOST=staging-komenci-weu.postgres.database.azure.com AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_PORT=5432 AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_DB_USERNAME=postgres@staging-komenci-weu diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts index c8a50de1f67..628b6fdf3a0 100644 --- a/packages/celotool/src/lib/env-utils.ts +++ b/packages/celotool/src/lib/env-utils.ts @@ -184,7 +184,8 @@ export enum DynamicEnvVar { KUBERNETES_CLUSTER_NAME = '{{ context }}_KUBERNETES_CLUSTER_NAME', KOMENCI_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_ADDRESS_AZURE_KEY_VAULTS', KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT = '{{ context }}_KOMENCI_ADDRESSES_FROM_MNEMONIC_COUNT', - KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT', + KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS', + KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULTS = '{{ context }}_KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULTS', KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = '{{ context }}_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT', KOMENCI_DB_HOST = '{{ context }}_KOMENCI_DB_HOST', KOMENCI_DB_PORT = '{{ context }}_KOMENCI_DB_PORT', diff --git a/packages/celotool/src/lib/komenci.ts b/packages/celotool/src/lib/komenci.ts index 2be71ca42c4..bdb0cbb55d8 100644 --- a/packages/celotool/src/lib/komenci.ts +++ b/packages/celotool/src/lib/komenci.ts @@ -97,7 +97,7 @@ const contextKomenciMnemonicIdentityConfigDynamicEnvVars: { const contextKomenciFoundationRewardsKeyVaultIdentityConfigDynamicEnvVars: { [k in keyof KomenciKeyVaultIdentityConfig]: DynamicEnvVar } = { - addressAzureKeyVaults: DynamicEnvVar.KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULT, + addressAzureKeyVaults: DynamicEnvVar.KOMENCI_FOUNDATION_REWARDS_ADDRESS_AZURE_KEY_VAULTS, } /** @@ -106,7 +106,7 @@ const contextKomenciFoundationRewardsKeyVaultIdentityConfigDynamicEnvVars: { const contextKomenciCeloLabsRewardsKeyVaultIdentityConfigDynamicEnvVars: { [k in keyof KomenciKeyVaultIdentityConfig]: DynamicEnvVar } = { - addressAzureKeyVaults: DynamicEnvVar.KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULT, + addressAzureKeyVaults: DynamicEnvVar.KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS, } /** @@ -355,9 +355,8 @@ async function komenciIdentityHelmParameters( function getKomenciConfig(context: string): KomenciConfig { return { relayerIdentities: getKomenciRelayerIdentities(context), - cLabsRewardsIdentities: getKomenciRewardIdentities(context, RewardType.Foundation), - // TODO: For Signup rewards - // foundationRewardsIdentities: getKomenciRewardIdentities(context, RewardType.CeloLabs), + cLabsRewardsIdentities: getKomenciRewardIdentities(context, RewardType.CeloLabs), + // foundationRewardsIdentities: getKomenciRewardIdentities(context, RewardType.Foundation), } } @@ -411,7 +410,7 @@ function getKomenciRewardIdentities(context: string, rewardType: RewardType): Ko return getAzureHsmKomenciIdentities(addressAzureKeyVaults) } - throw Error('No komenci identity env vars specified') + throw Error('No komenci reward identity env vars specified') } /** diff --git a/packages/helm-charts/komenci/templates/_helpers.tpl b/packages/helm-charts/komenci/templates/_helpers.tpl index 9cf6a682f49..f839094de47 100644 --- a/packages/helm-charts/komenci/templates/_helpers.tpl +++ b/packages/helm-charts/komenci/templates/_helpers.tpl @@ -9,6 +9,10 @@ The name of the deployment {{- .Values.environment.name -}}-onboarding {{- end -}} +{{- define "komenci-rewards-fullname" -}} +{{- .Values.environment.name -}}-rewards +{{- end -}} + {{/* Common labels that are recommended to be used by Helm and Kubernetes */}} @@ -48,6 +52,13 @@ Label specific to the komenci rewards component app.kubernetes.io/component: komenci-rewards {{- end -}} +{{/* +Label specific to the komenci rewards relayer component +*/}} +{{- define "komenci-rewards-relayer-component-label" -}} +app.kubernetes.io/component: komenci-rewards-relayer +{{- end -}} + {{/* The name of the azure identity binding for all relayers */}} diff --git a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml index c53e87568df..6a47285619c 100644 --- a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml @@ -110,8 +110,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name -{{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_TRY_COUNT" "dict" .Values.rewards.relayer.azureHsm "value_name" "initTryCount") | indent 8 }} -{{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_MAX_RETRY_BACKOFF_MS" "dict" .Values.rewards.relayer.azureHsm "value_name" "initMaxRetryBackoffMs") | indent 8 }} +{{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_TRY_COUNT" "dict" .Values.komenci.azureHsm "value_name" "initTryCount") | indent 8 }} +{{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_MAX_RETRY_BACKOFF_MS" "dict" .Values.komenci.azureHsm "value_name" "initMaxRetryBackoffMs") | indent 8 }} {{ include "common.env-var" (dict "name" "METRICS" "dict" .Values.rewards.relayer.metrics "value_name" "enabled") | indent 8 }} {{ include "common.env-var" (dict "name" "OVERRIDE_INDEX" "dict" .Values.rewards.relayer "value_name" "overrideIndex" "optional" true) | indent 8 }} {{ include "common.env-var" (dict "name" "PRIVATE_KEY_PATH" "dict" .Values.rewards.relayer "value_name" "privateKeyPath" "optional" true) | indent 8 }} @@ -138,4 +138,4 @@ spec: volumes: - name: private-key-volume secret: - secretName: pkey-secret + secretName: rewards-pkey-secret From e404a42a7adbe205b9ff941a7bcfa553cabbfd70 Mon Sep 17 00:00:00 2001 From: codyborn Date: Mon, 3 May 2021 21:10:17 -0700 Subject: [PATCH 05/16] Add missing pkey-secret file --- .../komenci/templates/rewards-pkey-secret.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml diff --git a/packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml b/packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml new file mode 100644 index 00000000000..b3f18d60509 --- /dev/null +++ b/packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: rewards-pkey-secret + labels: +{{ include "labels" . | indent 4 }} +type: Opaque +data: +{{ range $index, $identity := .Values.rewards.relayer.identities }} +{{ if (hasKey $identity "privateKey") }} + private-key-{{ $index }}: {{ $identity.privateKey }} +{{ end }} +{{ end }} From 5cdcc1380d72fca97c33f9ef2bc25d7f1453d658 Mon Sep 17 00:00:00 2001 From: codyborn Date: Tue, 4 May 2021 13:00:18 -0700 Subject: [PATCH 06/16] Validating --- .env.alfajores | 2 +- .../helm-charts/komenci/templates/pkey-secret.yaml | 13 ------------- .../komenci/templates/relayer-statefulset.yaml | 8 -------- .../komenci/templates/rewards-pkey-secret.yaml | 13 ------------- .../templates/rewards-relayer-statefulset.yaml | 10 +--------- 5 files changed, 2 insertions(+), 44 deletions(-) delete mode 100644 packages/helm-charts/komenci/templates/pkey-secret.yaml delete mode 100644 packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml diff --git a/.env.alfajores b/.env.alfajores index 13095364764..2316d9e5d67 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -153,7 +153,7 @@ CONTEXTS=azure-komenci-eastus,azure-komenci-westeu,azure-oracle-centralus # --- Komenci --- KOMENCI_DOCKER_IMAGE_REPOSITORY="celotestnet.azurecr.io/komenci/komenci" -KOMENCI_DOCKER_IMAGE_TAG="d78d4c25fcd04871a81c67e6107318c9dbd7ec03" +KOMENCI_DOCKER_IMAGE_TAG="7876a1622872789d051459e13b4fcc89725887d0" AZURE_KOMENCI_EASTUS_AZURE_KUBERNETES_RESOURCE_GROUP=staging-komenci-eastus AZURE_KOMENCI_EASTUS_KUBERNETES_CLUSTER_NAME=staging-komenci-eastus diff --git a/packages/helm-charts/komenci/templates/pkey-secret.yaml b/packages/helm-charts/komenci/templates/pkey-secret.yaml deleted file mode 100644 index 48a3fde41bf..00000000000 --- a/packages/helm-charts/komenci/templates/pkey-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: pkey-secret - labels: -{{ include "labels" . | indent 4 }} -type: Opaque -data: -{{ range $index, $identity := .Values.relayer.identities }} -{{ if (hasKey $identity "privateKey") }} - private-key-{{ $index }}: {{ $identity.privateKey }} -{{ end }} -{{ end }} diff --git a/packages/helm-charts/komenci/templates/relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/relayer-statefulset.yaml index 6162f5800d4..5ec83ac9f6b 100644 --- a/packages/helm-charts/komenci/templates/relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/relayer-statefulset.yaml @@ -132,11 +132,3 @@ spec: port: 3000 initialDelaySeconds: 15 periodSeconds: 20 - volumeMounts: - - name: private-key-volume - readOnly: true - mountPath: "/private-keys" - volumes: - - name: private-key-volume - secret: - secretName: pkey-secret diff --git a/packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml b/packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml deleted file mode 100644 index b3f18d60509..00000000000 --- a/packages/helm-charts/komenci/templates/rewards-pkey-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: rewards-pkey-secret - labels: -{{ include "labels" . | indent 4 }} -type: Opaque -data: -{{ range $index, $identity := .Values.rewards.relayer.identities }} -{{ if (hasKey $identity "privateKey") }} - private-key-{{ $index }}: {{ $identity.privateKey }} -{{ end }} -{{ end }} diff --git a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml index 6a47285619c..bbbbf819266 100644 --- a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml @@ -130,12 +130,4 @@ spec: tcpSocket: port: 3000 initialDelaySeconds: 15 - periodSeconds: 20 - volumeMounts: - - name: private-key-volume - readOnly: true - mountPath: "/private-keys" - volumes: - - name: private-key-volume - secret: - secretName: rewards-pkey-secret + periodSeconds: 20 \ No newline at end of file From aa5997d07b0010499c1d938ec0076bd0ff72a750 Mon Sep 17 00:00:00 2001 From: codyborn Date: Thu, 6 May 2021 20:19:21 -0700 Subject: [PATCH 07/16] WIP: Deployment successful --- packages/helm-charts/komenci/templates/_helpers.tpl | 4 ++++ .../komenci/templates/rewards-relayer-statefulset.yaml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/helm-charts/komenci/templates/_helpers.tpl b/packages/helm-charts/komenci/templates/_helpers.tpl index f839094de47..118e6454110 100644 --- a/packages/helm-charts/komenci/templates/_helpers.tpl +++ b/packages/helm-charts/komenci/templates/_helpers.tpl @@ -5,6 +5,10 @@ The name of the deployment {{- .Values.environment.name -}}-relayer {{- end -}} +{{- define "komenci-rewards-relayer-fullname" -}} +{{- .Values.environment.name -}}-rewards-relayer +{{- end -}} + {{- define "komenci-onboarding-fullname" -}} {{- .Values.environment.name -}}-onboarding {{- end -}} diff --git a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml index bbbbf819266..ca8b2cb78c4 100644 --- a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "name" . }} + name: {{ template "komenci-rewards-relayer-fullname" . }} labels: {{ include "labels" . | indent 4 }} {{ include "komenci-rewards-relayer-component-label" . | indent 4 }} @@ -16,7 +16,7 @@ spec: apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ template "name" . }} + name: {{ template "komenci-rewards-relayer-fullname" . }} labels: {{ include "labels" . | indent 4 }} {{ include "komenci-rewards-relayer-component-label" . | indent 4 }} From 632af7bf2b831dbd96059bc4596365e3e4daac31 Mon Sep 17 00:00:00 2001 From: codyborn Date: Tue, 11 May 2021 10:51:48 -0700 Subject: [PATCH 08/16] Add missing network param --- packages/helm-charts/komenci/templates/rewards-deployment.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/helm-charts/komenci/templates/rewards-deployment.yaml b/packages/helm-charts/komenci/templates/rewards-deployment.yaml index 1039ae80ac3..cf7c2ad9ea2 100644 --- a/packages/helm-charts/komenci/templates/rewards-deployment.yaml +++ b/packages/helm-charts/komenci/templates/rewards-deployment.yaml @@ -39,3 +39,4 @@ spec: {{ include "common.env-var" (dict "name" "DB_SSL" "dict" .Values.rewards.db "value_name" "ssl") | indent 10 }} {{ include "common.env-var" (dict "name" "RELAYER_HOST" "dict" .Values.rewards.relayer "value_name" "host") | indent 10 }} {{ include "common.env-var" (dict "name" "RELAYER_PORT" "dict" .Values.rewards.relayer "value_name" "port") | indent 10 }} +{{ include "common.env-var" (dict "name" "NETWORK" "dict" .Values.environment "value_name" "network") | indent 10 }} From 29840a7fbee531eeb1482eb96d4fa213a068278c Mon Sep 17 00:00:00 2001 From: codyborn Date: Thu, 13 May 2021 22:34:59 -0700 Subject: [PATCH 09/16] Add RBAC for rewards relayer --- .env.alfajores | 2 +- packages/celotool/src/lib/komenci.ts | 32 +++++++++++++------ .../komenci-rbac/templates/_helper.tpl | 8 +++++ .../komenci-rbac/templates/role.yaml | 14 ++++++++ .../komenci-rbac/templates/rolebinding.yaml | 16 ++++++++++ .../templates/service-account.yaml | 9 ++++++ .../komenci/templates/_helpers.tpl | 18 +++++++++-- .../templates/onboarding-deployment.yaml | 2 +- .../templates/relayer-statefulset.yaml | 2 +- .../komenci/templates/rewards-deployment.yaml | 2 +- .../rewards-relayer-statefulset.yaml | 13 +++----- packages/helm-charts/komenci/values.yaml | 30 +++++++++++++++++ 12 files changed, 124 insertions(+), 24 deletions(-) diff --git a/.env.alfajores b/.env.alfajores index 2316d9e5d67..73417114133 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -153,7 +153,7 @@ CONTEXTS=azure-komenci-eastus,azure-komenci-westeu,azure-oracle-centralus # --- Komenci --- KOMENCI_DOCKER_IMAGE_REPOSITORY="celotestnet.azurecr.io/komenci/komenci" -KOMENCI_DOCKER_IMAGE_TAG="7876a1622872789d051459e13b4fcc89725887d0" +KOMENCI_DOCKER_IMAGE_TAG="11f81e71201fb106928f6702993b7816d062dbf4" AZURE_KOMENCI_EASTUS_AZURE_KUBERNETES_RESOURCE_GROUP=staging-komenci-eastus AZURE_KOMENCI_EASTUS_KUBERNETES_CLUSTER_NAME=staging-komenci-eastus diff --git a/packages/celotool/src/lib/komenci.ts b/packages/celotool/src/lib/komenci.ts index bdb0cbb55d8..042ae91593d 100644 --- a/packages/celotool/src/lib/komenci.ts +++ b/packages/celotool/src/lib/komenci.ts @@ -197,8 +197,14 @@ async function helmParameters(celoEnv: string, context: string, useForno: boolea const rewardsRelayerCount = komenciConfig.cLabsRewardsIdentities.length const kubeServiceAccountSecretNames = await rbacServiceAccountSecretNames( celoEnv, + '', onboardingRelayerCount ) + const kubeRewardsServiceAccountSecretNames = await rbacServiceAccountSecretNames( + celoEnv, + 'rewards-', + rewardsRelayerCount + ) const databaseConfig = getContextDynamicEnvVarValues(contextDatabaseConfigDynamicEnvVars, context) const rewardDatabaseConfig = getContextDynamicEnvVarValues( @@ -291,6 +297,9 @@ async function helmParameters(celoEnv: string, context: string, useForno: boolea `--set rewards.relayer.metrics.enabled=true`, `--set rewards.relayer.metrics.prometheusPort=9090`, `--set rewards.relayer.host=${celoEnv + '-rewards-relayer'}`, + `--set kube.rewardsServiceAccountSecretNames='{${kubeRewardsServiceAccountSecretNames.join( + ',' + )}}'`, ] .concat( await komenciIdentityHelmParameters(context, komenciConfig.relayerIdentities, 'relayer') @@ -471,7 +480,7 @@ function getKomenciAzureIdentityName(keyVaultName: string, address: string) { async function installKomenciRBACHelmChart(celoEnv: string, context: string) { return installGenericHelmChart( celoEnv, - rbacReleaseName(celoEnv), + rbacReleaseName(celoEnv, ''), rbacHelmChartPath, rbacHelmParameters(celoEnv, context) ) @@ -480,29 +489,34 @@ async function installKomenciRBACHelmChart(celoEnv: string, context: string) { async function upgradeKomenciRBACHelmChart(celoEnv: string, context: string) { return upgradeGenericHelmChart( celoEnv, - rbacReleaseName(celoEnv), + rbacReleaseName(celoEnv, ''), rbacHelmChartPath, rbacHelmParameters(celoEnv, context) ) } function removeKomenciRBACHelmRelease(celoEnv: string) { - return removeGenericHelmChart(rbacReleaseName(celoEnv), celoEnv) + return removeGenericHelmChart(rbacReleaseName(celoEnv, ''), celoEnv) } function rbacHelmParameters(celoEnv: string, context: string) { const komenciConfig = getKomenciConfig(context) console.info(komenciConfig) - const replicas = komenciConfig.relayerIdentities.length - return [`--set environment.name=${celoEnv}`, `--set relayer.replicas=${replicas}`] + const relayerReplicas = komenciConfig.relayerIdentities.length + const rewardsRelayerReplicas = komenciConfig.cLabsRewardsIdentities.length + return [ + `--set environment.name=${celoEnv}`, + `--set relayer.replicas=${relayerReplicas}`, + `--set rewards.relayer.replicas=${rewardsRelayerReplicas}`, + ] } -function rbacReleaseName(celoEnv: string) { - return `${celoEnv}-komenci-rbac` +function rbacReleaseName(celoEnv: string, prefix: string) { + return `${celoEnv}-komenci-${prefix}rbac` } -async function rbacServiceAccountSecretNames(celoEnv: string, replicas: number) { - const names = [...Array(replicas).keys()].map((i) => `${rbacReleaseName(celoEnv)}-${i}`) +async function rbacServiceAccountSecretNames(celoEnv: string, prefix: string, replicas: number) { + const names = [...Array(replicas).keys()].map((i) => `${rbacReleaseName(celoEnv, prefix)}-${i}`) const [tokenName] = await execCmdWithExitOnFailure( `kubectl get serviceaccount --namespace=${celoEnv} ${names.join( ' ' diff --git a/packages/helm-charts/komenci-rbac/templates/_helper.tpl b/packages/helm-charts/komenci-rbac/templates/_helper.tpl index 77f405b1bf6..1527741e2c3 100644 --- a/packages/helm-charts/komenci-rbac/templates/_helper.tpl +++ b/packages/helm-charts/komenci-rbac/templates/_helper.tpl @@ -5,3 +5,11 @@ {{- define "komenci-pod-name" -}} {{- .Values.environment.name -}}-relayer-{{- .index -}} {{- end -}} + +{{- define "rewards-name" -}} +{{- .Values.environment.name -}}-komenci-rewards-rbac-{{- .index -}} +{{- end -}} + +{{- define "komenci-rewards-pod-name" -}} +{{- .Values.environment.name -}}-rewards-relayer-{{- .index -}} +{{- end -}} \ No newline at end of file diff --git a/packages/helm-charts/komenci-rbac/templates/role.yaml b/packages/helm-charts/komenci-rbac/templates/role.yaml index dd4be55079b..df04f4e72b0 100644 --- a/packages/helm-charts/komenci-rbac/templates/role.yaml +++ b/packages/helm-charts/komenci-rbac/templates/role.yaml @@ -11,3 +11,17 @@ rules: verbs: ["get", "patch"] --- {{ end }} + +{{ range $index, $e := until (.Values.rewards.relayer.replicas | int) }} +{{- $index_counter := (dict "Values" $.Values "index" $index) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "rewards-name" $index_counter }} +rules: +- apiGroups: [""] + resources: ["pods"] + resourceNames: ["{{ template "komenci-rewards-pod-name" $index_counter }}"] + verbs: ["get", "patch"] +--- +{{ end }} diff --git a/packages/helm-charts/komenci-rbac/templates/rolebinding.yaml b/packages/helm-charts/komenci-rbac/templates/rolebinding.yaml index dbc187d2d87..f9ebfb24826 100644 --- a/packages/helm-charts/komenci-rbac/templates/rolebinding.yaml +++ b/packages/helm-charts/komenci-rbac/templates/rolebinding.yaml @@ -13,3 +13,19 @@ subjects: name: {{ template "name" $index_counter }} --- {{ end }} + +{{ range $index, $e := until (.Values.rewards.relayer.replicas | int) }} +{{- $index_counter := (dict "Values" $.Values "index" $index) -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "rewards-name" $index_counter }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "rewards-name" $index_counter }} +subjects: +- kind: ServiceAccount + name: {{ template "rewards-name" $index_counter }} +--- +{{ end }} diff --git a/packages/helm-charts/komenci-rbac/templates/service-account.yaml b/packages/helm-charts/komenci-rbac/templates/service-account.yaml index 61cadbb621d..da8ea5730db 100644 --- a/packages/helm-charts/komenci-rbac/templates/service-account.yaml +++ b/packages/helm-charts/komenci-rbac/templates/service-account.yaml @@ -6,3 +6,12 @@ metadata: name: {{ template "name" $index_counter}} --- {{ end }} + +{{ range $index, $e := until (.Values.rewards.relayer.replicas | int) }} +{{- $index_counter := (dict "Values" $.Values "index" $index) -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "rewards-name" $index_counter}} +--- +{{ end }} diff --git a/packages/helm-charts/komenci/templates/_helpers.tpl b/packages/helm-charts/komenci/templates/_helpers.tpl index 118e6454110..716528b5058 100644 --- a/packages/helm-charts/komenci/templates/_helpers.tpl +++ b/packages/helm-charts/komenci/templates/_helpers.tpl @@ -5,7 +5,7 @@ The name of the deployment {{- .Values.environment.name -}}-relayer {{- end -}} -{{- define "komenci-rewards-relayer-fullname" -}} +{{- define "rewards-name" -}} {{- .Values.environment.name -}}-rewards-relayer {{- end -}} @@ -71,8 +71,22 @@ The name of the azure identity binding for all relayers {{- end -}} {{/* -The name of the azure identity for all oracles +The name of the azure identity binding for all rewards relayers +*/}} +{{- define "azure-rewards-identity-binding-name" -}} +{{- with .dot -}}{{ template "rewards-name" . }}{{- end -}}-{{ .index }}-identity-binding +{{- end -}} + +{{/* +The name of the azure identity for all relayers */}} {{- define "azure-identity-name" -}} {{- with .dot -}}{{ template "name" . }}{{- end -}}-{{ .index }}-identity +{{- end -}} + +{{/* +The name of the azure identity for all rewards relayers +*/}} +{{- define "azure-rewards-identity-name" -}} +{{- with .dot -}}{{ template "rewards-name" . }}{{- end -}}-{{ .index }}-identity {{- end -}} \ No newline at end of file diff --git a/packages/helm-charts/komenci/templates/onboarding-deployment.yaml b/packages/helm-charts/komenci/templates/onboarding-deployment.yaml index 6a66768a623..b6c1bf64733 100644 --- a/packages/helm-charts/komenci/templates/onboarding-deployment.yaml +++ b/packages/helm-charts/komenci/templates/onboarding-deployment.yaml @@ -27,7 +27,7 @@ spec: - bash - "-c" - | - node dist/apps/onboarding/main.js + node packages/apps/api/dist/main.js resources: {{- toYaml .Values.onboarding.resources | nindent 12 }} env: diff --git a/packages/helm-charts/komenci/templates/relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/relayer-statefulset.yaml index 5ec83ac9f6b..2a6d7b037b5 100644 --- a/packages/helm-charts/komenci/templates/relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/relayer-statefulset.yaml @@ -104,7 +104,7 @@ spec: ADDRESSES={{- range $index, $identity := .Values.relayer.identities -}}{{ $identity.address }},{{- end }} export WALLET_ADDRESS=`echo -n $ADDRESSES | cut -d ',' -f $((RID + 1))` - node dist/apps/relayer/main.js + node packages/apps/relayer/dist/main.js env: - name: REPLICA_NAME valueFrom: diff --git a/packages/helm-charts/komenci/templates/rewards-deployment.yaml b/packages/helm-charts/komenci/templates/rewards-deployment.yaml index cf7c2ad9ea2..82cc9d8e040 100644 --- a/packages/helm-charts/komenci/templates/rewards-deployment.yaml +++ b/packages/helm-charts/komenci/templates/rewards-deployment.yaml @@ -24,7 +24,7 @@ spec: - bash - "-c" - | - node dist/apps/rewards/main.js + node packages/apps/rewards/dist/main.js env: - name: REPLICA_NAME valueFrom: diff --git a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml index ca8b2cb78c4..8914d46c4be 100644 --- a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "komenci-rewards-relayer-fullname" . }} + name: {{ template "rewards-name" . }} labels: {{ include "labels" . | indent 4 }} {{ include "komenci-rewards-relayer-component-label" . | indent 4 }} @@ -16,7 +16,7 @@ spec: apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ template "komenci-rewards-relayer-fullname" . }} + name: {{ template "rewards-name" . }} labels: {{ include "labels" . | indent 4 }} {{ include "komenci-rewards-relayer-component-label" . | indent 4 }} @@ -67,7 +67,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - {{ range $index, $e := .Values.kube.serviceAccountSecretNames }} + {{ range $index, $e := .Values.kube.rewardsServiceAccountSecretNames }} - name: TOKEN_{{ $index }} valueFrom: secretKeyRef: @@ -90,10 +90,6 @@ spec: [[ $REPLICA_NAME =~ -([0-9]+)$ ]] || exit 1 RID=${BASH_REMATCH[1]} - # Set the private key path. If Azure HSM signing is specified, - # it will take precedence. - export PRIVATE_KEY_PATH="/private-keys/private-key-$RID" - # Get the correct key vault name. If this relayer's identity is not # using Azure HSM signing, the key vault name will be empty and ignored AZURE_KEY_VAULT_NAMES={{- range $index, $identity := .Values.rewards.relayer.identities -}}{{- if (hasKey $identity "azure" ) -}}{{ $identity.azure.keyVaultName | default "" }}{{- end }},{{- end }} @@ -104,7 +100,7 @@ spec: ADDRESSES={{- range $index, $identity := .Values.rewards.relayer.identities -}}{{ $identity.address }},{{- end }} export WALLET_ADDRESS=`echo -n $ADDRESSES | cut -d ',' -f $((RID + 1))` - node dist/apps/relayer/main.js + node packages/apps/relayer/dist/main.js env: - name: REPLICA_NAME valueFrom: @@ -114,7 +110,6 @@ spec: {{ include "common.env-var" (dict "name" "AZURE_HSM_INIT_MAX_RETRY_BACKOFF_MS" "dict" .Values.komenci.azureHsm "value_name" "initMaxRetryBackoffMs") | indent 8 }} {{ include "common.env-var" (dict "name" "METRICS" "dict" .Values.rewards.relayer.metrics "value_name" "enabled") | indent 8 }} {{ include "common.env-var" (dict "name" "OVERRIDE_INDEX" "dict" .Values.rewards.relayer "value_name" "overrideIndex" "optional" true) | indent 8 }} -{{ include "common.env-var" (dict "name" "PRIVATE_KEY_PATH" "dict" .Values.rewards.relayer "value_name" "privateKeyPath" "optional" true) | indent 8 }} {{ include "common.env-var" (dict "name" "PROMETHEUS_PORT" "dict" .Values.rewards.relayer.metrics "value_name" "prometheusPort") | indent 8 }} {{ include "common.env-var" (dict "name" "NODE_ENV" "dict" .Values.rewards.relayer "value_name" "node_env") | indent 8 }} {{ include "common.env-var" (dict "name" "RELAYER_PORT" "dict" .Values.rewards.relayer "value_name" "port") | indent 8 }} diff --git a/packages/helm-charts/komenci/values.yaml b/packages/helm-charts/komenci/values.yaml index 1afbed98a8b..cca9a0480f1 100644 --- a/packages/helm-charts/komenci/values.yaml +++ b/packages/helm-charts/komenci/values.yaml @@ -83,5 +83,35 @@ onboarding: synchronize: true ssl: true +rewards: + relayer: + node_env: production + image: + repository: celotestnet.azurecr.io/komenci/relayer + tag: dae43ddce108a73da07dce73875b980ff077c7d4 + replicas: 2 + port: 3000 + identities: + - address: 0x00454cac6dae53f8800f71395b9a174f07a784b1 + privateKey: 0x000 + azure: + id: defaultId + clientId: defaultClientId + keyVaultName: staging-komenci-eus + - address: 0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e + privateKey: 0x001 + azure: + id: defaultId1 + clientId: defaultClientId1 + keyVaultName: staging-komenci-eus + azureHsm: + initTryCount: 5 + initMaxRetryBackoffMs: 30000 + metrics: + enabled: true + prometheusPort: 9090 + walletType: azure-hsm + gasPriceUpdateIntervalMs: "1200000" + loggingAgent: credentials: eydleGFtcGxlJzogJ2NyZWRlbnRpYWxzJ30K # base64 credentials.json of a gcloud service account \ No newline at end of file From 0abb4a9bf51d73c95560be2e8e500d1d715964ee Mon Sep 17 00:00:00 2001 From: codyborn Date: Thu, 13 May 2021 23:00:20 -0700 Subject: [PATCH 10/16] Add identity binding for rewards relayers --- .../templates/azure-identity-binding.yaml | 13 +++++++++++++ .../komenci/templates/azure-identity.yaml | 16 ++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/packages/helm-charts/komenci/templates/azure-identity-binding.yaml b/packages/helm-charts/komenci/templates/azure-identity-binding.yaml index 455ed7b1c49..6160db69102 100644 --- a/packages/helm-charts/komenci/templates/azure-identity-binding.yaml +++ b/packages/helm-charts/komenci/templates/azure-identity-binding.yaml @@ -10,3 +10,16 @@ spec: --- {{ end }} {{ end }} + +{{- range $index, $identity := .Values.rewards.relayer.identities -}} +{{ if (hasKey $identity "azure") }} +apiVersion: "aadpodidentity.k8s.io/v1" +kind: AzureIdentityBinding +metadata: + name: {{ template "azure-rewards-identity-binding-name" (dict "dot" $ "index" $index) }} +spec: + azureIdentity: {{ template "azure-rewards-identity-name" (dict "dot" $ "index" $index) }} + selector: {{ template "azure-rewards-identity-binding-name" (dict "dot" $ "index" $index) }} +--- +{{ end }} +{{ end }} diff --git a/packages/helm-charts/komenci/templates/azure-identity.yaml b/packages/helm-charts/komenci/templates/azure-identity.yaml index 69098472a2d..b93930624c4 100644 --- a/packages/helm-charts/komenci/templates/azure-identity.yaml +++ b/packages/helm-charts/komenci/templates/azure-identity.yaml @@ -13,3 +13,19 @@ spec: --- {{ end }} {{ end }} + +{{- range $index, $identity := .Values.rewards.relayer.identities -}} +{{ if (hasKey $identity "azure") }} +apiVersion: aadpodidentity.k8s.io/v1 +kind: AzureIdentity +metadata: + name: {{ template "azure-rewards-identity-name" (dict "dot" $ "index" $index) }} + annotations: + aadpodidentity.k8s.io/Behavior: namespaced +spec: + type: 0 + resourceID: {{ $identity.azure.id }} + clientID: {{ $identity.azure.clientId }} +--- +{{ end }} +{{ end }} From 0055cfa4d6941698d77e38c8d56266024217b90d Mon Sep 17 00:00:00 2001 From: codyborn Date: Fri, 14 May 2021 15:45:35 -0700 Subject: [PATCH 11/16] Fix rbac secret issue --- packages/celotool/src/lib/komenci.ts | 6 +++++- packages/helm-charts/komenci/templates/_helpers.tpl | 6 +++--- .../komenci/templates/rewards-relayer-statefulset.yaml | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/celotool/src/lib/komenci.ts b/packages/celotool/src/lib/komenci.ts index 042ae91593d..db6c3501f84 100644 --- a/packages/celotool/src/lib/komenci.ts +++ b/packages/celotool/src/lib/komenci.ts @@ -517,10 +517,14 @@ function rbacReleaseName(celoEnv: string, prefix: string) { async function rbacServiceAccountSecretNames(celoEnv: string, prefix: string, replicas: number) { const names = [...Array(replicas).keys()].map((i) => `${rbacReleaseName(celoEnv, prefix)}-${i}`) + let jsonSecretPath = '"{.items[*].secrets[0][\'name\']}"' + if (names.length === 1) { + jsonSecretPath = '"{.secrets[0][\'name\']}"' + } const [tokenName] = await execCmdWithExitOnFailure( `kubectl get serviceaccount --namespace=${celoEnv} ${names.join( ' ' - )} -o=jsonpath="{.items[*].secrets[0]['name']}"` + )} -o=jsonpath=${jsonSecretPath}` ) const tokenNames = tokenName.trim().split(' ') return tokenNames diff --git a/packages/helm-charts/komenci/templates/_helpers.tpl b/packages/helm-charts/komenci/templates/_helpers.tpl index 716528b5058..df8bc9c4a72 100644 --- a/packages/helm-charts/komenci/templates/_helpers.tpl +++ b/packages/helm-charts/komenci/templates/_helpers.tpl @@ -5,7 +5,7 @@ The name of the deployment {{- .Values.environment.name -}}-relayer {{- end -}} -{{- define "rewards-name" -}} +{{- define "rewards-relayer-name" -}} {{- .Values.environment.name -}}-rewards-relayer {{- end -}} @@ -74,7 +74,7 @@ The name of the azure identity binding for all relayers The name of the azure identity binding for all rewards relayers */}} {{- define "azure-rewards-identity-binding-name" -}} -{{- with .dot -}}{{ template "rewards-name" . }}{{- end -}}-{{ .index }}-identity-binding +{{- with .dot -}}{{ template "rewards-relayer-name" . }}{{- end -}}-{{ .index }}-identity-binding {{- end -}} {{/* @@ -88,5 +88,5 @@ The name of the azure identity for all relayers The name of the azure identity for all rewards relayers */}} {{- define "azure-rewards-identity-name" -}} -{{- with .dot -}}{{ template "rewards-name" . }}{{- end -}}-{{ .index }}-identity +{{- with .dot -}}{{ template "rewards-relayer-name" . }}{{- end -}}-{{ .index }}-identity {{- end -}} \ No newline at end of file diff --git a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml index 8914d46c4be..70fc0e4a129 100644 --- a/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml +++ b/packages/helm-charts/komenci/templates/rewards-relayer-statefulset.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: Service metadata: - name: {{ template "rewards-name" . }} + name: {{ template "rewards-relayer-name" . }} labels: {{ include "labels" . | indent 4 }} {{ include "komenci-rewards-relayer-component-label" . | indent 4 }} @@ -16,7 +16,7 @@ spec: apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ template "rewards-name" . }} + name: {{ template "rewards-relayer-name" . }} labels: {{ include "labels" . | indent 4 }} {{ include "komenci-rewards-relayer-component-label" . | indent 4 }} From e30cb7a9cf284b2b2e6cab1eae0eef3481ea33e4 Mon Sep 17 00:00:00 2001 From: codyborn Date: Sun, 16 May 2021 17:37:49 -0700 Subject: [PATCH 12/16] Working version --- .env.alfajores | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.alfajores b/.env.alfajores index 73417114133..9abaa31b323 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -153,7 +153,7 @@ CONTEXTS=azure-komenci-eastus,azure-komenci-westeu,azure-oracle-centralus # --- Komenci --- KOMENCI_DOCKER_IMAGE_REPOSITORY="celotestnet.azurecr.io/komenci/komenci" -KOMENCI_DOCKER_IMAGE_TAG="11f81e71201fb106928f6702993b7816d062dbf4" +KOMENCI_DOCKER_IMAGE_TAG="fc8acf4c50b2b1bc3177ee0bbd82eff11e6dba98" AZURE_KOMENCI_EASTUS_AZURE_KUBERNETES_RESOURCE_GROUP=staging-komenci-eastus AZURE_KOMENCI_EASTUS_KUBERNETES_CLUSTER_NAME=staging-komenci-eastus From 52f1cbe091b8e8cf9d2e0f756b0639f3db24229b Mon Sep 17 00:00:00 2001 From: codyborn Date: Tue, 18 May 2021 15:11:46 -0700 Subject: [PATCH 13/16] working version --- .env.alfajores | 6 +++--- packages/helm-charts/komenci/values.yaml | 10 ++-------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.env.alfajores b/.env.alfajores index 9abaa31b323..82a3826afae 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -153,7 +153,7 @@ CONTEXTS=azure-komenci-eastus,azure-komenci-westeu,azure-oracle-centralus # --- Komenci --- KOMENCI_DOCKER_IMAGE_REPOSITORY="celotestnet.azurecr.io/komenci/komenci" -KOMENCI_DOCKER_IMAGE_TAG="fc8acf4c50b2b1bc3177ee0bbd82eff11e6dba98" +KOMENCI_DOCKER_IMAGE_TAG="04fb0ec48c7def2450cca9e1802caf4e0185b5fe" AZURE_KOMENCI_EASTUS_AZURE_KUBERNETES_RESOURCE_GROUP=staging-komenci-eastus AZURE_KOMENCI_EASTUS_KUBERNETES_CLUSTER_NAME=staging-komenci-eastus @@ -196,8 +196,8 @@ AZURE_KOMENCI_WESTEU_KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED=true #
:: AZURE_KOMENCI_EASTUS_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x00454cac6dae53f8800f71395b9a174f07a784b1:staging-komenci-eus,0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e:staging-komenci-eus AZURE_KOMENCI_WESTEU_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x0f812be74511b90ea6b2f80e77bea047e69a0b2a:staging-komenci-weu,0xb354d3d2908ba6a2b791683b0f454a38f69cb282:staging-komenci-weu -AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards -AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0x5226c3908b0db17ED553aEbC395dC685714453cb:staging-komenci-rewards +AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards +AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards # Celo Rewards AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 diff --git a/packages/helm-charts/komenci/values.yaml b/packages/helm-charts/komenci/values.yaml index cca9a0480f1..c0b4d60cace 100644 --- a/packages/helm-charts/komenci/values.yaml +++ b/packages/helm-charts/komenci/values.yaml @@ -92,18 +92,12 @@ rewards: replicas: 2 port: 3000 identities: - - address: 0x00454cac6dae53f8800f71395b9a174f07a784b1 + - address: 0xb04390478A57E3C2147599D5380434f25fa5234d privateKey: 0x000 azure: id: defaultId clientId: defaultClientId - keyVaultName: staging-komenci-eus - - address: 0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e - privateKey: 0x001 - azure: - id: defaultId1 - clientId: defaultClientId1 - keyVaultName: staging-komenci-eus + keyVaultName: staging-komenci-rewards azureHsm: initTryCount: 5 initMaxRetryBackoffMs: 30000 From eac77d90f6038eb50f462a9d1855dd81662998b6 Mon Sep 17 00:00:00 2001 From: codyborn Date: Fri, 4 Jun 2021 12:25:24 -1000 Subject: [PATCH 14/16] Rewards Prod env vars --- .env.rc1 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.env.rc1 b/.env.rc1 index 9a871662ece..54c07c1081d 100644 --- a/.env.rc1 +++ b/.env.rc1 @@ -449,6 +449,16 @@ AZURE_SEA_KOMENCI_DB_PORT=5432 AZURE_SEA_KOMENCI_DB_USERNAME=postgres@mainnet-komenci-southeastasia AZURE_SEA_KOMENCI_DB_PASSWORD_VAULT_NAME=mainnet-komenci-sea +AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_DB_HOST=mainnet-komenci-brazil.postgres.database.azure.com +AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_DB_PORT=5432 +AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_DB_USERNAME=postgres@mainnet-komenci-brazil +AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME=mainnet-komenci-brazil + +AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_DB_HOST=mainnet-komenci-brazil.postgres.database.azure.com +AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_DB_PORT=5432 +AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_DB_USERNAME=postgres@mainnet-komenci-brazil +AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_DB_PASSWORD_VAULT_NAME=mainnet-komenci-brazil + # App Secrets AZURE_KOMENCI_SOUTHBR_KOMENCI_APP_SECRETS_VAULT_NAME=mainnet-komenci-brazil AZURE_KOMENCI_SEA_KOMENCI_APP_SECRETS_VAULT_NAME=mainnet-komenci-sea @@ -463,6 +473,12 @@ AZURE_KOMENCI_SEA_KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED=false #
:: AZURE_KOMENCI_SOUTHBR_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x21888ae301658cdff7ce8c33cdf83a330a5e6273:mainnet-relayer0,0x1438128a2dcc645f0b9706350c1f5dad04845fe6:mainnet-relayer1,0x1e36bf42272a0693eba69332a6f623ce37694a27:mainnet-relayer2,0xd5afaaa7256c9eb86376c4214635dd56dffbd3a8:mainnet-relayer3,0xb09eba8bc1c8bedadd634a8219c0b09042170903:mainnet-relayer4 AZURE_KOMENCI_SEA_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x85a1e716608a84f455d7e07befb76c9b540ac040:mainnet-relayer5,0x2a094e77acf3faebb63279eb60e26d144b9048a2:mainnet-relayer6,0x2f23f9a8f68294a9d6b479c3dbe3dff4de510ced:mainnet-relayer7,0x3db3150c1267d3adeb7f960f3eef11c1dd47a38b:mainnet-relayer8,0xe170915ce32bb8e2ce2a4fcd9113e5298a2e10d2:mainnet-relayer9 +AZURE_KOMENCI_SOUTHBR_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0x198e0D8601AB509ABf1B0B99Fd8f234583Ef1309:mainnet-komenci-rewards0 +AZURE_KOMENCI_SEA_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xbDD68B64e288171B37F01346042BEe6Eb7dFAE4f:mainnet-komenci-rewards1 + +# Celo Rewards +AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 +AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 # Network AZURE_KOMENCI_SOUTHBR_KOMENCI_NETWORK=rc1 From 0c952c965f66618d593aca8aebbf194080fe9b2c Mon Sep 17 00:00:00 2001 From: Gonzalo Nardini Date: Thu, 15 Jul 2021 13:35:07 -0300 Subject: [PATCH 15/16] Expose rewards endpoint (#7968) * Update image & common chart dependency and set correcy rewards HSM address * Expose rewards service endpoints through the komenci ingress * Send Segment API key Co-authored-by: bowd --- .env.alfajores | 9 +++++---- packages/celotool/src/lib/env-utils.ts | 1 + packages/celotool/src/lib/komenci.ts | 6 ++++++ .../komenci/templates/onboarding-ingress.yaml | 6 +++++- .../komenci/templates/rewards-deployment.yaml | 5 +++++ .../komenci/templates/rewards-service.yaml | 14 ++++++++++++++ packages/helm-charts/komenci/values.yaml | 2 ++ 7 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 packages/helm-charts/komenci/templates/rewards-service.yaml diff --git a/.env.alfajores b/.env.alfajores index 76ca785ead7..91a0764ae97 100644 --- a/.env.alfajores +++ b/.env.alfajores @@ -250,7 +250,7 @@ AZURE_ODIS_EASTUS_3_PROM_SIDECAR_GCP_REGION=us-east1 # --- Komenci --- KOMENCI_DOCKER_IMAGE_REPOSITORY="celotestnet.azurecr.io/komenci/komenci" -KOMENCI_DOCKER_IMAGE_TAG="04fb0ec48c7def2450cca9e1802caf4e0185b5fe" +KOMENCI_DOCKER_IMAGE_TAG="08081d2d276a6fd0d420805f3bbe3866e866a63a" AZURE_KOMENCI_EASTUS_AZURE_KUBERNETES_RESOURCE_GROUP=staging-komenci-eastus AZURE_KOMENCI_EASTUS_KUBERNETES_CLUSTER_NAME=staging-komenci-eastus @@ -291,14 +291,15 @@ AZURE_KOMENCI_WESTEU_KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_ENABLED=true # Format should be a comma-separated sequence of: #
:: -AZURE_KOMENCI_EASTUS_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x00454cac6dae53f8800f71395b9a174f07a784b1:staging-komenci-eus,0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e:staging-komenci-eus,0xd7fc8227642bfab9aa927066e5952fece574f0d6:staging-komenci-eus,0x04a444af9a79b6784bcd57c50ba1e051ba536ed4:staging-komenci-eus,0xbb5932e6b6a588cd1c6764f50d1fe410e6a2d71e:staging-komenci-eus,0xc934bff63a0db800acdf7061eb5cc03211e7bccf:staging-komenci-eus,0x409832bd2d72017f12cfaa3d6dc0103767bb7e7e:staging-komenci-eus,0x75222b1aed66393fa43c6454000e097363d85c73:staging-komenci-eus,0xefbc10d42f77c778431043bd3a34b283f90f3979:staging-komenci-eus,0x70b69157973cd31dae5dc68ee1891b9eae379c42:staging-komenci-eus +AZURE_KOMENCI_EASTUS_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x00454cac6dae53f8800f71395b9a174f07a784b1:staging-komenci-eus,0xc6f0f9bfb1aed83620ece3eac0add98a65a8574e:staging-komenci-eus AZURE_KOMENCI_WESTEU_KOMENCI_ADDRESS_AZURE_KEY_VAULTS=0x0f812be74511b90ea6b2f80e77bea047e69a0b2a:staging-komenci-weu,0xb354d3d2908ba6a2b791683b0f454a38f69cb282:staging-komenci-weu -AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards -AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xb04390478A57E3C2147599D5380434f25fa5234d:staging-komenci-rewards +AZURE_KOMENCI_EASTUS_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xb04390478a57e3c2147599d5380434f25fa5234d:staging-komenci-rewards +AZURE_KOMENCI_WESTEU_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xb04390478a57e3c2147599d5380434f25fa5234d:staging-komenci-rewards # Celo Rewards AZURE_KOMENCI_EASTUS_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 AZURE_KOMENCI_WESTEU_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 +KOMENCI_SHOULD_SEND_REWARDS=true # Network AZURE_KOMENCI_EASTUS_KOMENCI_NETWORK=alfajores diff --git a/packages/celotool/src/lib/env-utils.ts b/packages/celotool/src/lib/env-utils.ts index c58017eef6e..a4e8875a91a 100644 --- a/packages/celotool/src/lib/env-utils.ts +++ b/packages/celotool/src/lib/env-utils.ts @@ -80,6 +80,7 @@ export enum envVar { KOMENCI_DOCKER_IMAGE_REPOSITORY = 'KOMENCI_DOCKER_IMAGE_REPOSITORY', KOMENCI_DOCKER_IMAGE_TAG = 'KOMENCI_DOCKER_IMAGE_TAG', KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_TOKEN = 'KOMENCI_RULE_CONFIG_CAPTCHA_BYPASS_TOKEN', + KOMENCI_SHOULD_SEND_REWARDS = 'KOMENCI_SHOULD_SEND_REWARDS', KOMENCI_UNUSED_KOMENCI_ADDRESSES = 'KOMENCI_UNUSED_KOMENCI_ADDRESSES', KUBECONFIG = 'KUBECONFIG', KUBERNETES_CLUSTER_NAME = 'KUBERNETES_CLUSTER_NAME', diff --git a/packages/celotool/src/lib/komenci.ts b/packages/celotool/src/lib/komenci.ts index db6c3501f84..842a4d5bcff 100644 --- a/packages/celotool/src/lib/komenci.ts +++ b/packages/celotool/src/lib/komenci.ts @@ -240,6 +240,10 @@ async function helmParameters(celoEnv: string, context: string, useForno: boolea vars.appSecretsKeyVault, 'LOGGER-SERVICE-ACCOUNT' ) + const segmentApiKey = await getPasswordFromKeyVaultSecret( + vars.appSecretsKeyVault, + 'SEGMENT-API-KEY' + ) const rewardServiceConfig = getContextDynamicEnvVarValues( contextKomenciRewardsServiceConfigDynamicEnvVars, context @@ -289,6 +293,8 @@ async function helmParameters(celoEnv: string, context: string, useForno: boolea `--set rewards.db.port=${rewardDatabaseConfig.port}`, `--set rewards.db.username=${rewardDatabaseConfig.username}`, `--set rewards.db.password=${rewardDatabasePassword}`, + `--set rewards.segmentApiKey=${segmentApiKey}`, + `--set rewards.shouldSendRewards=${fetchEnv(envVar.KOMENCI_SHOULD_SEND_REWARDS)}`, `--set rewards.metrics.enabled=true`, `--set rewards.metrics.prometheusPort=9090`, `--set rewards.relayer.replicas=${rewardsRelayerCount}`, diff --git a/packages/helm-charts/komenci/templates/onboarding-ingress.yaml b/packages/helm-charts/komenci/templates/onboarding-ingress.yaml index d5727df8f34..842c2870d60 100644 --- a/packages/helm-charts/komenci/templates/onboarding-ingress.yaml +++ b/packages/helm-charts/komenci/templates/onboarding-ingress.yaml @@ -14,7 +14,11 @@ spec: - host: {{ .Values.onboarding.publicHostname }} http: paths: + - path: /rewards + backend: + serviceName: {{ .Release.Namespace }}-rewards + servicePort: 3000 - path: / backend: serviceName: {{ .Release.Namespace }}-onboarding - servicePort: 3000 \ No newline at end of file + servicePort: 3000 diff --git a/packages/helm-charts/komenci/templates/rewards-deployment.yaml b/packages/helm-charts/komenci/templates/rewards-deployment.yaml index 82cc9d8e040..61f77c8d3b8 100644 --- a/packages/helm-charts/komenci/templates/rewards-deployment.yaml +++ b/packages/helm-charts/komenci/templates/rewards-deployment.yaml @@ -20,6 +20,9 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} image: {{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: Always + ports: + - name: http + containerPort: 3000 command: - bash - "-c" @@ -40,3 +43,5 @@ spec: {{ include "common.env-var" (dict "name" "RELAYER_HOST" "dict" .Values.rewards.relayer "value_name" "host") | indent 10 }} {{ include "common.env-var" (dict "name" "RELAYER_PORT" "dict" .Values.rewards.relayer "value_name" "port") | indent 10 }} {{ include "common.env-var" (dict "name" "NETWORK" "dict" .Values.environment "value_name" "network") | indent 10 }} +{{ include "common.env-var" (dict "name" "SEGMENT_API_KEY" "dict" .Values.rewards "value_name" "segmentApiKey") | indent 10 }} +{{ include "common.env-var" (dict "name" "SHOULD_SEND_REWARDS" "dict" .Values.rewards "value_name" "shouldSendRewards") | indent 10 }} diff --git a/packages/helm-charts/komenci/templates/rewards-service.yaml b/packages/helm-charts/komenci/templates/rewards-service.yaml new file mode 100644 index 00000000000..0653de5454a --- /dev/null +++ b/packages/helm-charts/komenci/templates/rewards-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "komenci-rewards-fullname" . }} + labels: +{{ include "labels" . | indent 4 }} +{{ include "komenci-rewards-component-label" . | indent 4 }} +spec: + clusterIP: None + selector: +{{ include "komenci-rewards-component-label" . | indent 4 }} + ports: + - name: http + port: 3000 diff --git a/packages/helm-charts/komenci/values.yaml b/packages/helm-charts/komenci/values.yaml index cf2577d1fcd..9f5b69577c6 100644 --- a/packages/helm-charts/komenci/values.yaml +++ b/packages/helm-charts/komenci/values.yaml @@ -87,6 +87,8 @@ onboarding: ssl: true rewards: + segmentApiKey: 'writeApiKey' + shouldSendRewards: false relayer: node_env: production image: From 032f95f0e8064d7f983e225ebcd68e979be560b9 Mon Sep 17 00:00:00 2001 From: Gonzalo Nardini Date: Thu, 15 Jul 2021 17:01:28 -0300 Subject: [PATCH 16/16] Final touches for rewards deployment --- .env.rc1 | 15 ++++++++------- packages/helm-charts/komenci/values.yaml | 7 +++++++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.env.rc1 b/.env.rc1 index 54c07c1081d..000700d1277 100644 --- a/.env.rc1 +++ b/.env.rc1 @@ -423,7 +423,7 @@ AZURE_ODIS_BRAZILSOUTH_A_PROM_SIDECAR_GCP_REGION=southamerica-east1-a # --- Komenci --- KOMENCI_DOCKER_IMAGE_REPOSITORY="celotestnet.azurecr.io/komenci/komenci" -KOMENCI_DOCKER_IMAGE_TAG="f1119833e7ad90ad741b2b509a1ad4ac647799ff" +KOMENCI_DOCKER_IMAGE_TAG="e220c5610e196a1d674edde0f24be0d5eca30c00" AZURE_KOMENCI_SOUTHBR_AZURE_KUBERNETES_RESOURCE_GROUP=mainnet-komenci-brazil AZURE_KOMENCI_SOUTHBR_KUBERNETES_CLUSTER_NAME=mainnet-komenci-brazil @@ -477,15 +477,16 @@ AZURE_KOMENCI_SOUTHBR_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0x198e0D AZURE_KOMENCI_SEA_KOMENCI_CELOLABS_REWARDS_ADDRESS_AZURE_KEY_VAULTS=0xbDD68B64e288171B37F01346042BEe6Eb7dFAE4f:mainnet-komenci-rewards1 # Celo Rewards -AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 -AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT = 1 +AZURE_KOMENCI_SOUTHBR_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT=1 +AZURE_KOMENCI_SEA_KOMENCI_REWARD_SERVICE_INSTANCE_COUNT=1 +KOMENCI_SHOULD_SEND_REWARDS=false # Network AZURE_KOMENCI_SOUTHBR_KOMENCI_NETWORK=rc1 AZURE_KOMENCI_SEA_KOMENCI_NETWORK=rc1 # For WalletConnect relay -WALLET_CONNECT_IMAGE_REPOSITORY = 'us.gcr.io/celo-testnet/walletconnect' -WALLET_CONNECT_IMAGE_TAG = '1472bcaad57e3746498f7a661c42ff5cf9acaf5a' -WALLET_CONNECT_REDIS_CLUSTER_ENABLED = false -WALLET_CONNECT_REDIS_CLUSTER_USEPASSWORD = false +WALLET_CONNECT_IMAGE_REPOSITORY='us.gcr.io/celo-testnet/walletconnect' +WALLET_CONNECT_IMAGE_TAG='1472bcaad57e3746498f7a661c42ff5cf9acaf5a' +WALLET_CONNECT_REDIS_CLUSTER_ENABLED=false +WALLET_CONNECT_REDIS_CLUSTER_USEPASSWORD=false diff --git a/packages/helm-charts/komenci/values.yaml b/packages/helm-charts/komenci/values.yaml index 9f5b69577c6..7992c7df226 100644 --- a/packages/helm-charts/komenci/values.yaml +++ b/packages/helm-charts/komenci/values.yaml @@ -111,6 +111,13 @@ rewards: prometheusPort: 9090 walletType: azure-hsm gasPriceUpdateIntervalMs: "1200000" + db: + host: komenci-komenci-postgresql + port: 5432 + username: 'postgres' + database: 'postgres' + synchronize: true + ssl: true loggingAgent: credentials: eydleGFtcGxlJzogJ2NyZWRlbnRpYWxzJ30K # base64 credentials.json of a gcloud service account \ No newline at end of file