Skip to content

fix: gcloud logs #11944

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sklppy88 merged 1 commit into from
Feb 12, 2025
Merged

fix: gcloud logs #11944

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion spartan/aztec-network/files/config/setup-service-addresses.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ else
fi

# Configure OTEL_COLLECTOR_ENDPOINT if not set in values file
if [ "${TELEMETRY:-false}" = "true" ] && [ "${OTEL_COLLECTOR_ENDPOINT}" = "" ] && [ "${USE_GCLOUD_OBSERVABILITY:-false}" = "false" ]; then
if [ "${TELEMETRY:-false}" = "true" ] && [ "${OTEL_COLLECTOR_ENDPOINT}" = "" ] && [ "${USE_GCLOUD_METRICS:-false}" = "false" ]; then
OTEL_COLLECTOR_PORT=${OTEL_COLLECTOR_PORT:-4318}
OTEL_COLLECTOR_ENDPOINT="http://metrics-opentelemetry-collector.metrics:$OTEL_COLLECTOR_PORT"
fi
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/_helpers.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,8 +167,10 @@ Service Address Setup Container
value: "{{ .Values.proverNode.service.nodePort }}"
- name: PROVER_BROKER_PORT
value: "{{ .Values.proverBroker.service.nodePort }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: SERVICE_NAME
value: {{ include "aztec-network.fullname" . }}
volumeMounts:
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/blob-sink.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,10 @@ spec:
value: "{{ .Values.blobSink.dataStoreConfig.dataDir }}"
- name: DATA_STORE_MAP_SIZE_KB
value: "{{ .Values.blobSink.dataStoreConfig.dataStoreMapSize }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
ports:
- containerPort: {{ .Values.blobSink.service.nodePort }}
resources:
Expand Down
12 changes: 8 additions & 4 deletions spartan/aztec-network/templates/boot-node.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,10 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
{{- end }}
containers:
- name: boot-node
Expand Down Expand Up @@ -247,8 +249,10 @@ spec:
value: "{{ .Values.storage.dataStoreMapSize }}"
- name: WS_DB_MAP_SIZE_KB
value: "{{ .Values.storage.worldStateMapSize }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
{{- if .Values.blobSink.enabled }}
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/deploy-l1-verifier.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,10 @@ spec:
value: "{{ .Values.proverNode.service.nodePort }}"
- name: SERVICE_NAME
value: {{ include "aztec-network.fullname" . }}
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
volumeMounts:
- name: config
mountPath: /shared/config
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/faucet.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,10 @@ spec:
fieldPath: metadata.name
- name: OTEL_SERVICE_NAME
value: faucet
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
ports:
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/prover-agent.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,10 @@ spec:
value: {{ join "," .Values.proverAgent.proofTypes | quote }}
- name: OTEL_RESOURCE_ATTRIBUTES
value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
resources:
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/prover-broker.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,10 @@ spec:
value: "{{ .Values.storage.dataStoreMapSize }}"
- name: OTEL_RESOURCE_ATTRIBUTES
value: service.name={{ .Release.Name }},service.namespace={{ .Release.Namespace }},service.version={{ .Chart.AppVersion }},environment={{ .Values.environment | default "production" }}
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
resources:
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/prover-node.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,10 @@ spec:
value: "{{ .Values.storage.dataStoreMapSize }}"
- name: WS_DB_MAP_SIZE_KB
value: "{{ .Values.storage.worldStateMapSize }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
{{- if .Values.blobSink.enabled }}
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/pxe.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,10 @@ spec:
value: "{{ .Values.pxe.logLevel }}"
- name: PXE_PROVER_ENABLED
value: "{{ .Values.aztec.realProofs }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
- name: L1_CHAIN_ID
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/setup-l2-contracts.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,8 @@ spec:
value: "{{ .Values.proverNode.service.nodePort }}"
- name: SERVICE_NAME
value: {{ include "aztec-network.fullname" . }}
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
{{ end }}
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/transaction-bot.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,8 +131,10 @@ spec:
value: "{{ .Values.bot.maxErrors }}"
- name: BOT_STOP_WHEN_UNHEALTHY
value: "{{ .Values.bot.stopIfUnhealthy }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
- name: L1_CHAIN_ID
Expand Down
6 changes: 4 additions & 2 deletions spartan/aztec-network/templates/validator.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,10 @@ spec:
value: "{{ .Values.storage.dataStoreMapSize }}"
- name: WS_DB_MAP_SIZE_KB
value: "{{ .Values.storage.worldStateMapSize }}"
- name: USE_GCLOUD_OBSERVABILITY
value: "{{ .Values.telemetry.useGcloudObservability }}"
- name: USE_GCLOUD_LOGGING
value: "{{ .Values.telemetry.useGcloudLogging }}"
- name: USE_GCLOUD_METRICS
value: "{{ .Values.telemetry.useGcloudMetrics }}"
- name: OTEL_EXCLUDE_METRICS
value: "{{ .Values.telemetry.excludeMetrics }}"
{{- if .Values.blobSink.enabled }}
Expand Down
3 changes: 2 additions & 1 deletion spartan/aztec-network/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ storage:
telemetry:
enabled: false
otelCollectorEndpoint:
useGcloudObservability: false
useGcloudLogging: true
useGcloudMetrics: false
excludeMetrics: "system"

images:
Expand Down
3 changes: 2 additions & 1 deletion yarn-project/foundation/src/config/env_var.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ export type EnvVar =
| 'NETWORK'
| 'NO_PXE'
| 'COIN_ISSUER_CONTRACT_ADDRESS'
| 'USE_GCLOUD_OBSERVABILITY'
| 'USE_GCLOUD_LOGGING'
| 'USE_GCLOUD_METRICS'
| 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT'
| 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'
| 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT'
Expand Down
6 changes: 3 additions & 3 deletions yarn-project/foundation/src/log/pino-logger.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,13 @@ const [logLevel, logFilters] = parseEnv(process.env.LOG_LEVEL, defaultLogLevel);
const customLevels = { verbose: 25 };

// Global pino options, tweaked for google cloud if running there.
const useGcloudObservability = parseBooleanEnv(process.env['USE_GCLOUD_OBSERVABILITY' satisfies EnvVar]);
const useGcloudLogging = parseBooleanEnv(process.env['USE_GCLOUD_LOGGING' satisfies EnvVar]);
const pinoOpts: pino.LoggerOptions<keyof typeof customLevels> = {
customLevels,
messageKey: 'msg',
useOnlyCustomLevels: false,
level: logLevel,
...(useGcloudObservability ? GoogleCloudLoggerConfig : {}),
...(useGcloudLogging ? GoogleCloudLoggerConfig : {}),
};

export const levels = {
Expand Down Expand Up @@ -148,7 +148,7 @@ const stdioTransport: pino.TransportTargetOptions = {
// this transport configured. Note that the target is defined as the export in the telemetry-client,
// since pino will load this transport separately on a worker thread, to minimize disruption to the main loop.
const otlpEndpoint = process.env['OTEL_EXPORTER_OTLP_LOGS_ENDPOINT' satisfies EnvVar];
const otlpEnabled = !!otlpEndpoint && !useGcloudObservability;
const otlpEnabled = !!otlpEndpoint && !useGcloudLogging;
const otelOpts = { levels };
const otelTransport: pino.TransportTargetOptions = {
target: '@aztec/telemetry-client/otel-pino-stream',
Expand Down
8 changes: 4 additions & 4 deletions yarn-project/telemetry-client/src/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';

export interface TelemetryClientConfig {
useGcloudObservability: boolean;
useGcloudMetrics: boolean;
metricsCollectorUrl?: URL;
tracesCollectorUrl?: URL;
logsCollectorUrl?: URL;
Expand All @@ -16,9 +16,9 @@ export interface TelemetryClientConfig {
}

export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientConfig> = {
useGcloudObservability: {
env: 'USE_GCLOUD_OBSERVABILITY',
description: 'Whether to use GCP observability',
useGcloudMetrics: {
env: 'USE_GCLOUD_METRICS',
Copy link
Contributor Author

@sklppy88 sklppy88 Feb 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this okay to be isolated like this ? It seems like if USE_GCLOUD_METRICS is not passed in, we use the full custom config, otherwise we use GCloud metrics only, without touching the logger provider.

Copy link
Collaborator

@just-mitch just-mitch Feb 12, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this is the exact behavior we want. The end result should be that in KIND tests, we are using full custom metrics and non-gcloud logs (should have nice formatting in Loki), and in real network deployments we have full custom metrics but nice formatting in gcloud.

description: 'Whether to use GCP metrics and traces',
...booleanConfigHelper(false),
},
metricsCollectorUrl: {
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/telemetry-client/src/otel.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ export class OpenTelemetryClient implements TelemetryClient {

public static createAndStart(config: TelemetryClientConfig, log: Logger): OpenTelemetryClient {
const resource = getOtelResource();
const factory = config.useGcloudObservability
const factory = config.useGcloudMetrics
? OpenTelemetryClient.getGcloudClientFactory(config)
: OpenTelemetryClient.getCustomClientFactory(config);

Expand Down
4 changes: 2 additions & 2 deletions yarn-project/telemetry-client/src/start.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ export function initTelemetryClient(config: TelemetryClientConfig): TelemetryCli
return telemetry;
}

if (config.metricsCollectorUrl || config.useGcloudObservability) {
log.info(`Using OpenTelemetry client ${config.useGcloudObservability ? 'with GCP' : 'with custom collector'}`);
if (config.metricsCollectorUrl || config.useGcloudMetrics) {
log.info(`Using OpenTelemetry client ${config.useGcloudMetrics ? 'with GCP' : 'with custom collector'}`);
telemetry = OpenTelemetryClient.createAndStart(config, log);
} else {
log.info('Using NoopTelemetryClient');
Expand Down
Loading