add error event to telemetry (#3750)

Author: FredKSchott

.changeset/silly-phones-watch.md

@@ -0,0 +1,6 @@
+---
+'astro': patch
+'@astrojs/telemetry': patch
+---
+
+Add basic error reporting to astro telemetry

packages/astro/src/cli/index.ts

@@ -1,24 +1,24 @@
 /* eslint-disable no-console */
 
 import { LogOptions } from '../core/logger/core.js';
-
-import { AstroTelemetry } from '@astrojs/telemetry';
 import * as colors from 'kleur/colors';
 import yargs from 'yargs-parser';
 import { z } from 'zod';
+import { telemetry } from '../events/index.js';
 import * as event from '../events/index.js';
-
 import add from '../core/add/index.js';
 import build from '../core/build/index.js';
 import { openConfig } from '../core/config.js';
 import devServer from '../core/dev/index.js';
 import { enableVerboseLogging, nodeLogDestination } from '../core/logger/node.js';
 import { formatConfigErrorMessage, formatErrorMessage, printHelp } from '../core/messages.js';
 import preview from '../core/preview/index.js';
-import { createSafeError } from '../core/util.js';
+import { createSafeError, ASTRO_VERSION } from '../core/util.js';
 import { check } from './check.js';
 import { openInBrowser } from './open.js';
 import * as telemetryHandler from './telemetry.js';
+import { collectErrorMetadata } from '../core/errors.js';
+import { eventError, eventConfigError } from '../events/index.js';
 
 type Arguments = yargs.Arguments;
 type CLICommand =
@@ -61,9 +61,6 @@ function printAstroHelp() {
 	});
 }
 
-// PACKAGE_VERSION is injected when we build and publish the astro package.
-const ASTRO_VERSION = process.env.PACKAGE_VERSION ?? 'development';
-
 /** Display --version flag */
 async function printVersion() {
 	console.log();
@@ -111,7 +108,6 @@ export async function cli(args: string[]) {
 	} else if (flags.silent) {
 		logging.level = 'silent';
 	}
-	const telemetry = new AstroTelemetry({ version: ASTRO_VERSION });
 
 	// Special CLI Commands: "add", "docs", "telemetry"
 	// These commands run before the user's config is parsed, and may have other special
@@ -120,19 +116,19 @@ export async function cli(args: string[]) {
 	switch (cmd) {
 		case 'add': {
 			try {
-				telemetry.record(event.eventCliSession({ cliCommand: cmd }));
+				telemetry.record(event.eventCliSession(cmd));
 				const packages = flags._.slice(3) as string[];
 				return await add(packages, { cwd: root, flags, logging, telemetry });
 			} catch (err) {
-				return throwAndExit(err);
+				return throwAndExit(cmd, err);
 			}
 		}
 		case 'docs': {
 			try {
-				telemetry.record(event.eventCliSession({ cliCommand: cmd }));
+				telemetry.record(event.eventCliSession(cmd));
 				return await openInBrowser('https://docs.astro.build/');
 			} catch (err) {
-				return throwAndExit(err);
+				return throwAndExit(cmd, err);
 			}
 		}
 		case 'telemetry': {
@@ -142,13 +138,13 @@ export async function cli(args: string[]) {
 				const subcommand = flags._[3]?.toString();
 				return await telemetryHandler.update(subcommand, { flags, telemetry });
 			} catch (err) {
-				return throwAndExit(err);
+				return throwAndExit(cmd, err);
 			}
 		}
 	}
 
 	const { astroConfig, userConfig } = await openConfig({ cwd: root, flags, cmd });
-	telemetry.record(event.eventCliSession({ cliCommand: cmd }, userConfig, flags));
+	telemetry.record(event.eventCliSession(cmd, userConfig, flags));
 
 	// Common CLI Commands:
 	// These commands run normally. All commands are assumed to have been handled
@@ -159,15 +155,15 @@ export async function cli(args: string[]) {
 				await devServer(astroConfig, { logging, telemetry });
 				return await new Promise(() => {}); // lives forever
 			} catch (err) {
-				return throwAndExit(err);
+				return throwAndExit(cmd, err);
 			}
 		}
 
 		case 'build': {
 			try {
 				return await build(astroConfig, { logging, telemetry });
 			} catch (err) {
-				return throwAndExit(err);
+				return throwAndExit(cmd, err);
 			}
 		}
 
@@ -181,21 +177,29 @@ export async function cli(args: string[]) {
 				const server = await preview(astroConfig, { logging, telemetry });
 				return await server.closed(); // keep alive until the server is closed
 			} catch (err) {
-				return throwAndExit(err);
+				return throwAndExit(cmd, err);
 			}
 		}
 	}
 
 	// No command handler matched! This is unexpected.
-	throwAndExit(new Error(`Error running ${cmd} -- no command found.`));
+	throwAndExit(cmd, new Error(`Error running ${cmd} -- no command found.`));
 }
 
 /** Display error and exit */
-function throwAndExit(err: unknown) {
+function throwAndExit(cmd: string, err: unknown) {
+	let telemetryPromise: Promise<any>;
 	if (err instanceof z.ZodError) {
 		console.error(formatConfigErrorMessage(err));
+		telemetryPromise = telemetry.record(eventConfigError({ cmd, err, isFatal: true }));
 	} else {
-		console.error(formatErrorMessage(createSafeError(err)));
+		const errorWithMetadata = collectErrorMetadata(createSafeError(err));
+		console.error(formatErrorMessage(errorWithMetadata));
+		telemetryPromise = telemetry.record(eventError({ cmd, err: errorWithMetadata, isFatal: true }));
 	}
-	process.exit(1);
+	// Wait for the telemetry event to send, then exit. Ignore an error.
+	telemetryPromise.catch(() => undefined).then(() => process.exit(1));
+	// Don't wait too long. Timeout the request faster than usual because the user is waiting.
+	// TODO: Investigate using an AbortController once we drop Node v14 support.
+	setTimeout(() => process.exit(1), 300);
 }

packages/astro/src/core/errors.ts

@@ -5,10 +5,19 @@ import type { ViteDevServer } from 'vite';
 import type { SSRError } from '../@types/astro';
 import { codeFrame, createSafeError } from './util.js';
 
+export enum AstroErrorCodes {
+	// 1xxx: Astro Runtime Errors
+	UnknownError = 1000,
+	ConfigError = 1001,
+	// 2xxx: Astro Compiler Errors
+	UnknownCompilerError = 2000,
+	UnknownCompilerCSSError = 2001,
+}
 export interface ErrorWithMetadata {
 	[name: string]: any;
 	message: string;
 	stack: string;
+	code?: number;
 	hint?: string;
 	id?: string;
 	frame?: string;

packages/astro/src/core/messages.ts

@@ -18,7 +18,7 @@ import type { AddressInfo } from 'net';
 import os from 'os';
 import { ZodError } from 'zod';
 import type { AstroConfig } from '../@types/astro';
-import { cleanErrorStack, collectErrorMetadata } from './errors.js';
+import { cleanErrorStack, collectErrorMetadata, ErrorWithMetadata } from './errors.js';
 import { emoji, getLocalAddress, padMultilineString } from './util.js';
 
 const PREFIX_PADDING = 6;
@@ -219,8 +219,7 @@ export function formatConfigErrorMessage(err: ZodError) {
 	)}`;
 }
 
-export function formatErrorMessage(_err: Error, args: string[] = []): string {
-	const err = collectErrorMetadata(_err);
+export function formatErrorMessage(err: ErrorWithMetadata, args: string[] = []): string {
 	args.push(`${bgRed(black(` error `))}${red(bold(padMultilineString(err.message)))}`);
 	if (err.hint) {
 		args.push(`  ${bold('Hint:')}`);

packages/astro/src/core/util.ts

@@ -8,6 +8,9 @@ import type { ErrorPayload } from 'vite';
 import type { AstroConfig } from '../@types/astro';
 import { removeTrailingForwardSlash } from './path.js';
 
+// process.env.PACKAGE_VERSION is injected when we build and publish the astro package.
+export const ASTRO_VERSION = process.env.PACKAGE_VERSION ?? 'development';
+
 /** Returns true if argument is an object of any prototype/class (but not null). */
 export function isObject(value: unknown): value is Record<string, any> {
 	return typeof value === 'object' && value != null;

packages/astro/src/events/error.ts

@@ -0,0 +1,75 @@
+import { ZodError } from 'zod';
+import { AstroErrorCodes, ErrorWithMetadata } from '../core/errors.js';
+
+const EVENT_ERROR = 'ASTRO_CLI_ERROR';
+
+interface ErrorEventPayload {
+	code: number | undefined;
+	isFatal: boolean;
+	plugin?: string | undefined;
+	cliCommand: string;
+	anonymousMessageHint?: string | undefined;
+}
+
+interface ConfigErrorEventPayload extends ErrorEventPayload {
+	isConfig: true;
+	configErrorPaths: string[];
+}
+
+/**
+ * This regex will grab a small snippet at the start of an error message.
+ * This was designed to stop capturing at the first sign of some non-message
+ * content like a filename, filepath, or any other code-specific value.
+ * We also trim this value even further to just a few words.
+ *
+ * Our goal is to remove this entirely before v1.0.0 is released, as we work
+ * to add a proper error code system (see AstroErrorCodes for examples).
+ *
+ * TODO(fks): Remove around v1.0.0 release.
+ */
+const ANONYMIZE_MESSAGE_REGEX = /^(\w| )+/;
+function anonymizeErrorMessage(msg: string): string | undefined {
+	const matchedMessage = msg.match(ANONYMIZE_MESSAGE_REGEX);
+	if (!matchedMessage || !matchedMessage[0]) {
+		return undefined;
+	}
+	return matchedMessage[0].trim().substring(0, 20);
+}
+
+export function eventConfigError({
+	err,
+	cmd,
+	isFatal,
+}: {
+	err: ZodError;
+	cmd: string;
+	isFatal: boolean;
+}): { eventName: string; payload: ConfigErrorEventPayload }[] {
+	const payload: ConfigErrorEventPayload = {
+		code: AstroErrorCodes.ConfigError,
+		isFatal,
+		isConfig: true,
+		cliCommand: cmd,
+		configErrorPaths: err.issues.map((issue) => issue.path.join('.')),
+	};
+	return [{ eventName: EVENT_ERROR, payload }];
+}
+
+export function eventError({
+	cmd,
+	err,
+	isFatal,
+}: {
+	err: ErrorWithMetadata;
+	cmd: string;
+	isFatal: boolean;
+}): { eventName: string; payload: ErrorEventPayload }[] {
+	const payload: ErrorEventPayload = {
+		code: err.code || AstroErrorCodes.UnknownError,
+		plugin: err.plugin,
+		cliCommand: cmd,
+		isFatal: isFatal,
+		anonymousMessageHint: anonymizeErrorMessage(err.message),
+	};
+	return [{ eventName: EVENT_ERROR, payload }];
+}

packages/astro/src/events/index.ts

@@ -1 +1,18 @@
+import { AstroTelemetry } from '@astrojs/telemetry';
+import { ASTRO_VERSION } from '../core/util.js';
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+
+function getViteVersion() {
+	try {
+		const { version } = require('vite/package.json');
+		return version;
+	} catch (e) {}
+	return undefined;
+}
+
+export const telemetry = new AstroTelemetry({ astroVersion: ASTRO_VERSION, viteVersion: getViteVersion()  });
+
+export * from './error.js';
 export * from './session.js';
+

packages/astro/src/events/session.ts

@@ -4,10 +4,6 @@ const require = createRequire(import.meta.url);
 
 const EVENT_SESSION = 'ASTRO_CLI_SESSION_STARTED';
 
-interface EventCliSession {
-	cliCommand: string;
-}
-
 interface ConfigInfo {
 	markdownPlugins: string[];
 	adapter: string | null;
@@ -26,23 +22,14 @@ interface ConfigInfo {
 		  };
 }
 
-interface EventCliSessionInternal extends EventCliSession {
-	nodeVersion: string;
-	viteVersion: string;
+interface EventPayload {
+	cliCommand: string;
 	config?: ConfigInfo;
 	configKeys?: string[];
 	flags?: string[];
 	optionalIntegrations?: number;
 }
 
-function getViteVersion() {
-	try {
-		const { version } = require('vite/package.json');
-		return version;
-	} catch (e) {}
-	return undefined;
-}
-
 const multiLevelKeys = new Set([
 	'build',
 	'markdown',
@@ -82,10 +69,10 @@ function configKeys(obj: Record<string, any> | undefined, parentKey: string): st
 }
 
 export function eventCliSession(
-	event: EventCliSession,
+	cliCommand: string,
 	userConfig?: AstroUserConfig,
 	flags?: Record<string, any>
-): { eventName: string; payload: EventCliSessionInternal }[] {
+): { eventName: string; payload: EventPayload }[] {
 	// Filter out falsy integrations
 	const configValues = userConfig
 		? {
@@ -117,13 +104,9 @@ export function eventCliSession(
 	// Filter out yargs default `_` flag which is the cli command
 	const cliFlags = flags ? Object.keys(flags).filter((name) => name != '_') : undefined;
 
-	const payload: EventCliSessionInternal = {
-		cliCommand: event.cliCommand,
-		// Versions
-		viteVersion: getViteVersion(),
-		nodeVersion: process.version.replace(/^v?/, ''),
+	const payload: EventPayload = {
+		cliCommand,
 		configKeys: userConfig ? configKeys(userConfig, '') : undefined,
-		// Config Values
 		config: configValues,
 		flags: cliFlags,
 	};

packages/astro/src/vite-plugin-astro-server/index.ts

@@ -6,7 +6,7 @@ import type { SSROptions } from '../core/render/dev/index';
 import { Readable } from 'stream';
 import stripAnsi from 'strip-ansi';
 import { call as callEndpoint } from '../core/endpoint/dev/index.js';
-import { fixViteErrorMessage } from '../core/errors.js';
+import { collectErrorMetadata, fixViteErrorMessage } from '../core/errors.js';
 import { error, info, LogOptions, warn } from '../core/logger/core.js';
 import * as msg from '../core/messages.js';
 import { appendForwardSlash } from '../core/path.js';
@@ -320,7 +320,8 @@ async function handleRequest(
 		}
 	} catch (_err) {
 		const err = fixViteErrorMessage(createSafeError(_err), viteServer);
-		error(logging, null, msg.formatErrorMessage(err));
+		const errorWithMetadata = collectErrorMetadata(_err);
+		error(logging, null, msg.formatErrorMessage(errorWithMetadata));
 		handle500Response(viteServer, origin, req, res, err);
 	}
 }