diff --git a/packages/playwright/src/common/annotationValidator.ts b/packages/playwright/src/common/annotationValidator.ts new file mode 100644 index 0000000000000..5be15e5f4f1dd --- /dev/null +++ b/packages/playwright/src/common/annotationValidator.ts @@ -0,0 +1,36 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { z } from 'zod'; + +const annotationSchema = z.object({ + type: z.string().min(1, 'Annotation type must be a non-empty string'), + description: z.string().optional(), +}); + + +export function validateAnnotations(annotations: any[]) { + return annotations.map((annotation, index) => { + try { + return annotationSchema.parse(annotation); + } catch (error) { + if (error instanceof z.ZodError) { + const issues = error.issues.map(issue => issue.message).join(', '); + throw new Error(`Invalid annotation at index ${index}: ${issues}`); + } + throw error; + } + }); +} diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index b7ab5648f2516..ae2c3f2694cc5 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -21,6 +21,7 @@ import { currentTestInfo, currentlyLoadingFileSuite, setCurrentlyLoadingFileSuit import { Suite, TestCase } from './test'; import { expect } from '../matchers/expect'; import { wrapFunctionWithLocation } from '../transform/transform'; +import { testDetailsSchema } from './validation'; import type { FixturesWithLocation } from './config'; import type { Fixtures, TestDetails, TestStepInfo, TestType } from '../../types/test'; @@ -310,13 +311,12 @@ function throwIfRunningInsideJest() { } function validateTestDetails(details: TestDetails, location: Location) { - const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []); - const annotations = originalAnnotations.map(annotation => ({ ...annotation, location })); - const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []); - for (const tag of tags) { - if (tag[0] !== '@') - throw new Error(`Tag must start with "@" symbol, got "${tag}" instead.`); - } + const validated = testDetailsSchema.parse(details); + + const originalAnnotations = Array.isArray(validated.annotation) ? validated.annotation : (validated.annotation ? [validated.annotation] : []); + const annotations = originalAnnotations.map(a => ({ ...a, location })); + + const tags = Array.isArray(validated.tag) ? validated.tag : (validated.tag ? [validated.tag] : []); return { annotations, tags }; } diff --git a/packages/playwright/src/common/validation.ts b/packages/playwright/src/common/validation.ts new file mode 100644 index 0000000000000..eba9f17bb993c --- /dev/null +++ b/packages/playwright/src/common/validation.ts @@ -0,0 +1,32 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { z } from 'zod'; + +export const annotationSchema = z.object({ + type: z.string().min(1, 'Annotation type must be a non-empty string'), + description: z.string().optional(), +}); + +const tagSchema = z.string().regex(/^@/, 'Tag must start with "@"'); + +export const testDetailsSchema = z.object({ + annotation: annotationSchema.or(annotationSchema.array()).optional(), + tag: tagSchema.or(tagSchema.array()).optional(), +}).strict(); + +export type Annotation = z.infer; +export type TestDetailsValidated = z.infer; diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 27ad9cc9148cd..ede2aa0239448 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -19,11 +19,13 @@ import path from 'path'; import { captureRawStack, monotonicTime, sanitizeForFilePath, stringifyStackFrames, currentZone, createGuid, escapeWithQuotes } from 'playwright-core/lib/utils'; +import { z } from 'zod'; import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutManager'; import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util'; import { TestTracing } from './testTracing'; import { testInfoError } from './util'; import { wrapFunctionWithLocation } from '../transform/transform'; +import { annotationSchema } from '../common/validation'; import type { RunnableDescription } from './timeoutManager'; import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test'; @@ -226,6 +228,27 @@ export class TestInfoImpl implements TestInfo { configurable: true }); + const annotationsPush = this.annotations.push.bind(this.annotations); + const validatedAnnotationsPush = (...annotations: TestAnnotation[]) => { + const validated = annotations.map((a, idx) => { + try { + return annotationSchema.parse(a); + } catch (e) { + if (e instanceof z.ZodError) + throw new Error(`Invalid annotation at index ${idx}: ${(e as z.ZodError).issues.map(i => i.message).join(', ')}`); + throw e; + } + }); + return annotationsPush(...validated); + }; + + Object.defineProperty(this.annotations, 'push', { + value: validatedAnnotationsPush, + writable: true, + enumerable: false, + configurable: true + }); + this._tracing = new TestTracing(this, workerParams.artifactsDir); this.skip = wrapFunctionWithLocation((location, ...args) => this._modifier('skip', location, args)); diff --git a/tests/playwright-test/basic.spec.ts b/tests/playwright-test/basic.spec.ts index 73ce104ba011a..a2d5364232a4e 100644 --- a/tests/playwright-test/basic.spec.ts +++ b/tests/playwright-test/basic.spec.ts @@ -662,3 +662,58 @@ test('should report serialization error', async ({ runInlineTest }) => { expect(result.passed).toBe(0); expect(result.output).toContain('Expected: {\"a\": [Function a]}'); }); + +test('annotation validation missing type', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', { annotation: { description: 'x' } }, async () => {}); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Invalid annotation'); +}); + +test('annotation validation non string type', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', { annotation: { type: 123 } }, async () => {}); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Invalid annotation'); +}); + +test('annotation validation empty type', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', { annotation: { type: '' } }, async () => {}); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Invalid annotation'); +}); + +test('annotation runtime validation missing type', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', async () => { test.info().annotations.push({ description: 'x' }); }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Invalid annotation'); +}); + +test('annotation runtime validation valid', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test } from '@playwright/test'; + test('t', async () => { test.info().annotations.push({ type: 'bug' }); }); + ` + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); +});