feat: add zod-based validation (#37771)

packages/playwright/src/common/annotationValidator.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { z } from 'zod';
+
+const annotationSchema = z.object({
+  type: z.string().min(1, 'Annotation type must be a non-empty string'),
+  description: z.string().optional(),
+});
+
+
+export function validateAnnotations(annotations: any[]) {
+  return annotations.map((annotation, index) => {
+    try {
+      return annotationSchema.parse(annotation);
+    } catch (error) {
+      if (error instanceof z.ZodError) {
+        const issues = error.issues.map(issue => issue.message).join(', ');
+        throw new Error(`Invalid annotation at index ${index}: ${issues}`);
+      }
+      throw error;
+    }
+  });
+}

packages/playwright/src/common/testType.ts

@@ -21,6 +21,7 @@ import { currentTestInfo, currentlyLoadingFileSuite, setCurrentlyLoadingFileSuit
 import { Suite, TestCase } from './test';
 import { expect } from '../matchers/expect';
 import { wrapFunctionWithLocation } from '../transform/transform';
+import { testDetailsSchema } from './validation';
 
 import type { FixturesWithLocation } from './config';
 import type { Fixtures, TestDetails, TestStepInfo, TestType } from '../../types/test';
@@ -310,13 +311,12 @@ function throwIfRunningInsideJest() {
 }
 
 function validateTestDetails(details: TestDetails, location: Location) {
-  const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);
-  const annotations = originalAnnotations.map(annotation => ({ ...annotation, location }));
-  const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []);
-  for (const tag of tags) {
-    if (tag[0] !== '@')
-      throw new Error(`Tag must start with "@" symbol, got "${tag}" instead.`);
-  }
+  const validated = testDetailsSchema.parse(details);
+
+  const originalAnnotations = Array.isArray(validated.annotation) ? validated.annotation : (validated.annotation ? [validated.annotation] : []);
+  const annotations = originalAnnotations.map(a => ({ ...a, location }));
+
+  const tags = Array.isArray(validated.tag) ? validated.tag : (validated.tag ? [validated.tag] : []);
   return { annotations, tags };
 }
 

packages/playwright/src/common/validation.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+export const annotationSchema = z.object({
+  type: z.string().min(1, 'Annotation type must be a non-empty string'),
+  description: z.string().optional(),
+});
+
+const tagSchema = z.string().regex(/^@/, 'Tag must start with "@"');
+
+export const testDetailsSchema = z.object({
+  annotation: annotationSchema.or(annotationSchema.array()).optional(),
+  tag: tagSchema.or(tagSchema.array()).optional(),
+}).strict();
+
+export type Annotation = z.infer<typeof annotationSchema>;
+export type TestDetailsValidated = z.infer<typeof testDetailsSchema>;

packages/playwright/src/worker/testInfo.ts

@@ -19,11 +19,13 @@ import path from 'path';
 
 import { captureRawStack, monotonicTime, sanitizeForFilePath, stringifyStackFrames, currentZone, createGuid, escapeWithQuotes } from 'playwright-core/lib/utils';
 
+import { z } from 'zod';
 import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutManager';
 import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util';
 import { TestTracing } from './testTracing';
 import { testInfoError } from './util';
 import { wrapFunctionWithLocation } from '../transform/transform';
+import { annotationSchema } from '../common/validation';
 
 import type { RunnableDescription } from './timeoutManager';
 import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';
@@ -226,6 +228,27 @@ export class TestInfoImpl implements TestInfo {
       configurable: true
     });
 
+    const annotationsPush = this.annotations.push.bind(this.annotations);
+    const validatedAnnotationsPush = (...annotations: TestAnnotation[]) => {
+      const validated = annotations.map((a, idx) => {
+        try {
+          return annotationSchema.parse(a);
+        } catch (e) {
+          if (e instanceof z.ZodError)
+            throw new Error(`Invalid annotation at index ${idx}: ${(e as z.ZodError).issues.map(i => i.message).join(', ')}`);
+          throw e;
+        }
+      });
+      return annotationsPush(...validated);
+    };
+
+    Object.defineProperty(this.annotations, 'push', {
+      value: validatedAnnotationsPush,
+      writable: true,
+      enumerable: false,
+      configurable: true
+    });
+
     this._tracing = new TestTracing(this, workerParams.artifactsDir);
 
     this.skip = wrapFunctionWithLocation((location, ...args) => this._modifier('skip', location, args));

tests/playwright-test/basic.spec.ts

@@ -662,3 +662,58 @@ test('should report serialization error', async ({ runInlineTest }) => {
   expect(result.passed).toBe(0);
   expect(result.output).toContain('Expected: {\"a\": [Function a]}');
 });
+
+test('annotation validation missing type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      import { test } from '@playwright/test';
+      test('t', { annotation: { description: 'x' } }, async () => {});
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Invalid annotation');
+});
+
+test('annotation validation non string type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      import { test } from '@playwright/test';
+      test('t', { annotation: { type: 123 } }, async () => {});
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Invalid annotation');
+});
+
+test('annotation validation empty type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      import { test } from '@playwright/test';
+      test('t', { annotation: { type: '' } }, async () => {});
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Invalid annotation');
+});
+
+test('annotation runtime validation missing type', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      import { test } from '@playwright/test';
+      test('t', async () => { test.info().annotations.push({ description: 'x' }); });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('Invalid annotation');
+});
+
+test('annotation runtime validation valid', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      import { test } from '@playwright/test';
+      test('t', async () => { test.info().annotations.push({ type: 'bug' }); });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});