chore: use zod to validate annotations

packages/playwright-core/ThirdPartyNotices.txt

@@ -48,6 +48,7 @@ This project incorporates components from the projects listed below. The origina
 -	[email protected] (https://github.com/eemeli/yaml)
 -	[email protected] (https://github.com/thejoshwolfe/yauzl)
 -	[email protected] (https://github.com/thejoshwolfe/yazl)
+-	[email protected] (https://github.com/colinhacks/zod)
 
 %% [email protected] NOTICES AND INFORMATION BEGIN HERE
 =========================================
@@ -1127,8 +1128,34 @@ SOFTWARE.
 =========================================
 END OF [email protected] AND INFORMATION
 
+%% [email protected] NOTICES AND INFORMATION BEGIN HERE
+=========================================
+MIT License
+
+Copyright (c) 2025 Colin McDonnell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+=========================================
+END OF [email protected] AND INFORMATION
+
 SUMMARY BEGIN HERE
 =========================================
-Total Packages: 44
+Total Packages: 45
 =========================================
 END OF SUMMARY

packages/playwright-core/bundles/utils/package.json

@@ -21,7 +21,8 @@
     "signal-exit": "3.0.7",
     "socks-proxy-agent": "8.0.5",
     "ws": "8.17.1",
-    "yaml": "^2.6.0"
+    "yaml": "^2.6.0",
+    "zod": "^3.25.76"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",

packages/playwright-core/bundles/utils/src/utilsBundleImpl.ts

@@ -57,12 +57,15 @@ export const progress = progressLibrary;
 
 export { SocksProxyAgent } from 'socks-proxy-agent';
 
-import yamlLibrary from 'yaml';
-export const yaml = yamlLibrary;
-
 // @ts-ignore
 import wsLibrary, { WebSocketServer, Receiver, Sender } from 'ws';
 export const ws = wsLibrary;
 export const wsServer = WebSocketServer;
 export const wsReceiver = Receiver;
 export const wsSender = Sender;
+
+import yamlLibrary from 'yaml';
+export const yaml = yamlLibrary;
+
+import zodLibrary from 'zod';
+export const zod = zodLibrary;

packages/playwright-core/src/utilsBundle.ts

@@ -30,12 +30,13 @@ export const program: typeof import('../bundles/utils/node_modules/commander').p
 export const ProgramOption: typeof import('../bundles/utils/node_modules/commander').Option = require('./utilsBundleImpl').ProgramOption;
 export const progress: typeof import('../bundles/utils/node_modules/@types/progress') = require('./utilsBundleImpl').progress;
 export const SocksProxyAgent: typeof import('../bundles/utils/node_modules/socks-proxy-agent').SocksProxyAgent = require('./utilsBundleImpl').SocksProxyAgent;
-export const yaml: typeof import('../bundles/utils/node_modules/yaml') = require('./utilsBundleImpl').yaml;
-export type { Range as YAMLRange, Scalar as YAMLScalar, YAMLError, YAMLMap, YAMLSeq } from '../bundles/utils/node_modules/yaml';
 export const ws: typeof import('../bundles/utils/node_modules/@types/ws') = require('./utilsBundleImpl').ws;
 export const wsServer: typeof import('../bundles/utils/node_modules/@types/ws').WebSocketServer = require('./utilsBundleImpl').wsServer;
 export const wsReceiver = require('./utilsBundleImpl').wsReceiver;
 export const wsSender = require('./utilsBundleImpl').wsSender;
+export const yaml: typeof import('../bundles/utils/node_modules/yaml') = require('./utilsBundleImpl').yaml;
+export type { Range as YAMLRange, Scalar as YAMLScalar, YAMLError, YAMLMap, YAMLSeq } from '../bundles/utils/node_modules/yaml';
+export const zod: typeof import('../bundles/utils/node_modules/zod') = require('./utilsBundleImpl').zod;
 export type { Command } from '../bundles/utils/node_modules/commander';
 export type { EventEmitter as WebSocketEventEmitter, RawData as WebSocketRawData, WebSocket, WebSocketServer } from '../bundles/utils/node_modules/@types/ws';
 

packages/playwright/src/common/testType.ts

@@ -21,6 +21,7 @@ import { currentTestInfo, currentlyLoadingFileSuite, setCurrentlyLoadingFileSuit
 import { Suite, TestCase } from './test';
 import { expect } from '../matchers/expect';
 import { wrapFunctionWithLocation } from '../transform/transform';
+import { validateTestDetails } from './validators';
 
 import type { FixturesWithLocation } from './config';
 import type { Fixtures, TestDetails, TestStepInfo, TestType } from '../../types/test';
@@ -309,17 +310,6 @@ function throwIfRunningInsideJest() {
   }
 }
 
-function validateTestDetails(details: TestDetails, location: Location) {
-  const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);
-  const annotations = originalAnnotations.map(annotation => ({ ...annotation, location }));
-  const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []);
-  for (const tag of tags) {
-    if (tag[0] !== '@')
-      throw new Error(`Tag must start with "@" symbol, got "${tag}" instead.`);
-  }
-  return { annotations, tags };
-}
-
 export const rootTestType = new TestTypeImpl([]);
 
 export function mergeTests(...tests: TestType<any, any>[]) {

packages/playwright/src/common/validators.ts

@@ -0,0 +1,70 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { zod } from 'playwright-core/lib/utilsBundle';
+
+import type { TestAnnotation, TestDetailsAnnotation } from 'packages/playwright/types/test';
+import type { Location } from '../../types/testReporter';
+import type { ZodError } from 'zod';
+
+const testAnnotationSchema = zod.object({
+  type: zod.string(),
+  description: zod.string().optional(),
+});
+
+const testDetailsSchema = zod.object({
+  tag: zod.union([
+    zod.string().optional(),
+    zod.array(zod.string())
+  ]).transform(val => Array.isArray(val) ? val : val !== undefined ? [val] : []).refine(val => val.every(v => v.startsWith('@')), {
+    message: "Tag must start with '@'"
+  }),
+  annotation: zod.union([
+    testAnnotationSchema,
+    zod.array(testAnnotationSchema).optional()
+  ]).transform(val => Array.isArray(val) ? val : val !== undefined ? [val] : []),
+});
+
+export function validateTestAnnotation(annotation: unknown): TestAnnotation {
+  try {
+    return testAnnotationSchema.parse(annotation);
+  } catch (error) {
+    throwZodError(error);
+  }
+}
+
+type ValidTestDetails = {
+  tags: string[];
+  annotations: (TestDetailsAnnotation & { location: Location })[];
+  location: Location;
+};
+
+export function validateTestDetails(details: unknown, location: Location): ValidTestDetails {
+  try {
+    const parsedDetails = testDetailsSchema.parse(details);
+    return {
+      annotations: parsedDetails.annotation.map(a => ({ ...a, location })),
+      tags: parsedDetails.tag,
+      location,
+    };
+  } catch (error) {
+    throwZodError(error);
+  }
+}
+
+function throwZodError(error: any): never {
+  throw new Error((error as ZodError).issues.map(i => i.message).join('\n'));
+}

tests/playwright-test/test-tag.spec.ts

@@ -148,7 +148,7 @@ test('should enforce @ symbol', async ({ runInlineTest }) => {
     `
   });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`Error: Tag must start with "@" symbol, got "foo" instead.`);
+  expect(result.output).toContain(`Error: Tag must start with '@'`);
 });
 
 test('should be included in testInfo', async ({ runInlineTest }, testInfo) => {