feat: annotation API (#7953)

Author: sheremet-va

.gitignore

@@ -32,3 +32,4 @@ test/cli/fixtures/browser-multiple/basic-*
 # exclude static html reporter folder
 test/browser/html/
 test/core/html/
+.vitest-attachments

docs/.vitepress/config.ts

@@ -516,6 +516,10 @@ function guide(): DefaultTheme.SidebarItem[] {
       text: 'Test Context',
       link: '/guide/test-context',
     },
+    {
+      text: 'Test Annotations',
+      link: '/guide/test-annotations',
+    },
     {
       text: 'Environment',
       link: '/guide/environment',

docs/advanced/api/reporters.md

@@ -15,6 +15,7 @@ Vitest has its own test run lifecycle. These are represented by reporter's metho
       - [`onHookStart(beforeAll)`](#onhookstart)
       - [`onHookEnd(beforeAll)`](#onhookend)
         - [`onTestCaseReady`](#ontestcaseready)
+          - [`onTestAnnotate`](#ontestannotate) <Version>3.2.0</Version>
           - [`onHookStart(beforeEach)`](#onhookstart)
           - [`onHookEnd(beforeEach)`](#onhookend)
           - [`onHookStart(afterEach)`](#onhookstart)
@@ -317,3 +318,16 @@ function onTestCaseResult(testCase: TestCase): Awaitable<void>
 This method is called when the test has finished running or was just skipped. Note that this will be called after the `afterEach` hook is finished, if there are any.
 
 At this point, [`testCase.result()`](/advanced/api/test-case#result) will have non-pending state.
+
+## onTestAnnotate <Version>3.2.0</Version> {#ontestannotate}
+
+```ts
+function onTestAnnotate(
+  testCase: TestCase,
+  annotation: TestAnnotation,
+): Awaitable<void>
+```
+
+The `onTestAnnotate` hook is associated with the [`context.annotate`](/guide/test-context#annotate) method. When `annotate` is invoked, Vitest serialises it and sends the same attachment to the main thread where reporter can interact with it.
+
+If the path is specified, Vitest stores it in a separate directory (configured by [`attachmentsDir`](/config/#attachmentsdir)) and modifies the `path` property to reference it.

docs/config/index.md

@@ -2645,3 +2645,10 @@ Polling timeout in milliseconds
 - **Default:** `false`
 
 Always print console traces when calling any `console` method. This is useful for debugging.
+
+### attachmentsDir <Version>3.2.0</Version>
+
+- **Type:** `string`
+- **Default:** `'.vitest-attachments'`
+
+Directory path for storing attachments created by [`context.annotate`](/guide/test-context#annotate) relative to the project root.

docs/guide/test-annotations.md

@@ -0,0 +1,98 @@
+---
+title: Test Annotations | Guide
+outline: deep
+---
+
+# Test Annotations
+
+Vitest supports annotating your tests with custom messages and files via the [`context.annotate`](/guide/test-context#annotate) API. These annotations will be attached to the test case and passed down to reporters in the [`onTestAnnotate`](/advanced/api/reporters#ontestannotate) hook.
+
+```ts
+test('hello world', async ({ annotate }) => {
+  await annotate('this is my test')
+
+  if (condition) {
+    await annotate('this should\'ve errored', 'error')
+  }
+
+  const file = createTestSpecificFile()
+  await annotate('creates a file', { body: file })
+})
+```
+
+Depending on your reporter, you will see these annotations differently.
+
+## Built-in Reporters
+### default
+
+The `default` reporter prints annotations only if the test has failed:
+
+```
+  ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯
+
+  FAIL  example.test.js > an example of a test with annotation
+Error: thrown error
+  ❯ example.test.js:11:21
+      9 |    await annotate('annotation 1')
+      10|    await annotate('annotation 2', 'warning')
+      11|    throw new Error('thrown error')
+        |          ^
+      12|  })
+
+  ❯ example.test.js:9:15 notice
+    ↳ annotation 1
+  ❯ example.test.js:10:15 warning
+    ↳ annotation 2
+
+  ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯
+```
+
+### verbose
+
+In a TTY terminal, the `verbose` reporter works similarly to the `default` reporter. However, in a non-TTY environment, the `verbose` reporter will also print annotations after every test.
+
+```
+✓ example.test.js > an example of a test with annotation
+
+  ❯ example.test.js:9:15 notice
+    ↳ annotation 1
+  ❯ example.test.js:10:15 warning
+    ↳ annotation 2
+
+```
+
+### html
+
+The HTML reporter shows annotations the same way the UI does. You can see the annotation on the line where it was called. At the moment, if the annotation wasn't called in a test file, you cannot see it in the UI. We are planning to support a separate test summary view where it will be visible.
+
+<img alt="Vitest UI" img-light src="/annotations-html-light.png">
+<img alt="Vitest UI" img-dark src="/annotations-html-dark.png">
+
+### junit
+
+The `junit` reporter lists annotations inside the testcase's `properties` tag. The JUnit reporter will ignore all attachments and will print only the type and the message.
+
+```xml
+<testcase classname="basic/example.test.js" name="an example of a test with annotation" time="0.14315">
+    <properties>
+        <property name="notice" value="the message of the annotation">
+        </property>
+    </properties>
+</testcase>
+```
+
+### github-actions
+
+The `github-actions` reporter will print the annotation as a [notice message](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-a-notice-message) by default. You can configure the `type` by passing down the second argument as `notice`, `warning` or `error`. If type is none of these, Vitest will show the message as a notice.
+
+<img alt="GitHub Actions" img-light src="/annotations-gha-light.png">
+<img alt="GitHub Actions" img-dark src="/annotations-gha-dark.png">
+
+### tap
+
+The `tap` and `tap-flat` reporters print annotations as diagnostic messages on a new line starting with a `#` symbol. They will ignore all attachments and will print only the type and message:
+
+```
+ok 1 - an example of a test with annotation # time=143.15ms
+    # notice: the message of the annotation
+```

docs/guide/test-context.md

@@ -79,7 +79,30 @@ it('math is hard', ({ skip, mind }) => {
 })
 ```
 
-#### `context.signal` <Version>3.2.0</Version> {#context-signal}
+#### `annotate` <Version>3.2.0</Version> {#annotate}
+
+```ts
+function annotate(
+  message: string,
+  attachment?: TestAttachment,
+): Promise<TestAnnotation>
+
+function annotate(
+  message: string,
+  type?: string,
+  attachment?: TestAttachment,
+): Promise<TestAnnotation>
+```
+
+Add a [test annotation](/guide/test-annotations) that will be displayed by your [reporter](/config/#reporter).
+
+```ts
+test('annotations API', async ({ annotate }) => {
+  await annotate('https://github.com/vitest-dev/vitest/pull/7953', 'issues')
+})
+```
+
+#### `signal` <Version>3.2.0</Version> {#signal}
 
 An [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) that can be aborted by Vitest. The signal is aborted in these situations: