Merge pull request #5724 from Arize-ai/prompts

feat(prompts)!: prompt template management
Arize-ai · Feb 19, 2025 · 5be8bf5 · 5be8bf5
2 parents 242dc7f + 2ecac95
commit 5be8bf5
Show file tree

Hide file tree

Showing 560 changed files with 52,065 additions and 6,415 deletions.
diff --git a/.github/workflows/playwright.yaml b/.github/workflows/playwright.yaml
@@ -2,9 +2,9 @@ name: Playwright Tests
 
 on:
   push:
-    branches: [main, sessions]
+    branches: [main, prompts]
   pull_request:
-    branches: [main, sessions]
+    branches: [main, prompts]
 jobs:
   e2e-test:
     timeout-minutes: 60
@@ -17,13 +17,10 @@ jobs:
       # despite always installing it on cache-hit.
       CI_PLAYWRIGHT_SKIP_WEBKIT: true
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        py: [3.9, 3.12, 3.13]
-        os: [ubuntu-latest, macos-latest, macos-13]
-        exclude:
-          - py: 3.13
-            os: macos-13
+        py: [3.13]
+        os: [ubuntu-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
       - name: Install PNPM

diff --git a/.github/workflows/python-CI.yml b/.github/workflows/python-CI.yml
@@ -112,6 +112,7 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.phoenix_evals == 'true' }}
     strategy:
+      fail-fast: false
       matrix:
         py: [3.9, 3.12]
         os: [ubuntu-latest, windows-latest, macos-13]
@@ -307,7 +308,7 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
-            tests/
+            packages/phoenix-client/
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -342,7 +343,11 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
-            tests/
+            tests/unit/
+            tests/conftest.py
+            tests/mypy.ini
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -382,6 +387,8 @@ jobs:
             src/phoenix/
             tests/unit/
             tests/conftest.py
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -423,7 +430,10 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
+            packages/phoenix-client/
             tests/integration/
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -484,7 +494,10 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
+            packages/phoenix-client/
             tests/integration/
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -501,3 +514,37 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run integration tests
         run: uvx --with tox-uv tox run -e integration_tests -- -ra -x -n auto --reruns 5
+
+  phoenix-client-canary-tests-sdk:
+    name: Phoenix Client Canary Tests for Third-Party SDKs
+    runs-on: ubuntu-latest
+    needs: changes
+    if: ${{ needs.changes.outputs.phoenix_client == 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        py: [ 3.9 ]
+        pkg: [ openai, anthropic, google_generativeai ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            requirements/
+            packages/phoenix-client/
+      - name: Set up Python ${{ matrix.py }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py }}
+      - name: Set up `uv`
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: 0.5.15
+          enable-cache: true
+          cache-dependency-glob: |
+            pyproject.toml
+            requirements/ci.txt
+            requirements/canary/sdk/${{ matrix.pkg }}.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run canary tests for ${{ matrix.pkg }}
+        run: uvx --with tox-uv tox run -e phoenix_client_canary_tests_sdk_${{ matrix.pkg }} -- -ra -x
diff --git a/.github/workflows/typescript-packages-CI.yml b/.github/workflows/typescript-packages-CI.yml
@@ -2,10 +2,11 @@ name: Typescript Packages CI
 
 on:
   push:
-    branches: [main, js]
+    branches: [main, prompts]
   pull_request:
     paths:
       - "js/**"
+      - "schemas/**"
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 

diff --git a/.github/workflows/typescript-packages-publish-experimental.yml b/.github/workflows/typescript-packages-publish-experimental.yml
@@ -0,0 +1,56 @@
+name: Publish Any Pull Request
+# TODO: Only publish on pull requests by arize team members
+on:
+  pull_request:
+
+jobs:
+  # JOB to run change detection
+  changes:
+    runs-on: ubuntu-latest
+    # Required permissions
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      workflow_file: ${{ steps.filter.outputs.workflow_file }}
+      phoenix-client: ${{ steps.filter.outputs.phoenix-client }}
+    steps:
+      # For pull requests it's not necessary to checkout the code
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: |
+            phoenix-client:
+              - 'js/packages/phoenix-client/**'
+            workflow_file:
+              - '.github/workflows/typescript-packages-publish-experimental.yml'
+
+  publish-experimental-packages:
+    needs: changes
+    if: ${{ needs.changes.outputs.phoenix-client == 'true' || needs.changes.outputs.workflow_file == 'true' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # workaround for broken corepack https://github.com/nodejs/corepack/issues/612
+      - run: |
+          npm i -g corepack@latest
+          corepack enable
+        working-directory: ./js
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+
+      - name: Install dependencies
+        working-directory: ./js
+        run: pnpm install
+
+      - name: Build
+        working-directory: ./js
+        run: pnpm -r build
+
+      - name: Publish
+        working-directory: ./js
+        run: pnpx pkg-pr-new publish ./packages/phoenix-client
diff --git a/.gitignore b/.gitignore
@@ -25,6 +25,9 @@ coverage.xml
 
 # devtools
 pyrightconfig.json
+!/packages/phoenix-client/src/phoenix/client/helpers/sdk/pyrightconfig.json
+!/packages/phoenix-client/tests/canary/sdk/pyrightconfig.json
+!/tests/integration/pyrightconfig.json
 
 # Demo data
 tutorials/internal/demo_llama_index/*.json
@@ -35,6 +38,24 @@ examples/agent_framework_comparison/utils/saved_traces/*.parquet
 .conda
 .venv
 
+# Deno Notebook Environment
+js/examples/notebooks/**/deno.lock
+
+# These can be generated by code refactoring via IntelliJ,
+# but they should be excluded from their Python packages, as
+# they would clobber the same files in phoenix when they are
+# installed as submodules.
+/packages/__init__.py
+/packages/phoenix-client/__init__.py
+/packages/phoenix-client/src/__init__.py
+/packages/phoenix-client/src/phoenix/__init__.py
+/packages/phoenix-evals/__init__.py
+/packages/phoenix-evals/src/__init__.py
+/packages/phoenix-evals/src/phoenix/__init__.py
+/packages/phoenix-otel/__init__.py
+/packages/phoenix-otel/src/__init__.py
+/packages/phoenix-otel/src/phoenix/__init__.py
+
 # Symbolic links
 # Note that this can affect hatch build for those packages. That's why
 # we need to un-ignore it if it's an actual folder (i.e. with the trailing /).

diff --git a/README.md b/README.md
@@ -53,30 +53,30 @@ Phoenix container images are available via [Docker Hub](https://hub.docker.com/r
 
 ## Features
 
-| Key Features                                                                                                     | Availability   |
-| ---------------------------------------------------------------------------------------------------------------- | -------------- |
-| [Tracing](https://docs.arize.com/phoenix/tracing/concepts-tracing/what-are-traces)                               | ✅             |
-| [Evaluation](https://docs.arize.com/phoenix/evaluation/llm-evals)                                                | ✅             |
-| [Retrieval (RAG) Analysis](https://docs.arize.com/phoenix/tracing/use-cases-tracing/rag-evaluation)              | ✅             |
-| [Datasets](https://docs.arize.com/phoenix/datasets-and-experiments/overview-datasets)                            | ✅             |
-| [Fine-Tuning Export](https://docs.arize.com/phoenix/datasets-and-experiments/how-to-datasets/exporting-datasets) | ✅             |
-| [Annotations](https://docs.arize.com/phoenix/tracing/concepts-tracing/how-to-annotate-traces)                    | ✅             |
-| [Human Feedback](https://docs.arize.com/phoenix/tracing/how-to-tracing/capture-feedback)                         | ✅             |
-| [Experiments](https://docs.arize.com/phoenix/datasets-and-experiments/how-to-experiments/run-experiments)        | ✅             |
-| [Embeddings Analysis](https://docs.arize.com/phoenix/inferences/phoenix-inferences)                              | ✅             |
-| [Data Export](https://docs.arize.com/phoenix/tracing/how-to-tracing/extract-data-from-spans)                     | ✅             |
-| REST API                                                                                                         | ✅             |
-| GraphQL API                                                                                                      | ✅             |
-| Data Retention                                                                                                   | Customizable   |
-| [Authentication](https://docs.arize.com/phoenix/deployment/authentication)                                       | ✅             |
-| [Social Login](https://docs.arize.com/phoenix/deployment/authentication#configuring-oauth2-identity-providers)   | ✅             |
-| [RBAC](https://docs.arize.com/phoenix/deployment/authentication#permissions)                                     | ✅             |
-| Projects                                                                                                         | ✅             |
-| [Self-Hosting](https://docs.arize.com/phoenix/deployment)                                                        | ✅             |
-| Jupyter Notebooks                                                                                                | ✅             |
-| [Prompt Playground](https://docs.arize.com/phoenix/prompt-engineering/overview-prompts)                          | ✅             |
-| [Sessions](https://docs.arize.com/phoenix/tracing/how-to-tracing/setup-sessions)                                 | ✅             |
-| Prompt Management                                                                                                | Coming soon ⏱️ |
+| Key Features                                                                                                     | Availability |
+| ---------------------------------------------------------------------------------------------------------------- | ------------ |
+| [Tracing](https://docs.arize.com/phoenix/tracing/concepts-tracing/what-are-traces)                               | ✅           |
+| [Evaluation](https://docs.arize.com/phoenix/evaluation/llm-evals)                                                | ✅           |
+| [Retrieval (RAG) Analysis](https://docs.arize.com/phoenix/tracing/use-cases-tracing/rag-evaluation)              | ✅           |
+| [Datasets](https://docs.arize.com/phoenix/datasets-and-experiments/overview-datasets)                            | ✅           |
+| [Fine-Tuning Export](https://docs.arize.com/phoenix/datasets-and-experiments/how-to-datasets/exporting-datasets) | ✅           |
+| [Annotations](https://docs.arize.com/phoenix/tracing/concepts-tracing/how-to-annotate-traces)                    | ✅           |
+| [Human Feedback](https://docs.arize.com/phoenix/tracing/how-to-tracing/capture-feedback)                         | ✅           |
+| [Experiments](https://docs.arize.com/phoenix/datasets-and-experiments/how-to-experiments/run-experiments)        | ✅           |
+| [Embeddings Analysis](https://docs.arize.com/phoenix/inferences/phoenix-inferences)                              | ✅           |
+| [Data Export](https://docs.arize.com/phoenix/tracing/how-to-tracing/extract-data-from-spans)                     | ✅           |
+| REST API                                                                                                         | ✅           |
+| GraphQL API                                                                                                      | ✅           |
+| Data Retention                                                                                                   | Customizable |
+| [Authentication](https://docs.arize.com/phoenix/deployment/authentication)                                       | ✅           |
+| [Social Login](https://docs.arize.com/phoenix/deployment/authentication#configuring-oauth2-identity-providers)   | ✅           |
+| [RBAC](https://docs.arize.com/phoenix/deployment/authentication#permissions)                                     | ✅           |
+| Projects                                                                                                         | ✅           |
+| [Self-Hosting](https://docs.arize.com/phoenix/deployment)                                                        | ✅           |
+| Jupyter Notebooks                                                                                                | ✅           |
+| [Prompt Playground](https://docs.arize.com/phoenix/prompt-engineering/overview-prompts)                          | ✅           |
+| [Sessions](https://docs.arize.com/phoenix/tracing/how-to-tracing/setup-sessions)                                 | ✅           |
+| Prompt Management                                                                                                | ✅           |
 
 ## Tracing Integrations
 
@@ -131,6 +131,6 @@ See the [migration guide](./MIGRATION.md) for a list of breaking changes.
 
 Copyright 2024 Arize AI, Inc. All Rights Reserved.
 
-Portions of this code are patent protected by one or more U.S. Patents. See [IP_NOTICE](https://github.com/Arize-ai/phoenix/blob/main/IP_NOTICE).
+Portions of this code are patent protected by one or more U.S. Patents. See the [IP_NOTICE](https://github.com/Arize-ai/phoenix/blob/main/IP_NOTICE).
 
 This software is licensed under the terms of the Elastic License 2.0 (ELv2). See [LICENSE](https://github.com/Arize-ai/phoenix/blob/main/LICENSE).
diff --git a/app/.eslintrc.js b/app/.eslintrc.js
@@ -98,6 +98,26 @@ module.exports = {
         name: "theme",
         module: "@arizeai/components",
       },
+      {
+        name: "RadioGroup",
+        module: "@arizeai/components",
+        use: "@phoenix/components",
+      },
+      {
+        name: "Radio",
+        module: "@arizeai/components",
+        use: "@phoenix/components",
+      },
+      {
+        name: "TextField",
+        module: "@arizeai/components",
+        use: "@phoenix/components",
+      },
+      {
+        name: "TextArea",
+        module: "@arizeai/components",
+        use: "@phoenix/components",
+      },
     ],
     "no-duplicate-imports": "error",
   },

diff --git a/app/.storybook/main.ts b/app/.storybook/main.ts
@@ -1,4 +1,5 @@
 import type { StorybookConfig } from "@storybook/react-vite";
+import { mergeConfig } from "vite";
 import { resolve } from "path";
 
 const config: StorybookConfig = {
@@ -7,7 +8,7 @@ const config: StorybookConfig = {
     "@storybook/addon-essentials",
     "@chromatic-com/storybook",
     "@storybook/addon-interactions",
-    "@storybook/addon-themes",
+    "react-docgen-typescript",
   ],
   framework: {
     name: "@storybook/react-vite",
@@ -16,5 +17,19 @@ const config: StorybookConfig = {
   typescript: {
     reactDocgen: "react-docgen-typescript",
   },
+  async viteFinal(config, { configType }) {
+    // return the customized config
+    return mergeConfig(config, {
+      // customize the Vite config here
+      optimizeDeps: {
+        include: ["@storybook/addon-interactions"],
+      },
+      resolve: {
+        alias: {
+          "@phoenix": resolve(__dirname, "../src"),
+        },
+      },
+    });
+  },
 };
 export default config;
diff --git a/app/.storybook/preview-body.html b/app/.storybook/preview-body.html
@@ -1,6 +1,8 @@
 <style>
-body {
-    font-family: 'Roboto', sans-serif;
+  body,
+  .docs-story {
+    font-family: "Roboto", sans-serif;
     color: var(--ac-global-text-color-900);
+    background-color: var(--ac-global-color-grey-75);
   }
-</style>
+</style>