fix: don't swallow errors, make datasets processable (#9469)

* fix: don't swallow errors, make datasets processable

* make it be a type import

* changeset

* fix types

* fix tests

Author: mikeldking

js/.changeset/fancy-items-mate.md

@@ -0,0 +1,5 @@
+---
+"@arizeai/phoenix-client": patch
+---
+
+don't swallow errors, allow for incomplete datasets (e.g. just imputs)

js/packages/phoenix-client/src/client.ts

@@ -1,4 +1,7 @@
-import createOpenApiClient, { type ClientOptions } from "openapi-fetch";
+import createOpenApiClient, {
+  type Middleware,
+  type ClientOptions,
+} from "openapi-fetch";
 import type {
   paths as oapiPathsV1,
   components as oapiComponentsV1,
@@ -49,6 +52,20 @@ export const getMergedOptions = ({
   };
 };
 
+/**
+ * Middleware to take non-successful API calls throw instead of being swallowed
+ */
+const middleware: Middleware = {
+  onResponse({ response }) {
+    if (!response.ok) {
+      // Will produce error messages like "https://example.org/api/v1/example: 404 Not Found".
+      throw new Error(
+        `${response.url}: ${response.status} ${response.statusText}`
+      );
+    }
+  },
+};
+
 /**
  * Create a Phoenix client.
  *
@@ -79,8 +96,10 @@ export const createClient = (
   } = {}
 ) => {
   const mergedOptions = getMergedOptions(config);
+  const openApiClient = createOpenApiClient<pathsV1>(mergedOptions);
+  openApiClient.use(middleware);
   return {
-    ...createOpenApiClient<pathsV1>(mergedOptions),
+    ...openApiClient,
     config: mergedOptions,
   };
 };

js/packages/phoenix-client/src/datasets/appendDatasetExamples.ts

@@ -33,7 +33,7 @@ export async function appendDatasetExamples({
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
   const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const metadata = examples.map((example) => example.metadata ?? {});
   let datasetName: string;
   if ("datasetName" in dataset) {
     datasetName = dataset.datasetName;

js/packages/phoenix-client/src/datasets/createDataset.ts

@@ -34,8 +34,8 @@ export async function createDataset({
 }: CreateDatasetParams): Promise<CreateDatasetResponse> {
   const client = _client || createClient();
   const inputs = examples.map((example) => example.input);
-  const outputs = examples.map((example) => example.output ?? {}); // Treat null as an empty object
-  const metadata = examples.map((example) => example.metadata);
+  const outputs = examples.map((example) => example?.output ?? {}); // Treat null as an empty object
+  const metadata = examples.map((example) => example?.metadata ?? {});
   const createDatasetResponse = await client.POST("/v1/datasets/upload", {
     params: {
       query: {

js/packages/phoenix-client/src/experiments/runExperiment.ts

@@ -699,7 +699,7 @@ async function runEvaluator({
         input: example.input,
         output: run.output ?? null,
         expected: example.output,
-        metadata: example.metadata,
+        metadata: example?.metadata,
       });
       thisEval.result = result;
       logger.info(

js/packages/phoenix-client/src/types/datasets.ts

@@ -47,8 +47,8 @@ export interface DatasetExamples {
  */
 export interface Example {
   input: Record<string, unknown>;
-  output: Record<string, unknown> | null;
-  metadata: Record<string, unknown>;
+  output?: Record<string, unknown> | null;
+  metadata?: Record<string, unknown> | null;
 }
 
 /**

js/packages/phoenix-client/src/types/experiments.ts

@@ -67,7 +67,7 @@ export type EvaluatorParams = {
   /**
    * Metadata associated with the Dataset Example
    */
-  metadata?: Record<string, unknown>;
+  metadata?: Example["metadata"];
 };
 
 export type Evaluator = {

js/packages/phoenix-client/test/prompts/createPrompt.test.ts

@@ -32,6 +32,7 @@ vi.mock("openapi-fetch", () => ({
       },
       error: null,
     }),
+    use: () => {},
   }),
 }));
 

js/packages/phoenix-client/test/spans/addDocumentAnnotation.test.ts

@@ -13,6 +13,7 @@ vi.mock("openapi-fetch", () => ({
       },
       error: null,
     }),
+    use: () => {},
   }),
 }));
 

js/packages/phoenix-client/test/spans/addSpanAnnotation.test.ts

@@ -13,6 +13,7 @@ vi.mock("openapi-fetch", () => ({
       },
       error: null,
     }),
+    use: () => {},
   }),
 }));