Change upload API spec to use the common APIPromise response shape

blainekasten · blainekasten · commit fdf80b26b34c · 2025-11-17T14:48:35.000-06:00
diff --git a/src/lib/check-file.ts b/src/lib/check-file.ts
@@ -463,9 +463,7 @@ async function _check_csv(file: string, purpose: FilePurpose | string): Promise<
       }
     }
 
-    report_dict.update = _check_samples_count(file, report_dict, idx);
-    Object.assign(report_dict, report_dict.update);
-    delete (report_dict as any).update;
+    Object.assign(report_dict, _check_samples_count(file, report_dict, idx));
     report_dict.load_csv = true;
   } catch (e) {
     report_dict.load_csv = false;
@@ -573,7 +571,9 @@ async function _check_jsonl(file: string, purpose: FilePurpose | string): Promis
         } else if (current_format === DatasetFormat.CONVERSATION) {
           const message_column = JSONL_REQUIRED_COLUMNS_MAP[DatasetFormat.CONVERSATION][0];
           const require_assistant = purpose !== 'eval';
-          validate_messages(json_line[message_column], idx, require_assistant);
+          // @ts-ignore - just bad typescript not being great at record access
+          const messages = json_line[message_column];
+          validate_messages(messages, idx, require_assistant);
         } else {
           for (const column of JSONL_REQUIRED_COLUMNS_MAP[current_format]) {
             if (typeof json_line[column] !== 'string') {
diff --git a/src/lib/upload.ts b/src/lib/upload.ts
@@ -1,21 +1,13 @@
 // Upload file to server using /files API
 
 import { readEnv } from '../internal/utils/env';
-import fs from 'fs';
+import fs from 'fs/promises';
+import { createReadStream } from 'fs';
 import * as path from 'path';
-import { FilePurpose } from '../resources';
+import { FilePurpose, FileRetrieveResponse } from '../resources';
 import { checkFile } from './check-file';
-
-export interface FileResponse {
-  id: string;
-  object: string;
-  type: 'jsonl' | 'parquet';
-  purpose: 'fine-tune';
-  filename: string;
-  bytes: number;
-  line_count: number;
-  processed: boolean;
-}
+import { Together } from '../client';
+import { APIPromise } from '../core/api-promise';
 
 export interface ErrorResponse {
   message: string;
@@ -27,114 +19,105 @@ const failedUploadMessage = {
 
 const baseURL = readEnv('TOGETHER_API_BASE_URL') || 'https://api.together.xyz/v1';
 
-export async function upload(
+export function upload(
+  client: Together,
   fileName: string,
   purpose: FilePurpose,
   check: boolean = true,
-): Promise<FileResponse | ErrorResponse> {
-  if (!fs.existsSync(fileName)) {
-    return {
-      message: 'File does not exists',
-    };
-  }
-
-  const fileType = path.extname(fileName);
-  if (fileType !== '.jsonl' && fileType !== '.parquet') {
-    return {
-      message: 'File type must be either .jsonl or .parquet',
-    };
-  }
-
-  if (check) {
-    const checkResponse = await checkFile(fileName, purpose);
-    if (!checkResponse.is_check_passed) {
-      return {
-        message: checkResponse.message || `verification of ${fileName} failed with some unknown reason`,
-      };
-    }
-  }
-
-  // steps to do
-  // 1. check if file exists
-  // 2. get signed upload url
-  // 3. upload file
-  const apiKey = readEnv('TOGETHER_API_KEY');
-
-  if (!apiKey) {
-    return {
-      message: 'API key is required',
-    };
-  }
-
-  const getSigned = baseURL + '/files';
-
-  try {
-    const params = new URLSearchParams({
-      file_name: fileName,
-      purpose: purpose,
-    });
-    const fullUrl = `${getSigned}?${params}`;
-    const r = await fetch(fullUrl, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/x-www-form-urlencoded',
-        Authorization: `Bearer ${apiKey}`,
-      },
-      redirect: 'manual',
-      body: params.toString(),
-    });
-
-    if (r.status !== 302) {
-      return failedUploadMessage;
-    }
-
-    const uploadUrl = r.headers.get('location') || '';
-    if (!uploadUrl || uploadUrl === '') {
-      return failedUploadMessage;
-    }
-    const fileId = r.headers.get('x-together-file-id') || '';
-    if (!fileId || fileId === '') {
-      return failedUploadMessage;
-    }
-
-    const fileStream = fs.createReadStream(fileName);
-    const fileSize = fs.statSync(fileName).size;
-
-    // upload the file to uploadUrl
-    const uploadResponse = await fetch(uploadUrl, {
-      method: 'PUT',
-      headers: {
-        'Content-Type': 'application/octet-stream',
-        'Content-Length': `${fileSize}`,
-      },
-      body: fileStream,
-    });
-
-    if (uploadResponse.status !== 200) {
-      return {
-        message: `failed to upload file (${uploadResponse.statusText}) status code ${uploadResponse.status}`,
-      };
-    }
-
-    return {
-      id: fileId,
-      object: 'file',
-      type: 'jsonl',
-      purpose: 'fine-tune',
-      filename: fileName,
-      bytes: fileSize,
-      line_count: 0,
-      processed: true,
-    };
-  } catch (error) {
-    if (error instanceof Error && 'status' in error && error.status) {
-      return {
-        message: `failed to upload file with status ${error.status}`,
-      };
-    }
-
-    return {
-      message: 'failed to upload file',
-    };
-  }
+): APIPromise<FileRetrieveResponse> {
+  return new APIPromise<FileRetrieveResponse>(
+    client,
+    new Promise(async (resolve, reject) => {
+      let fileSize = 0;
+
+      try {
+        const stat = await fs.stat(fileName);
+        fileSize = stat.size;
+      } catch {
+        reject(new Error('File does not exists'));
+      }
+
+      const fileType = path.extname(fileName).replace('.', '');
+      if (fileType !== 'jsonl' && fileType !== 'parquet' && fileType !== 'csv') {
+        return {
+          message: 'File type must be either .jsonl, .parquet, or .csv',
+        };
+      }
+
+      if (check) {
+        const checkResponse = await checkFile(fileName, purpose);
+        if (!checkResponse.is_check_passed) {
+          reject(checkResponse.message || `verification of ${fileName} failed with some unknown reason`);
+        }
+      }
+
+      try {
+        const params = new URLSearchParams({
+          file_name: fileName,
+          file_type: fileType,
+          purpose: purpose,
+        });
+        const fullUrl = `${baseURL}/files?${params}`;
+        const r = await fetch(fullUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            Authorization: `Bearer ${client.apiKey}`,
+          },
+          redirect: 'manual',
+          body: params.toString(),
+        });
+
+        if (r.status !== 302) {
+          return reject(failedUploadMessage);
+        }
+
+        const uploadUrl = r.headers.get('location') || '';
+        if (!uploadUrl || uploadUrl === '') {
+          return reject(failedUploadMessage);
+        }
+        const fileId = r.headers.get('x-together-file-id') || '';
+        if (!fileId || fileId === '') {
+          return reject(failedUploadMessage);
+        }
+
+        const fileStream = createReadStream(fileName);
+
+        // upload the file to uploadUrl
+        const uploadResponse = await fetch(uploadUrl, {
+          method: 'PUT',
+          headers: {
+            'Content-Type': 'application/octet-stream',
+            'Content-Length': fileSize.toString(),
+          },
+          body: fileStream,
+        });
+
+        // uploadResponse.
+
+        if (uploadResponse.status !== 200) {
+          return reject({
+            message: `failed to upload file (${uploadResponse.statusText}) status code ${uploadResponse.status}`,
+          });
+        }
+
+        const data = await client.files.retrieve(fileId).asResponse();
+
+        // Forcing the shape into the APIResponse interface
+        resolve({
+          controller: new AbortController(),
+          requestLogID: '',
+          retryOfRequestLogID: undefined,
+          startTime: Date.now(),
+          options: {
+            method: 'post',
+            path: '/files',
+          },
+          response: data,
+        });
+      } catch (error) {
+        reject(error);
+      }
+    }),
+  );
 }
diff --git a/src/resources/files.ts b/src/resources/files.ts
@@ -68,8 +68,8 @@ export class Files extends APIResource {
   /**
    * Upload a file.
    */
-  upload(file: string, purpose: FilePurpose, check: boolean = true): ReturnType<typeof upload> {
-    return upload(file, purpose, check);
+  upload(file: string, purpose: FilePurpose, check: boolean = true): APIPromise<FileRetrieveResponse> {
+    return upload(this._client, file, purpose, check);
   }
 }
 
diff --git a/tests/lib/check-file.test.ts b/tests/lib/check-file.test.ts
@@ -292,10 +292,7 @@ describe('checkFile', () => {
         const file = path.join(tmpDir, 'missing_field_in_conversation.jsonl');
         const content = [
           {
-            messages: [
-              { role: 'user', content: 'Hi' },
-              { role: 'assistant' },
-            ],
+            messages: [{ role: 'user', content: 'Hi' }, { role: 'assistant' }],
           },
         ];
         fs.writeFileSync(file, content.map((item) => JSON.stringify(item)).join('\n'));
@@ -309,11 +306,7 @@ describe('checkFile', () => {
         const file = path.join(tmpDir, 'wrong_turn_type.jsonl');
         const content = [
           {
-            messages: [
-              'Hi!',
-              { role: 'user', content: 'Hi' },
-              { role: 'assistant' },
-            ],
+            messages: ['Hi!', { role: 'user', content: 'Hi' }, { role: 'assistant' }],
           },
         ];
         fs.writeFileSync(file, content.map((item) => JSON.stringify(item)).join('\n'));
@@ -427,4 +420,3 @@ describe('checkFile', () => {
     });
   });
 });
-
diff --git a/tests/lib/upload.test.ts b/tests/lib/upload.test.ts
@@ -0,0 +1,59 @@
+// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import { tmpdir } from 'os';
+import Together from 'together-ai';
+
+const client = new Together({
+  apiKey: 'My API Key',
+  baseURL: process.env['TEST_API_BASE_URL'] ?? 'http://127.0.0.1:4010',
+});
+
+describe('Files', () => {
+  const originalFetch = globalThis.fetch;
+  const mockFetch = jest.fn();
+
+  beforeEach(() => {
+    mockFetch.mockReset();
+    globalThis.fetch = mockFetch;
+  });
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+  });
+
+  test('upload', async () => {
+    const file = path.join(tmpdir(), 'valid.jsonl');
+    const content = [{ text: 'Hello, world!' }, { text: 'How are you?' }];
+    await fs.writeFile(file, content.map((item) => JSON.stringify(item)).join('\n'));
+
+    // Mock the Signed URL Response
+    mockFetch.mockResolvedValueOnce(
+      new Response('', {
+        status: 302,
+        headers: {
+          location: 's3://bucket/signed-url',
+          'x-together-file-id': 'signed-url-file-id',
+        },
+      }),
+    );
+
+    // Actual file upload response is a 200
+    mockFetch.mockResolvedValueOnce(
+      new Response('', {
+        status: 200,
+      }),
+    );
+
+    const responsePromise = client.files.upload(file, 'fine-tune', false);
+
+    const rawResponse = await responsePromise.asResponse();
+    expect(rawResponse).toBeInstanceOf(Response);
+    const response = await responsePromise;
+    expect(response).not.toBeInstanceOf(Response);
+    const dataAndResponse = await responsePromise.withResponse();
+    expect(dataAndResponse.data).toBe(response);
+    expect(dataAndResponse.response).toBe(rawResponse);
+  });
+});

Original file line number	Diff line number	Diff line change
`@@ -68,8 +68,8 @@ export class Files extends APIResource {`
`68`	`68`	`/**`
`69`	`69`	`* Upload a file.`
`70`	`70`	`*/`
`71`		`- upload(file: string, purpose: FilePurpose, check: boolean = true): ReturnType<typeof upload> {`
`72`		`- return upload(file, purpose, check);`
	`71`	`+ upload(file: string, purpose: FilePurpose, check: boolean = true): APIPromise<FileRetrieveResponse> {`
	`72`	`+ return upload(this._client, file, purpose, check);`
`73`	`73`	`}`
`74`	`74`	`}`
`75`	`75`