diff --git a/code/package-lock.json b/code/package-lock.json index 0367663..49adb3d 100644 --- a/code/package-lock.json +++ b/code/package-lock.json @@ -9,7 +9,7 @@ "version": "1.1.6", "license": "ISC", "dependencies": { - "@devrev/ts-adaas": "1.9.0", + "@devrev/ts-adaas": "1.11.0", "@devrev/typescript-sdk": "1.1.63", "axios": "^1.9.0", "dotenv": "^16.0.3", @@ -1874,9 +1874,9 @@ } }, "node_modules/@devrev/ts-adaas": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@devrev/ts-adaas/-/ts-adaas-1.9.0.tgz", - "integrity": "sha512-x5fEWW/7dKojgVwbc4biJbqkIrtGuLc0tkKwbZjpbeCRv+TSvT2U1e2LWD2BkqQyK+unMRyr9wxeLYu1UboRPg==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@devrev/ts-adaas/-/ts-adaas-1.11.0.tgz", + "integrity": "sha512-6KkPO42397liEFRTGLT526MUfXw4RIBYIWGTJQG9/CDbah72dalXDSyUezj5x8IWjIVMaIMHbFJtlbt/85hqeA==", "license": "ISC", "dependencies": { "@devrev/typescript-sdk": "^1.1.59", @@ -1885,23 +1885,9 @@ "form-data": "^4.0.4", "js-jsonl": "^1.1.1", "ts-node": "^10.9.2", - "typescript": "^5.3.3", "yargs": "^17.7.2" } }, - "node_modules/@devrev/ts-adaas/node_modules/typescript": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", - "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, "node_modules/@devrev/typescript-sdk": { "version": "1.1.63", "resolved": "https://registry.npmjs.org/@devrev/typescript-sdk/-/typescript-sdk-1.1.63.tgz", diff --git a/code/package.json b/code/package.json index 04f7d90..6122d7a 100644 --- a/code/package.json +++ b/code/package.json @@ -56,7 +56,7 @@ "yargs": "^17.6.2" }, "dependencies": { - "@devrev/ts-adaas": "1.9.0", + "@devrev/ts-adaas": "1.11.0", "@devrev/typescript-sdk": "1.1.63", "axios": "^1.9.0", "dotenv": "^16.0.3", diff --git a/code/src/functions/extraction/workers/attachments-extraction.ts b/code/src/functions/extraction/workers/attachments-extraction.ts index cc55892..82f13fa 100644 --- a/code/src/functions/extraction/workers/attachments-extraction.ts +++ b/code/src/functions/extraction/workers/attachments-extraction.ts @@ -5,11 +5,11 @@ import { ExternalSystemAttachmentStreamingResponse, ExtractorEventType, processTask, - serializeAxiosError, } from '@devrev/ts-adaas'; // TODO: Replace with function for fetching attachment streams from the -// external system. This function should return a stream of the attachment data. +// external system. This function should return either a stream of the +// attachment data, a delay or an error. async function getFileStream({ item, }: ExternalSystemAttachmentStreamingParams): Promise { @@ -20,22 +20,28 @@ async function getFileStream({ responseType: 'stream', headers: { 'Accept-Encoding': 'identity', + timeout: 30000, }, }); return { httpStream: fileStreamResponse }; } catch (error) { if (axios.isAxiosError(error)) { - console.warn(`Error while fetching attachment ${id} from URL.`, serializeAxiosError(error)); - console.warn('Failed attachment metadata', item); - } else { - console.warn(`Error while fetching attachment ${id} from URL.`, error); - console.warn('Failed attachment metadata', item); + if (error?.response?.status === 429) { + const retryAfter = error.response?.headers['retry-after']; + return { delay: retryAfter }; + } else { + return { + error: { + message: `Error while fetching attachment ${id} from URL. Error code: ${error.response?.status}. Error message: ${error.response?.data.message}.`, + }, + }; + } } return { error: { - message: `Failed to fetch attachment ${id} from URL.`, + message: `Unknown error while fetching attachment ${id} from URL. Error: ${error}.`, }, }; } @@ -46,6 +52,10 @@ processTask({ try { const response = await adapter.streamAttachments({ stream: getFileStream, + + // TODO: If needed you can specify how many attachments to stream at + // once. Minimum is 1 and maximum is 50. + // batchSize: 10, }); if (response?.delay) {