Merge pull request #1430 from remotion-dev/lift-lambda-payload-limit

packages/docs/docs/lambda/checklist.md

@@ -27,10 +27,6 @@ Make sure your AWS user only has as many permissions as needed and store your cr
 
 Familiarize yourself with the [AWS burst limit](https://docs.aws.amazon.com/lambda/latest/dg/invocation-scaling.html). Essentially, you need to avoid a quick spike in video renders that will cause the burst limit to take effect. If you need to scale beyond the burst limit, consider scaling across multiple regions as the burst limit only applies for a certain region. Another strategy to consider is creating multiple sub-accounts in your AWS organization as the burst limit only affects a single account.
 
-### AWS payload limit
-
-The maximum payload for invoking a Lambda function is 256KB. Ensure that in your application, the `inputProps` payload does not exceed this amount and introduce validation and error handling if necessary.
-
 ### Selecting the right concurrency
 
 If you are using the [`framesPerLambda`](/docs/lambda/rendermediaonlambda#framesperlambda) option, make sure that for each video you render, the parameter is set in a way that it stays within the allowed bounds (no more than 200 lambda functions per render). See: [Concurrency](/docs/lambda/concurrency)

packages/lambda/src/api/render-media-on-lambda.ts

@@ -14,6 +14,7 @@ import {LambdaRoutines} from '../shared/constants';
 import type {DownloadBehavior} from '../shared/content-disposition-header';
 import {convertToServeUrl} from '../shared/convert-to-serve-url';
 import {getCloudwatchStreamUrl} from '../shared/get-cloudwatch-stream-url';
+import {serializeInputProps} from '../shared/serialize-input-props';
 import {validateDownloadBehavior} from '../shared/validate-download-behavior';
 import {validateFramesPerLambda} from '../shared/validate-frames-per-lambda';
 import type {LambdaCodec} from '../shared/validate-lambda-codec';
@@ -123,6 +124,11 @@ export const renderMediaOnLambda = async ({
 	validateDownloadBehavior(downloadBehavior);
 
 	const realServeUrl = await convertToServeUrl(serveUrl, region);
+	const serializedInputProps = await serializeInputProps({
+		inputProps,
+		region,
+		type: 'video-or-audio',
+	});
 	try {
 		const res = await callLambda({
 			functionName,
@@ -131,7 +137,7 @@ export const renderMediaOnLambda = async ({
 				framesPerLambda: framesPerLambda ?? null,
 				composition,
 				serveUrl: realServeUrl,
-				inputProps: inputProps ?? {},
+				inputProps: serializedInputProps,
 				codec: actualCodec,
 				imageFormat: imageFormat ?? 'jpeg',
 				crf,

packages/lambda/src/api/render-still-on-lambda.ts

@@ -11,6 +11,7 @@ import {DEFAULT_MAX_RETRIES, LambdaRoutines} from '../shared/constants';
 import type {DownloadBehavior} from '../shared/content-disposition-header';
 import {convertToServeUrl} from '../shared/convert-to-serve-url';
 import {getCloudwatchStreamUrl} from '../shared/get-cloudwatch-stream-url';
+import {serializeInputProps} from '../shared/serialize-input-props';
 
 export type RenderStillOnLambdaInput = {
 	region: AwsRegion;
@@ -78,14 +79,21 @@ export const renderStillOnLambda = async ({
 	downloadBehavior,
 }: RenderStillOnLambdaInput): Promise<RenderStillOnLambdaOutput> => {
 	const realServeUrl = await convertToServeUrl(serveUrl, region);
+
+	const serializedInputProps = await serializeInputProps({
+		inputProps,
+		region,
+		type: 'still',
+	});
+
 	try {
 		const res = await callLambda({
 			functionName,
 			type: LambdaRoutines.still,
 			payload: {
 				composition,
 				serveUrl: realServeUrl,
-				inputProps,
+				inputProps: serializedInputProps,
 				imageFormat,
 				envVariables,
 				quality,

packages/lambda/src/functions/launch.ts

@@ -4,6 +4,7 @@ import fs from 'fs';
 import {Internals} from 'remotion';
 import {VERSION} from 'remotion/version';
 import {getLambdaClient} from '../shared/aws-clients';
+import {cleanupSerializedInputProps} from '../shared/cleanup-serialized-input-props';
 import type {
 	EncodingProgress,
 	LambdaPayload,
@@ -560,6 +561,12 @@ const innerLaunchHandler = async (params: LambdaPayload, options: Options) => {
 		jobs,
 	});
 
+	const cleanupSerializedInputPropsProm = cleanupSerializedInputProps({
+		bucketName: params.bucketName,
+		region: getCurrentRegionInFunction(),
+		serialized: params.inputProps,
+	});
+
 	const outputUrl = getOutputUrlFromMetadata(
 		renderMetadata,
 		params.bucketName,
@@ -574,7 +581,9 @@ const innerLaunchHandler = async (params: LambdaPayload, options: Options) => {
 		contents,
 		errorExplanations: await errorExplanationsProm,
 		timeToEncode: encodingStop - encodingStart,
-		timeToDelete: await deletProm,
+		timeToDelete: (
+			await Promise.all([deletProm, cleanupSerializedInputPropsProm])
+		).reduce((a, b) => a + b, 0),
 		outputFile: {
 			lastModified: Date.now(),
 			size: outputSize.size,

packages/lambda/src/functions/renderer.ts

@@ -12,6 +12,7 @@ import {
 	lambdaTimingsKey,
 	RENDERER_PATH_TOKEN,
 } from '../shared/constants';
+import {deserializeInputProps} from '../shared/deserialize-input-props';
 import type {
 	ChunkTimingData,
 	ObjectChunkTimingData,
@@ -41,6 +42,13 @@ const renderHandler = async (
 		throw new Error('Params must be renderer');
 	}
 
+	const inputPropsPromise = deserializeInputProps({
+		bucketName: params.bucketName,
+		expectedBucketOwner: options.expectedBucketOwner,
+		region: getCurrentRegionInFunction(),
+		serialized: params.inputProps,
+	});
+
 	const browserInstance = await getBrowserInstance(
 		RenderInternals.isEqualOrBelowLogLevel(params.logLevel, 'verbose'),
 		params.chromiumOptions ?? {}
@@ -80,6 +88,7 @@ const renderHandler = async (
 
 	const downloads: Record<string, number> = {};
 
+	const inputProps = await inputPropsPromise;
 	await new Promise<void>((resolve, reject) => {
 		renderMedia({
 			composition: {
@@ -90,7 +99,7 @@ const renderHandler = async (
 				width: params.width,
 			},
 			imageFormat: params.imageFormat,
-			inputProps: params.inputProps,
+			inputProps,
 			frameRange: params.frameRange,
 			onProgress: ({renderedFrames, encodedFrames, stitchStage}) => {
 				if (

packages/lambda/src/functions/still.ts

@@ -7,6 +7,7 @@ import {VERSION} from 'remotion/version';
 import {estimatePrice} from '../api/estimate-price';
 import {getOrCreateBucket} from '../api/get-or-create-bucket';
 import {getLambdaClient} from '../shared/aws-clients';
+import {cleanupSerializedInputProps} from '../shared/cleanup-serialized-input-props';
 import type {
 	LambdaPayload,
 	LambdaPayloads,
@@ -17,6 +18,7 @@ import {
 	MAX_EPHEMERAL_STORAGE_IN_MB,
 	renderMetadataKey,
 } from '../shared/constants';
+import {deserializeInputProps} from '../shared/deserialize-input-props';
 import {getServeUrlHash} from '../shared/make-s3-url';
 import {randomHash} from '../shared/random-hash';
 import {validateDownloadBehavior} from '../shared/validate-download-behavior';
@@ -78,6 +80,13 @@ const innerStillHandler = async (
 			lambdaParams.chromiumOptions ?? {}
 		),
 	]);
+	const inputPropsPromise = deserializeInputProps({
+		bucketName,
+		expectedBucketOwner: options.expectedBucketOwner,
+		region: getCurrentRegionInFunction(),
+		serialized: lambdaParams.inputProps,
+	});
+
 	const outputDir = RenderInternals.tmpDir('remotion-render-');
 
 	const outputPath = path.join(outputDir, 'output');
@@ -131,6 +140,7 @@ const innerStillHandler = async (
 		customCredentials: null,
 	});
 
+	const inputProps = await inputPropsPromise;
 	await renderStill({
 		composition,
 		output: outputPath,
@@ -142,7 +152,7 @@ const innerStillHandler = async (
 			durationInFrames: composition.durationInFrames,
 		}),
 		imageFormat: lambdaParams.imageFormat as StillImageFormat,
-		inputProps: lambdaParams.inputProps,
+		inputProps,
 		overwrite: false,
 		puppeteerInstance: browserInstance,
 		quality: lambdaParams.quality,
@@ -170,7 +180,15 @@ const innerStillHandler = async (
 		downloadBehavior: lambdaParams.downloadBehavior,
 		customCredentials,
 	});
-	await fs.promises.rm(outputPath, {recursive: true});
+
+	await Promise.all([
+		fs.promises.rm(outputPath, {recursive: true}),
+		cleanupSerializedInputProps({
+			bucketName,
+			region: getCurrentRegionInFunction(),
+			serialized: lambdaParams.inputProps,
+		}),
+	]);
 
 	const estimatedPrice = estimatePrice({
 		durationInMiliseconds: Date.now() - start + 100,

packages/lambda/src/shared/cleanup-serialized-input-props.ts

@@ -0,0 +1,28 @@
+import type {AwsRegion} from '../client';
+import {lambdaDeleteFile} from '../functions/helpers/io';
+import type {SerializedInputProps} from './constants';
+import {inputPropsKey} from './constants';
+
+export const cleanupSerializedInputProps = async ({
+	serialized,
+	region,
+	bucketName,
+}: {
+	serialized: SerializedInputProps;
+	region: AwsRegion;
+	bucketName: string;
+}): Promise<number> => {
+	if (serialized.type === 'payload') {
+		return 0;
+	}
+
+	const time = Date.now();
+	await lambdaDeleteFile({
+		bucketName,
+		key: inputPropsKey(serialized.hash),
+		region,
+		customCredentials: null,
+	});
+
+	return Date.now() - time;
+};

packages/lambda/src/shared/constants.ts

@@ -180,6 +180,10 @@ export const postRenderDataKey = (renderId: string) => {
 	return `${rendersPrefix(renderId)}/post-render-metadata.json`;
 };
 
+export const inputPropsKey = (hash: string) => {
+	return `input-props/${hash}.json`;
+};
+
 export const RENDERER_PATH_TOKEN = 'remotion-bucket';
 export const CONCAT_FOLDER_TOKEN = 'remotion-concat';
 export const REMOTION_CONCATED_TOKEN = 'remotion-concated-token';
@@ -199,6 +203,16 @@ type WebhookOption = null | {
 	secret: string | null;
 };
 
+export type SerializedInputProps =
+	| {
+			type: 'bucket-url';
+			hash: string;
+	  }
+	| {
+			type: 'payload';
+			payload: unknown;
+	  };
+
 export type LambdaPayloads = {
 	info: {
 		type: LambdaRoutines.info;
@@ -208,7 +222,7 @@ export type LambdaPayloads = {
 		serveUrl: string;
 		composition: string;
 		framesPerLambda: number | null;
-		inputProps: unknown;
+		inputProps: SerializedInputProps;
 		codec: LambdaCodec;
 		imageFormat: ImageFormat;
 		crf: number | undefined;
@@ -241,7 +255,7 @@ export type LambdaPayloads = {
 		composition: string;
 		framesPerLambda: number | null;
 		bucketName: string;
-		inputProps: unknown;
+		inputProps: SerializedInputProps;
 		renderId: string;
 		imageFormat: ImageFormat;
 		codec: LambdaCodec;
@@ -288,7 +302,7 @@ export type LambdaPayloads = {
 		width: number;
 		durationInFrames: number;
 		retriesLeft: number;
-		inputProps: unknown;
+		inputProps: SerializedInputProps;
 		renderId: string;
 		imageFormat: ImageFormat;
 		codec: Exclude<Codec, 'h264'>;
@@ -312,7 +326,7 @@ export type LambdaPayloads = {
 		type: LambdaRoutines.still;
 		serveUrl: string;
 		composition: string;
-		inputProps: unknown;
+		inputProps: SerializedInputProps;
 		imageFormat: ImageFormat;
 		envVariables: Record<string, string> | undefined;
 		attempt: number;
@@ -351,7 +365,7 @@ export type RenderMetadata = {
 	usesOptimizationProfile: boolean;
 	type: 'still' | 'video';
 	imageFormat: ImageFormat;
-	inputProps: unknown;
+	inputProps: SerializedInputProps;
 	framesPerLambda: number;
 	memorySizeInMb: number;
 	lambdaVersion: string;

packages/lambda/src/shared/deserialize-input-props.ts

@@ -0,0 +1,39 @@
+import type {AwsRegion} from '../client';
+import {lambdaReadFile} from '../functions/helpers/io';
+import type {SerializedInputProps} from './constants';
+import {inputPropsKey} from './constants';
+import {streamToString} from './stream-to-string';
+
+export const deserializeInputProps = async ({
+	serialized,
+	region,
+	bucketName,
+	expectedBucketOwner,
+}: {
+	serialized: SerializedInputProps;
+	region: AwsRegion;
+	bucketName: string;
+	expectedBucketOwner: string;
+}): Promise<unknown> => {
+	if (serialized.type === 'payload') {
+		return {
+			inputProps: serialized.payload,
+		};
+	}
+
+	try {
+		const response = await lambdaReadFile({
+			bucketName,
+			expectedBucketOwner,
+			key: inputPropsKey(serialized.hash),
+			region,
+		});
+
+		const body = await streamToString(response);
+		const payload = JSON.parse(body);
+
+		return payload;
+	} catch (err) {
+		throw new Error('Failed to parse input props that were');
+	}
+};