Skip to content
This repository was archived by the owner on Jun 28, 2022. It is now read-only.

Commit

Permalink
feat: upload group of files on s3
Browse files Browse the repository at this point in the history
  • Loading branch information
arantespp committed Nov 13, 2020
1 parent 7e2dcc8 commit ea7eaf8
Showing 1 changed file with 34 additions and 16 deletions.
50 changes: 34 additions & 16 deletions packages/cli/src/deploy/s3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,7 @@ export const getContentMetadata = (
};
case 'jpeg':
case 'jpg':
return {
ContentType: 'image/jpeg',
ContentEncoding: 'binary',
};
return {};
case 'mjs':
case 'js':
return {
Expand Down Expand Up @@ -147,8 +144,6 @@ export const uploadFileToS3 = async ({
throw new Error('file or filePath must be defined');
}

log.info(logPrefix, `Uploading file to ${bucket}/${key}...`);

let params: S3.PutObjectRequest = {
Bucket: bucket,
Key: key,
Expand Down Expand Up @@ -201,17 +196,40 @@ export const uploadDirectoryToS3 = async ({
},
);

const allFiles = allFilesAndDirectories.filter((item) =>
fs.lstatSync(item).isFile(),
);
const allFiles = allFilesAndDirectories
/**
* Remove directories.
*/
.filter((item) => fs.lstatSync(item).isFile());

for (const [index, file] of allFiles.entries()) {
log.info(logPrefix, `Upload ${index + 1}/${allFiles.length}`);
await uploadFileToS3({
bucket,
key: path.relative(directory, file),
filePath: file,
});
const GROUP_MAX_LENGTH = 63;

const numberOfGroups = Math.ceil(allFiles.length / GROUP_MAX_LENGTH);

/**
* Divide all files and create "numberOfGroups" groups of files whose max
* length is GROUP_MAX_LENGTH.
*/
const aoaOfFiles = allFiles.reduce<string[][]>((acc, file, index) => {
const groupIndex = index % numberOfGroups;
if (!acc[groupIndex]) {
acc[groupIndex] = [];
}
acc[index % numberOfGroups].push(file);
return acc;
}, []);

for (const [index, groupOfFiles] of aoaOfFiles.entries()) {
log.info(logPrefix, `Uploading group ${index + 1}/${aoaOfFiles.length}...`);
await Promise.all(
groupOfFiles.map((file) =>
uploadFileToS3({
bucket,
key: path.relative(directory, file),
filePath: file,
}),
),
);
}
};

Expand Down

0 comments on commit ea7eaf8

Please sign in to comment.