Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MS api throttling mitigation #203

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

MS api throttling mitigation #203

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 67 additions & 73 deletions src/VideoUtils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,116 +45,110 @@ function durationToTotalChunks(duration: string): number {
}


export async function getVideoInfo(videoGuids: Array<string>, session: Session, subtitles?: boolean): Promise<Array<Video>> {
let metadata: Array<Video> = [];
export async function getVideoInfo(videoGuid: string, session: Session, subtitles?: boolean): Promise<Video> {
// template elements
let title: string;
let duration: string;
let publishDate: string;
let publishTime: string;
let author: string;
let authorEmail: string;
let uniqueId: string;
// final video path (here for consistency with typedef)
const outPath = '';
// ffmpeg magic (abstraction of FFmpeg timemark)
let totalChunks: number;
// various sources
let playbackUrl: string;
let posterImageUrl: string;
let captionsUrl: string | undefined;

const apiClient: ApiClient = ApiClient.getInstance(session);

/* TODO: change this to a single guid at a time to ease our footprint on the
MSS servers or we get throttled after 10 sequential reqs */
for (const guid of videoGuids) {
let response: AxiosResponse<any> | undefined =
await apiClient.callApi('videos/' + guid + '?$expand=creator', 'get');
let response: AxiosResponse<any> | undefined =
await apiClient.callApi('videos/' + videoGuid + '?$expand=creator', 'get');

title = sanitizeWindowsName(response?.data['name']);
title = sanitizeWindowsName(response?.data['name']);

duration = isoDurationToString(response?.data.media['duration']);
duration = isoDurationToString(response?.data.media['duration']);

publishDate = publishedDateToString(response?.data['publishedDate']);
publishDate = publishedDateToString(response?.data['publishedDate']);

publishTime = publishedTimeToString(response?.data['publishedDate']);
publishTime = publishedTimeToString(response?.data['publishedDate']);

author = response?.data['creator'].name;
author = response?.data['creator'].name;

authorEmail = response?.data['creator'].mail;
authorEmail = response?.data['creator'].mail;

uniqueId = '#' + guid.split('-')[0];
uniqueId = '#' + videoGuid.split('-')[0];

totalChunks = durationToTotalChunks(response?.data.media['duration']);
totalChunks = durationToTotalChunks(response?.data.media['duration']);

playbackUrl = response?.data['playbackUrls']
.filter((item: { [x: string]: string; }) =>
item['mimeType'] == 'application/vnd.apple.mpegurl')
.map((item: { [x: string]: string }) => {
return item['playbackUrl'];
})[0];
playbackUrl = response?.data['playbackUrls']
.filter((item: { [x: string]: string; }) =>
item['mimeType'] == 'application/vnd.apple.mpegurl')
.map((item: { [x: string]: string }) => {
return item['playbackUrl'];
})[0];

posterImageUrl = response?.data['posterImage']['medium']['url'];
posterImageUrl = response?.data['posterImage']['medium']['url'];

if (subtitles) {
let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${guid}/texttracks`, 'get');
if (subtitles) {
let captions: AxiosResponse<any> | undefined = await apiClient.callApi(`videos/${videoGuid}/texttracks`, 'get');

if (!captions?.data.value.length) {
captionsUrl = undefined;
}
else if (captions?.data.value.length === 1) {
logger.info(`Found subtitles for ${title}. \n`);
captionsUrl = captions?.data.value.pop().url;
}
else {
const index: number = promptUser(captions.data.value.map((item: { language: string; autoGenerated: string; }) => {
return `[${item.language}] autogenerated: ${item.autoGenerated}`;
}));
captionsUrl = captions.data.value[index].url;
}
if (!captions?.data.value.length) {
captionsUrl = undefined;
}
else if (captions?.data.value.length === 1) {
logger.info(`Found subtitles for ${title}. \n`);
captionsUrl = captions?.data.value.pop().url;
}
else {
const index: number = promptUser(captions.data.value.map((item: { language: string; autoGenerated: string; }) => {
return `[${item.language}] autogenerated: ${item.autoGenerated}`;
}));
captionsUrl = captions.data.value[index].url;
}

metadata.push({
title: title,
duration: duration,
publishDate: publishDate,
publishTime: publishTime,
author: author,
authorEmail: authorEmail,
uniqueId: uniqueId,
outPath: outPath,
totalChunks: totalChunks, // Abstraction of FFmpeg timemark
playbackUrl: playbackUrl,
posterImageUrl: posterImageUrl,
captionsUrl: captionsUrl
});
}

return metadata;
return {
title: title,
duration: duration,
publishDate: publishDate,
publishTime: publishTime,
author: author,
authorEmail: authorEmail,
uniqueId: uniqueId,
outPath: outPath,
totalChunks: totalChunks,
playbackUrl: playbackUrl,
posterImageUrl: posterImageUrl,
captionsUrl: captionsUrl
};
}


export function createUniquePath(videos: Array<Video>, outDirs: Array<string>, template: string, format: string, skip?: boolean): Array<Video> {

videos.forEach((video: Video, index: number) => {
let title: string = template;
let finalTitle: string;
const elementRegEx = RegExp(/{(.*?)}/g);
let match = elementRegEx.exec(template);
export function createUniquePath(video: Video, outDir: string, template: string, format: string, skip?: boolean): Video {

while (match) {
let value = video[match[1] as keyof Video] as string;
title = title.replace(match[0], value);
match = elementRegEx.exec(template);
}
let title: string = template;
let finalTitle: string;
const elementRegEx = RegExp(/{(.*?)}/g);
let match = elementRegEx.exec(template);

let i = 0;
finalTitle = title;
while (match) {
let value = video[match[1] as keyof Video] as string;
Copy link
Owner

@snobu snobu Aug 15, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(____/)
( ͡ ͡° ͜ ʖ ͡ ͡°)
\╭☞ \╭☞. Love the keyof T here.

title = title.replace(match[0], value);
match = elementRegEx.exec(template);
}

while (!skip && fs.existsSync(path.join(outDirs[index], finalTitle + '.' + format))) {
finalTitle = `${title}.${++i}`;
}
let i = 0;
finalTitle = title;

while (!skip && fs.existsSync(path.join(outDir, finalTitle + '.' + format))) {
finalTitle = `${title}.${++i}`;
}

video.outPath = path.join(outDirs[index], finalTitle + '.' + format);
});
video.outPath = path.join(outDir, finalTitle + '.' + format);

return videos;
return video;
}
25 changes: 12 additions & 13 deletions src/destreamer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,38 +119,37 @@ async function DoInteractiveLogin(url: string, username?: string): Promise<Sessi
}


async function downloadVideo(videoGUIDs: Array<string>, outputDirectories: Array<string>, session: Session): Promise<void> {
async function downloadVideo(videoGuidArray: Array<string>, outputDirectoryArray: Array<string>, session: Session): Promise<void> {

logger.info('Fetching videos info... \n');
const videos: Array<Video> = createUniquePath (
await getVideoInfo(videoGUIDs, session, argv.closedCaptions),
outputDirectories, argv.outputTemplate, argv.format, argv.skip
for (const [index, videoGuid] of videoGuidArray.entries()) {
logger.info(`Fetching video's #${index} info... \n`);

const video: Video = createUniquePath (
await getVideoInfo(videoGuid, session, argv.closedCaptions),
outputDirectoryArray[index], argv.outputTemplate, argv.format, argv.skip
);

if (argv.simulate) {
videos.forEach((video: Video) => {
if (argv.simulate) {
logger.info(
'\nTitle: '.green + video.title +
'\nOutPath: '.green + video.outPath +
'\nPublished Date: '.green + video.publishDate +
'\nPlayback URL: '.green + video.playbackUrl +
((video.captionsUrl) ? ('\nCC URL: '.green + video.captionsUrl) : '')
);
});

return;
}

for (const [index, video] of videos.entries()) {
continue;
}

if (argv.skip && fs.existsSync(video.outPath)) {
logger.info(`File already exists, skipping: ${video.outPath} \n`);

continue;
}

if (argv.keepLoginCookies && index !== 0) {
logger.info('Trying to refresh token...');
session = await refreshSession('https://web.microsoftstream.com/video/' + videoGUIDs[index]);
session = await refreshSession('https://web.microsoftstream.com/video/' + videoGuidArray[index]);
}

const pbar: cliProgress.SingleBar = new cliProgress.SingleBar({
Expand Down