Merge pull request #198 from vitalygashkov/next

Fixed Okko captcha/antibot triggering; support `x-subrip` mime type for subtitles

package.json

@@ -1,6 +1,6 @@
 {
   "name": "streamyx",
-  "version": "4.0.0-beta.58",
+  "version": "4.0.0-beta.59",
   "author": "Vitaly Gashkov <[email protected]>",
   "description": "Powerful media downloader",
   "main": "dist/src/app/main.js",

src/core/lib/http.ts

@@ -22,17 +22,53 @@ const HTTP_METHOD = {
 
 const USER_AGENTS = {
   chromeWindows:
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
   chromeMacOS:
-    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
   chromeLinux:
-    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36',
   smartTv:
     'Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) 76.0.3809.146/6.0 TV Safari/537.36',
   tizen:
     'Mozilla/5.0 (Linux; U; Tizen 2.0; en-us) AppleWebKit/537.1 (KHTML, like Gecko) Mobile TizenBrowser/2.0',
 };
 
+const COMMON_HEADERS = {
+  Accept:
+    'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+  'Accept-Encoding': 'gzip, deflate, br, zstd',
+  'Accept-Language': 'ru-RU,ru;q=0.9,en-NL;q=0.8,en-US;q=0.7,en;q=0.6,vi;q=0.5',
+  'Sec-Ch-Ua-Mobile': '?0',
+  'Upgrade-Insecure-Requests': '1',
+  'Sec-Fetch-Site': 'none',
+  'Sec-Fetch-Mode': 'navigate',
+  'Sec-Fetch-User': '?1',
+  'Sec-Fetch-Dest': 'document',
+};
+
+const DEFAULT_HEADERS_MAP = {
+  darwin: {
+    ...COMMON_HEADERS,
+    'User-Agent': USER_AGENTS.chromeMacOS,
+    'Sec-Ch-Ua': '"Google Chrome";v="127", "Chromium";v="127", "Not.A/Brand";v="24"',
+    'Sec-Ch-Ua-Platform': '"macOS"',
+  },
+  linux: {
+    ...COMMON_HEADERS,
+    'User-Agent': USER_AGENTS.chromeLinux,
+    'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="127", "Google Chrome";v="127"',
+    'Sec-Ch-Ua-Platform': '"Linux"',
+  },
+  win32: {
+    ...COMMON_HEADERS,
+    'User-Agent': USER_AGENTS.chromeWindows,
+    'Sec-Ch-Ua': '"Not/A)Brand";v="8", "Chromium";v="127", "Google Chrome";v="127"',
+    'Sec-Ch-Ua-Platform': '"Windows"',
+  },
+};
+
+const DEFAULT_HEADERS = DEFAULT_HEADERS_MAP[process.platform as 'darwin' | 'linux' | 'win32'];
+
 const parseUrlFromResource = (resource: string | URL | Request) =>
   resource instanceof Request
     ? new URL(resource.url)
@@ -72,7 +108,7 @@ class Http implements IHttp {
   #proxy?: string | null;
 
   constructor({ proxy }: { proxy?: string | null } = {}) {
-    this.headers = { 'User-Agent': USER_AGENTS.tizen };
+    this.headers = DEFAULT_HEADERS;
     this.cookies = [];
     this.#sessions = new Map();
     this.#retryThreshold = 3;
@@ -133,7 +169,7 @@ class Http implements IHttp {
         followRedirect: !redirect || redirect === 'follow',
         proxyUrl: this.#proxy || undefined,
         ...options,
-        headers: { ...this.headers, ...options.headers },
+        headers: { ...this.headers, ...options.headers, ...DEFAULT_HEADERS },
         useHeaderGenerator: true,
         headerGeneratorOptions: { browsers: ['chrome'] },
         http2: true,
@@ -156,7 +192,7 @@ class Http implements IHttp {
       for (const cookie of headers['set-cookie'] || []) newHeaders.append('set-cookie', cookie);
       this.appendCookies(response.headers['set-cookie'] || '');
       delete response.headers['set-cookie'];
-      return new Response(response.body, {
+      return new Response(response.rawBody, {
         headers: newHeaders,
         status: status,
       });
@@ -201,14 +237,17 @@ class Http implements IHttp {
       }
       page
         .evaluate(
-          (resource, init) => {
+          (resource, init: globalThis.RequestInit) => {
             const fetchData = async () => {
-              const response = await globalThis.fetch(
-                resource as globalThis.RequestInfo,
-                init as globalThis.RequestInit
-              );
+              const initBody = init.body as string | { type: 'Buffer'; data: number[] };
+              const body =
+                typeof initBody === 'object' && initBody.type === 'Buffer'
+                  ? Uint8Array.from(initBody.data)
+                  : init.body;
+              init.body = body;
+              const response = await globalThis.fetch(resource as globalThis.RequestInfo, init);
               return {
-                body: await response.text(),
+                body: new Uint8Array(await response.arrayBuffer()),
                 init: {
                   headers: response.headers as unknown as Headers,
                   status: response.status,
@@ -221,7 +260,11 @@ class Http implements IHttp {
           resource,
           init
         )
-        .then(({ body, init }) => (isWaitingForRedirect ? {} : resolve(new Response(body, init))))
+        .then(({ body, init }) => {
+          return isWaitingForRedirect
+            ? {}
+            : resolve(new Response(Buffer.from(Object.values(body)), init));
+        })
         .catch((e) => {
           logger.debug(`Error while evaluate browser fetch: ${e?.message}`);
           return { body: null, init: undefined };

src/core/lib/service.ts

@@ -2,7 +2,7 @@ import { RunArgs } from './args';
 import { IHttp } from './http';
 import { IPrompt } from './prompt';
 import { createStore } from './store';
-import { Http } from './http';
+import { Http, http } from './http';
 import { default as fs } from './fs';
 import { logger as log } from './logger';
 import { prompt } from './prompt';
@@ -81,7 +81,7 @@ export type Plugin<T = unknown> = (streamyx: StreamyxCore) => PluginInstance<T>;
 
 export const create = (name: string): StreamyxCore => ({
   log,
-  http: new Http(),
+  http,
   prompt,
   fs,
   store: createStore(name),
@@ -120,6 +120,7 @@ export const registerService = <T extends RegisterService<T>>(
     core: StreamyxCore;
   };
   const name = instance.name;
+  core.http = new Http();
   core.store = createStore(name);
   core.log.debug(`Service registered: ${name}`);
   instance.core = core;