From a63c9a61747b1662008a24908cd27552c6fea803 Mon Sep 17 00:00:00 2001 From: Artemy Date: Wed, 6 Dec 2023 11:44:38 +0300 Subject: [PATCH 1/5] ci: format --- .eslintrc.json | 30 ++++---- .github/dependabot.yml | 8 +-- .prettierrc | 5 ++ docker-compose.yml | 6 +- package-lock.json | 16 +++++ package.json | 4 +- src/app.ts | 52 +++++++------- src/config/config.service.ts | 6 +- src/config/main.ts | 2 +- src/errors/api.ts | 18 ++--- src/errors/handler.ts | 25 +++---- src/errors/main.ts | 31 +++----- src/handlers/google.ts | 30 ++++---- src/handlers/handler-input.ts | 7 +- src/handlers/handler.interface.ts | 10 +-- src/handlers/main.ts | 41 +++++------ src/handlers/readability.ts | 12 ++-- src/handlers/stackoverflow/main.ts | 42 +++++------ src/handlers/stackoverflow/post-parser.ts | 6 +- src/handlers/stackoverflow/questions-posts.ts | 18 ++--- src/publicConfig.ts | 4 +- src/routes/api/parse.ts | 14 ++-- src/routes/api/raw-html.ts | 14 ++-- src/routes/browser/get.ts | 18 ++--- src/routes/browser/index.ts | 12 ++-- src/routes/browser/proxy.ts | 18 ++--- src/types/axios.ts | 8 +-- src/types/handlers.ts | 4 +- src/types/requests/api.ts | 40 +++++------ src/types/requests/browser.ts | 44 ++++++------ src/utils/generate.ts | 13 ++-- src/utils/http.ts | 10 +-- src/utils/islocal.ts | 70 +++++++++---------- src/utils/replace-href.ts | 70 +++++++------------ static/form.css | 2 +- 35 files changed, 342 insertions(+), 368 deletions(-) create mode 100644 .prettierrc diff --git a/.eslintrc.json b/.eslintrc.json index 6751428..380b417 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,20 +1,14 @@ { - "env": { - "browser": true, - "es2021": true - }, - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended" - ], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "plugins": [ - "@typescript-eslint" - ], - "rules": { - } + "env": { + "browser": true, + "es2021": true + }, + "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"], + "rules": {} } diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5d5040e..3cba37f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,8 +5,8 @@ version: 2 updates: - - package-ecosystem: "npm" # See documentation for possible values - directory: "/" # Location of package manifests - target-branch: "dependabot" + - package-ecosystem: 'npm' # See documentation for possible values + directory: '/' # Location of package manifests + target-branch: 'dependabot' schedule: - interval: "weekly" + interval: 'weekly' diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..65261d6 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,5 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "singleQuote": true +} diff --git a/docker-compose.yml b/docker-compose.yml index 533ca5f..273f5d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ -version: "3" +version: '3' services: txtdot: image: ghcr.io/txtdot/txtdot:latest ports: - - "8080:8080" + - '8080:8080' restart: unless-stopped volumes: - - ".env:/app/.env" + - '.env:/app/.env' diff --git a/package-lock.json b/package-lock.json index 8fa04d1..f88aa04 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "clean-css-cli": "^5.6.2", "copyfiles": "^2.4.1", "eslint": "^8.47.0", + "prettier": "^3.1.0", "tsc-watch": "^6.0.4", "typescript": "^5.1.6" } @@ -2616,6 +2617,21 @@ "node": ">= 0.8.0" } }, + "node_modules/prettier": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.1.0.tgz", + "integrity": "sha512-TQLvXjq5IAibjh8EpBIkNKxO749UEWABoiIZehEPiY4GNpVdhaFKqSTu+QrlU6D2dPAfubRmtJTi4K4YkQ5eXw==", + "dev": true, + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/process": { "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", diff --git a/package.json b/package.json index 5a1d16a..9e32b08 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "clean-css-cli": "^5.6.2", "copyfiles": "^2.4.1", "eslint": "^8.47.0", + "prettier": "^3.1.0", "tsc-watch": "^6.0.4", "typescript": "^5.1.6" }, @@ -41,7 +42,8 @@ "start": "cd ./dist && node ./src/app.js", "start:docker": "node ./src/app.js", "clean-css": "cleancss --batch static/*.css -o dist/static --batch-suffix \"\"", - "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"" + "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"", + "format": "prettier --write ." }, "keywords": [], "authors": [ diff --git a/src/app.ts b/src/app.ts index 70a935a..e4037cf 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,21 +1,21 @@ -import path from "path"; +import path from 'path'; -import Fastify from "fastify"; -import fastifyStatic from "@fastify/static"; -import fastifyView from "@fastify/view"; -import fastifySwagger from "@fastify/swagger"; -import fastifySwaggerUi from "@fastify/swagger-ui"; -import ejs from "ejs"; +import Fastify from 'fastify'; +import fastifyStatic from '@fastify/static'; +import fastifyView from '@fastify/view'; +import fastifySwagger from '@fastify/swagger'; +import fastifySwaggerUi from '@fastify/swagger-ui'; +import ejs from 'ejs'; -import indexRoute from "./routes/browser/index"; -import getRoute from "./routes/browser/get"; -import proxyRoute from "./routes/browser/proxy"; -import parseRoute from "./routes/api/parse"; -import rawHtml from "./routes/api/raw-html"; +import indexRoute from './routes/browser/index'; +import getRoute from './routes/browser/get'; +import proxyRoute from './routes/browser/proxy'; +import parseRoute from './routes/api/parse'; +import rawHtml from './routes/api/raw-html'; -import publicConfig from "./publicConfig"; -import errorHandler from "./errors/handler"; -import getConfig from "./config/main"; +import publicConfig from './publicConfig'; +import errorHandler from './errors/handler'; +import getConfig from './config/main'; class App { async init() { @@ -27,8 +27,8 @@ class App { }); fastify.register(fastifyStatic, { - root: path.join(process.cwd(), "static"), - prefix: "/static/", + root: path.join(process.cwd(), 'static'), + prefix: '/static/', }); fastify.register(fastifyView, { @@ -41,32 +41,28 @@ class App { await fastify.register(fastifySwagger, { swagger: { info: { - title: "TXTDot API", + title: 'TXTDot API', description: publicConfig.description, version: publicConfig.version, }, - } + }, }); - await fastify.register(fastifySwaggerUi, { routePrefix: "/doc" }); + await fastify.register(fastifySwaggerUi, { routePrefix: '/doc' }); } fastify.register(indexRoute); fastify.register(getRoute); - if (config.proxy_res) - fastify.register(proxyRoute); + if (config.proxy_res) fastify.register(proxyRoute); fastify.register(parseRoute); fastify.register(rawHtml); fastify.setErrorHandler(errorHandler); - fastify.listen( - { host: config.host, port: config.port }, - (err) => { - err && console.log(err); - } - ); + fastify.listen({ host: config.host, port: config.port }, (err) => { + err && console.log(err); + }); } } diff --git a/src/config/config.service.ts b/src/config/config.service.ts index 3ff93e3..c492768 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -1,4 +1,4 @@ -import { config } from "dotenv"; +import { config } from 'dotenv'; export class ConfigService { public readonly host: string; @@ -10,7 +10,7 @@ export class ConfigService { constructor() { config(); - this.host = process.env.HOST || "0.0.0.0"; + this.host = process.env.HOST || '0.0.0.0'; this.port = Number(process.env.PORT) || 8080; this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false); @@ -21,6 +21,6 @@ export class ConfigService { parseBool(value: string | undefined, def: boolean): boolean { if (!value) return def; - return value === "true" || value === "1"; + return value === 'true' || value === '1'; } } diff --git a/src/config/main.ts b/src/config/main.ts index 37dcad3..ea4e219 100644 --- a/src/config/main.ts +++ b/src/config/main.ts @@ -1,4 +1,4 @@ -import { ConfigService } from "./config.service"; +import { ConfigService } from './config.service'; let configSvc: ConfigService | undefined; diff --git a/src/errors/api.ts b/src/errors/api.ts index 4d0728c..8515501 100644 --- a/src/errors/api.ts +++ b/src/errors/api.ts @@ -5,28 +5,28 @@ export interface IApiError { } export const errorSchema = { - type: "object", + type: 'object', properties: { code: { - type: "number", - description: "HTTP error code", + type: 'number', + description: 'HTTP error code', }, name: { - type: "string", - description: "Exception class name", + type: 'string', + description: 'Exception class name', }, message: { - type: "string", - description: "Exception message", + type: 'string', + description: 'Exception message', }, }, }; export const errorResponseSchema = { - type: "object", + type: 'object', properties: { data: { - type: "object", + type: 'object', nullable: true, }, error: errorSchema, diff --git a/src/errors/handler.ts b/src/errors/handler.ts index 5d28722..f5cf4c3 100644 --- a/src/errors/handler.ts +++ b/src/errors/handler.ts @@ -1,16 +1,16 @@ -import { FastifyReply, FastifyRequest } from "fastify"; -import { NotHtmlMimetypeError, TxtDotError } from "./main"; -import { getFastifyError } from "./validation"; +import { FastifyReply, FastifyRequest } from 'fastify'; +import { NotHtmlMimetypeError, TxtDotError } from './main'; +import { getFastifyError } from './validation'; -import { IGetSchema } from "../types/requests/browser"; -import getConfig from "../config/main"; +import { IGetSchema } from '../types/requests/browser'; +import getConfig from '../config/main'; export default function errorHandler( error: Error, req: FastifyRequest, reply: FastifyReply ) { - if (req.originalUrl.startsWith("/api/")) { + if (req.originalUrl.startsWith('/api/')) { return apiErrorHandler(error, reply); } @@ -43,26 +43,23 @@ function apiErrorHandler(error: Error, reply: FastifyReply) { function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) { if (getFastifyError(error)?.statusCode === 400) { - return reply.code(400).view("/templates/error.ejs", { + return reply.code(400).view('/templates/error.ejs', { url, code: 400, description: `Invalid parameter specified: ${error.message}`, - }) + }); } if (error instanceof TxtDotError) { - return reply.code(error.code).view("/templates/error.ejs", { + return reply.code(error.code).view('/templates/error.ejs', { url, code: error.code, description: error.description, - proxyBtn: ( - error instanceof NotHtmlMimetypeError && - getConfig().proxy_res - ), + proxyBtn: error instanceof NotHtmlMimetypeError && getConfig().proxy_res, }); } - return reply.code(500).view("/templates/error.ejs", { + return reply.code(500).view('/templates/error.ejs', { url, code: 500, description: `${error.name}: ${error.message}`, diff --git a/src/errors/main.ts b/src/errors/main.ts index 81f983a..2513a9d 100644 --- a/src/errors/main.ts +++ b/src/errors/main.ts @@ -1,15 +1,11 @@ -import getConfig from "../config/main"; +import getConfig from '../config/main'; export abstract class TxtDotError extends Error { code: number; name: string; description: string; - constructor( - code: number, - name: string, - description: string, - ) { + constructor(code: number, name: string, description: string) { super(description); this.code = code; this.name = name; @@ -19,21 +15,13 @@ export abstract class TxtDotError extends Error { export class EngineParseError extends TxtDotError { constructor(message: string) { - super( - 422, - "EngineParseError", - `Parse error: ${message}`, - ); + super(422, 'EngineParseError', `Parse error: ${message}`); } } export class LocalResourceError extends TxtDotError { constructor() { - super( - 403, - "LocalResourceError", - "Proxying local resources is forbidden.", - ); + super(403, 'LocalResourceError', 'Proxying local resources is forbidden.'); } } @@ -41,12 +29,11 @@ export class NotHtmlMimetypeError extends TxtDotError { constructor() { super( 421, - "NotHtmlMimetypeError", - "Received non-HTML content, " + ( - getConfig().proxy_res ? - "use proxy instead of parser." : - "proxying is disabled by the instance admin." - ), + 'NotHtmlMimetypeError', + 'Received non-HTML content, ' + + (getConfig().proxy_res + ? 'use proxy instead of parser.' + : 'proxying is disabled by the instance admin.') ); } } diff --git a/src/handlers/google.ts b/src/handlers/google.ts index 6783d8a..392edd5 100644 --- a/src/handlers/google.ts +++ b/src/handlers/google.ts @@ -1,6 +1,6 @@ -import { HandlerInput } from "./handler-input"; -import { IHandlerOutput } from "./handler.interface"; -import { EngineParseError } from "../errors/main"; +import { HandlerInput } from './handler-input'; +import { IHandlerOutput } from './handler.interface'; +import { EngineParseError } from '../errors/main'; export default async function google( input: HandlerInput @@ -8,18 +8,18 @@ export default async function google( const window = input.parseDom().window; const googleAnchors = [ - ...window.document.querySelectorAll("a[jsname=UWckNb]"), + ...window.document.querySelectorAll('a[jsname=UWckNb]'), ] as HTMLAnchorElement[]; if (!googleAnchors) { throw new EngineParseError( - "Failed to find anchors in search result [google]" + 'Failed to find anchors in search result [google]' ); } const results = googleAnchors .map((a: HTMLAnchorElement): GoogleProps => { - const parsedHref = new URL(new URL(a.href).searchParams.get("url")!); + const parsedHref = new URL(new URL(a.href).searchParams.get('url')!); return { href: a.href!, siteName: parsedHref.hostname, @@ -43,7 +43,7 @@ export default async function google( }); const search = window.document.getElementById( - "APjFqb" + 'APjFqb' ) as HTMLTextAreaElement; const searchForm = ` @@ -54,18 +54,18 @@ export default async function google( `; return { - content: `${searchForm}${content.join("")}`, - textContent: textContent.join("\n"), + content: `${searchForm}${content.join('')}`, + textContent: textContent.join('\n'), }; } export const GoogleDomains = [ - "google.*", - "google.co.*", - "google.com.*", - "www.google.*", - "www.google.co.*", - "www.google.com.*", + 'google.*', + 'google.co.*', + 'google.com.*', + 'www.google.*', + 'www.google.co.*', + 'www.google.com.*', ]; interface GoogleProps { diff --git a/src/handlers/handler-input.ts b/src/handlers/handler-input.ts index ad510b9..b930bbd 100644 --- a/src/handlers/handler-input.ts +++ b/src/handlers/handler-input.ts @@ -1,14 +1,11 @@ -import { JSDOM } from "jsdom"; +import { JSDOM } from 'jsdom'; export class HandlerInput { private data: string; private url: string; private dom?: JSDOM; - constructor( - data: string, - url: string, - ) { + constructor(data: string, url: string) { this.data = data; this.url = url; } diff --git a/src/handlers/handler.interface.ts b/src/handlers/handler.interface.ts index 1bcd7c2..2b5bd8f 100644 --- a/src/handlers/handler.interface.ts +++ b/src/handlers/handler.interface.ts @@ -6,19 +6,19 @@ export interface IHandlerOutput { } export const handlerSchema = { - type: "object", + type: 'object', properties: { content: { - type: "string", + type: 'string', }, textContent: { - type: "string", + type: 'string', }, title: { - type: "string", + type: 'string', }, lang: { - type: "string", + type: 'string', }, }, }; diff --git a/src/handlers/main.ts b/src/handlers/main.ts index 01c79f7..bf9bf93 100644 --- a/src/handlers/main.ts +++ b/src/handlers/main.ts @@ -1,30 +1,30 @@ -import { IHandlerOutput } from "./handler.interface"; -import { Engines, EngineFunction, EnginesMatch } from "../types/handlers"; -import axios from "../types/axios"; +import { IHandlerOutput } from './handler.interface'; +import { Engines, EngineFunction, EnginesMatch } from '../types/handlers'; +import axios from '../types/axios'; -import micromatch from "micromatch"; +import micromatch from 'micromatch'; -import { JSDOM } from "jsdom"; -import DOMPurify from "dompurify"; +import { JSDOM } from 'jsdom'; +import DOMPurify from 'dompurify'; -import { Readable } from "stream"; +import { Readable } from 'stream'; -import readability from "./readability"; -import google, { GoogleDomains } from "./google"; -import stackoverflow, { StackOverflowDomains } from "./stackoverflow/main"; +import readability from './readability'; +import google, { GoogleDomains } from './google'; +import stackoverflow, { StackOverflowDomains } from './stackoverflow/main'; -import isLocalResource from "../utils/islocal"; +import isLocalResource from '../utils/islocal'; -import { LocalResourceError, NotHtmlMimetypeError } from "../errors/main"; -import { HandlerInput } from "./handler-input"; -import { decodeStream, parseEncodingName } from "../utils/http"; -import replaceHref from "../utils/replace-href"; +import { LocalResourceError, NotHtmlMimetypeError } from '../errors/main'; +import { HandlerInput } from './handler-input'; +import { decodeStream, parseEncodingName } from '../utils/http'; +import replaceHref from '../utils/replace-href'; export default async function handlePage( url: string, // remote URL requestUrl: URL, // proxy URL engine?: string, - redirectPath: string = "get", + redirectPath: string = 'get' ): Promise { const urlObj = new URL(url); @@ -34,18 +34,15 @@ export default async function handlePage( const response = await axios.get(url); const data: Readable = response.data; - const mime: string | undefined = response.headers["content-type"]?.toString(); + const mime: string | undefined = response.headers['content-type']?.toString(); - if (mime && mime.indexOf("text/html") === -1) { + if (mime && mime.indexOf('text/html') === -1) { throw new NotHtmlMimetypeError(); } const handler = getFallbackEngine(urlObj.hostname, engine); const output = await handler( - new HandlerInput( - await decodeStream(data, parseEncodingName(mime)), - url, - ) + new HandlerInput(await decodeStream(data, parseEncodingName(mime)), url) ); // post-process diff --git a/src/handlers/readability.ts b/src/handlers/readability.ts index a22cc66..f8e2fcb 100644 --- a/src/handlers/readability.ts +++ b/src/handlers/readability.ts @@ -1,16 +1,16 @@ -import { Readability } from "@mozilla/readability"; -import { HandlerInput } from "./handler-input"; -import { IHandlerOutput } from "./handler.interface"; -import { EngineParseError } from "../errors/main"; +import { Readability } from '@mozilla/readability'; +import { HandlerInput } from './handler-input'; +import { IHandlerOutput } from './handler.interface'; +import { EngineParseError } from '../errors/main'; export default async function readability( - input: HandlerInput, + input: HandlerInput ): Promise { const reader = new Readability(input.parseDom().window.document); const parsed = reader.parse(); if (!parsed) { - throw new EngineParseError("Failed to parse [readability]"); + throw new EngineParseError('Failed to parse [readability]'); } return { diff --git a/src/handlers/stackoverflow/main.ts b/src/handlers/stackoverflow/main.ts index f1b9c6c..f69344d 100644 --- a/src/handlers/stackoverflow/main.ts +++ b/src/handlers/stackoverflow/main.ts @@ -1,30 +1,30 @@ -import { HandlerInput } from "../handler-input"; -import { IHandlerOutput } from "../handler.interface"; -import { EngineParseError } from "../../errors/main"; -import qPostsHandler from "./questions-posts"; +import { HandlerInput } from '../handler-input'; +import { IHandlerOutput } from '../handler.interface'; +import { EngineParseError } from '../../errors/main'; +import qPostsHandler from './questions-posts'; export default async function stackoverflow( - input: HandlerInput, + input: HandlerInput ): Promise { const window = input.parseDom().window; const url = new URL(window.location.href); - const path = url.pathname.split("/").filter((p) => p !== ""); + const path = url.pathname.split('/').filter((p) => p !== ''); let result: IHandlerOutput = { - content: "", - textContent: "", - title: "", - lang: "", + content: '', + textContent: '', + title: '', + lang: '', }; - if (path[0] === "questions") { + if (path[0] === 'questions') { if (path.length === 3) { result = await qPostsHandler(window); } else if (path.length === 1) { - result.content = "questions"; + result.content = 'questions'; } else { - throw new EngineParseError("Invalid URL [stackoverflow]"); + throw new EngineParseError('Invalid URL [stackoverflow]'); } } @@ -32,12 +32,12 @@ export default async function stackoverflow( } export const StackOverflowDomains = [ - "stackoverflow.com", - "*.stackoverflow.com", - "*.stackexchange.com", - "askubuntu.com", - "stackapps.com", - "mathoverflow.net", - "superuser.com", - "serverfault.com", + 'stackoverflow.com', + '*.stackoverflow.com', + '*.stackexchange.com', + 'askubuntu.com', + 'stackapps.com', + 'mathoverflow.net', + 'superuser.com', + 'serverfault.com', ]; diff --git a/src/handlers/stackoverflow/post-parser.ts b/src/handlers/stackoverflow/post-parser.ts index e5e6999..7b06a3e 100644 --- a/src/handlers/stackoverflow/post-parser.ts +++ b/src/handlers/stackoverflow/post-parser.ts @@ -1,9 +1,9 @@ export default function postParser(el: Element | null): string { if (!el) { - return ""; + return ''; } - const body = el.querySelector(".js-post-body")?.innerHTML || ""; - const voteCount = el.querySelector(".js-vote-count")?.textContent || ""; + const body = el.querySelector('.js-post-body')?.innerHTML || ''; + const voteCount = el.querySelector('.js-vote-count')?.textContent || ''; return `

${voteCount} votes

${body}`; } diff --git a/src/handlers/stackoverflow/questions-posts.ts b/src/handlers/stackoverflow/questions-posts.ts index 3d011f8..a429cf5 100644 --- a/src/handlers/stackoverflow/questions-posts.ts +++ b/src/handlers/stackoverflow/questions-posts.ts @@ -1,26 +1,26 @@ -import { DOMWindow } from "jsdom"; -import { IHandlerOutput } from "../handler.interface"; -import postParser from "./post-parser"; +import { DOMWindow } from 'jsdom'; +import { IHandlerOutput } from '../handler.interface'; +import postParser from './post-parser'; export default async function qPostsHandler( window: DOMWindow ): Promise { - const questionEl = window.document.getElementById("question"); + const questionEl = window.document.getElementById('question'); const question = postParser(questionEl); const title = - window.document.querySelector(".question-hyperlink")?.innerHTML || ""; + window.document.querySelector('.question-hyperlink')?.innerHTML || ''; - const allAnswers = [...window.document.querySelectorAll(".answer")]; + const allAnswers = [...window.document.querySelectorAll('.answer')]; const answers = allAnswers.map((a) => postParser(a)); return { content: `${question}
${answers.length} answers
${answers.join( - "
" + '
' )}`, - textContent: "question", + textContent: 'question', title, - lang: "en", + lang: 'en', }; } diff --git a/src/publicConfig.ts b/src/publicConfig.ts index 355f63c..04daf30 100644 --- a/src/publicConfig.ts +++ b/src/publicConfig.ts @@ -1,5 +1,5 @@ export default { - version: "1.4.0", + version: '1.4.0', description: - "txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts", + 'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts', }; diff --git a/src/routes/api/parse.ts b/src/routes/api/parse.ts index 5d51014..bff8af4 100644 --- a/src/routes/api/parse.ts +++ b/src/routes/api/parse.ts @@ -1,13 +1,17 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import { EngineRequest, IParseSchema, parseSchema } from "../../types/requests/api"; +import { + EngineRequest, + IParseSchema, + parseSchema, +} from '../../types/requests/api'; -import handlePage from "../../handlers/main"; -import { generateRequestUrl } from "../../utils/generate"; +import handlePage from '../../handlers/main'; +import { generateRequestUrl } from '../../utils/generate'; export default async function parseRoute(fastify: FastifyInstance) { fastify.get( - "/api/parse", + '/api/parse', { schema: parseSchema }, async (request: EngineRequest) => { return { diff --git a/src/routes/api/raw-html.ts b/src/routes/api/raw-html.ts index b9d9de5..4deb197 100644 --- a/src/routes/api/raw-html.ts +++ b/src/routes/api/raw-html.ts @@ -1,16 +1,16 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import { IParseSchema, rawHtmlSchema } from "../../types/requests/api"; +import { IParseSchema, rawHtmlSchema } from '../../types/requests/api'; -import handlePage from "../../handlers/main"; -import { generateRequestUrl } from "../../utils/generate"; +import handlePage from '../../handlers/main'; +import { generateRequestUrl } from '../../utils/generate'; export default async function rawHtml(fastify: FastifyInstance) { fastify.get( - "/api/raw-html", + '/api/raw-html', { schema: rawHtmlSchema }, async (request, reply) => { - reply.type("text/html; charset=utf-8"); + reply.type('text/html; charset=utf-8'); return ( await handlePage( request.query.url, @@ -20,7 +20,7 @@ export default async function rawHtml(fastify: FastifyInstance) { request.originalUrl ), request.query.engine, - "api/raw-html" + 'api/raw-html' ) ).content; } diff --git a/src/routes/browser/get.ts b/src/routes/browser/get.ts index ef94428..4ee62f2 100644 --- a/src/routes/browser/get.ts +++ b/src/routes/browser/get.ts @@ -1,12 +1,12 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import { GetSchema, IGetSchema } from "../../types/requests/browser"; -import handlePage from "../../handlers/main"; -import { generateRequestUrl } from "../../utils/generate"; +import { GetSchema, IGetSchema } from '../../types/requests/browser'; +import handlePage from '../../handlers/main'; +import { generateRequestUrl } from '../../utils/generate'; export default async function getRoute(fastify: FastifyInstance) { fastify.get( - "/get", + '/get', { schema: GetSchema }, async (request, reply) => { const remoteUrl = request.query.url; @@ -22,12 +22,12 @@ export default async function getRoute(fastify: FastifyInstance) { engine ); - if (request.query.format === "text") { - reply.type("text/plain; charset=utf-8"); + if (request.query.format === 'text') { + reply.type('text/plain; charset=utf-8'); return parsed.textContent; } else { - reply.type("text/html; charset=utf-8"); - return reply.view("/templates/get.ejs", { parsed, remoteUrl }); + reply.type('text/html; charset=utf-8'); + return reply.view('/templates/get.ejs', { parsed, remoteUrl }); } } ); diff --git a/src/routes/browser/index.ts b/src/routes/browser/index.ts index 1d69770..83ee81b 100644 --- a/src/routes/browser/index.ts +++ b/src/routes/browser/index.ts @@ -1,11 +1,11 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import publicConfig from "../../publicConfig"; -import { engineList } from "../../handlers/main"; -import { indexSchema } from "../../types/requests/browser"; +import publicConfig from '../../publicConfig'; +import { engineList } from '../../handlers/main'; +import { indexSchema } from '../../types/requests/browser'; export default async function indexRoute(fastify: FastifyInstance) { - fastify.get("/", { schema: indexSchema }, async (_, reply) => { - return reply.view("/templates/index.ejs", { publicConfig, engineList }); + fastify.get('/', { schema: indexSchema }, async (_, reply) => { + return reply.view('/templates/index.ejs', { publicConfig, engineList }); }); } diff --git a/src/routes/browser/proxy.ts b/src/routes/browser/proxy.ts index 4947abf..20aa0c4 100644 --- a/src/routes/browser/proxy.ts +++ b/src/routes/browser/proxy.ts @@ -1,17 +1,19 @@ -import { FastifyInstance } from "fastify"; -import { IProxySchema, ProxySchema } from "../../types/requests/browser"; -import axios from "../../types/axios"; +import { FastifyInstance } from 'fastify'; +import { IProxySchema, ProxySchema } from '../../types/requests/browser'; +import axios from '../../types/axios'; export default async function proxyRoute(fastify: FastifyInstance) { fastify.get( - "/proxy", + '/proxy', { schema: ProxySchema }, async (request, reply) => { const response = await axios.get(request.query.url); - const mime: string | undefined = response.headers["content-type"]?.toString(); - const clen: string | undefined = response.headers["content-length"]?.toString(); - mime && reply.header("Content-Type", mime); - clen && reply.header("Content-Length", Number(clen)); + const mime: string | undefined = + response.headers['content-type']?.toString(); + const clen: string | undefined = + response.headers['content-length']?.toString(); + mime && reply.header('Content-Type', mime); + clen && reply.header('Content-Length', Number(clen)); return reply.send(response.data); } ); diff --git a/src/types/axios.ts b/src/types/axios.ts index 947ccfa..6e4628b 100644 --- a/src/types/axios.ts +++ b/src/types/axios.ts @@ -1,9 +1,9 @@ -import axios from "axios"; +import axios from 'axios'; export default axios.create({ headers: { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0", + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0', }, - responseType: "stream", + responseType: 'stream', }); diff --git a/src/types/handlers.ts b/src/types/handlers.ts index 846eeda..58138c7 100644 --- a/src/types/handlers.ts +++ b/src/types/handlers.ts @@ -1,5 +1,5 @@ -import { HandlerInput } from "../handlers/handler-input"; -import { IHandlerOutput } from "../handlers/handler.interface"; +import { HandlerInput } from '../handlers/handler-input'; +import { IHandlerOutput } from '../handlers/handler.interface'; export interface Engines { [key: string]: EngineFunction; diff --git a/src/types/requests/api.ts b/src/types/requests/api.ts index 0472505..db8ee91 100644 --- a/src/types/requests/api.ts +++ b/src/types/requests/api.ts @@ -1,8 +1,8 @@ -import { FastifySchema, FastifyRequest } from "fastify"; -import { IApiError, errorResponseSchema } from "../../errors/api"; -import { handlerSchema } from "../../handlers/handler.interface"; -import { engineList } from "../../handlers/main"; -import { FromSchema } from "json-schema-to-ts"; +import { FastifySchema, FastifyRequest } from 'fastify'; +import { IApiError, errorResponseSchema } from '../../errors/api'; +import { handlerSchema } from '../../handlers/handler.interface'; +import { engineList } from '../../handlers/main'; +import { FromSchema } from 'json-schema-to-ts'; export interface IApiResponse { data?: T; @@ -10,38 +10,38 @@ export interface IApiResponse { } export const parseQuerySchema = { - type: "object", - required: ["url"], + type: 'object', + required: ['url'], properties: { url: { - type: "string", - description: "URL", + type: 'string', + description: 'URL', }, engine: { - type: "string", - enum: [...engineList, ""], + type: 'string', + enum: [...engineList, ''], }, }, } as const; export const parseSchema: FastifySchema = { - description: "Parse the page and get all data from the engine", + description: 'Parse the page and get all data from the engine', querystring: parseQuerySchema, response: { - "2xx": { - type: "object", + '2xx': { + type: 'object', properties: { data: handlerSchema, error: { - type: "object", + type: 'object', nullable: true, }, }, }, - "4xx": errorResponseSchema, - "5xx": errorResponseSchema, + '4xx': errorResponseSchema, + '5xx': errorResponseSchema, }, - produces: ["text/json"], + produces: ['text/json'], }; export interface IParseSchema { @@ -49,9 +49,9 @@ export interface IParseSchema { } export const rawHtmlSchema: FastifySchema = { - description: "Parse the page and get raw HTML from the engine", + description: 'Parse the page and get raw HTML from the engine', querystring: parseQuerySchema, - produces: ["text/html"], + produces: ['text/html'], }; export type EngineRequest = FastifyRequest<{ diff --git a/src/types/requests/browser.ts b/src/types/requests/browser.ts index 4507b11..4504b0c 100644 --- a/src/types/requests/browser.ts +++ b/src/types/requests/browser.ts @@ -1,6 +1,6 @@ -import { FastifySchema } from "fastify"; -import { engineList } from "../../handlers/main"; -import { FromSchema } from "json-schema-to-ts"; +import { FastifySchema } from 'fastify'; +import { engineList } from '../../handlers/main'; +import { FromSchema } from 'json-schema-to-ts'; export interface IGetSchema { Querystring: IGetQuerySchema; @@ -11,52 +11,52 @@ export interface IProxySchema { } export const getQuerySchema = { - type: "object", - required: ["url"], + type: 'object', + required: ['url'], properties: { url: { - type: "string", - description: "URL", + type: 'string', + description: 'URL', }, format: { - type: "string", - enum: ["text", "html", ""], - default: "html", + type: 'string', + enum: ['text', 'html', ''], + default: 'html', }, engine: { - type: "string", - enum: [...engineList, ""], + type: 'string', + enum: [...engineList, ''], }, }, } as const; export type IGetQuerySchema = FromSchema; export const proxyQuerySchema = { - type: "object", - required: ["url"], + type: 'object', + required: ['url'], properties: { url: { - type: "string", - description: "URL", + type: 'string', + description: 'URL', }, - } + }, } as const; export type IProxyQuerySchema = FromSchema; export const indexSchema = { hide: true, - produces: ["text/html"], + produces: ['text/html'], }; export const GetSchema: FastifySchema = { - description: "Get page", + description: 'Get page', hide: true, querystring: getQuerySchema, - produces: ["text/html", "text/plain"], + produces: ['text/html', 'text/plain'], }; export const ProxySchema: FastifySchema = { - description: "Proxy resource", + description: 'Proxy resource', hide: true, querystring: proxyQuerySchema, -} +}; diff --git a/src/utils/generate.ts b/src/utils/generate.ts index a275585..0d58678 100644 --- a/src/utils/generate.ts +++ b/src/utils/generate.ts @@ -10,23 +10,20 @@ export function generateParserUrl( requestUrl: URL, href: string, engine?: string, - redirect_url: string = "get" + redirect_url: string = 'get' ): string { const parsedHref = new URL(href); - const hash = parsedHref.hash; // save #hash - parsedHref.hash = ""; // remove + const hash = parsedHref.hash; // save #hash + parsedHref.hash = ''; // remove const urlParam = `?url=${encodeURIComponent(parsedHref.toString())}`; - const engineParam = engine ? `&engine=${engine}` : ""; + const engineParam = engine ? `&engine=${engine}` : ''; return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`; } -export function generateProxyUrl( - requestUrl: URL, - href: string, -): string { +export function generateProxyUrl(requestUrl: URL, href: string): string { const urlParam = `?url=${encodeURIComponent(href)}`; return `${requestUrl.origin}/proxy${urlParam}`; } diff --git a/src/utils/http.ts b/src/utils/http.ts index 86e436a..b5eecea 100644 --- a/src/utils/http.ts +++ b/src/utils/http.ts @@ -1,12 +1,12 @@ -import { Readable } from "stream"; -import iconv from "iconv-lite"; +import { Readable } from 'stream'; +import iconv from 'iconv-lite'; export async function decodeStream( data: Readable, - charset: string = "utf-8", + charset: string = 'utf-8' ): Promise { const strm = data.pipe(iconv.decodeStream(charset)) as IconvStream; - return await new Promise(resolve => { + return await new Promise((resolve) => { strm.collect((_err: Error, body: string) => { resolve(body); }); @@ -16,7 +16,7 @@ export async function decodeStream( export function parseEncodingName(ctype?: string): string { const match = ctype?.match(/charset=([A-Za-z0-9-]+)$/); if (!match) { - return "utf-8"; + return 'utf-8'; } return match[1]; } diff --git a/src/utils/islocal.ts b/src/utils/islocal.ts index c997c01..752cb7f 100644 --- a/src/utils/islocal.ts +++ b/src/utils/islocal.ts @@ -1,45 +1,43 @@ -import dns from "dns"; -import ipRangeCheck from "ip-range-check"; +import dns from 'dns'; +import ipRangeCheck from 'ip-range-check'; const subnets = [ - "0.0.0.0/8", - "127.0.0.0/8", - "10.0.0.0/8", - "100.64.0.0/10", - "169.254.0.0/16", - "172.16.0.0/12", - "192.0.0.0/24", - "192.0.2.0/24", - "192.88.99.0/24", - "192.168.0.0/16", - "198.18.0.0/15", - "198.51.100.0/24", - "203.0.113.0/24", - "224.0.0.0/4", - "233.252.0.0/24", - "240.0.0.0/4", - "255.255.255.255/32", - "::/128", - "::1/128", - "::ffff:0:0/96", - "::ffff:0:0:0/96", - "64:ff9b::/96", - "64:ff9b:1::/48", - "100::/64", - "2001:0000::/32", - "2001:20::/28", - "2001:db8::/32", - "2002::/16", - "fc00::/7", - "fe80::/64", - "ff00::/8", + '0.0.0.0/8', + '127.0.0.0/8', + '10.0.0.0/8', + '100.64.0.0/10', + '169.254.0.0/16', + '172.16.0.0/12', + '192.0.0.0/24', + '192.0.2.0/24', + '192.88.99.0/24', + '192.168.0.0/16', + '198.18.0.0/15', + '198.51.100.0/24', + '203.0.113.0/24', + '224.0.0.0/4', + '233.252.0.0/24', + '240.0.0.0/4', + '255.255.255.255/32', + '::/128', + '::1/128', + '::ffff:0:0/96', + '::ffff:0:0:0/96', + '64:ff9b::/96', + '64:ff9b:1::/48', + '100::/64', + '2001:0000::/32', + '2001:20::/28', + '2001:db8::/32', + '2002::/16', + 'fc00::/7', + 'fe80::/64', + 'ff00::/8', ]; export default async function isLocalResource(url: URL): Promise { // Resolve domain name - const addr = ( - await dns.promises.lookup(url.hostname) - ).address; + const addr = (await dns.promises.lookup(url.hostname)).address; // Check if IP is in local network return ipRangeCheck(addr, subnets); diff --git a/src/utils/replace-href.ts b/src/utils/replace-href.ts index b964870..d4ea6d2 100644 --- a/src/utils/replace-href.ts +++ b/src/utils/replace-href.ts @@ -1,72 +1,54 @@ -import { JSDOM } from "jsdom"; -import { generateParserUrl, generateProxyUrl } from "./generate"; -import getConfig from "../config/main"; +import { JSDOM } from 'jsdom'; +import { generateParserUrl, generateProxyUrl } from './generate'; +import getConfig from '../config/main'; export default function replaceHref( dom: JSDOM, requestUrl: URL, engine?: string, - redirectPath: string = "get", + redirectPath: string = 'get' ) { const doc = dom.window.document; const parserUrl = (href: string) => - href.startsWith("http") ? generateParserUrl( - requestUrl, - href, - engine, - redirectPath, - ) : href; + href.startsWith('http') + ? generateParserUrl(requestUrl, href, engine, redirectPath) + : href; const proxyUrl = (href: string) => - href.startsWith("http") ? generateProxyUrl( - requestUrl, - href, - ) : href; + href.startsWith('http') ? generateProxyUrl(requestUrl, href) : href; - modifyLinks( - doc.getElementsByTagName("a"), - "href", - parserUrl, - ); - modifyLinks( - doc.querySelectorAll("frame,iframe"), - "src", - parserUrl, - ); + modifyLinks(doc.getElementsByTagName('a'), 'href', parserUrl); + modifyLinks(doc.querySelectorAll('frame,iframe'), 'src', parserUrl); if (getConfig().proxy_res) { modifyLinks( - doc.querySelectorAll("img,image,video,audio,embed,track,source"), - "src", - proxyUrl, + doc.querySelectorAll('img,image,video,audio,embed,track,source'), + 'src', + proxyUrl ); - modifyLinks( - doc.getElementsByTagName("object"), - "data", - proxyUrl, - ); + modifyLinks(doc.getElementsByTagName('object'), 'data', proxyUrl); - const sources = doc.querySelectorAll("source,img"); + const sources = doc.querySelectorAll('source,img'); for (const source of sources) { // split srcset by comma // @ts-ignore - if (!source.srcset) - continue; + if (!source.srcset) continue; // @ts-ignore - source.srcset = source.srcset.split(",").map( - (src: string) => { + source.srcset = source.srcset + .split(',') + .map((src: string) => { // split src by space - const parts = src.trim().split(" "); + const parts = src.trim().split(' '); try { // first part is URL // (srcset="http 200w 1x,...") parts[0] = proxyUrl(parts[0]); - } catch (_err) { } + } catch (_err) {} // join by space after splitting - return parts.join(" "); - } - ).join(","); // join by comma + return parts.join(' '); + }) + .join(','); // join by comma } } } @@ -74,12 +56,12 @@ export default function replaceHref( function modifyLinks( nodeList: NodeListOf | HTMLCollectionOf, property: string, - generateLink: (value: string) => string, + generateLink: (value: string) => string ) { for (const node of nodeList) { try { // @ts-ignore node[property] = generateLink(node[property]); - } catch (_err) { } + } catch (_err) {} } } diff --git a/static/form.css b/static/form.css index 8dd12af..b5bed02 100644 --- a/static/form.css +++ b/static/form.css @@ -31,7 +31,7 @@ label { #url { width: 100%; - height: 100%; /* shrink to #submit height */ + height: 100%; /* shrink to #submit height */ outline: none; border: 0; From 2da620fdec385fbcc4ac77f04e5f95bf1bd5e508 Mon Sep 17 00:00:00 2001 From: Artemy Date: Wed, 13 Dec 2023 18:09:42 +0300 Subject: [PATCH 2/5] Create format-check.yml --- .github/workflows/format-check.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/format-check.yml diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml new file mode 100644 index 0000000..ad57a28 --- /dev/null +++ b/.github/workflows/format-check.yml @@ -0,0 +1,19 @@ +name: format-check +on: pull_request +jobs: + check-formatting: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: "14" + + - name: Install dependencies + run: npm install + + - name: Check formatting + run: npm run format:check From a8b0225571bfe03c4f98c8ae85631e451ad65b5c Mon Sep 17 00:00:00 2001 From: Artemy Date: Wed, 13 Dec 2023 18:14:15 +0300 Subject: [PATCH 3/5] Update package.json --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 9e32b08..aa0d5df 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ "start:docker": "node ./src/app.js", "clean-css": "cleancss --batch static/*.css -o dist/static --batch-suffix \"\"", "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"", - "format": "prettier --write ." + "format": "prettier --write .", + "format:check": "prettier --check ." }, "keywords": [], "authors": [ From f4061faa33f3dccf8057d8192db957d24447966a Mon Sep 17 00:00:00 2001 From: Artemy Date: Wed, 13 Dec 2023 18:16:50 +0300 Subject: [PATCH 4/5] format --- .github/workflows/format-check.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/format-check.yml b/.github/workflows/format-check.yml index ad57a28..6ea15f5 100644 --- a/.github/workflows/format-check.yml +++ b/.github/workflows/format-check.yml @@ -10,7 +10,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v2 with: - node-version: "14" + node-version: '14' - name: Install dependencies run: npm install From 46550bcba7143ca0bc7815d3cf6fab958b21ad51 Mon Sep 17 00:00:00 2001 From: Artemy Date: Thu, 14 Dec 2023 11:15:34 +0300 Subject: [PATCH 5/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 794f245..b2fd2a5 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ HTTP proxy that parses only text, links and pictures from pages reducing internet traffic, removing ads and heavy scripts. Uses [Mozilla's readability.js](https://github.com/mozilla/readability), -[JSDOM](https://github.com/jsdom/jsdom), +[🔗 linkedom](https://github.com/WebReflection/linkedom), [Fastify web framework](https://github.com/fastify/fastify). ## Installation