[feat] json feed loader

core/download.ts

@@ -4,6 +4,7 @@ export interface TextResponse {
   readonly headers: Headers
   readonly ok: boolean
   parse(): Document | XMLDocument
+  parseJson(): null | unknown
   readonly status: number
   readonly text: string
   readonly url: string
@@ -15,6 +16,14 @@ export interface DownloadTask {
   text(...args: Parameters<typeof request>): Promise<TextResponse>
 }
 
+function getContentMediaType(headers: Headers): string {
+  let contentType = headers.get('content-type') ?? 'text/html'
+  if (contentType.includes(';')) {
+    contentType = contentType.split(';')[0]!
+  }
+  return contentType
+}
+
 export function createTextResponse(
   text: string,
   other: Partial<Omit<TextResponse, 'ok' | 'text'>> = {}
@@ -27,10 +36,8 @@ export function createTextResponse(
     ok: status >= 200 && status < 300,
     parse() {
       if (!bodyCache) {
-        let parseType = headers.get('content-type') ?? 'text/html'
-        if (parseType.includes(';')) {
-          parseType = parseType.split(';')[0]!
-        }
+        let parseType = getContentMediaType(headers)
+
         if (parseType.includes('+xml')) {
           parseType = 'application/xml'
         }
@@ -51,6 +58,24 @@ export function createTextResponse(
       }
       return bodyCache
     },
+    parseJson() {
+      let parseType = getContentMediaType(headers)
+
+      if (
+        parseType !== 'application/json' &&
+        parseType !== 'application/feed+json'
+      ) {
+        return null
+      }
+
+      try {
+        return JSON.parse(text)
+      } catch (e) {
+        // eslint-disable-next-line no-console
+        console.error('Parse JSON error', e)
+        return null
+      }
+    },
     status,
     text,
     url: other.url ?? 'https://example.com'

core/loader/index.ts

@@ -1,6 +1,7 @@
 import type { DownloadTask, TextResponse } from '../download.js'
 import type { PostsPage } from '../posts-page.js'
 import { atom } from './atom.js'
+import { jsonFeed } from './json-feed.js'
 import { rss } from './rss.js'
 
 export type Loader = {
@@ -13,6 +14,7 @@ export type Loader = {
 
 export const loaders = {
   atom,
+  jsonFeed,
   rss
 }
 

core/loader/json-feed.ts

@@ -0,0 +1,137 @@
+import type { TextResponse } from '../download.js'
+import type { OriginPost } from '../post.js'
+import { createPostsPage } from '../posts-page.js'
+import type { Loader } from './index.js'
+import { findAnchorHrefs, findLinksByType, toTime } from './utils.js'
+
+// https://www.jsonfeed.org/version/1.1/
+export type Author = {
+  avatar?: string
+  name?: string
+  url?: string
+}
+
+export type Item = {
+  /** deprecated from 1.1 version */
+  author?: Author
+  authors?: Author[]
+  banner_image?: string
+  content_html?: string
+  content_text?: string
+  date_modified?: string
+  date_published?: string
+  external_url?: string
+  id: string
+  image?: string
+  summary?: string
+  tags?: string[]
+  title?: string
+  url?: string
+}
+
+export type JsonFeed = {
+  /** deprecated from 1.1 version */
+  author?: Author
+  authors?: Author[]
+  description?: string
+  favicon?: string
+  feed_url?: string
+  home_page_url?: string
+  icon?: string
+  items: Item[]
+  next_url?: string
+  title: string
+  user_comment?: string
+  version: string
+}
+
+type ValidationRules = {
+  [key: string]: (value: unknown) => boolean
+}
+
+function isObjValid<ValidatedType>(
+  obj: unknown,
+  rules: ValidationRules
+): obj is ValidatedType {
+  if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
+    return false
+  }
+
+  let objRecord = obj as Record<string, unknown>
+
+  for (let field in rules) {
+    if (!(field in objRecord) || !rules[field]!(objRecord[field])) {
+      // eslint-disable-next-line no-console
+      console.error(
+        `Value ${objRecord[field]} of object field ${field} is not valid`
+      )
+      return false
+    }
+  }
+
+  return true
+}
+
+let existJsonFeedVersions = ['1', '1.1']
+
+let jsonFeedValidationRules: ValidationRules = {
+  items: (value: unknown): boolean => Array.isArray(value),
+  title: (value: unknown): boolean => typeof value === 'string',
+  version: (value: unknown): boolean => {
+    if (typeof value !== 'string' || !value.includes('jsonfeed')) return false
+    let version = value.split('/').pop()
+    return existJsonFeedVersions.includes(version!)
+  }
+}
+
+function parsePosts(text: TextResponse): OriginPost[] {
+  let parsedJson = text.parseJson()
+  if (!isObjValid<JsonFeed>(parsedJson, jsonFeedValidationRules)) return []
+
+  return parsedJson.items.map(item => ({
+    full: (item.content_html || item.content_text) ?? undefined,
+    intro: item.summary ?? undefined,
+    media: [],
+    originId: item.id,
+    publishedAt: toTime(item.date_published) ?? undefined,
+    title: item.title,
+    url: item.url ?? undefined
+  }))
+}
+
+export const jsonFeed: Loader = {
+  getMineLinksFromText(text) {
+    let linksByType = findLinksByType(text, 'application/feed+json')
+    if (linksByType.length === 0) {
+      linksByType = findLinksByType(text, 'application/json')
+    }
+
+    return [...linksByType, ...findAnchorHrefs(text, /feeds?\.json/i)]
+  },
+
+  getPosts(task, url, text) {
+    if (text) {
+      return createPostsPage(parsePosts(text), undefined)
+    } else {
+      return createPostsPage(undefined, async () => {
+        return [parsePosts(await task.text(url)), undefined]
+      })
+    }
+  },
+
+  getSuggestedLinksFromText(text) {
+    return [new URL('/feed.json', new URL(text.url).origin).href]
+  },
+
+  isMineText(text) {
+    let parsedJson = text.parseJson()
+    if (isObjValid<JsonFeed>(parsedJson, jsonFeedValidationRules)) {
+      return parsedJson.title
+    }
+    return false
+  },
+
+  isMineUrl() {
+    return undefined
+  }
+}

core/messages/preview/en.ts

@@ -8,7 +8,7 @@ export const previewMessages = i18n('preview', {
     'Feeds were not found on this website.\n\n' +
     'Please check URL and [open an issue] if it’s correct.',
   searchGuide:
-    'For now we support RSS and Atom feeds.\n\n' +
+    'For now we support RSS, Atom and JSONFeed feeds.\n\n' +
     'Social networks are coming soon, but you can use RSS wrappers for them.',
   title: 'Add',
   unloadable: 'Can’t open this website',

core/test/download.test.ts

@@ -152,6 +152,19 @@ test('parses content', async () => {
     headers: new Headers({ 'content-type': 'application/xml' })
   })
   equal(broken.parse().textContent, null)
+
+  let json = createTextResponse('{ "version": "1.1", "title": "test_title" }', {
+    headers: new Headers({ 'content-type': 'application/json' })
+  })
+  equal((json.parseJson() as { title: string; version: string }).version, '1.1')
+
+  let brokenJson = createTextResponse(
+    '{ "items": [], "version": 1.1", "title": "test_title" }',
+    {
+      headers: new Headers({ 'content-type': 'application/json' })
+    }
+  )
+  equal(brokenJson.parseJson(), null)
 })
 
 test('has helper to ignore abort errors', async () => {