RSS feed error handling #15476
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,7 @@ | ||
| import { parseString } from "xml2js" | ||
|
|
||
| import { RSS_DISPLAY_COUNT } from "../constants" | ||
| import type { AtomElement, AtomResult, RSSItem, RSSResult } from "../types" | ||
| import { isValidDate } from "../utils/date" | ||
|
|
||
| /** | ||
|
|
@@ -18,108 +13,125 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { | |
| const urls = Array.isArray(xmlUrl) ? xmlUrl : [xmlUrl] | ||
| const allItems: RSSItem[][] = [] | ||
| for (const url of urls) { | ||
| try { | ||
| const response = (await fetchXml(url)) as RSSResult | AtomResult | ||
|
|
||
| if ("rss" in response) { | ||
| const [mainChannel] = response.rss.channel | ||
| const [source] = mainChannel.title | ||
| const [sourceUrl] = mainChannel.link | ||
| const channelImage = mainChannel.image | ||
| ? mainChannel.image[0].url[0] | ||
| : "" | ||
|
|
||
| const parsedRssItems = mainChannel.item | ||
| // Filter out items with invalid dates | ||
| .filter((item) => { | ||
| if (!item.pubDate) return false | ||
| const [pubDate] = item.pubDate | ||
| return isValidDate(pubDate) | ||
| }) | ||
| // Sort by pubDate (most recent is first in array | ||
| .sort((a, b) => { | ||
| const dateA = new Date(a.pubDate[0]) | ||
| const dateB = new Date(b.pubDate[0]) | ||
| return dateB.getTime() - dateA.getTime() | ||
| }) | ||
| // Map to RSSItem object | ||
| .map((item) => { | ||
| const getImgSrc = () => { | ||
| if (item["content:encoded"]) | ||
| return item["content:encoded"][0].match( | ||
| /https?:\/\/[^"]*?\.(jpe?g|png|webp)/g | ||
| )?.[0] | ||
| if (item.enclosure) return item.enclosure[0].$.url | ||
| if (item["media:content"]) return item["media:content"][0].$.url | ||
| return channelImage | ||
| } | ||
| return { | ||
| pubDate: item.pubDate[0], | ||
| title: item.title[0], | ||
| link: item.link[0], | ||
| imgSrc: getImgSrc(), | ||
| source, | ||
| sourceUrl, | ||
| sourceFeedUrl: url, | ||
| } | ||
| }) | ||
|
|
||
| allItems.push(parsedRssItems) | ||
| } else if ("feed" in response) { | ||
| const [source] = response.feed.title | ||
| const [sourceUrl] = response.feed.id | ||
| const feedImage = response.feed.icon?.[0] | ||
|
|
||
| const parsedAtomItems = response.feed.entry | ||
| // Filter out items with invalid dates | ||
| .filter((entry) => { | ||
| if (!entry.updated) return false | ||
| const [published] = entry.updated | ||
| return isValidDate(published) | ||
| }) | ||
| // Sort by published (most recent is first in array | ||
| .sort((a, b) => { | ||
| const dateA = new Date(a.updated[0]) | ||
| const dateB = new Date(b.updated[0]) | ||
| return dateB.getTime() - dateA.getTime() | ||
| }) | ||
| // Map to RSSItem object | ||
| .map((entry) => { | ||
| const getString = (el?: AtomElement[]): string => { | ||
| if (!el) return "" | ||
| const [firstEl] = el | ||
| if (typeof firstEl === "string") return firstEl | ||
| return firstEl._ || "" | ||
| } | ||
| const getHref = (): string => { | ||
| if (!entry.link) { | ||
| console.warn(`No link found for RSS url: ${url}`) | ||
| return "" | ||
| } | ||
| const link = entry.link[0] | ||
| if (typeof link === "string") return link | ||
| return link.$.href || "" | ||
| } | ||
| const getImgSrc = (): string => { | ||
| const imgRegEx = /https?:\/\/[^"]*?\.(jpe?g|png|webp)/g | ||
| const contentMatch = getString(entry.content).match(imgRegEx) | ||
| if (contentMatch) return contentMatch[0] | ||
| const summaryMatch = getString(entry.summary).match(imgRegEx) | ||
| if (summaryMatch) return summaryMatch[0] | ||
| return feedImage || "" | ||
| } | ||
| return { | ||
| pubDate: entry.updated[0], | ||
| title: getString(entry.title), | ||
| link: getHref(), | ||
| imgSrc: getImgSrc(), | ||
| source, | ||
| sourceUrl, | ||
| sourceFeedUrl: url, | ||
| } | ||
| }) | ||
|
|
||
| allItems.push(parsedAtomItems) | ||
| } else { | ||
| throw new Error( | ||
| `Error parsing XML, invalid RSSResult or AtomResult type: ${url}` | ||
| ) | ||
| } | ||
| } catch (error) { | ||
| console.error(error instanceof Error ? error.message : error) | ||
| // Do not break build for single fetch failure | ||
| continue | ||
| } | ||
| } | ||
|
|
||
| // Only break build if insufficient number of items fetched | ||
| if (allItems.length < RSS_DISPLAY_COUNT) | ||
| throw new Error("Insufficient number of RSS items fetched") | ||
|
Comment on lines
+131
to
+132
Member
Author
There was a problem hiding this comment. This isn't actually breaking the build as-is, since it is contained inside an api function call... 🤔 cc: @pettinarip
Member
Author
There was a problem hiding this comment. Okay, with the addition of |
||
|
|
||
| return allItems | ||
|
Member
There was a problem hiding this comment. this function fetch will never throw, is that the intention? if we want to break the build, we should
Member
Author
There was a problem hiding this comment. I don't want a singular feed to break the build (ie. recent example where one blog changed their feed link)... these cases should just skip over that feed. BUT, in the event something happens where we don't get ANY feed results, then that should break the build... which was the intent of throwing an error at the end if there aren't enough results. |
||
| } | ||
|
|
||
| /** | ||
|
|
@@ -135,16 +147,12 @@ export const fetchXml = async (url: string) => { | |
| credentials: "omit", // Don't send or receive cookies | ||
| }) | ||
| const xml = await response.text() | ||
| return await new Promise<Record<string, unknown>>((resolve, reject) => { | ||
| parseString(xml, (err, result) => { | ||
| err ? reject(err) : resolve(result) | ||
| }) | ||
| }) | ||
| } catch (error) { | ||
| throw new Error(`Error fetching or parsing XML: ${url}`) | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Forces homepage to builds paths at build, which allows
throw Errorto break the build