From 13c6f02a4d73976331f15bc482e60ac027c52916 Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Sun, 18 May 2025 16:39:51 -0700 Subject: [PATCH 01/10] refactor: use try/catches during rss build --- src/lib/api/fetchPosts.ts | 69 +++++++----- src/lib/api/fetchRSS.ts | 217 +++++++++++++++++++------------------- 2 files changed, 149 insertions(+), 137 deletions(-) diff --git a/src/lib/api/fetchPosts.ts b/src/lib/api/fetchPosts.ts index 31b1bea95a6..827379ee217 100644 --- a/src/lib/api/fetchPosts.ts +++ b/src/lib/api/fetchPosts.ts @@ -4,36 +4,47 @@ import { fetchXml } from "./fetchRSS" export const fetchAttestantPosts = async () => { const BASE_URL = "https://www.attestant.io/posts/" - const htmlData = (await fetchXml(BASE_URL)) as HTMLResult + const allItems: RSSItem[] = [] - // Extract div containing list of posts from deeply nested HTML structure - const postsContainer = - htmlData.html.body[0].div[0].div[1].div[0].div[0].div[0].div + try { + const htmlData = (await fetchXml(BASE_URL)) as HTMLResult - const posts: RSSItem[] = postsContainer - .map(({ a }) => { - const [ - { - $: { href }, - h4: [{ _: title }], - div: [{ _: content }, { _: pubDate }], - }, - ] = a - const { href: link } = new URL(href, BASE_URL) - return { - title, - link, - content, - source: "Attestant", - sourceUrl: BASE_URL, - sourceFeedUrl: BASE_URL, - imgSrc: "/images/attestant-logo.svg", - pubDate, - } - }) - .sort( - (a: RSSItem, b: RSSItem) => - new Date(b.pubDate).getTime() - new Date(a.pubDate).getTime() + // Extract div containing list of posts from deeply nested HTML structure + const postsContainer = + htmlData.html.body[0].div[0].div[1].div[0].div[0].div[0].div + + const sortedPosts = postsContainer + .map(({ a }) => { + const [ + { + $: { href }, + h4: [{ _: title }], + div: [{ _: content }, { _: pubDate }], + }, + ] = a + const { href: link } = new URL(href, BASE_URL) + return { + title, + link, + content, + source: "Attestant", + sourceUrl: BASE_URL, + sourceFeedUrl: BASE_URL, + imgSrc: "/images/attestant-logo.svg", + pubDate, + } + }) + .sort( + (a: RSSItem, b: RSSItem) => + new Date(b.pubDate).getTime() - new Date(a.pubDate).getTime() + ) + allItems.push(...sortedPosts) + } catch (error) { + console.error( + "Error fetching Attestant posts:", + error instanceof Error ? error.message : error ) - return posts + } + + return allItems } diff --git a/src/lib/api/fetchRSS.ts b/src/lib/api/fetchRSS.ts index 09d6ee43d88..14c5795d5eb 100644 --- a/src/lib/api/fetchRSS.ts +++ b/src/lib/api/fetchRSS.ts @@ -1,12 +1,6 @@ import { parseString } from "xml2js" -import type { - AtomElement, - AtomResult, - RSSChannel, - RSSItem, - RSSResult, -} from "../types" +import type { AtomElement, AtomResult, RSSItem, RSSResult } from "../types" import { isValidDate } from "../utils/date" /** @@ -18,108 +12,118 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { const urls = Array.isArray(xmlUrl) ? xmlUrl : [xmlUrl] const allItems: RSSItem[][] = [] for (const url of urls) { - const response = (await fetchXml(url)) as RSSResult | AtomResult + try { + const response = (await fetchXml(url)) as RSSResult | AtomResult - if ("rss" in response) { - const [mainChannel] = response.rss.channel as RSSChannel[] - const [source] = mainChannel.title - const [sourceUrl] = mainChannel.link - const channelImage = mainChannel.image ? mainChannel.image[0].url[0] : "" + if ("rss" in response) { + const [mainChannel] = response.rss.channel + const [source] = mainChannel.title + const [sourceUrl] = mainChannel.link + const channelImage = mainChannel.image + ? mainChannel.image[0].url[0] + : "" - const parsedRssItems = mainChannel.item - // Filter out items with invalid dates - .filter((item) => { - if (!item.pubDate) return false - const [pubDate] = item.pubDate - return isValidDate(pubDate) - }) - // Sort by pubDate (most recent is first in array - .sort((a, b) => { - const dateA = new Date(a.pubDate[0]) - const dateB = new Date(b.pubDate[0]) - return dateB.getTime() - dateA.getTime() - }) - // Map to RSSItem object - .map((item) => { - const getImgSrc = () => { - if (url.includes("medium.com/feed/")) - return item["content:encoded"]?.[0].match( - /https?:\/\/[^"]*?\.(jpe?g|png|webp)/g - ) - if (item.enclosure) return item.enclosure[0].$.url - if (item["media:content"]) return item["media:content"][0].$.url - return channelImage - } - return { - pubDate: item.pubDate[0], - title: item.title[0], - link: item.link[0], - imgSrc: getImgSrc(), - source, - sourceUrl, - sourceFeedUrl: url, - } as RSSItem - }) + const parsedRssItems = mainChannel.item + // Filter out items with invalid dates + .filter((item) => { + if (!item.pubDate) return false + const [pubDate] = item.pubDate + return isValidDate(pubDate) + }) + // Sort by pubDate (most recent is first in array + .sort((a, b) => { + const dateA = new Date(a.pubDate[0]) + const dateB = new Date(b.pubDate[0]) + return dateB.getTime() - dateA.getTime() + }) + // Map to RSSItem object + .map((item) => { + const getImgSrc = () => { + if (url.includes("medium.com/feed/")) + return item["content:encoded"]?.[0].match( + /https?:\/\/[^"]*?\.(jpe?g|png|webp)/g + ) + if (item.enclosure) return item.enclosure[0].$.url + if (item["media:content"]) return item["media:content"][0].$.url + return channelImage + } + return { + pubDate: item.pubDate[0], + title: item.title[0], + link: item.link[0], + imgSrc: getImgSrc(), + source, + sourceUrl, + sourceFeedUrl: url, + } + }) - allItems.push(parsedRssItems) - } else if ("feed" in response) { - const [source] = response.feed.title - const [sourceUrl] = response.feed.id - const feedImage = response.feed.icon?.[0] + allItems.push(parsedRssItems) + } else if ("feed" in response) { + const [source] = response.feed.title + const [sourceUrl] = response.feed.id + const feedImage = response.feed.icon?.[0] - const parsedAtomItems = response.feed.entry - // Filter out items with invalid dates - .filter((entry) => { - if (!entry.updated) return false - const [published] = entry.updated - return isValidDate(published) - }) - // Sort by published (most recent is first in array - .sort((a, b) => { - const dateA = new Date(a.updated[0]) - const dateB = new Date(b.updated[0]) - return dateB.getTime() - dateA.getTime() - }) - // Map to RSSItem object - .map((entry) => { - const getString = (el?: AtomElement[]): string => { - if (!el) return "" - const [firstEl] = el - if (typeof firstEl === "string") return firstEl - return firstEl._ || "" - } - const getHref = (): string => { - if (!entry.link) { - console.warn(`No link found for RSS url: ${url}`) - return "" + const parsedAtomItems = response.feed.entry + // Filter out items with invalid dates + .filter((entry) => { + if (!entry.updated) return false + const [published] = entry.updated + return isValidDate(published) + }) + // Sort by published (most recent is first in array + .sort((a, b) => { + const dateA = new Date(a.updated[0]) + const dateB = new Date(b.updated[0]) + return dateB.getTime() - dateA.getTime() + }) + // Map to RSSItem object + .map((entry) => { + const getString = (el?: AtomElement[]): string => { + if (!el) return "" + const [firstEl] = el + if (typeof firstEl === "string") return firstEl + return firstEl._ || "" + } + const getHref = (): string => { + if (!entry.link) { + console.warn(`No link found for RSS url: ${url}`) + return "" + } + const link = entry.link[0] + if (typeof link === "string") return link + return link.$.href || "" + } + const getImgSrc = (): string => { + const imgRegEx = /https?:\/\/[^"]*?\.(jpe?g|png|webp)/g + const contentMatch = getString(entry.content).match(imgRegEx) + if (contentMatch) return contentMatch[0] + const summaryMatch = getString(entry.summary).match(imgRegEx) + if (summaryMatch) return summaryMatch[0] + return feedImage || "" } - const link = entry.link[0] - if (typeof link === "string") return link - return link.$.href || "" - } - const getImgSrc = (): string => { - const imgRegEx = /https?:\/\/[^"]*?\.(jpe?g|png|webp)/g - const contentMatch = getString(entry.content).match(imgRegEx) - if (contentMatch) return contentMatch[0] - const summaryMatch = getString(entry.summary).match(imgRegEx) - if (summaryMatch) return summaryMatch[0] - return feedImage || "" - } - return { - pubDate: entry.updated[0], - title: getString(entry.title), - link: getHref(), - imgSrc: getImgSrc(), - source, - sourceUrl, - sourceFeedUrl: url, - } as RSSItem - }) + return { + pubDate: entry.updated[0], + title: getString(entry.title), + link: getHref(), + imgSrc: getImgSrc(), + source, + sourceUrl, + sourceFeedUrl: url, + } + }) - allItems.push(parsedAtomItems) + allItems.push(parsedAtomItems) + } + } catch (error) { + console.error( + `Failed to fetch or parse RSS feed from ${url}:`, + error instanceof Error ? error.message : error + ) + continue } } - return allItems as RSSItem[][] + return allItems } /** @@ -132,14 +136,11 @@ export const fetchXml = async (url: string) => { try { const response = await fetch(url) const xml = await response.text() - let returnObject: Record = {} - parseString(xml, (err, result) => { - if (err) { - throw err // Throw the error to be caught by the outer try-catch - } - returnObject = result + return await new Promise>((resolve, reject) => { + parseString(xml, (err, result) => { + err ? reject(err) : resolve(result) + }) }) - return returnObject } catch (error) { console.error("Error fetching or parsing XML:", url, error) throw error From 02da378309e74d94d45ee35bb4ff32b16af79b42 Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Mon, 19 May 2025 08:53:39 -0700 Subject: [PATCH 02/10] debug: add intentional broken link for testing --- src/lib/constants.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/constants.ts b/src/lib/constants.ts index d6b0aecbe46..a18e6e2ee0a 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -199,7 +199,7 @@ export const COMMUNITY_BLOGS: CommunityBlog[] = [ }, { href: "https://ethpandaops.io/posts/", - feed: "https://ethpandaops.io/posts/rss.xml", + feed: "https://ethpandaops.io/posts/testing-error-revert-me.xml", }, { href: "https://ethstaker.cc/blog", @@ -232,7 +232,7 @@ export const COMMUNITY_BLOGS: CommunityBlog[] = [ }, { href: "http://geodework.com/blog", - feed: "http://geodework.com/feed.xml", + feed: "http://geodework.com/testing-error-revert-me.xml", }, ] From 0a7e55b5cb044a226bb849bcc2a7cb62f7fe056f Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Mon, 19 May 2025 15:29:01 -0700 Subject: [PATCH 03/10] fix: error logging Pass individual fetch/parse errors from child process and console.warn with parent, continuing processes. Add additional guard to make sure enough blog items were fetch; if less then break build. --- src/lib/api/fetchRSS.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lib/api/fetchRSS.ts b/src/lib/api/fetchRSS.ts index 14c5795d5eb..2d297fedacd 100644 --- a/src/lib/api/fetchRSS.ts +++ b/src/lib/api/fetchRSS.ts @@ -1,5 +1,6 @@ import { parseString } from "xml2js" +import { RSS_DISPLAY_COUNT } from "../constants" import type { AtomElement, AtomResult, RSSItem, RSSResult } from "../types" import { isValidDate } from "../utils/date" @@ -116,13 +117,14 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { allItems.push(parsedAtomItems) } } catch (error) { - console.error( - `Failed to fetch or parse RSS feed from ${url}:`, - error instanceof Error ? error.message : error - ) + console.error(error instanceof Error ? error.message : error) continue } } + + if (allItems.length < RSS_DISPLAY_COUNT) + throw new Error("Insufficient number of RSS items fetched") + return allItems } @@ -142,7 +144,6 @@ export const fetchXml = async (url: string) => { }) }) } catch (error) { - console.error("Error fetching or parsing XML:", url, error) - throw error + throw new Error(`Error fetching or parsing XML: ${url}`) } } From 8f4031994dbf8358c7c4a88d7d9e27beeab9a5e9 Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Tue, 20 May 2025 11:52:24 -0700 Subject: [PATCH 04/10] fix: throw error for invalid type matching handles cases where fetch returns html, but invalid RSS/XML feed --- src/lib/api/fetchRSS.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lib/api/fetchRSS.ts b/src/lib/api/fetchRSS.ts index 2d297fedacd..578d92643fc 100644 --- a/src/lib/api/fetchRSS.ts +++ b/src/lib/api/fetchRSS.ts @@ -115,6 +115,10 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { }) allItems.push(parsedAtomItems) + } else { + throw new Error( + `Error parsing XML, invalid RSSResult or AtomResult type: ${url}` + ) } } catch (error) { console.error(error instanceof Error ? error.message : error) From 7fcca626f90f77ea071d17c6841d97c895807a1b Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Tue, 20 May 2025 11:53:40 -0700 Subject: [PATCH 05/10] Revert "debug: add intentional broken link for testing" This reverts commit 02da378309e74d94d45ee35bb4ff32b16af79b42. --- src/lib/constants.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/constants.ts b/src/lib/constants.ts index a18e6e2ee0a..d6b0aecbe46 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -199,7 +199,7 @@ export const COMMUNITY_BLOGS: CommunityBlog[] = [ }, { href: "https://ethpandaops.io/posts/", - feed: "https://ethpandaops.io/posts/testing-error-revert-me.xml", + feed: "https://ethpandaops.io/posts/rss.xml", }, { href: "https://ethstaker.cc/blog", @@ -232,7 +232,7 @@ export const COMMUNITY_BLOGS: CommunityBlog[] = [ }, { href: "http://geodework.com/blog", - feed: "http://geodework.com/testing-error-revert-me.xml", + feed: "http://geodework.com/feed.xml", }, ] From 9a1d824fe92a3a6c6d072f2fc8196ff6fd84aac7 Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Tue, 20 May 2025 11:54:10 -0700 Subject: [PATCH 06/10] fix: use https --- src/lib/constants.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/constants.ts b/src/lib/constants.ts index d6b0aecbe46..083a0dc1419 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -231,8 +231,8 @@ export const COMMUNITY_BLOGS: CommunityBlog[] = [ feed: "https://medium.com/feed/ethereum-cat-herders", }, { - href: "http://geodework.com/blog", - feed: "http://geodework.com/feed.xml", + href: "https://geodework.com/blog", + feed: "https://geodework.com/feed.xml", }, ] From 360c43955efb27ce4be3d0cb43222774ccc9ee3c Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Tue, 20 May 2025 13:39:26 -0700 Subject: [PATCH 07/10] revert: throwing error on few rss items does not break build as intended, only breaks front end --- src/lib/api/fetchRSS.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/lib/api/fetchRSS.ts b/src/lib/api/fetchRSS.ts index 578d92643fc..c9ca7b37a68 100644 --- a/src/lib/api/fetchRSS.ts +++ b/src/lib/api/fetchRSS.ts @@ -1,6 +1,5 @@ import { parseString } from "xml2js" -import { RSS_DISPLAY_COUNT } from "../constants" import type { AtomElement, AtomResult, RSSItem, RSSResult } from "../types" import { isValidDate } from "../utils/date" @@ -125,10 +124,6 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { continue } } - - if (allItems.length < RSS_DISPLAY_COUNT) - throw new Error("Insufficient number of RSS items fetched") - return allItems } From 7142a0ab22abcee74d5b67fb06d4e79b0ec93ddf Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Wed, 21 May 2025 07:51:23 -0700 Subject: [PATCH 08/10] Revert "revert: throwing error on few rss items" This reverts commit 360c43955efb27ce4be3d0cb43222774ccc9ee3c. --- src/lib/api/fetchRSS.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/lib/api/fetchRSS.ts b/src/lib/api/fetchRSS.ts index c9ca7b37a68..578d92643fc 100644 --- a/src/lib/api/fetchRSS.ts +++ b/src/lib/api/fetchRSS.ts @@ -1,5 +1,6 @@ import { parseString } from "xml2js" +import { RSS_DISPLAY_COUNT } from "../constants" import type { AtomElement, AtomResult, RSSItem, RSSResult } from "../types" import { isValidDate } from "../utils/date" @@ -124,6 +125,10 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { continue } } + + if (allItems.length < RSS_DISPLAY_COUNT) + throw new Error("Insufficient number of RSS items fetched") + return allItems } From 6e9692e70e6f386ff86f0384a5e6feaa4a4bc196 Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Tue, 27 May 2025 10:35:15 -0700 Subject: [PATCH 09/10] feat: build homepage statically --- app/[locale]/page.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/[locale]/page.tsx b/app/[locale]/page.tsx index 9e2713e92b8..0e5df96a926 100644 --- a/app/[locale]/page.tsx +++ b/app/[locale]/page.tsx @@ -22,6 +22,7 @@ import { BLOG_FEEDS, BLOGS_WITHOUT_FEED, CALENDAR_DISPLAY_COUNT, + LOCALES_CODES, RSS_DISPLAY_COUNT, } from "@/lib/constants" @@ -127,6 +128,8 @@ const Page = async ({ params }: { params: Promise<{ locale: Lang }> }) => { ) } +export const generateStaticParams = async () => LOCALES_CODES + export async function generateMetadata() { const t = await getTranslations() From 0a5739f0831de7f49f0e58cf51720e32b8066094 Mon Sep 17 00:00:00 2001 From: Paul Wackerow <54227730+wackerow@users.noreply.github.com> Date: Tue, 27 May 2025 10:35:24 -0700 Subject: [PATCH 10/10] chore: add comments --- src/lib/api/fetchRSS.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib/api/fetchRSS.ts b/src/lib/api/fetchRSS.ts index 578d92643fc..2cc000f5e3f 100644 --- a/src/lib/api/fetchRSS.ts +++ b/src/lib/api/fetchRSS.ts @@ -122,10 +122,12 @@ export const fetchRSS = async (xmlUrl: string | string[]) => { } } catch (error) { console.error(error instanceof Error ? error.message : error) + // Do not break build for single fetch failure continue } } + // Only break build if insufficient number of items fetched if (allItems.length < RSS_DISPLAY_COUNT) throw new Error("Insufficient number of RSS items fetched")