FIX FlowiseAI#2617 Cherio Web Crawler doesn't work with large sites (F…

…lowiseAI#2678) * FIX FlowiseAI#2617 Big sites scan error * FIX FlowiseAI#2617 Big sites scan error - review fix --------- Co-authored-by: Ahmed Osman <[email protected]>
BarOfSoapOnWednesday · Jul 5, 2024 · 90558ca · 90558ca
1 parent b1e3878
commit 90558ca
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts b/packages/components/nodes/documentloaders/Cheerio/Cheerio.ts
@@ -131,7 +131,11 @@ class Cheerio_DocumentLoaders implements INode {
 
         async function cheerioLoader(url: string): Promise<any> {
             try {
-                let docs = []
+                let docs: IDocument[] = []
+                if (url.endsWith('.pdf')) {
+                    if (process.env.DEBUG === 'true') options.logger.info(`CheerioWebBaseLoader does not support PDF files: ${url}`)
+                    return docs
+                }
                 const loader = new CheerioWebBaseLoader(url, params)
                 if (textSplitter) {
                     docs = await loader.loadAndSplit(textSplitter)
@@ -141,6 +145,7 @@ class Cheerio_DocumentLoaders implements INode {
                 return docs
             } catch (err) {
                 if (process.env.DEBUG === 'true') options.logger.error(`error in CheerioWebBaseLoader: ${err.message}, on page: ${url}`)
+                return []
             }
         }