fix: fix amazon scrapping (#92)

* fix: fix amazon scrapping * fix: fix URL pattern matching
rows · May 13, 2024 · a47b7d9 · a47b7d9
1 parent 09f26c5
commit a47b7d9
Show file tree

Hide file tree

Showing 9 changed files with 13 additions and 15 deletions.
diff --git a/src/scrappers/amazon.yml b/src/scrappers/amazon.yml
@@ -1,4 +1,4 @@
-url: 'https://www.amazon.*/s?k=*'
+url: 'https://www.amazon.*' 
 header: Amazon search results
 listElementsQuery: '[class*="sg-"][data-cel-widget*="search_result_"]'
 elementParser:

diff --git a/src/scrappers/g2-search.yml b/src/scrappers/g2-search.yml
@@ -1,4 +1,4 @@
-url: https://www.g2.com/search
+url: https://www.g2.com/search*
 header: 'G2 search results'
 listElementsQuery: '[class*="paper mb-1"]'
 elementParser:

diff --git a/src/scrappers/linkedin-groups.yml b/src/scrappers/linkedin-groups.yml
@@ -1,4 +1,4 @@
-url: https://www.linkedin.com/groups/
+url: https://www.linkedin.com/groups/*
 header: Linkedin groups
 listElementsQuery: .scaffold-finite-scroll__content > div > ul > li
 elementParser:

diff --git a/src/scrappers/linkedin-jobs.yml b/src/scrappers/linkedin-jobs.yml
@@ -1,4 +1,4 @@
-url: https://www.linkedin.com/jobs/search/
+url: https://www.linkedin.com/jobs/search/*
 header: Linkedin jobs search results
 listElementsQuery: .jobs-search-results__list-item
 elementParser:

diff --git a/src/scrappers/tik-tok-accounts.yml b/src/scrappers/tik-tok-accounts.yml
@@ -1,4 +1,4 @@
-url: https://www.tiktok.com/search/user
+url: https://www.tiktok.com/search/user*
 header: TikTok Search Results
 listElementsQuery: '[class*="DivPanelContainer"] > [class*="-DivLink"]'
 elementParser:

diff --git a/src/scrappers/tik-tok-video.yml b/src/scrappers/tik-tok-video.yml
@@ -1,4 +1,4 @@
-url: https://www.tiktok.com/search/video
+url: https://www.tiktok.com/search/video*
 header: TikTok Search Results
 listElementsQuery: '[class*="-DivItemContainerForSearch"]'
 elementParser:

diff --git a/src/scrappers/yellow-pages.yml b/src/scrappers/yellow-pages.yml
@@ -1,4 +1,4 @@
-url: https://www.yellowpages.com/search
+url: https://www.yellowpages.com/search*
 listElementsQuery: .result
 elementParser:
   - title: Logo

diff --git a/src/scrappers/youtube.yml b/src/scrappers/youtube.yml
@@ -1,7 +1,7 @@
 url:
-  - https://www.youtube.com/feed/history
-  - https://www.youtube.com/results
-  - https://www.youtube.com/playlist
+  - https://www.youtube.com/feed/history*
+  - https://www.youtube.com/results*
+  - https://www.youtube.com/playlist*
 listElementsQuery: 'ytd-video-renderer, ytd-playlist-video-renderer, ytd-rich-item-renderer'
 elementParser:
   - title: Video thumbnail

diff --git a/src/utils/urlUtils.ts b/src/utils/urlUtils.ts
@@ -2,9 +2,7 @@ export function urlMatchesPatternUrl(url: string, patternURL: string): boolean {
   if (!patternURL) {
     return false;
   }
-  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-  // @ts-ignore
-  const pattern = new URLPattern(patternURL);
 
-  return pattern.test(url);
-}
+  const regex = new RegExp('^' + patternURL.replace(/\*/g, '.*') + '$');
+  return regex.test(url);
+}