diff --git a/.github/workflows/lints.yaml b/.github/workflows/lints.yaml index 5e3ebf5..e70c3dd 100644 --- a/.github/workflows/lints.yaml +++ b/.github/workflows/lints.yaml @@ -26,9 +26,10 @@ jobs: - name: Run Pylint static code analyser run: | pip install coverage pylint - pylint src/facere_sensum test + pylint src/facere_sensum test fsy.py - name: Run Bandit security analyser run: | pip install bandit bandit -r src/facere_sensum - bandit -r test \ No newline at end of file + bandit -r test + bandit fsy.py \ No newline at end of file diff --git a/examples/config_customsearch.json b/examples/config_customsearch.json new file mode 100644 index 0000000..d8f873e --- /dev/null +++ b/examples/config_customsearch.json @@ -0,0 +1,18 @@ +{ + "log": "log.csv", + "metrics": [ + { + "id": "spartan race", + "source": "customsearch", + "priority": 0.7, + "num": 5, + "URL": "https://www.spartan.com/" + }, + { + "id": "obstacle course racing", + "source": "customsearch", + "priority": 0.3, + "URL": "https://www.spartan.com/" + } + ] +} diff --git a/fsy.py b/fsy.py new file mode 100644 index 0000000..ff88c3a --- /dev/null +++ b/fsy.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: MIT + +''' +facere-sensum debug launcher. +Need to keep this separate to make sure fs.py is imported as a module and +not used as the main script. +''' + +from facere_sensum import fs + +fs.main() diff --git a/requirements.txt b/requirements.txt index 1975934..ee23ca9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ + google-api-python-client>=2.93.0 numpy>=1.24.3 pandas>=2.0.1 \ No newline at end of file diff --git a/src/facere_sensum/connectors/customsearch.py b/src/facere_sensum/connectors/customsearch.py new file mode 100644 index 0000000..5be5034 --- /dev/null +++ b/src/facere_sensum/connectors/customsearch.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: MIT + +''' +Data connector for Google Custom Search API. +''' + +from googleapiclient.discovery import build +from facere_sensum import fs + +# Default for number of search results to consider. +_NUM = 50 + +_auth = fs.auth['Google'] +_cse = build('customsearch', 'v1', developerKey=_auth['custom search API key']).cse() # pylint: disable=E1101 + +def invoke_cse(query, start): # pragma: no cover + ''' + Invoke Custom Search API with specified query and index of the first result to return. + Keep this function separate so that testing scripts can substitute with a mockup. + ''' + return _cse.list(q=query, cx=_auth['search engine ID'], start=start).execute() + +def get_raw(metric): + ''' + Get raw metric score for Google Custom Search API: + rank of the query or zero, if it didn't appear in search results. + 'metric' is the metric JSON description. + ''' + query = metric['q'] if 'q' in metric else metric['id'] + num = metric['num'] if 'num' in metric else _NUM + url = metric['URL'] + + start = 1 + while num > 0: + res = invoke_cse(query, start) + + for (index,item) in enumerate(res['items'][:num]): + if item['link'] == url: + return start+index + + if 'nextPage' not in res['queries']: + print('Warning (Google Custom Search API connector): ' \ + f'query "{query}" produced small number of search results') + return 0 + + start += 10 + num -= 10 + return 0 + +def get_value(metric): + ''' + Get standard (i.e., normalized) metric score for Google Custom Search API. + 'metric' is the metric JSON description. + ''' + raw = get_raw(metric) + metric_id = metric['id'] + metric_outcome = str(raw) if raw else 'not found' + print(f' - {metric_id}: {metric_outcome}') + + if raw: + num = metric['num'] if 'num' in metric else _NUM + return (num+1-raw) / num + + return 0 diff --git a/src/facere_sensum/fs.py b/src/facere_sensum/fs.py index e7ed087..40bea7a 100644 --- a/src/facere_sensum/fs.py +++ b/src/facere_sensum/fs.py @@ -143,15 +143,17 @@ def main(): if args.auth: try : - with open(args.auth, encoding='utf-8') as auth: - auth = json.load(auth) + with open(args.auth, encoding='utf-8') as auth_file: + # Put authentication config in global scope + # for all other modules to access as necessary. + globals()['auth'] = json.load(auth_file) except FileNotFoundError: print('Authentication config file \''+args.auth+'\' not found. Exiting.') sys.exit(1) try: - with open(args.config, encoding='utf-8') as config: - config = json.load(config) + with open(args.config, encoding='utf-8') as config_file: + config = json.load(config_file) except FileNotFoundError: print('Project config file \''+args.config+'\' not found. Exiting.') sys.exit(1) @@ -168,6 +170,3 @@ def main(): 'Please submit an issue at https://github.com/lunarserge/facere-sensum/issues/new', 'with the command that led here.') sys.exit(1) - -if __name__ == '__main__': - main() diff --git a/test/input/customsearch.json b/test/input/customsearch.json new file mode 100644 index 0000000..239e01d --- /dev/null +++ b/test/input/customsearch.json @@ -0,0 +1,575 @@ +{ + "kind": "customsearch#search", + "url": { + "type": "application/json", + "template": "https://www.googleapis.com/customsearch/v1?q={searchTerms}&num={count?}&start={startIndex?}&lr={language?}&safe={safe?}&cx={cx?}&sort={sort?}&filter={filter?}&gl={gl?}&cr={cr?}&googlehost={googleHost?}&c2coff={disableCnTwTranslation?}&hq={hq?}&hl={hl?}&siteSearch={siteSearch?}&siteSearchFilter={siteSearchFilter?}&exactTerms={exactTerms?}&excludeTerms={excludeTerms?}&linkSite={linkSite?}&orTerms={orTerms?}&relatedSite={relatedSite?}&dateRestrict={dateRestrict?}&lowRange={lowRange?}&highRange={highRange?}&searchType={searchType}&fileType={fileType?}&rights={rights?}&imgSize={imgSize?}&imgType={imgType?}&imgColorType={imgColorType?}&imgDominantColor={imgDominantColor?}&alt=json" + }, + "queries": { + "request": [ + { + "title": "Google Custom Search - obstacle course racing", + "totalResults": "32800000", + "searchTerms": "obstacle course racing", + "count": 10, + "startIndex": 1, + "inputEncoding": "utf8", + "outputEncoding": "utf8", + "safe": "off", + "cx": "d488dea99655943fe" + } + ], + "nextPage": [ + { + "title": "Google Custom Search - obstacle course racing", + "totalResults": "32800000", + "searchTerms": "obstacle course racing", + "count": 10, + "startIndex": 11, + "inputEncoding": "utf8", + "outputEncoding": "utf8", + "safe": "off", + "cx": "d488dea99655943fe" + } + ] + }, + "context": { + "title": "Serge's Search" + }, + "searchInformation": { + "searchTime": 0.320836, + "formattedSearchTime": "0.32", + "totalResults": "32800000", + "formattedTotalResults": "32,800,000" + }, + "items": [ + { + "kind": "customsearch#result", + "title": "Obstacle course racing - Wikipedia", + "htmlTitle": "Obstacle course racing - Wikipedia", + "link": "https://en.wikipedia.org/wiki/Obstacle_course_racing", + "displayLink": "en.wikipedia.org", + "snippet": "Obstacle course racing (OCR) is a sport in which a competitor, traveling on foot, must overcome various physical challenges in the form of obstacles.", + "htmlSnippet": "Obstacle course racing (OCR) is a sport in which a competitor, traveling on foot, must overcome various physical challenges in the form of obstacles.", + "cacheId": "fFxpvxmZxD0J", + "formattedUrl": "https://en.wikipedia.org/wiki/Obstacle_course_racing", + "htmlFormattedUrl": "https://en.wikipedia.org/wiki/Obstacle_course_racing", + "pagemap": { + "hcard": [ + { + "fn": "Obstacle course racing" + } + ], + "metatags": [ + { + "referrer": "origin", + "og:image": "https://upload.wikimedia.org/wikipedia/commons/thumb/0/04/Obstacle_race_1.jpg/1200px-Obstacle_race_1.jpg", + "theme-color": "#eaecf0", + "og:image:width": "1200", + "og:type": "website", + "viewport": "width=device-width, initial-scale=1.0, user-scalable=yes, minimum-scale=0.25, maximum-scale=5.0", + "og:title": "Obstacle course racing - Wikipedia", + "og:image:height": "800", + "format-detection": "telephone=no" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "The OCR World Championships | October 5-9, 2023", + "htmlTitle": "The OCR World Championships | October 5-9, 2023", + "link": "https://ocrworldchampionships.com/", + "displayLink": "ocrworldchampionships.com", + "snippet": "Since its inaugural event in 2014, the OCR World Championships has served as the premier independent world championships for the sport of Obstacle Course\u00a0...", + "htmlSnippet": "Since its inaugural event in 2014, the OCR World Championships has served as the premier independent world championships for the sport of Obstacle Course ...", + "cacheId": "mPEIpV9HiuoJ", + "formattedUrl": "https://ocrworldchampionships.com/", + "htmlFormattedUrl": "https://ocrworldchampionships.com/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSF2Cm2Q4hULl27dxw0uj6fdWPq9OHbUH2IveTe1AOTRjexqTBH7XSjBNz9", + "width": "275", + "height": "183" + } + ], + "organization": [ + { + "logo": "https://ocrworld.wpenginepowered.com/wp-content/uploads/2020/04/home-logo.svg", + "url": "Home" + }, + { + "logo": "https://ocrworld.wpenginepowered.com/wp-content/uploads/2020/02/black-logo@2x.svg", + "url": "Home" + } + ], + "metatags": [ + { + "og:image": "https://ocrworldchampionships.com/wp-content/uploads/2019/09/49649536892_96af079960_c-1.jpg", + "og:type": "website", + "og:image:width": "799", + "twitter:card": "summary_large_image", + "og:site_name": "OCR World Championships", + "og:title": "The OCR World Championships | October 5-9, 2023", + "og:image:height": "533", + "og:image:type": "image/jpeg", + "msapplication-tileimage": "https://ocrworldchampionships.com/wp-content/uploads/2020/04/cropped-index-270x270.png", + "og:description": "The OCR World Championships are the only independent global championships for the sport of Obstacle Course Racing.", + "article:publisher": "https://www.facebook.com/OCRWorldChampionships", + "article:modified_time": "2023-01-19T15:35:31+00:00", + "viewport": "width=device-width, initial-scale=1", + "og:locale": "en_US", + "og:url": "https://ocrworldchampionships.com/" + } + ], + "cse_image": [ + { + "src": "https://ocrworldchampionships.com/wp-content/uploads/2019/09/49649536892_96af079960_c-1.jpg" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Tough Mudder USA | Mud Run & Obstacle Race", + "htmlTitle": "Tough Mudder USA | Mud Run & Obstacle Race", + "link": "https://toughmudder.com/", + "displayLink": "toughmudder.com", + "snippet": "From 5K to 15 miles, Tough Mudder is your best chance to test your teamwork, conquer best-in-class obstacles, and let your inner party animal go wild.", + "htmlSnippet": "From 5K to 15 miles, Tough Mudder is your best chance to test your teamwork, conquer best-in-class obstacles, and let your inner party animal go wild.", + "cacheId": "UlaL39uA6l4J", + "formattedUrl": "https://toughmudder.com/", + "htmlFormattedUrl": "https://toughmudder.com/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn3.gstatic.com/images?q=tbn:ANd9GcStUAnaZRlxnwzYiPT27KGDa3MGA9AMfxPDuyoU3QEGmNebU9lEYjeBO5E_", + "width": "348", + "height": "145" + } + ], + "metatags": [ + { + "og:image": "https://toughmudder.com/wp-content/uploads/2022/05/Worlds-Best_Desktop-HERO-web-scaled.jpg", + "og:type": "website", + "og:image:width": "2560", + "twitter:card": "summary_large_image", + "og:site_name": "Tough Mudder", + "og:title": "Tough Mudder USA | Mud Run & Obstacle Race", + "og:image:height": "1067", + "og:image:type": "image/jpeg", + "msapplication-tileimage": "https://toughmudder.com/wp-content/uploads/2021/12/Tough-Mudder-Favicon_new_2022.png", + "og:description": "From 5K to 15 miles, Tough Mudder is your best chance to test your teamwork, conquer best-in-class obstacles, and let your inner party animal go wild.", + "article:modified_time": "2023-07-11T11:22:38+00:00", + "viewport": "width=device-width, initial-scale=1", + "og:locale": "en_US", + "og:url": "https://toughmudder.com/" + } + ], + "cse_image": [ + { + "src": "https://toughmudder.com/wp-content/uploads/2022/05/Worlds-Best_Desktop-HERO-web-scaled.jpg" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Spartan Race | Become Unbreakable", + "htmlTitle": "Spartan Race | Become Unbreakable", + "link": "https://www.spartan.com/", + "displayLink": "www.spartan.com", + "snippet": "Spartan is an extreme wellness platform helping humans become UNBREAKABLE. Commit to Races, Shop Merchandise & Train to be Unbreakable.", + "htmlSnippet": "Spartan is an extreme wellness platform helping humans become UNBREAKABLE. Commit to Races, Shop Merchandise & Train to be Unbreakable.", + "cacheId": "dDikWufJvm8J", + "formattedUrl": "https://www.spartan.com/", + "htmlFormattedUrl": "https://www.spartan.com/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcT9gciiNiLHRw7mJHW0J44hXcTowm7EBxNEHDkf-0hOU2NRAfrNOJ91BtM", + "width": "301", + "height": "167" + } + ], + "organization": [ + { + "url": "Spartan Race | Home" + } + ], + "metatags": [ + { + "og:image": "https://cdn.shopify.com/s/files/1/1321/0493/files/Meta_Image_56f176eb-5c59-498a-b281-96d243d9a4f4.jpg?v=1677259204", + "theme-color": "#000000", + "og:type": "website", + "twitter:card": "summary_large_image", + "twitter:title": "Spartan Race | Commit To Your Unbreakable Journey", + "og:image:width": "1200", + "supported-color-schemes": "light", + "og:site_name": "Spartan Race", + "og:title": "Spartan Race | Commit To Your Unbreakable Journey", + "shopify-checkout-api-token": "ed9167d08d7523055fce0a3b16670119", + "og:image:height": "668", + "google": "notranslate", + "color-scheme": "light", + "og:description": "Spartan is an extreme wellness platform helping humans become UNBREAKABLE. Find a race near you and commit to an event. Shop performance & commemorative gear. Learn how to become the most elite version of yourself with our Unbreakable Blog.", + "og:image:secure_url": "https://cdn.shopify.com/s/files/1/1321/0493/files/Meta_Image_56f176eb-5c59-498a-b281-96d243d9a4f4.jpg?v=1677259204", + "facebook-domain-verification": "m6v0apekod6yg8riyrx3r19epu50wf", + "twitter:site": "@spartanrace", + "viewport": "width=device-width, initial-scale=1.0, height=device-height, minimum-scale=1.0, user-scalable=0", + "twitter:description": "Spartan is an extreme wellness platform helping humans become UNBREAKABLE. Find a race near you and commit to an event. Shop performance & commemorative gear. Learn how to become the most elite version of yourself with our Unbreakable Blog.", + "shopify-digital-wallet": "/13210493/digital_wallets/dialog", + "seomanager": "6.2", + "og:url": "https://www.spartan.com/" + } + ], + "cse_image": [ + { + "src": "https://cdn.shopify.com/s/files/1/1321/0493/files/Meta_Image_56f176eb-5c59-498a-b281-96d243d9a4f4.jpg?v=1677259204" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Rugged Maniac Obstacle Course Races", + "htmlTitle": "Rugged Maniac Obstacle Course Races", + "link": "https://ruggedmaniac.com/", + "displayLink": "ruggedmaniac.com", + "snippet": "Simply put, Rugged Maniac is the greatest obstacle course race ever concocted. With the most obstacles per mile of any race on the planet,\u00a0...", + "htmlSnippet": "Simply put, Rugged Maniac is the greatest obstacle course race ever concocted. With the most obstacles per mile of any race on the planet, ...", + "cacheId": "6QjJaHQUXfIJ", + "formattedUrl": "https://ruggedmaniac.com/", + "htmlFormattedUrl": "https://ruggedmaniac.com/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSYsXzCGOzxQb-pFODf1bBlyYCHN3rTA8Q7GadgXTq6XjAs0_zd87sKxXE", + "width": "303", + "height": "166" + } + ], + "metatags": [ + { + "og:image": "https://ruggedmaniac.com/wp-content/uploads/2021/11/Hero-image-01-1000px.png", + "viewport": "width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0", + "msapplication-tileimage": "https://ruggedmaniac.com/wp-content/uploads/2022/11/cropped-MicrosoftTeams-image-43-270x270.png" + } + ], + "cse_image": [ + { + "src": "https://ruggedmaniac.com/wp-content/uploads/2021/11/Hero-image-01-1000px.png" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Obstacle Course Races This Spring: Spartan/Tough Mudder ...", + "htmlTitle": "Obstacle Course Races This Spring: Spartan/Tough Mudder ...", + "link": "https://www.spartan.com/blogs/unbreakable-race-stories/obstacle-course-races-this-spring", + "displayLink": "www.spartan.com", + "snippet": "Nov 29, 2022 ... Get ready for a Spartan Sprint 5K obstacle race that will change your life. This is our signature race distance. A fast-paced adventure through\u00a0...", + "htmlSnippet": "Nov 29, 2022 ... Get ready for a Spartan Sprint 5K obstacle race that will change your life. This is our signature race distance. A fast-paced adventure through ...", + "cacheId": "UiOhrUUJrLIJ", + "formattedUrl": "https://www.spartan.com/blogs/...race.../obstacle-course-races-this-spring", + "htmlFormattedUrl": "https://www.spartan.com/blogs/...race.../obstacle-course-races-this-spring", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn1.gstatic.com/images?q=tbn:ANd9GcSUkHZiXR47pzdKTIkmqnoLXbuuOuaB4iw2tn-g-LeoJCEwQ_RadMhagVOl", + "width": "300", + "height": "168" + } + ], + "organization": [ + { + "url": "Spartan Race | Home" + } + ], + "metatags": [ + { + "og:image": "https://www.spartan.com/cdn/shop/articles/220506MT-Saturday-MBahrenfuss-117_1024x1024.jpg?v=1669750807", + "theme-color": "#000000", + "og:type": "article", + "twitter:card": "summary_large_image", + "twitter:title": "22 Epic Obstacle Course Races You NEED to Try This Spring", + "supported-color-schemes": "light", + "og:site_name": "Spartan Race", + "og:title": "22 Epic Obstacle Course Races You NEED to Try This Spring", + "shopify-checkout-api-token": "ed9167d08d7523055fce0a3b16670119", + "google": "notranslate", + "color-scheme": "light", + "og:description": "Ready to challenge yourself this year and get out of your comfort zone? Here is our official 2023 schedule of obstacle course races this spring race season.", + "og:image:secure_url": "https://www.spartan.com/cdn/shop/articles/220506MT-Saturday-MBahrenfuss-117_1024x1024.jpg?v=1669750807", + "facebook-domain-verification": "m6v0apekod6yg8riyrx3r19epu50wf", + "twitter:site": "@spartanrace", + "viewport": "width=device-width, initial-scale=1.0, height=device-height, minimum-scale=1.0, user-scalable=0", + "twitter:description": "Ready to challenge yourself this year and get out of your comfort zone? Here is our official 2023 schedule of obstacle course races this spring race season.", + "shopify-digital-wallet": "/13210493/digital_wallets/dialog", + "seomanager": "6.2", + "og:url": "https://www.spartan.com/blogs/unbreakable-race-stories/obstacle-course-races-this-spring" + } + ], + "cse_image": [ + { + "src": "https://www.spartan.com/cdn/shop/articles/220506MT-Saturday-MBahrenfuss-117_1024x1024.jpg?v=1669750807" + } + ], + "article": [ + { + "articlebody": "22 Epic Obstacle Course Races You NEED to Try This Spring By The Spartan Editors \u00b7 November 29, 2022 Share Facebook Share on Facebook Tweet Tweet on Twitter Pin it Pin on Pinterest Whatsapp..." + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Obstacle Course Races - The Best Events for Runners to Try", + "htmlTitle": "Obstacle Course Races - The Best Events for Runners to Try", + "link": "https://www.runnersworld.com/training/g22144071/best-obstacle-course-races/", + "displayLink": "www.runnersworld.com", + "snippet": "Oct 21, 2021 ... Obstacle Course Races Every Runner Should Try \u00b7 Spartan Race \u00b7 Tough Mudder \u00b7 BoneFrog Challenge \u00b7 Savage Race \u00b7 Epic Series Obstacle Challenge.", + "htmlSnippet": "Oct 21, 2021 ... Obstacle Course Races Every Runner Should Try · Spartan Race · Tough Mudder · BoneFrog Challenge · Savage Race · Epic Series Obstacle Challenge.", + "cacheId": "EiYE335ZgMEJ", + "formattedUrl": "https://www.runnersworld.com/training/.../best-obstacle-course-races/", + "htmlFormattedUrl": "https://www.runnersworld.com/training/.../best-obstacle-course-races/", + "pagemap": { + "speakablespecification": [ + { + "cssselector": ".content-hed" + } + ], + "metatags": [ + { + "og:image": "https://hips.hearstapps.com/hmg-prod/images/gettyimages-158386364-1634824513.jpg?crop=1.00xw:0.553xh;0,0.0742xh&resize=1200:*", + "theme-color": "#000000", + "og:image:width": "1200", + "twitter:card": "summary_large_image", + "article:published_time": "2021-10-21T14:28:00Z", + "og:site_name": "Runner's World", + "sailthru.tags": "training,Training", + "sailthru.excerpt": "

Bored with your same old marathon training? Well, there's good news. There\u2019s a way to breathe some new life into your running regimen\u2014signing up for an obstacle course race.

These races take regular running courses and pepper them with obstacle stations\u2014everything from monkey bars to penalty burpees\u2014that challenge every muscle in your body. Think of obstacle races as a happy blend of cardio and strength training that provide you with an unforgettable race-day experience.

\u2192Learn how to prep for your first OCR here!

Many obstacle course races faced the impact of COVID-19 with cancellations or postponements, but have since returned to form, ", + "sailthru.contenttype": "listicle", + "title": "Obstacle Course Races - The Best Events for Runners to Try", + "og:description": "Get ready to challenge your body in a whole new way.", + "article:publisher": "https://www.facebook.com/RunnersWorld/", + "twitter:image": "https://hips.hearstapps.com/hmg-prod/images/gettyimages-158386364-1634824513.jpg?crop=1.00xw:0.553xh;0,0.0742xh&resize=640:*", + "next-head-count": "47", + "msapplication-tap-highlight": "no", + "twitter:site": "@runnersworld", + "article:modified_time": "2023-02-06T17:13:06Z", + "sailthru.socialtitle": "Anyone Who Considers Themselves a Runner Needs to Try These 6 Obstacle Course Races", + "sailthru.date": "2021-10-21 14:28:00", + "og:type": "website", + "thumbnail": "https://hips.hearstapps.com/hmg-prod/images/gettyimages-158386364-1634824513.jpg?crop=0.909xw:1.00xh;0.0465xw,0&resize=320:*", + "article:section": "Training", + "x-ua-compatible": "IE=edge,chrom=1", + "m1": ".content-hed", + "m2": ".content-dek p", + "og:title": "Anyone Who Considers Themselves a Runner Needs to Try These 6 Obstacle Course Races", + "auto-publish": "never", + "og:image:height": "600", + "sailthru.image.thumb": "https://hips.hearstapps.com/hmg-prod/images/gettyimages-158386364-1634824513.jpg?crop=0.909xw:1.00xh;0.0465xw,0", + "fb:app_id": "424005050993003", + "viewport": "width=device-width, initial-scale=1.0", + "og:url": "https://www.runnersworld.com/training/g22144071/best-obstacle-course-races/", + "sailthru.image.full": "https://hips.hearstapps.com/hmg-prod/images/gettyimages-158386364-1634824513.jpg?crop=0.909xw:1.00xh;0.0465xw,0&resize=320:*", + "article:opinion": "false" + } + ], + "listitem": [ + { + "item": "Training", + "name": "Training", + "position": "1" + }, + { + "item": "Obstacle Course Races Every Runner Should Try", + "name": "Obstacle Course Races Every Runner Should Try", + "position": "2" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "USAOCR, the National Governing Body for Obstacle Course Racing", + "htmlTitle": "USAOCR, the National Governing Body for Obstacle Course Racing", + "link": "https://usaocr.org/", + "displayLink": "usaocr.org", + "snippet": "USA Obstacle Course Racing (USAOCR) is the National Governing Body for Obstacle Course Racing (OCR) and events in the United States of America.", + "htmlSnippet": "USA Obstacle Course Racing (USAOCR) is the National Governing Body for Obstacle Course Racing (OCR) and events in the United States of America.", + "cacheId": "2JBQXMDob-AJ", + "formattedUrl": "https://usaocr.org/", + "htmlFormattedUrl": "https://usaocr.org/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQOGGeE3ZjiRHFnUVuHrcRfXfkWmgRX6D2ul3PwJdOfX1I0haetWrxU5Uo", + "width": "248", + "height": "204" + } + ], + "metatags": [ + { + "og:image": "https://img1.wsimg.com/isteam/ip/9ce670b9-0b57-4abd-8b63-79c2c0b7ef54/USAOCR.png", + "og:type": "website", + "twitter:card": "summary", + "twitter:title": "USAOCR", + "theme-color": "#2175FF", + "og:site_name": "USAOCR", + "author": "USAOCR", + "og:title": "USAOCR", + "og:description": "The National Governing Body for Obstacle Course Racing\n in the United States of America", + "twitter:image": "https://img1.wsimg.com/isteam/ip/9ce670b9-0b57-4abd-8b63-79c2c0b7ef54/USAOCR.png", + "twitter:image:alt": "USAOCR", + "viewport": "width=device-width, initial-scale=1", + "twitter:description": "USA Obstacle Course Racing", + "og:locale": "en_US", + "og:url": "https://usaocr.org/" + } + ], + "cse_image": [ + { + "src": "https://img1.wsimg.com/isteam/ip/9ce670b9-0b57-4abd-8b63-79c2c0b7ef54/USAOCR.png" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "\u200b7 Obstacle Course Races That Will Seriously Test Your Fitness ...", + "htmlTitle": "\u200b7 Obstacle Course Races That Will Seriously Test Your Fitness ...", + "link": "https://www.menshealth.com/fitness/g19524677/7-obstacle-course-races-to-try/", + "displayLink": "www.menshealth.com", + "snippet": "Jul 10, 2017 ... 7 Obstacle Course Races That Will Seriously Test Your Fitness ; Spartan Race. spartan race ; Zombie Mud Run. zombie mud run ; Warrior Dash. warrior\u00a0...", + "htmlSnippet": "Jul 10, 2017 ... 7 Obstacle Course Races That Will Seriously Test Your Fitness ; Spartan Race. spartan race ; Zombie Mud Run. zombie mud run ; Warrior Dash. warrior ...", + "cacheId": "K5B73v_SI2gJ", + "formattedUrl": "https://www.menshealth.com/fitness/.../7-obstacle-course-races-to-try/", + "htmlFormattedUrl": "https://www.menshealth.com/fitness/.../7-obstacle-course-races-to-try/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcRs2QhRr67aEa-oVOWz2lVntxI_9FW_C_TePZUTAZuQi14YCm4LpmCs1Q4", + "width": "282", + "height": "179" + } + ], + "thumbnail": [ + { + "src": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-warrior-dash-1506732588.jpg?crop=0.636xw:1xh;center,top&resize=320:*" + } + ], + "speakablespecification": [ + { + "cssselector": ".content-hed" + } + ], + "metatags": [ + { + "og:image": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-warrior-dash-1506732588.jpg?crop=1xw:0.786xh;center,top&resize=1200:*", + "theme-color": "#000000", + "og:image:width": "1200", + "twitter:card": "summary_large_image", + "article:published_time": "2017-07-10T18:30:52Z", + "og:site_name": "Men's Health", + "sailthru.tags": "fitness,Fitness", + "sailthru.excerpt": "

Here's my belief: the notion of straight-up running kind of sucks to most of us. Why spend money on running around on hot pavement when you could just... not? (To be fair, this study says an hour of running could actually extend your life) But if you want to get out and enjoy the outdoors while getting a serious workout in, it may be high time you look into signing up for an obstacle course race.

\n

The appeal of obstacle course racing, to most, is that you hardly notice you're running, because you're more focused on dealing with intense, oftentimes military-inspired obstacles that will seriously challenge your mind and body.

\n

Signing up for a race can be a great motivator to get to the gym for training so you don't completely die out on the course. Signing u", + "sailthru.contenttype": "gallery", + "title": "\u200b7 Obstacle Course Races That Will Seriously Test Your Fitness | Men\u2019s Health", + "og:description": "\u200bThese races will make you forget you're running and actually kick your butt", + "article:publisher": "https://www.facebook.com/MensHealth", + "twitter:image": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-warrior-dash-1506732588.jpg?crop=1xw:0.786xh;center,top&resize=640:*", + "next-head-count": "47", + "msapplication-tap-highlight": "no", + "twitter:site": "@MensHealthMag", + "article:modified_time": "2022-06-03T18:08:17Z", + "sailthru.socialtitle": "7 Obstacle Course Races That Will Seriously Test Your Fitness", + "sailthru.date": "2017-07-10 18:30:52", + "og:type": "website", + "thumbnail": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-warrior-dash-1506732588.jpg?crop=0.636xw:1xh;center,top&resize=320:*", + "article:section": "Fitness", + "x-ua-compatible": "IE=edge,chrom=1", + "m1": ".content-hed", + "m2": ".content-dek p", + "og:title": "7 Obstacle Course Races That Will Seriously Test Your Fitness", + "og:image:height": "600", + "sailthru.image.thumb": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-warrior-dash-1506732588.jpg?crop=0.636xw:1xh;center,top", + "fb:app_id": "128455567236258", + "viewport": "width=device-width, initial-scale=1.0", + "og:url": "https://www.menshealth.com/fitness/g19524677/7-obstacle-course-races-to-try/", + "sailthru.image.full": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-warrior-dash-1506732588.jpg?crop=0.636xw:1xh;center,top&resize=320:*", + "article:opinion": "false" + } + ], + "cse_image": [ + { + "src": "https://hips.hearstapps.com/hmg-prod/images/701/obstacle-course-races-test-fitness-tough-mudder-1506732585.jpg?resize=480:*" + } + ], + "listitem": [ + { + "item": "Fitness", + "name": "Fitness", + "position": "1" + }, + { + "item": "7 Obstacle Course Races That Will Seriously Test Your Fitness", + "name": "7 Obstacle Course Races That Will Seriously Test Your Fitness", + "position": "2" + } + ] + } + }, + { + "kind": "customsearch#result", + "title": "Obstacle Racing Media: Homepage", + "htmlTitle": "Obstacle Racing Media: Homepage", + "link": "https://www.obstacleracingmedia.com/", + "displayLink": "www.obstacleracingmedia.com", + "snippet": "Spartan Tea The minds behind Spartan Race set out to make a tea for Spartans and came up with using the herb Sidiritis. Why did they\u2026 October 26, 2020\u00a0...", + "htmlSnippet": "Spartan Tea The minds behind Spartan Race set out to make a tea for Spartans and came up with using the herb Sidiritis. Why did they\u2026 October 26, 2020 ...", + "cacheId": "RzzvLotsrWMJ", + "formattedUrl": "https://www.obstacleracingmedia.com/", + "htmlFormattedUrl": "https://www.obstacleracingmedia.com/", + "pagemap": { + "cse_thumbnail": [ + { + "src": "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcSqo9J2wZVN7FIfPa-esF2I7nXGIPXpL0KQ6V2Mp5A-ck16x23w3z8H53ER", + "width": "300", + "height": "168" + } + ], + "metatags": [ + { + "application-name": "Obstacle Racing Media", + "msapplication-tilecolor": "#FFFFFF", + "og:type": "website", + "twitter:card": "summary", + "twitter:title": "Homepage - Obstacle Racing Media", + "msapplication-square70x70logo": "https://obstacleracingmedia.com/mstile-70x70.png", + "og:site_name": "Obstacle Racing Media", + "tec-api-origin": "https://www.obstacleracingmedia.com", + "og:title": "Homepage - Obstacle Racing Media", + "msapplication-wide310x150logo": "https://obstacleracingmedia.com/mstile-310x150.png", + "msapplication-tileimage": "https://obstacleracingmedia.com/mstile-144x144.png", + "twitter:creator": "@ObstacleMedia", + "msapplication-square150x150logo": "https://obstacleracingmedia.com/mstile-150x150.png", + "twitter:site": "@ObstacleMedia", + "viewport": "width=device-width, initial-scale=1", + "msapplication-square310x310logo": "https://obstacleracingmedia.com/mstile-310x310.png", + "og:locale": "en_US", + "og:url": "https://www.obstacleracingmedia.com/", + "tec-api-version": "v1" + } + ], + "cse_image": [ + { + "src": "https://i.ytimg.com/vi/nMAHk__TVbk/mqdefault.jpg" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/test/t_connectors/t_customsearch.py b/test/t_connectors/t_customsearch.py new file mode 100644 index 0000000..aa63578 --- /dev/null +++ b/test/t_connectors/t_customsearch.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: MIT + +''' +Data connector for Google Custom Search API - testing support. +''' + +from os import path +import json +from facere_sensum.connectors import customsearch + +# Load mock response from the Custom Search API. +with open(path.join('test', 'input', 'customsearch.json'), encoding='utf-8') as cs_file: + _cse_result = json.load(cs_file) + +def _mock_up_data(): + ''' + Mock Google Custom Search API response. + ''' + customsearch.invoke_cse = lambda term, start: _cse_result + +def _test(metric, expected_raw, expected_value): + ''' + Test Google Custom Search API data connector with one of the test metrics + against its expected outcomes. + Return True if the test was successful, False otherwise. + ''' + _mock_up_data() + if customsearch.get_raw(metric) != expected_raw: + return False + + _mock_up_data() + return customsearch.get_value(metric) == expected_value + +def test(): + ''' + Test Google Custom Search API data connector. + Return True if the test was successful, False otherwise. + ''' + # Metric from examples. + with open(path.join('examples', 'config_customsearch.json'), encoding='utf-8') as config_file: + metric1 = json.load(config_file)['metrics'][1] + + # Metric to test for target URL that doesn't appear in search results. + metric2 = { + 'id': 'obstacle course racing', + 'source': 'customsearch', + 'num': 10, + 'URL': 'https://www.notfound.com/' + } + + return _test(metric1, 4, 0.94) & _test(metric2, 0, 0) diff --git a/test/t_connectors/t_user.py b/test/t_connectors/t_user.py deleted file mode 100644 index 884560c..0000000 --- a/test/t_connectors/t_user.py +++ /dev/null @@ -1,15 +0,0 @@ -# SPDX-License-Identifier: MIT - -''' -Data connector for direct user input - testing support. -''' - -import facere_sensum.connectors.user as user_connector - -def mock_up_data(data): - ''' - Mocks direct user input. - 'data' is a list of values to be used instead of the actual user input. - ''' - data = (item for item in data) - user_connector.get_value = lambda metric: next(data) diff --git a/test/test.py b/test/test.py index 4502f08..b52fd5a 100644 --- a/test/test.py +++ b/test/test.py @@ -7,11 +7,12 @@ import os import sys import subprocess # nosec B404 +import importlib import json import shutil import unittest -from t_connectors import t_user from facere_sensum import fs +from facere_sensum.connectors import user as user_connector # Generate paths for test files. _CONFIG_PATH = os.path.join('examples', 'config_personal.json') @@ -32,6 +33,14 @@ def _logs_equal(log1, log2): with open(log2, encoding='utf8') as file2: return file1.readlines() == file2.readlines() +def _mock_up_direct_user_input(data): + ''' + Mock direct user input. + 'data' is a list of values to be used instead of the actual user input. + ''' + data = (item for item in data) + user_connector.get_value = lambda metric: next(data) + class Test(unittest.TestCase): ''' Test cases. @@ -51,20 +60,33 @@ def test_score_combined(self): shutil.copy(_REF_BASE_PATH, _LOG_PATH) # Minimal extreme: all metrics are zero. - t_user.mock_up_data([0,0,0]) + _mock_up_direct_user_input([0,0,0]) fs.command_update(_CONFIG, _LOG_PATH, 'A') # Maximal extreme: all metrics are one. - t_user.mock_up_data([1,1,1]) + _mock_up_direct_user_input([1,1,1]) fs.command_update(_CONFIG, _LOG_PATH, 'B') # Various values for metrics. - t_user.mock_up_data([.25,.5,.75]) + _mock_up_direct_user_input([.25,.5,.75]) fs.command_update(_CONFIG, _LOG_PATH, 'C') # Compare with a reference. self.assertTrue(_logs_equal(_LOG_PATH, _REF_UPDATED_PATH)) + def test_connectors(self): + ''' + Test data connectors. + ''' + # Load sample authentication config file so that all the connectors can load. + # Connector loading only needs JSON scheme, not actual credentials. + with open('auth.json', encoding='utf-8') as auth_file: + fs.auth = json.load(auth_file) + + for file_name in os.listdir(os.path.join('test', 't_connectors')): + if file_name.startswith('t_') and file_name.endswith('.py'): + self.assertTrue(importlib.import_module('t_connectors.t_'+file_name[2:-3]).test()) + def _test_integration(descr, args, ref): ''' Run an integration test. @@ -73,8 +95,7 @@ def _test_integration(descr, args, ref): 'ref' expected output. ''' print(descr, end=': ') - res = subprocess.run(['python', - os.path.join('src', 'facere_sensum', 'fs.py')] + args, + res = subprocess.run(['python', 'fsy.py'] + args, check=False, capture_output=True, text=True).stdout # nosec B603 if res == ref: print('OK')