add logging function to this tool

Author: yuukidach

.gitignore

@@ -194,5 +194,6 @@ Temporary Items
 secret.yaml
 dataset/*
 .vscode/
+*.log
 
 # End of https://www.toptal.com/developers/gitignore/api/python,venv,vscode

src/twitchanal/cli.py

@@ -1,4 +1,6 @@
 import click
+import logging
+import os
 from twitchanal.secret.secret import save_id_secret
 from twitchanal.collect.save import collect_data
 
@@ -37,9 +39,22 @@ def save_user(dir):
     help=
     'Whether to collect extra info like `peek viewers`, `peek channels` and so on for top games.'
 )
-def collect(dir: str, timestamp: bool, num: int, extra: bool):
+@click.option('--debug/--no-debug',
+              default=False,
+              help='Run in debug mode or not.')
+def collect(dir: str, timestamp: bool, num: int, extra: bool, debug: bool):
     """ Collect data for analysis
     """
+    try:
+        os.remove('twitchanal.log')
+    except OSError:
+        pass
+
+    if debug:
+        lv = logging.DEBUG
+    else:
+        lv = logging.INFO
+    logging.basicConfig(filename='twitchanal.log', level=lv)
     collect_data(dir, timestamp, num, extra)
 
 

src/twitchanal/collect/fetch.py

@@ -2,6 +2,7 @@
 import requests
 import time
 import random
+import logging
 from typing import List
 from termcolor import colored, cprint
 from twitchAPI import Twitch
@@ -97,7 +98,7 @@ def fetch_url(url: str, hint: str = ""):
         page = requests.get(url, headers=HAEDER)
         if page.status_code == 200:
             break
-        print('Busy. Try again...')
+        logging.info(url.split('/')[-1] + ' busy. Try again...')
         time.sleep(random.uniform(1.6, 3.0))
 
     html = BeautifulSoup(page.text, 'html.parser')

src/twitchanal/collect/save.py

@@ -1,6 +1,7 @@
 import os
 import time
 import pandas as pd
+import logging
 from typing import NoReturn
 from twitchAPI.twitch import Twitch
 from twitchAPI.oauth import UserAuthenticator