update etl scripts

etl/scripts/etl.py

@@ -19,7 +19,7 @@
 source_dir = '../source/'
 output_dir = '../../'
 
-data_csv = os.path.join(source_dir, 'WDIData.csv')
+data_csv = os.path.join(source_dir, 'WDICSV.csv')
 country_csv = os.path.join(source_dir, 'WDICountry.csv')
 series_csv = os.path.join(source_dir, 'WDISeries.csv')
 groups_xls = os.path.join(source_dir, 'CLASS.xlsx')
@@ -53,7 +53,7 @@ def extract_economy_entities(countries: pd.DataFrame, domains: pd.DataFrame, gro
                                       sets=sets_list,
                                       props={'name': name})
 
-    grouped = groups.groupby(by=['CountryCode'])
+    grouped = groups.groupby(by='CountryCode')
     for eco, df in grouped:
         eco_groups = df['GroupCode'].values.tolist()
         eco_id = to_concept_id(eco)

etl/scripts/update_source.py

@@ -4,13 +4,19 @@
 
 from zipfile import ZipFile
 # from ddf_utils.factory import WorldBankLoader
-from ddf_utils.factory.common import download
+# from ddf_utils.factory.common import download
+import requests
+
+
+def download(url, outpath):
+    response = requests.get(url, allow_redirects=True)
+    open(outpath, 'wb').write(response.content)
 
 
 source_dir = '../source/'
 # classification files
 # see https://datahelpdesk.worldbank.org/knowledgebase/articles/906519-world-bank-country-and-lending-groups
-zip_file = "http://databank.worldbank.org/data/download/WDI_csv.zip"
+zip_file = "https://databank.worldbank.org/data/download/WDI_CSV.zip"
 url_class_xls = 'http://databank.worldbank.org/data/download/site-content/CLASS.xlsx'
 url_oghist_xls = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xlsx'