diff --git a/.env.example b/.env.example index 74d8dd6..1c34dd2 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,8 @@ AWS_PROFILE=verify-audit-billing-dev RP_REPORT_OUTPUT_BUCKET=govukverify-hub-integration-billing-reports PIWIK_API_TOKEN= +GOOGLE_AUTH_PRIVATE_KEY_ID= +# GOOGLE_AUTH_PRIVATE_KEY=(cannot be set in `.env.`, must be defined in shell) +GOOGLE_AUTH_CLIENT_EMAIL= +GOOGLE_AUTH_CLIENT_ID= +GOOGLE_AUTH_CLIENT_CERT_URL= diff --git a/performance/config.py b/performance/config.py index a8f68c2..5499e6a 100644 --- a/performance/config.py +++ b/performance/config.py @@ -22,6 +22,8 @@ class Config: PIWIK_LIMIT = '-1' PIWIK_BASE_URL = 'https://analytics-hub-prod-a-dmz.ida.digital.cabinet-office.gov.uk/index.php' DEFAULT_OUTPUT_PATH = os.path.join(BASE_DIR, 'output') + # This is only used if Google auth credentials aren't already present in environment variables See + # `performance.gsheets.get_pygsheets_client` for implementation details. GSHEETS_CREDENTIALS_FILE = os.path.join( VERIFY_DATA_PIPELINE_CONFIG_PATH, 'credentials', 'google_sheets_credentials.json') diff --git a/performance/gsheets.py b/performance/gsheets.py new file mode 100644 index 0000000..b9e0bdf --- /dev/null +++ b/performance/gsheets.py @@ -0,0 +1,35 @@ +import json +import os +import pygsheets +import tempfile + +from performance.env import check_get_env +from performance import prod_config as config + + +def get_pygsheets_client(): + google_auth_private_key_id = os.getenv('GOOGLE_AUTH_PRIVATE_KEY_ID') + if not google_auth_private_key_id: + return pygsheets.authorize(service_file=config.GSHEETS_CREDENTIALS_FILE) + + creds = { + "type": "service_account", + "project_id": "verify-performance", + "private_key_id": google_auth_private_key_id, + # Private key cannot be set in `.env` at the moment and must therefore be defined in the + # shell when running locally. + "private_key": check_get_env('GOOGLE_AUTH_PRIVATE_KEY'), + "client_email": check_get_env('GOOGLE_AUTH_CLIENT_EMAIL'), + "client_id": check_get_env('GOOGLE_AUTH_CLIENT_ID'), + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": check_get_env('GOOGLE_AUTH_CLIENT_CERT_URL') + } + temp = tempfile.NamedTemporaryFile(delete=False, mode="w+t") + temp.write(json.dumps(creds)) + temp.close() + try: + return pygsheets.authorize(service_file=temp.name) + finally: + os.unlink(temp.name) diff --git a/performance/reports/rp.py b/performance/reports/rp.py index 99cd9a3..371513d 100644 --- a/performance/reports/rp.py +++ b/performance/reports/rp.py @@ -1,16 +1,14 @@ import os import pandas -import pygsheets +import performance import performance.piwik as piwik import performance.billing as billing - from performance import prod_config as config -import performance +from performance.gsheets import get_pygsheets_client from performance.reports.tests import conftest - RP_REPORT_COLUMNS = [ 'rp', 'all_referrals_with_intent', @@ -74,7 +72,7 @@ def test_upload(gsheets_key, date_start): # because if you want to test the upload process, you don't want to have to wait around # for Piwik - any data will do. config = performance.config.TestConfig() - pygsheets_client = pygsheets.authorize(service_file=config.GSHEETS_CREDENTIALS_FILE) + pygsheets_client = get_pygsheets_client() for rp_info in config.rp_information.values(): rp_info['sheet_key'] = gsheets_key @@ -119,7 +117,7 @@ def export_metrics_to_csv(df_export, report_output_path, date_start): def export_metrics_to_google_sheets(df_export, date_start): - pygsheets_client = pygsheets.authorize(service_file=config.GSHEETS_CREDENTIALS_FILE) + pygsheets_client = get_pygsheets_client() exporter = GoogleSheetsRelyingPartyReportExporter(config, pygsheets_client) exporter.export(df_export, column_heading=date_start)