-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* new version * fix the prod to local script * first attempt at adding environments * Pretty much works but need to look at github API backoff because that's failing now * a couple of fixes - may improve performance * refactoring the environments to use helm config as the deciding factor * correct the summary * add more to the summary * bit more info * finally no major errors (I think) * bit of light document updating * add some docco * mystery solved - alertSeverity was missing -prod * add force option (maybe run it once a week?) * tidying up the scripts a bit * ignore .venv directory * tweaks to various environment bits * remove environment if there aren't any for a component * timezone oddities * retain existing alertmanager config if it's not available * tidy up unnecessary bits * crontab bits * tidy up some bits and update the README.md * add functionality to preserve non-discovered fields (build_image_tag) * retain build_image_tag * fix circleci exception * add the dockerfile components
- Loading branch information
1 parent
6239623
commit dff4494
Showing
35 changed files
with
5,318 additions
and
1,504 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,10 @@ | ||
| # dotenv environment variables file | ||
| .env* | ||
|
|
||
| .venv/ | ||
| .python-version | ||
| .idea | ||
| .vscode | ||
| **/Chart.lock | ||
| __pycache__/ | ||
| **/.DS_Store | ||
| github_discovery.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| import os | ||
| from classes.github import GithubSession | ||
|
|
||
| gh_params = { | ||
| 'app_id': int(os.getenv('GITHUB_APP_ID')), | ||
| 'app_installation_id': int(os.getenv('GITHUB_APP_INSTALLATION_ID')), | ||
| 'app_private_key': os.getenv('GITHUB_APP_PRIVATE_KEY'), | ||
| } | ||
| gh = GithubSession(gh_params) | ||
| print(gh.get_rate_limit()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| import requests | ||
| import yaml | ||
| import json | ||
| import logging | ||
|
|
||
|
|
||
| class AlertmanagerData: | ||
| def __init__(self, am_params, log_level=logging.INFO): | ||
| # Needs custom logging because of a bit of a mess later on | ||
| logging.basicConfig( | ||
| format='[%(asctime)s] %(levelname)s %(threadName)s %(message)s', level=log_level | ||
| ) | ||
| self.log = logging.getLogger(__name__) | ||
| self.url = am_params['url'] | ||
| self.get_alertmanager_data() | ||
|
|
||
| def get_alertmanager_data(self): | ||
| self.json_config_data = None | ||
| try: | ||
| response = requests.get(self.url, verify=False, timeout=5) | ||
| if response.status_code == 200: | ||
| alertmanager_data = response.json() | ||
| config_data = alertmanager_data['config'] | ||
| formatted_config_data = config_data['original'].replace('\\n', '\n') | ||
| yaml_config_data = yaml.safe_load(formatted_config_data) | ||
| self.json_config_data = json.loads(json.dumps(yaml_config_data)) | ||
| self.log.info('Successfully fetched Alertmanager data') | ||
| else: | ||
| self.log.error(f'Error: {response.status_code}') | ||
|
|
||
| except requests.exceptions.SSLError as e: | ||
| self.log.error(f'SSL Error: {e}') | ||
|
|
||
| except requests.exceptions.RequestException as e: | ||
| self.log.error(f'Request Error: {e}') | ||
|
|
||
| except json.JSONDecodeError as e: | ||
| self.log.error(f'JSON Decode Error: {e}') | ||
|
|
||
| except Exception as e: | ||
| self.log.error(f'Error getting data from Alertmanager: {e}') | ||
|
|
||
| def find_channel_by_severity_label(self, alert_severity_label): | ||
| # Find the receiver name for the given severity | ||
| receiver_name = '' | ||
| if self.json_config_data is None: | ||
| return '' | ||
|
|
||
| for route in self.json_config_data['route']['routes']: | ||
| if route['match'].get('severity') == alert_severity_label: | ||
| receiver_name = route['receiver'] | ||
| break | ||
| # Find the channel for the receiver name | ||
| if receiver_name: | ||
| for receiver in self.json_config_data['receivers']: | ||
| if receiver['name'] == receiver_name: | ||
| slack_configs = receiver.get('slack_configs', []) | ||
| if slack_configs: | ||
| return slack_configs[0].get('channel') | ||
| else: | ||
| return '' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| import requests | ||
| import logging | ||
| from includes.utils import update_dict | ||
|
|
||
|
|
||
| class CircleCI: | ||
| def __init__(self, params, log_level=logging.INFO): | ||
| # Needs custom logging because of a bit of a mess later on | ||
| logging.basicConfig( | ||
| format='[%(asctime)s] %(levelname)s %(threadName)s %(message)s', level=log_level | ||
| ) | ||
| self.log_level = log_level | ||
| self.log = logging.getLogger(__name__) | ||
| self.url = params['url'] | ||
| self.headers = { | ||
| 'Circle-Token': params['token'], | ||
| 'Content-Type': 'application/json', | ||
| 'Accept': 'application/json', | ||
| } | ||
|
|
||
| def test_connection(self): | ||
| try: | ||
| response = requests.get( | ||
| f'{self.url}/hmpps-project-bootstrap', headers=self.headers, timeout=10 | ||
| ) | ||
| response.raise_for_status() | ||
| self.log.info(f'CircleCI API: {response.status_code}') | ||
| return True | ||
| except Exception as e: | ||
| self.log.critical(f'Unable to connect to the CircleCI API: {e}') | ||
| return None | ||
|
|
||
| def get_trivy_scan_json_data(self, project_name): | ||
| self.log.debug(f'Getting trivy scan data for {project_name}') | ||
|
|
||
| project_url = f'{self.url}{project_name}' | ||
| output_json_content = {} | ||
| try: | ||
| response = requests.get(project_url, headers=self.headers, timeout=30) | ||
| artifacts_url = None | ||
| for build_info in response.json(): | ||
| workflows = build_info.get('workflows', {}) | ||
| workflow_name = workflows.get('workflow_name', {}) | ||
| job_name = build_info.get('workflows', {}).get('job_name') | ||
| if workflow_name == 'security' and job_name == 'hmpps/trivy_latest_scan': | ||
| latest_build_num = build_info['build_num'] | ||
| artifacts_url = f'{project_url}/{latest_build_num}/artifacts' | ||
| break | ||
|
|
||
| if artifacts_url: | ||
| self.log.debug('Getting artifact URLs from CircleCI') | ||
| response = requests.get(artifacts_url, headers=self.headers, timeout=30) | ||
|
|
||
| artifact_urls = response.json() | ||
| if output_json_url := next( | ||
| ( | ||
| artifact['url'] | ||
| for artifact in artifact_urls | ||
| if 'results.json' in artifact['url'] | ||
| ), | ||
| None, | ||
| ): | ||
| self.log.debug('Fetching artifacts from CircleCI data') | ||
| # do not use DEBUG logging for this request | ||
| logging.getLogger('urllib3').setLevel(logging.INFO) | ||
| response = requests.get(output_json_url, headers=self.headers, timeout=30) | ||
| logging.getLogger('urllib3').setLevel(self.log_level) | ||
| output_json_content = response.json() | ||
|
|
||
| except Exception as e: | ||
| self.log.debug(f'Error: {e}') | ||
|
|
||
| return output_json_content | ||
|
|
||
| def get_circleci_orb_version(self, circleci_config): | ||
| versions_data = {} | ||
| try: | ||
| cirleci_orbs = circleci_config['orbs'] | ||
| for key, value in cirleci_orbs.items(): | ||
| if 'ministryofjustice/hmpps' in value: | ||
| hmpps_orb_version = value.split('@')[1] | ||
| update_dict(versions_data, 'circleci', {'hmpps_orb': hmpps_orb_version}) | ||
| self.log.debug(f'hmpps orb version: {hmpps_orb_version}') | ||
| except Exception: | ||
| self.log.debug('No hmpps orb version found') | ||
| return versions_data |
Oops, something went wrong.