269-deliver-notifications: Adds alerts with Gov UK Notify

269-deliver-notifications: Adds elastalert custom alerter tests

.env.example

@@ -0,0 +1,5 @@
+export GITHUB_WEBHOOKSECRET=blah
+export ELASTICSEARCH_URL="http://localhost:9200"
+export GOVUK_NOTIFY_API_KEY=<_update_me_>
+export GOVUK_NOTIFY_TEMPLATE_ID=<_update_me_>
+export NOTIFICATION_EMAILS=[email protected]

.gitignore

@@ -1,4 +1,4 @@
-.env*
+.env
 
 scanrepo
 release

README.md

@@ -46,21 +46,44 @@ these/pems/are/ok/*.pem
 
 
 -----------------------------------------------------------
-### Notifications
+## Notifications
 Work in progress.
 
-#### Local testing of notifications
+
+### Local Testing
+
+#### Set environment variables needed
+Create `env` file and update environment variables.
+```
+cp .env{.example,}
+vi .env
 ```
-# launch server and elasticsearch
-. .env && docker-compose up -d server elasticsearch
+
+#### Launch containers
+```
+# launch server first
+. .env && docker-compose up -d server
 
 # Elastalert expects the elasticsearch index it is monitoring to exist otherwise it will error. The server creates the index in elasticsearch the first time it writes a log.
 make test-run-offenses
 
 # launch elastalert
-docker-compose up -d elastalert
+docker-compose up elastalert
+```
+
+#### Debugging
+```
+docker exec -it <elastalert_container_hash> sh
 
 # run elastalert test rule utility within elastalert container
-docker exec -it <elastlaert_container_hash> sh
 elastalert-test-rule --config $ELASTALERT_CONFIG --count-only "$RULES_DIRECTORY/new_violation.yaml"
+elastalert-test-rule --alert --config $ELASTALERT_CONFIG "$RULES_DIRECTORY/new_violation.yaml"
+
+# run elastalert in debug mode
+elastalert --config "$ELASTALERT_CONFIG" --rule "$RULES_DIRECTORY/new_violation.yaml" --debug
+```
+
+#### Logs
+```
+tail -f /log/elastalert_new_violation_rule.log
 ```

docker-compose.yml

@@ -30,12 +30,16 @@ services:
       retries: 5
 
   elastalert:
+    # privileged needed for ntpd
+    privileged: true
     build:
       context: elastalert
     environment:
+      - GOVUK_NOTIFY_API_KEY
+      - GOVUK_NOTIFY_TEMPLATE_ID
+      - NOTIFICATION_EMAILS
       - ELASTICSEARCH_HOST=elasticsearch
       - ELASTICSEARCH_PORT=9200
-      - [email protected],[email protected]
     depends_on:
       elasticsearch:
         condition: service_healthy

elastalert/.gitignore

@@ -0,0 +1,105 @@
+env
+
+# Created by https://www.gitignore.io/api/python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# End of https://www.gitignore.io/api/python

elastalert/Dockerfile

@@ -1,5 +1,12 @@
 FROM iron/python:2
 
+RUN mkdir -p /var/empty
+ # Set this environment variable to true to set timezone on container start.
+ENV SET_CONTAINER_TIMEZONE true
+# Default container timezone as found under the directory /usr/share/zoneinfo/.
+ENV CONTAINER_TIMEZONE Europe/London
+# URL from which to download Elastalert.
+
 WORKDIR /opt
 RUN apk update && \
     apk upgrade && \
@@ -16,16 +23,23 @@ RUN apk update && \
     mv e* elastalert
 
 # Install Elastalert.
-WORKDIR /opt/elastalert
+ENV ELASTALERT_ROOT /opt/elastalert
+WORKDIR ${ELASTALERT_ROOT}
 RUN python setup.py install && \
     pip install -e . && \
+    pip install notifications-python-client && \
     pip uninstall twilio --yes && \
     pip install twilio==6.0.0
 
+WORKDIR /opt
 ENV RULES_DIRECTORY /opt/rules
 COPY rules ${RULES_DIRECTORY}
 ENV ELASTALERT_CONFIG /opt/elastalert_config.yaml
 COPY elastalert_config.yaml ${ELASTALERT_CONFIG}
+COPY modules ${ELASTALERT_ROOT}/elastalert_modules
+
+
+
 
 COPY start-elastalert.sh /opt/
 RUN chmod +x /opt/start-elastalert.sh

elastalert/Makefile

@@ -0,0 +1,25 @@
+env:
+	@python3 -m venv env
+
+deps-update:
+	@pip install -r requirements-to-freeze.txt --upgrade
+	@pip freeze > requirements.txt
+
+deps:
+	@pip install -r requirements.txt
+	@pre-commit install
+
+clean:
+	@pip uninstall -yr requirements.txt
+	@pip freeze > requirements.txt
+
+autopep8:
+	@autopep8 . --recursive --in-place --pep8-passes 2000 --verbose
+
+autopep8-stats:
+	@pep8 --quiet --statistics .
+
+test:
+	@pytest tests -vv
+
+.PHONY: deps lint test* debug clean

elastalert/elastalert_config.yaml

@@ -1,7 +1,3 @@
-# This is the folder that contains the rule yaml files
-# Any .yaml file will be loaded as a rule
-rules_folder: /opt/rules
-
 # How often ElastAlert will query Elasticsearch
 # The unit can be anything from weeks to seconds
 run_every:
@@ -12,13 +8,6 @@ run_every:
 buffer_time:
   minutes: 15
 
-# The Elasticsearch hostname for metadata writeback
-# Note that every rule can have its own Elasticsearch host
-es_host:
-
-# The Elasticsearch port
-es_port:
-
 # The index on es_host which is used for metadata storage
 # This can be a unmapped index, but it is recommended that you run
 # elastalert-create-index to set a mapping

elastalert/email.tmpl

@@ -0,0 +1,11 @@
+Message: ((Message))
+Timestamp: ((Timestamp))
+
+Filename: ((Filename))
+Reason: ((Reason))
+Organisation: ((Organisation))
+Repo: ((Repo))
+URL: ((URL))
+
+Elasticsearch Index: ((ElasticsearchIndex))
+Elasticsearch ID: ((ElasticsearchId))

elastalert/modules/govuknotify.py

@@ -0,0 +1,68 @@
+import os
+from elastalert.alerts import Alerter, BasicMatchString
+from notifications_python_client.notifications import NotificationsAPIClient
+
+
+class GovNotifyAlerter(Alerter):
+
+    required_options = set(['log_file_path', 'email'])
+
+    def __init__(self, rule):
+        Alerter.__init__(self, rule)
+        self.template_id = os.environ['GOVUK_NOTIFY_TEMPLATE_ID']
+        self.email_addresses = os.environ['NOTIFICATION_EMAILS'].split(',')
+        api_key = os.environ['GOVUK_NOTIFY_API_KEY']
+        self.notifications_client = NotificationsAPIClient(api_key)
+
+    @staticmethod
+    def _generate_personalisation(match_items):
+        personalisation = {}
+        for i, v in enumerate(match_items):
+            if v[0] == 'Message':
+                personalisation['Message'] = v[1]
+            elif v[0] == 'Timestamp':
+                personalisation['Timestamp'] = v[1]
+            elif v[0] == '_index':
+                personalisation['ElasticsearchIndex'] = v[1]
+            elif v[0] == '_id':
+                personalisation['ElasticsearchId'] = v[1]
+            elif v[0] == 'Data':
+                personalisation['Filename'] = v[1]['filename']
+                personalisation['Reason'] = v[1]['reason']
+                personalisation['Organisation'] = v[1]['organisation']
+                personalisation['Repo'] = v[1]['repo']
+                personalisation['URL'] = v[1]['url']
+        return personalisation
+
+    def _send_notification(self, email_address, personalisation):
+        return self.notifications_client.send_email_notification(
+            email_address=email_address,
+            template_id=self.template_id,
+            personalisation=personalisation,
+            reference=None
+        )
+
+    def alert(self, matches):
+        # Matches is a list of match dictionaries.
+        # It contains more than one match when the alert has
+        # the aggregation option set
+        for match in matches:
+            personalisation = self._generate_personalisation(match.items())
+            for email_address in self.email_addresses:
+                self._send_notification(
+                    email_address, personalisation)
+
+            with open(self.rule['log_file_path'], 'a') as output_file:
+                # basic_match_string will transform the match into the default
+                # human readable string format
+                # https://github.com/Yelp/elastalert/blob/3931d7feaf0d07b6531fb53042b9284bb46712ce/elastalert/alerts.py#L128
+                match_string = str(BasicMatchString(self.rule, match))
+                output_file.write(match_string)
+
+    # get_info is called after an alert is sent to get
+    # data that is written back to Elasticsearch in the field "alert_info"
+    # It should return a dict of information relevant to what the alert does
+    def get_info(self):
+        return {'type': 'GovUK Notify Alerter',
+                'email': self.rule['email'],
+                'log_file_path': self.rule['log_file_path']}