Enhancements to JIRA v1 output

conf/lambda.json

@@ -54,7 +54,7 @@
     "concurrency_limit": 10,
     "memory": 128,
     "timeout": 300,
-    "file_format": null,
+    "file_format": "parquet",
     "log_level": "info"
   },
   "classifier_config": {},

conf/schemas/cloudtrail.json

@@ -22,26 +22,31 @@
   "cloudtrail:events": {
     "schema": {
       "additionalEventData": {},
+      "addendum": {},
       "apiVersion": "string",
       "awsRegion": "string",
       "errorCode": "string",
       "errorMessage": "string",
+      "eventCategory": "string",
       "eventID": "string",
       "eventName": "string",
       "eventSource": "string",
       "eventTime": "string",
       "eventType": "string",
       "eventVersion": "string",
+      "insightDetails": {},
       "managementEvent": "boolean",
       "readOnly": "boolean",
       "recipientAccountId": "string",
       "requestID": "string",
       "requestParameters": {},
       "resources": [],
       "responseElements": {},
+      "sessionCredentialFromConsole": "boolean",
       "serviceEventDetails": {},
       "sharedEventID": "string",
       "sourceIPAddress": "string",
+      "tlsDetails": {},
       "userAgent": "string",
       "userIdentity": {},
       "vpcEndpointId": "string"
@@ -50,17 +55,22 @@
     "configuration": {
       "json_path": "Records[*]",
       "optional_top_level_keys": [
+        "addendum",
         "additionalEventData",
         "apiVersion",
         "errorCode",
         "errorMessage",
+        "eventCategory",
+        "insightDetails",
         "managementEvent",
         "requestID",
         "readOnly",
         "resources",
         "serviceEventDetails",
+        "sessionCredentialFromConsole",
         "sharedEventID",
         "sourceIPAddress",
+        "tlsDetails",
         "userAgent",
         "vpcEndpointId"
       ]

docs/source/getting-started.rst

@@ -103,10 +103,6 @@ Deploy
   python manage.py configure aws_account_id 111111111111  # Replace with your 12-digit AWS account ID
   python manage.py configure prefix <value>               # Choose a unique name prefix (alphanumeric characters only)
 
-.. note::
-
-  * Update the ``file_format`` value in ``conf/lambda.json``. Valid options are ``parquet`` or ``json``. The default value will be parquet in a future release, but this must be manually configured at this time.
-
   .. code-block:: bash
 
     "athena_partitioner_config": {

docs/source/outputs.rst

@@ -45,7 +45,7 @@ Adding a new configuration for a currently supported service is handled using ``
 
   ``<SERVICE_NAME>`` above should be one of the following supported service identifiers.
   ``aws-cloudwatch-log``, ``aws-firehose``, ``aws-lambda``, ``aws-lambda-v2``, ``aws-s3``,
-  ``aws-sns``, ``aws-sqs``, ``carbonblack``, ``github``, ``jira``, ``komand``, ``pagerduty``,
+  ``aws-sns``, ``aws-sqs``, ``carbonblack``, ``github``, ``jira``, ``jira-v2``, ``komand``, ``pagerduty``,
   ``pagerduty-incident``, ``pagerduty-v2``, ``phantom``, ``slack``
 
 For example:

requirements.txt

@@ -39,10 +39,10 @@ boto==2.49.0
 botocore==1.17.29
 cachetools==4.1.1
 certifi==2020.6.20
-cffi==1.14.1
+cffi==1.14.4
 cfn-lint==0.34.0
 chardet==3.0.4
-cryptography==3.0
+cryptography==3.2
 decorator==4.4.2
 docker==4.2.2
 docopt==0.6.2
@@ -55,7 +55,7 @@ google-api-core==1.22.0
 google-auth==1.19.2
 google-auth-httplib2==0.0.4
 googleapis-common-protos==1.52.0
-httplib2==0.18.1
+httplib2==0.19.0
 idna==2.8
 imagesize==1.2.0
 importlib-metadata==1.7.0

rules/community/cloudwatch_events/cloudtrail_critical_api_calls.py

@@ -11,6 +11,7 @@
     'DeleteCluster',
     # CloudTrail
     'DeleteTrail',
+    'PutEventSelectors',
     'UpdateTrail',
     'StopLogging',
     # AWS Config

streamalert/__init__.py

@@ -1,2 +1,2 @@
 """StreamAlert version."""
-__version__ = '3.4.0'
+__version__ = '3.4.1'

streamalert/alert_processor/outputs/jira.py

@@ -44,6 +44,7 @@ class JiraOutput(OutputDispatcher):
     def __init__(self, *args, **kwargs):
         OutputDispatcher.__init__(self, *args, **kwargs)
         self._base_url = None
+        self._verify_ssl = False
         self._auth_cookie = None
 
     @classmethod
@@ -76,9 +77,11 @@ def get_user_defined_properties(cls):
              OutputProperty(description='the Jira password',
                             mask_input=True,
                             cred_requirement=True)),
+            # Example: "https://jira.mywebsite.com"
             ('url',
-             OutputProperty(description='the Jira url',
+             OutputProperty(description='the Jira REST API base url',
                             mask_input=True,
+                            input_restrictions={' '},  # include this or ":" will be invalid
                             cred_requirement=True)),
             ('project_key',
              OutputProperty(description='the Jira project key',
@@ -92,6 +95,41 @@ def get_user_defined_properties(cls):
              OutputProperty(description='the Jira aggregation behavior to aggregate '
                                         'alerts by rule name (yes/no)',
                             mask_input=False,
+                            cred_requirement=True)),
+            # When aggregation is enabled, it will fuzzy-search any JIRA ticket that best-matches
+            # the "summary ~ ..." statement, within the project key. For each matching rule,
+            # instead of creating new JIRA tasks over and over, it will instead opt to append a
+            # comment to a similar(ish) JIRA task.
+            #
+            # However, this can result in  very long-lived JIRA tickets getting tons of comments
+            # appended on. This optional parameter allows users to specify an additional JQL clause
+            # to filter out these older tickets, encouraging new JIRA tasks to be created from
+            # time to time. It can also be used to increase the accuracy of finding the parent
+            # task (maybe filtering on a component) in case you find the StreamAlert integration
+            # is appending comments to unrelated issues.
+            #
+            # Example: A highly effective JQL suffix is "created > startOfWeek(-1w)"
+            ('aggregation_additional_jql',
+             OutputProperty(description='when aggregation is enabled, provide additional JQL '
+                                        'clause to filter out older/outdated issues',
+                            mask_input=False,
+                            input_restrictions={},
+                            cred_requirement=True)),
+            ('ssl_verify',
+             OutputProperty(description='do clientside ssl cert verification (yes/no)',
+                            mask_input=False,
+                            cred_requirement=True)),
+            # For example, if your JIRA project requires a custom field called "custom_field_1",
+            # you can set the following json-encoded string in this:
+            # {"custom_field_1": {"value": "FooBar"}}
+            #
+            # These fields are DEFAULT values. You can still override them using the
+            # @jira.additional_fields publisher parameter.
+            ('additional_required_issue_fields',
+             OutputProperty(description='when a JIRA project has additional required fields, '
+                                        'provide them here, as a json-encoded string',
+                            mask_input=False,
+                            input_restrictions={},
                             cred_requirement=True))
         ])
 
@@ -127,7 +165,7 @@ def _search_jira(self, jql, fields=None, max_results=100, validate_query=True):
             resp = self._get_request_retry(search_url,
                                            params=params,
                                            headers=self._get_headers(),
-                                           verify=False)
+                                           verify=self._verify_ssl)
         except OutputRequestFailure:
             return []
 
@@ -152,7 +190,7 @@ def _create_comment(self, issue_id, comment):
             resp = self._post_request_retry(comment_url,
                                             data={'body': comment},
                                             headers=self._get_headers(),
-                                            verify=False)
+                                            verify=self._verify_ssl)
         except OutputRequestFailure:
             return False
 
@@ -175,7 +213,7 @@ def _get_comments(self, issue_id):
         try:
             resp = self._get_request_retry(comment_url,
                                            headers=self._get_headers(),
-                                           verify=False)
+                                           verify=self._verify_ssl)
         except OutputRequestFailure:
             return []
 
@@ -185,17 +223,24 @@ def _get_comments(self, issue_id):
 
         return response.get('comments', [])
 
-    def _get_existing_issue(self, issue_summary, project_key):
+    def _get_existing_issue(self, issue_summary, project_key, additional_jql):
         """Find an existing Jira issue based on the issue summary
 
         Args:
             issue_summary (str): The Jira issue summary
             project_key (str): The Jira project to search
+            additional_jql (str): Additional JQL statement to filter by
 
         Returns:
             int: ID of the found issue or False if existing issue does not exist
         """
-        jql = 'summary ~ "{}" and project="{}"'.format(issue_summary, project_key)
+        jql = 'summary ~ "{}" and project="{}"{}'.format(
+            issue_summary,
+            project_key,
+            " AND {}".format(additional_jql) if additional_jql else ""
+        )
+
+        LOGGER.debug('Aggregation using JQL: (%s)', jql)
         resp = self._search_jira(jql, fields=['id', 'summary'], max_results=1)
         jira_id = False
 
@@ -206,7 +251,7 @@ def _get_existing_issue(self, issue_summary, project_key):
 
         return jira_id
 
-    def _create_issue(self, summary, project_key, issue_type, description):
+    def _create_issue(self, summary, project_key, issue_type, description, additional_fields):
         """Create a Jira issue to write alerts to. Alert is written to the "description"
         field of an issue.
 
@@ -215,6 +260,11 @@ def _create_issue(self, summary, project_key, issue_type, description):
             project_key (str): The Jira project key which issues will be associated with
             issue_type (str): The type of issue being created
             description (str): The body of text which describes the issue
+            additional_fields (dict):
+                Additional fields to set with the integration. This can vary greatly from
+                project to project, so be wary of which fields are available. You can use the
+                /issue/createmeta?projectKeys=CSIRT endpoint to discover which fields are available
+                (and which ones are required) for your specific project.
 
         Returns:
             int: ID of the created issue or False if unsuccessful
@@ -229,14 +279,15 @@ def _create_issue(self, summary, project_key, issue_type, description):
                 'description': description,
                 'issuetype': {
                     'name': issue_type
-                }
+                },
+                **additional_fields
             }
         }
         try:
             resp = self._post_request_retry(issue_url,
                                             data=issue_body,
                                             headers=self._get_headers(),
-                                            verify=False)
+                                            verify=self._verify_ssl)
         except OutputRequestFailure:
             return False
 
@@ -264,7 +315,7 @@ def _establish_session(self, username, password):
             resp = self._post_request_retry(login_url,
                                             data=auth_info,
                                             headers=self._get_default_headers(),
-                                            verify=False)
+                                            verify=self._verify_ssl)
         except OutputRequestFailure:
             LOGGER.error("Failed to authenticate to Jira")
             return False
@@ -291,6 +342,18 @@ def _dispatch(self, alert, descriptor):
                     so it supports their custom markdown-like formatting and respects newline
                     characters (e.g. \n).
 
+            - @jira.additional_fields (dict):
+                    A structure of additional fields to add to Create Issue API calls. For example,
+                    if you have a custom field for severity, you could specify it in this dict
+                    like so:
+
+                        {
+                          "custom_field_1122": {
+                            "value": "Low"
+                          }
+                        }
+
+
         Args:
             alert (Alert): Alert instance which triggered a rule
             descriptor (str): Output descriptor
@@ -307,7 +370,11 @@ def _dispatch(self, alert, descriptor):
         # Presentation defaults
         default_issue_summary = 'StreamAlert {}'.format(alert.rule_name)
         default_alert_body = '{{code:JSON}}{}{{code}}'.format(
-            json.dumps(publication, sort_keys=True)
+            json.dumps(
+                publication,
+                indent=2,
+                sort_keys=True,
+            )
         )
 
         # True Presentation values
@@ -318,6 +385,7 @@ def _dispatch(self, alert, descriptor):
         comment_id = None
 
         self._base_url = creds['url']
+        self._verify_ssl = creds.get('verify_ssl', '').lower() == 'yes'
         self._auth_cookie = self._establish_session(creds['username'], creds['password'])
 
         # Validate successful authentication
@@ -327,7 +395,11 @@ def _dispatch(self, alert, descriptor):
         # If aggregation is enabled, attempt to add alert to an existing issue. If a
         # failure occurs in this block, creation of a new Jira issue will be attempted.
         if creds.get('aggregate', '').lower() == 'yes':
-            issue_id = self._get_existing_issue(issue_summary, creds['project_key'])
+            issue_id = self._get_existing_issue(
+                issue_summary,
+                creds['project_key'],
+                creds.get('aggregation_additional_jql', '')
+            )
             if issue_id:
                 comment_id = self._create_comment(issue_id, description)
                 if comment_id:
@@ -340,10 +412,22 @@ def _dispatch(self, alert, descriptor):
                              issue_id)
 
         # Create a new Jira issue
+        required_fields_json = creds.get('additional_required_issue_fields')
+        additional_required_fields = (
+            json.loads(required_fields_json)
+            if required_fields_json
+            else {}
+        )
+
+        additional_fields = {
+            **additional_required_fields,
+            **publication.get('@jira.additional_fields', {}),
+        }
         issue_id = self._create_issue(issue_summary,
                                       creds['project_key'],
                                       creds['issue_type'],
-                                      description)
+                                      description,
+                                      additional_fields)
         if issue_id:
             LOGGER.debug('Sending alert to a new Jira issue %s', issue_id)