Merge pull request #4 from demisto/master

Update content pack

.circleci/content_release_vars.sh

@@ -1,3 +1,3 @@
-export CONTENT_VERSION="21.2.1"
+export CONTENT_VERSION="21.3.0"
 export SERVER_VERSION="6.0.0"
-export GIT_SHA1="86f1e4c0a1ec79e2e2b48a07a8c8798e59bb93f1" # guardrails-disable-line disable-secrets-detection
+export GIT_SHA1="2e46e448868d480f623a382dea6baed7059e1522" # guardrails-disable-line disable-secrets-detection

.github/workflows/project_manager_daily.yml

@@ -0,0 +1,23 @@
+name: Manage Contribution Board - Daily
+on:
+  schedule:
+    - cron:  '0 0 * * *'
+jobs:
+  manage_project_board:
+    runs-on: ubuntu-latest
+    if: github.repository == 'demisto/content'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - name: Get project manager
+        run: |
+          pip install --upgrade pip
+          pip install https://github.com/demisto/github-automation/archive/ContentFeatureBranch.zip\#egg\=github-automation
+      - name: Manage project
+        run: |
+          github-automation manage -c .github/project_conf/contributions.ini
+        env:
+          GITHUB_TOKEN: ${{ secrets.CONTENTBOT_BOARD_UPDATE_TOKEN }}

.github/workflows/project_manager_hourly.yml

@@ -0,0 +1,23 @@
+name: Manage Contribution Board - Daily
+on:
+  schedule:
+    - cron:  '0 * * * *'
+jobs:
+  manage_project_board:
+    runs-on: ubuntu-latest
+    if: github.repository == 'demisto/content'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.7'
+      - name: Get project manager
+        run: |
+          pip install --upgrade pip
+          pip install https://github.com/demisto/github-automation/archive/ContentFeatureBranch.zip\#egg\=github-automation
+      - name: Manage project
+        run: |
+          github-automation manage -c .github/project_conf/contributions.ini
+        env:
+          GITHUB_TOKEN: ${{ secrets.CONTENTBOT_BOARD_UPDATE_TOKEN }}	

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule.py

@@ -5,7 +5,6 @@
 ''' IMPORTS '''
 import urllib3
 import requests
-from dateutil.parser import parse
 from typing import Optional, Pattern, List
 
 # disable insecure warnings
@@ -14,6 +13,7 @@
 ''' GLOBALS '''
 TAGS = 'feedTags'
 TLP_COLOR = 'trafficlightprotocol'
+DATE_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
 
 
 class Client(BaseClient):
@@ -280,9 +280,14 @@ def custom_fields_creator(self, attributes: dict):
         return created_custom_fields
 
 
-def datestring_to_millisecond_timestamp(datestring):
-    date = parse(str(datestring))
-    return int(date.timestamp() * 1000)
+def datestring_to_server_format(date_string: str) -> str:
+    """
+    formats a datestring to the ISO-8601 format which the server expects to recieve
+    :param date_string: Date represented as a tring
+    :return: ISO-8601 date string
+    """
+    parsed_date = dateparser.parse(date_string, settings={'TIMEZONE': 'UTC'})
+    return parsed_date.strftime(DATE_FORMAT)
 
 
 def get_indicator_fields(line, url, feed_tags: list, tlp_color: Optional[str], client: Client):
@@ -365,12 +370,10 @@ def fetch_indicators_command(client, feed_tags, tlp_color, itype, auto_detect, *
                 attributes, value = get_indicator_fields(line, url, feed_tags, tlp_color, client)
                 if value:
                     if 'lastseenbysource' in attributes.keys():
-                        attributes['lastseenbysource'] = datestring_to_millisecond_timestamp(
-                            attributes['lastseenbysource'])
+                        attributes['lastseenbysource'] = datestring_to_server_format(attributes['lastseenbysource'])
 
                     if 'firstseenbysource' in attributes.keys():
-                        attributes['firstseenbysource'] = datestring_to_millisecond_timestamp(
-                            attributes['firstseenbysource'])
+                        attributes['firstseenbysource'] = datestring_to_server_format(attributes['firstseenbysource'])
                     indicator_type = determine_indicator_type(
                         client.feed_url_to_config.get(url, {}).get('indicator_type'), itype, auto_detect, value)
                     indicator_data = {

Packs/ApiModules/Scripts/HTTPFeedApiModule/HTTPFeedApiModule_test.py

@@ -1,4 +1,4 @@
-from HTTPFeedApiModule import get_indicators_command, Client, datestring_to_millisecond_timestamp, feed_main
+from HTTPFeedApiModule import get_indicators_command, Client, datestring_to_server_format, feed_main
 import requests_mock
 import demistomock as demisto
 
@@ -127,17 +127,29 @@ def test_custom_fields_creator():
     assert "old_filed2" not in custom_fields.keys()
 
 
-def test_datestring_to_millisecond_timestamp():
-    datesting1 = "2020-02-10 13:39:14"
-    datesting2 = "2020-02-10T13:39:14"
-    datesting3 = "2020-02-10 13:39:14.123"
-    datesting4 = "2020-02-10T13:39:14.123"
-    datesting5 = "2020-02-10T13:39:14Z"
-    assert 1581341954000 == datestring_to_millisecond_timestamp(datesting1)
-    assert 1581341954000 == datestring_to_millisecond_timestamp(datesting2)
-    assert 1581341954000 == datestring_to_millisecond_timestamp(datesting5)
-    assert 1581341954123 == datestring_to_millisecond_timestamp(datesting3)
-    assert 1581341954123 == datestring_to_millisecond_timestamp(datesting4)
+def test_datestring_to_server_format():
+    """
+    Given
+    - A string represting a date.
+
+    When
+    - running datestring_to_server_format on the date.
+
+    Then
+    - Ensure the datestring is converted to the ISO-8601 format.
+    """
+    datestring1 = "2020-02-10 13:39:14"
+    datestring2 = "2020-02-10T13:39:14"
+    datestring3 = "2020-02-10 13:39:14.123"
+    datestring4 = "2020-02-10T13:39:14.123"
+    datestring5 = "2020-02-10T13:39:14Z"
+    datestring6 = "2020-11-01T04:16:13-04:00"
+    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring1)
+    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring2)
+    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring3)
+    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring4)
+    assert '2020-02-10T13:39:14Z' == datestring_to_server_format(datestring5)
+    assert '2020-11-01T08:16:13Z' == datestring_to_server_format(datestring6)
 
 
 def test_get_feed_config():

Packs/ApiModules/Scripts/MicrosoftApiModule/MicrosoftApiModule.py

@@ -107,6 +107,7 @@ def http_request(
         Returns:
             Response from api according to resp_type. The default is `json` (dict or list).
         """
+        ok_codes = (200, 201, 202, 204, 206, 404)
         token = self.get_access_token(resource=resource, scope=scope)
         default_headers = {
             'Authorization': f'Bearer {token}',
@@ -117,7 +118,7 @@ def http_request(
         if headers:
             default_headers.update(headers)
         response = super()._http_request(   # type: ignore[misc]
-            *args, resp_type="response", headers=default_headers, **kwargs)
+            *args, resp_type="response", headers=default_headers, **kwargs, ok_codes=ok_codes)
 
         # 206 indicates Partial Content, reason will be in the warning header.
         # In that case, logs with the warning header will be written.
@@ -127,6 +128,14 @@ def http_request(
         if is_response_empty_and_successful and return_empty_response:
             return response
 
+        # Handle 404 errors instead of raising them as exceptions:
+        if response.status_code == 404:
+            try:
+                error_message = response.json()
+            except Exception:
+                error_message = 'Not Found - 404 Response'
+            raise NotFoundError(error_message)
+
         try:
             if resp_type == 'json':
                 return response.json()
@@ -531,3 +540,14 @@ def device_auth_request(self) -> str:
             return_error(f'Error in Microsoft authorization: {str(e)}')
         set_integration_context({'device_code': response_json.get('device_code')})
         return response_json.get('user_code', '')
+
+
+class NotFoundError(Exception):
+    """Exception raised for 404 - Not Found errors.
+
+    Attributes:
+        message -- explanation of the error
+    """
+
+    def __init__(self, message):
+        self.message = message

Packs/ApiModules/Scripts/MicrosoftApiModule/MicrosoftApiModule_test.py

@@ -1,5 +1,5 @@
 from requests import Response
-from MicrosoftApiModule import MicrosoftClient
+from MicrosoftApiModule import *
 import demistomock as demisto
 import pytest
 import datetime
@@ -67,6 +67,29 @@ def test_error_parser(mocker):
     assert response == 'code: message'
 
 
+def test_page_not_found_error(mocker):
+    """
+    Given:
+        - The http_request command for making MS API calls.
+    When:
+        - The response returned is a 404 response.
+    Then:
+        - Validate that the exception is handled in the http_request function of MicrosoftClient.
+    """
+    error_404 = Response()
+    error_404._content = b'{"error": {"code": "Request_ResourceNotFound", "message": "Resource ' \
+                         b'"NotExistingUser does not exist."}}'
+    error_404.status_code = 404
+    client = self_deployed_client()
+    mocker.patch.object(BaseClient, '_http_request', return_value=error_404)
+    mocker.patch.object(client, 'get_access_token')
+
+    try:
+        client.http_request()
+    except Exception as e:  # Validate that a `NotFoundError` was raised
+        assert type(e).__name__ == 'NotFoundError'
+
+
 def test_epoch_seconds(mocker):
     mocker.patch.object(MicrosoftClient, '_get_utcnow', return_value=datetime.datetime(2019, 12, 24, 14, 12, 0, 586636))
     mocker.patch.object(MicrosoftClient, '_get_utcfromtimestamp', return_value=datetime.datetime(1970, 1, 1, 0, 0))

Packs/ArcherRSA/Integrations/ArcherV2/ArcherV2.py

@@ -13,7 +13,7 @@
 
 FETCH_PARAM_ID_KEY = 'field_time_id'
 LAST_FETCH_TIME_KEY = 'last_fetch'
-OCCURRED_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+OCCURRED_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
 
 REQUEST_HEADERS = {
     'Accept': 'application/json,text/html,application/xhtml +xml,application/xml;q=0.9,*/*;q=0.8',
@@ -124,7 +124,8 @@ def search_records_by_report_soap_request(token, report_guid):
 
 def search_records_soap_request(
         token, app_id, display_fields, field_id, field_name, search_value, date_operator='',
-        numeric_operator='', max_results=10
+        numeric_operator='', max_results=10,
+        sort_type: str = 'Ascending'
 ):
     request_body = '<?xml version="1.0" encoding="UTF-8"?>' + \
                    '<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" ' \
@@ -135,8 +136,8 @@ def search_records_soap_request(
                    f'            <sessionToken>{token}</sessionToken>' + \
                    '            <searchOptions>' + \
                    '                <![CDATA[<SearchReport>' + \
-                   '                <PageSize>100</PageSize>' + \
-                   '                <PageNumber>1</PageNumber>' + \
+                   f'                <PageSize>{max_results}</PageSize>' + \
+                   '                 <PageNumber>1</PageNumber>' + \
                    f'                <MaxRecordCount>{max_results}</MaxRecordCount>' + \
                    '                <ShowStatSummaries>false</ShowStatSummaries>' + \
                    f'                <DisplayFields>{display_fields}</DisplayFields>' + \
@@ -181,6 +182,14 @@ def search_records_soap_request(
                         '</Conditions>' + \
                         '</Filter>'
 
+    if field_id:
+        request_body += '<SortFields>' + \
+                        '    <SortField>' + \
+                        f'        <Field>{field_id}</Field>' + \
+                        f'        <SortType>{sort_type}</SortType>' + \
+                        '    </SortField >' + \
+                        '</SortFields>'
+
     request_body += ' </Criteria></SearchReport>]]>' + \
                     '</searchOptions>' + \
                     '<pageNumber>1</pageNumber>' + \
@@ -434,6 +443,7 @@ def record_to_incident(
     def search_records(
             self, app_id, fields_to_display=None, field_to_search='', search_value='',
             numeric_operator='', date_operator='', max_results=10,
+            sort_type: str = 'Ascending'
     ):
         demisto.debug(f'searching for records {field_to_search}:{search_value}')
         if fields_to_display is None:
@@ -463,7 +473,8 @@ def search_records(
             field_id=search_field_id, field_name=search_field_name,
             numeric_operator=numeric_operator,
             date_operator=date_operator, search_value=search_value,
-            max_results=max_results
+            max_results=max_results,
+            sort_type=sort_type,
         )
 
         if not res:
@@ -1038,6 +1049,7 @@ def search_records_command(client: Client, args: Dict[str, str]):
     fields_to_display = argToList(args.get('fieldsToDisplay'))
     fields_to_get = argToList(args.get('fieldsToGet'))
     full_data = args.get('fullData', 'true') == 'true'
+    sort_type = 'Descending' if argToBoolean(args.get('isDescending', 'false')) else 'Ascending'
 
     if fields_to_get and 'Id' not in fields_to_get:
         fields_to_get.append('Id')
@@ -1052,7 +1064,8 @@ def search_records_command(client: Client, args: Dict[str, str]):
 
     records, raw_res = client.search_records(
         app_id, fields_to_get, field_to_search, search_value,
-        numeric_operator, date_operator, max_results=max_results
+        numeric_operator, date_operator, max_results=max_results,
+        sort_type=sort_type,
     )
 
     records = list(map(lambda x: x['record'], records))
@@ -1137,7 +1150,7 @@ def fetch_incidents(
     fields_to_display = argToList(params.get('fields_to_fetch'))
     fields_to_display.append(date_field)
     # API Call
-    records, raw_res = client.search_records(
+    records, _ = client.search_records(
         app_id, fields_to_display, date_field,
         from_time.strftime(OCCURRED_FORMAT),
         date_operator='GreaterThan',

Packs/ArcherRSA/Integrations/ArcherV2/ArcherV2.yml

@@ -506,6 +506,12 @@ script:
       - 'False'
       required: true
       secret: false
+    - name: isDescending
+      auto: PREDEFINED
+      predefined:
+      - "true"
+      - "false"
+      description: Whether to order by descending order.
     deprecated: false
     description: Searches for records in the given application.
     execution: false
@@ -542,7 +548,7 @@ script:
     description: prints Archer's integration cache.
     execution: false
     name: archer-print-cache
-  dockerimage: demisto/python3:3.9.1.14969
+  dockerimage: demisto/python3:3.9.1.15759
   feed: false
   isfetch: true
   longRunning: false