Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cs falcon fetch limit issue #29411

Merged
merged 21 commits into from
Sep 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -529,7 +529,7 @@ def detection_to_incident(detection):

incident = {
'name': 'Detection ID: ' + str(detection.get('detection_id')),
'occurred': str(detection.get('created_timestamp')),
'occurred': str(detection.get('first_behavior')),
'rawJSON': json.dumps(detection),
'severity': severity_string_to_int(detection.get('max_severity_displayname'))
}
Expand Down Expand Up @@ -1306,7 +1306,7 @@ def get_fetch_detections(last_created_timestamp=None, filter_arg=None, offset: i
if filter_arg:
params['filter'] = filter_arg
elif last_created_timestamp:
params['filter'] = f"created_timestamp:>'{last_created_timestamp}'"
params['filter'] = f"first_behavior:>'{last_created_timestamp}'"
elif last_updated_timestamp:
params['filter'] = f"date_updated:>'{last_updated_timestamp}'"

Expand Down Expand Up @@ -2500,6 +2500,21 @@ def migrate_last_run(last_run: dict[str, str] | list[dict]) -> list[dict]:
return [updated_last_run_detections, updated_last_run_incidents, {}]


def sort_incidents_summaries_by_ids_order(ids_order, full_incidents, id_field):
""" sort incidents list by the order that ids_order list has

Args:
ids_order: list of ids
full_incidents: list of incidents
id_field: name of the id field
Returns:
list[dict]: New last run object.
"""
incidents_by_id = {i[id_field]: i for i in full_incidents}
incidents = [incidents_by_id[i] for i in ids_order]
return incidents


def fetch_incidents():
incidents: list = []
detections: list = []
Expand Down Expand Up @@ -2527,7 +2542,7 @@ def fetch_incidents():
incident_type = 'detection'
fetch_query = demisto.params().get('fetch_query')
if fetch_query:
fetch_query = f"created_timestamp:>'{start_fetch_time}'+{fetch_query}"
fetch_query = f"first_behavior:>'{start_fetch_time}'+{fetch_query}"
detections_ids = demisto.get(get_fetch_detections(filter_arg=fetch_query, limit=fetch_limit), 'resources')
else:
detections_ids = demisto.get(get_fetch_detections(last_created_timestamp=start_fetch_time, limit=fetch_limit),
Expand All @@ -2536,18 +2551,22 @@ def fetch_incidents():
raw_res = get_detections_entities(detections_ids)

if raw_res is not None and "resources" in raw_res:
for detection in demisto.get(raw_res, "resources"):
full_detections = demisto.get(raw_res, "resources")
sorted_detections = sort_incidents_summaries_by_ids_order(ids_order=detections_ids,
full_incidents=full_detections,
id_field='detection_id')
for detection in sorted_detections:
detection['incident_type'] = incident_type
demisto.debug(
f"CrowdStrikeFalconMsg: Detection {detection['detection_id']} "
f"was fetched which was created in {detection['created_timestamp']}")
f"was fetched which was created in {detection['first_behavior']}")
incident = detection_to_incident(detection)

detections.append(incident)

detections = filter_incidents_by_duplicates_and_limit(incidents_res=detections,
last_run=current_fetch_info_detections,
fetch_limit=fetch_limit, id_field='name')
fetch_limit=INCIDENTS_PER_FETCH, id_field='name')

for detection in detections:
occurred = dateparser.parse(detection["occurred"])
Expand Down Expand Up @@ -2583,13 +2602,17 @@ def fetch_incidents():
if incidents_ids:
raw_res = get_incidents_entities(incidents_ids)
if raw_res is not None and "resources" in raw_res:
for incident in demisto.get(raw_res, "resources"):
full_incidents = demisto.get(raw_res, "resources")
sorted_incidents = sort_incidents_summaries_by_ids_order(ids_order=incidents_ids,
full_incidents=full_incidents,
id_field='incident_id')
for incident in sorted_incidents:
incident['incident_type'] = incident_type
incident_to_context = incident_to_incident_context(incident)
incidents.append(incident_to_context)

incidents = filter_incidents_by_duplicates_and_limit(incidents_res=incidents, last_run=current_fetch_info_incidents,
fetch_limit=fetch_limit, id_field='name')
fetch_limit=INCIDENTS_PER_FETCH, id_field='name')
for incident in incidents:
occurred = dateparser.parse(incident["occurred"])
if occurred:
Expand Down Expand Up @@ -2617,14 +2640,18 @@ def fetch_incidents():
if idp_detections_ids:
raw_res = get_idp_detection_entities(idp_detections_ids)
if "resources" in raw_res:
for idp_detection in demisto.get(raw_res, "resources"):
full_detections = demisto.get(raw_res, "resources")
sorted_detections = sort_incidents_summaries_by_ids_order(ids_order=idp_detections_ids,
full_incidents=full_detections,
id_field='composite_id')
for idp_detection in sorted_detections:
idp_detection['incident_type'] = IDP_DETECTION
idp_detection_to_context = idp_detection_to_incident_context(idp_detection)
idp_detections.append(idp_detection_to_context)

idp_detections = filter_incidents_by_duplicates_and_limit(incidents_res=idp_detections,
last_run=current_fetch_info_idp_detections,
fetch_limit=fetch_limit, id_field='name')
fetch_limit=INCIDENTS_PER_FETCH, id_field='name')
updated_last_run = update_last_run_object(last_run=current_fetch_info_idp_detections, incidents=idp_detections,
fetch_limit=fetch_limit,
start_fetch_time=start_fetch_time, end_fetch_time=end_fetch_time,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,7 @@ script:
- description: Any commands run against an offline-queued session will be queued up and executed when the host comes online.
name: queue_offline
defaultValue: false
- description: A comma-separated list of host agent IDs to run commands for. (Can be retrieved by running the 'cs-falcon-search-device' command.)
- description: A comma-separated list of host agent IDs to run commands for. (Can be retrieved by running the 'cs-falcon-search-device' command.).
name: host_ids
required: true
- description: The type of command to run.
Expand All @@ -423,7 +423,7 @@ script:
required: true
- auto: PREDEFINED
defaultValue: read
description: 'The scope to run the command for. Possible values are: "read", "write", and "admin". (NOTE: In order to run the CrowdStrike RTR `put` command, it is necessary to pass `scope=admin`.)'
description: 'The scope to run the command for. Possible values are: "read", "write", and "admin". (NOTE: In order to run the CrowdStrike RTR `put` command, it is necessary to pass `scope=admin`.).'
name: scope
predefined:
- read
Expand Down Expand Up @@ -491,7 +491,7 @@ script:
- description: The file entry ID to upload.
name: entry_id
required: true
description: Uploads a file to the CrowdStrike cloud. (Can be used for the RTR 'put' command.)
description: Uploads a file to the CrowdStrike cloud. (Can be used for the RTR 'put' command.).
name: cs-falcon-upload-file
- arguments:
- description: The ID of the file to delete. (The ID of the file can be retrieved by running the 'cs-falcon-list-files' command).
Expand All @@ -500,7 +500,7 @@ script:
description: Deletes a file based on the provided ID. Can delete only one file at a time.
name: cs-falcon-delete-file
- arguments:
- description: A comma-separated list of file IDs to get. (The list of file IDs can be retrieved by running the 'cs-falcon-list-files' command.)
- description: A comma-separated list of file IDs to get. (The list of file IDs can be retrieved by running the 'cs-falcon-list-files' command.).
name: file_id
required: true
description: Returns files based on the provided IDs. These files are used for the RTR 'put' command.
Expand Down Expand Up @@ -531,7 +531,7 @@ script:
description: The full name of the file.
type: String
- contextPath: CrowdStrike.File.Permission
description: 'The permission type of the file. Possible values are: "private", which is used only by the user who uploaded it, "group", which is used by all RTR Admins, and "public", which is used by all active-responders and RTR admins'
description: 'The permission type of the file. Possible values are: "private", which is used only by the user who uploaded it, "group", which is used by all RTR Admins, and "public", which is used by all active-responders and RTR admins.'
type: String
- contextPath: CrowdStrike.File.SHA256
description: The SHA-256 hash of the file.
Expand Down Expand Up @@ -595,7 +595,7 @@ script:
description: The size of the file in bytes.
type: Number
- arguments:
- description: A comma-separated list of script IDs to return. (The script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.)
- description: A comma-separated list of script IDs to return. (The script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.).
name: script_id
required: true
description: Returns custom scripts based on the provided ID. Used for the RTR 'runscript' command.
Expand Down Expand Up @@ -641,7 +641,7 @@ script:
description: Whether the user has write access to the script.
type: Boolean
- arguments:
- description: The script ID to delete. (Script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.)
- description: The script ID to delete. (Script IDs can be retrieved by running the 'cs-falcon-list-scripts' command.).
name: script_id
required: true
description: Deletes a custom-script based on the provided ID. Can delete only one script at a time.
Expand Down Expand Up @@ -692,7 +692,7 @@ script:
- arguments:
- description: The name of the script to run.
name: script_name
- description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.)
- description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.).
name: host_ids
required: true
- description: The PowerShell script code to run.
Expand Down Expand Up @@ -1290,7 +1290,7 @@ script:
description: The identity of the user/process who last updated the IOC.
type: string
- contextPath: CrowdStrike.NextPageToken
description: A pagination token used with the limit parameter to manage pagination of results
description: A pagination token used with the limit parameter to manage pagination of results.
- arguments:
- auto: PREDEFINED
description: 'The IOC type to retrieve. Possible values are: "sha256", "sha1", "md5", "domain", "ipv4", and "ipv6". Either ioc_id or ioc_type and value must be provided.'
Expand Down Expand Up @@ -2274,7 +2274,7 @@ script:
- description: The ID of the host group.
name: host_group_id
required: true
- description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.)
- description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.).
isArray: true
name: host_ids
required: true
Expand Down Expand Up @@ -2309,7 +2309,7 @@ script:
- description: The ID of the host group.
name: host_group_id
required: true
- description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.)
- description: A comma-separated list of host agent IDs to run commands. (The list of host agent IDs can be retrieved by running the 'cs-falcon-search-device' command.).
isArray: true
name: host_ids
required: true
Expand Down Expand Up @@ -2627,7 +2627,7 @@ script:
description: Updates the remote incident or detection with local incident or detection changes. This method is only used for debugging purposes and will not update the current incident or detection.
name: update-remote-system
- arguments:
- description: Limit the vulnerabilities returned to specific properties. Each value must be enclosed in single quotes and placed immediately after the colon with no space. For example, 'filter=status:'open'+cve.id:['CVE-2013-3900','CVE-2021-1675']'
- description: Limit the vulnerabilities returned to specific properties. Each value must be enclosed in single quotes and placed immediately after the colon with no space. For example, 'filter=status:'open'+cve.id:['CVE-2013-3900','CVE-2021-1675']'.
name: filter
- description: Unique agent identifier (AID) of a sensor.
name: aid
Expand All @@ -2652,7 +2652,7 @@ script:
- description: Type of host a sensor is running on.
name: host_type
isArray: true
- description: Filter for vulnerabilities based on the number of days since a host last connected to CrowdStrike Falcon. Enter a numeric value from 3 to 45 to indicate the number of days you want to look back. Example- last_seen_within:10
- description: Filter for vulnerabilities based on the number of days since a host last connected to CrowdStrike Falcon. Enter a numeric value from 3 to 45 to indicate the number of days you want to look back. Example- last_seen_within:10.
name: last_seen_within
- auto: PREDEFINED
description: Indicates if the vulnerability is suppressed by a suppression rule.
Expand Down Expand Up @@ -3864,7 +3864,7 @@ script:
description: A unique identifier for the scan profile used in the scan.
type: String
- contextPath: CrowdStrike.ODSScanHost.host_id
description: A unique identifier for the host that was scanned
description: A unique identifier for the host that was scanned.
type: String
- contextPath: CrowdStrike.ODSScanHost.host_scan_id
description: A unique identifier for the scan that was performed on the host.
Expand All @@ -3882,10 +3882,10 @@ script:
description: The number of files that were skipped during the scan.
type: Number
- contextPath: CrowdStrike.ODSScanHost.status
description: The status of the scan. (e.g., "completed", "pending", "cancelled", "running", or "failed")
description: The status of the scan. (e.g., "completed", "pending", "cancelled", "running", or "failed").
type: String
- contextPath: CrowdStrike.ODSScanHost.severity
description: A severity score assigned to the scan, ranging from 0 to 100
description: A severity score assigned to the scan, ranging from 0 to 100.
type: Number
- contextPath: CrowdStrike.ODSScanHost.started_on
description: The date and time when the scan was started.
Expand Down Expand Up @@ -4375,7 +4375,7 @@ script:
- contextPath: CrowdStrike.IDPEntity.EmailAddresses
description: The identity entity email address.
type: String
dockerimage: demisto/py3-tools:1.0.0.72621
dockerimage: demisto/py3-tools:1.0.0.73055
isfetch: true
ismappable: true
isremotesyncin: true
Expand Down
Loading