Added support for custom formatter helpers #444 (#3377)

data/formatters/browser.yaml

@@ -76,6 +76,9 @@ boolean_helpers:
 - input_attribute: 'url_hidden'
   output_attribute: 'url_hidden_string'
   value_if_true: '(URL hidden)'
+custom_helpers:
+- identifier: 'chrome_history_typed_count'
+  output_attribute: 'url_typed_string'
 enumeration_helpers:
 - input_attribute: 'page_transition_type'
   output_attribute: 'page_transition'
@@ -126,6 +129,11 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'chrome:preferences:content_settings:exceptions'
+custom_helpers:
+- identifier: 'chrome_preferences_primary_url'
+  output_attribute: 'primary_url'
+- identifier: 'chrome_preferences_secondary_url'
+  output_attribute: 'secondary_url'
 message:
 - 'Permission {permission}'
 - 'used by {primary_url}'
@@ -253,6 +261,11 @@ short_message: '{title}'
 ---
 type: 'conditional'
 data_type: 'firefox:places:page_visited'
+custom_helpers:
+- identifier: 'firefox_history_typed_count'
+  output_attribute: 'url_typed_string'
+- identifier: 'firefox_history_url_hidden'
+  output_attribute: 'url_hidden_string'
 enumeration_helpers:
 - input_attribute: 'visit_type'
   output_attribute: 'transition_string'
@@ -290,6 +303,9 @@ boolean_helpers:
 - input_attribute: 'recovered'
   output_attribute: 'recovered_string'
   value_if_true: '[Recovered Entry]'
+custom_helpers:
+- identifier: 'msiecf_cached_path'
+  output_attribute: 'cached_file_path'
 message:
 - 'Cached file: {cached_file_path}'
 - 'Cached file size: {cached_file_size}'
@@ -315,6 +331,11 @@ boolean_helpers:
 - input_attribute: 'recovered'
   output_attribute: 'recovered_string'
   value_if_true: '[Recovered Entry]'
+custom_helpers:
+- identifier: 'msiecf_cached_path'
+  output_attribute: 'cached_file_path'
+- identifier: 'msiecf_http_headers'
+  output_attribute: 'http_headers'
 message:
 - 'Location: {url}'
 - 'Number of hits: {number_of_hits}'

data/formatters/generic.yaml

@@ -183,6 +183,11 @@ short_message: '{filename}'
 ---
 type: 'conditional'
 data_type: 'fs:ntfs:usn_change'
+custom_helpers:
+- identifier: 'ntfs_file_reference'
+  output_attribute: 'file_reference'
+- identifier: 'ntfs_parent_file_reference'
+  output_attribute: 'parent_file_reference'
 flags_helpers:
 - input_attribute: 'update_reason_flags'
   output_attribute: 'update_reason'
@@ -245,6 +250,13 @@ boolean_helpers:
 - input_attribute: 'is_allocated'
   output_attribute: 'unallocated'
   value_if_false: 'unallocated'
+custom_helpers:
+- identifier: 'ntfs_file_reference'
+  output_attribute: 'file_reference'
+- identifier: 'ntfs_parent_file_reference'
+  output_attribute: 'parent_file_reference'
+- identifier: 'ntfs_path_hints'
+  output_attribute: 'path_hints'
 enumeration_helpers:
 - input_attribute: 'attribute_type'
   output_attribute: 'attribute_name'

data/formatters/windows.yaml

@@ -161,6 +161,9 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'windows:lnk:link'
+custom_helpers:
+- identifier: 'windows_shortcut_linked_path'
+  output_attribute: 'linked_path'
 message:
 - '[{description}]'
 - 'File size: {file_size}'
@@ -224,6 +227,11 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'windows:prefetch:execution'
+custom_helpers:
+- identifier: 'windows_prefetch_path_hints'
+  output_attribute: 'path_hints'
+- identifier: 'windows_prefetch_volumes_string'
+  output_attribute: 'volumes_string'
 message:
 - 'Prefetch'
 - '[{executable}] was executed -'
@@ -330,6 +338,9 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'windows:registry:key_value'
+custom_helpers:
+- identifier: 'windows_registry_values'
+  output_attribute: 'values'
 message:
 - '[{key_path}]'
 - '{values}'
@@ -637,6 +648,9 @@ short_message: '[{key_path}] {entries}'
 ---
 type: 'conditional'
 data_type: 'windows:shell_item:file_entry'
+custom_helpers:
+- identifier: 'shell_item_file_entry_name'
+  output_attribute: 'file_entry_name'
 message:
 - 'Name: {name}'
 - 'Long name: {long_name}'

docs/sources/user/Output-and-formatting.md

@@ -96,6 +96,7 @@ An event formatter is defined as a set of attributes:
 
 * "data_type"; required event data type.
 * "boolean_helpers"; optional boolean helpers.
+* "custom_helpers"; optional custom helpers.
 * "enumeration_helpers"; optional enumeration helpers.
 * "message"; required formatter message string, for a basic type, or list of messages string pieces, for a conditional type.
 * "separator"; optional conditional message string piece separator, the default is a single space.
@@ -143,14 +144,42 @@ short_message:
 - '{path}'
 ```
 
-boolean helpers are defined as a set of attributes:
+Boolean helpers are defined as a set of attributes:
 
-* "input_attribute"; required name of the attribute which the value that needs to be mapped is read from.
-* "output_attribute"; required name of the attribute which the mapped value is written to.
+* "input_attribute"; required name of the attribute which the value is read from.
+* "output_attribute"; required name of the attribute which the formatted value is written to.
 * "default_value"; optional default value if there is no corresponding mapping in "values".
 * "value_if_false"; optional output value if the boolean input value is False.
 * "value_if_true"; optional output value if the boolean input value is True.
 
+#### Custom helpers
+
+Custom helpers can be defined to map a value of an event attribute to custom
+formatting code.
+
+```
+type: 'conditional'
+data_type: 'fs:stat:ntfs'
+custom_helpers:
+- identifier: 'ntfs_file_reference'
+  output_attribute: 'file_reference'
+message:
+- '{display_name}'
+- 'File reference: {file_reference}'
+short_message:
+- '{filename}'
+- '{file_reference}'
+```
+
+Here `ntfs_file_reference` references the `NTFSFileReferenceFormatterHelper`,
+which is defined in `plaso/formatters/file_system.py`.
+
+Custom helpers are defined as a set of attributes:
+
+* "identifier"; required identifier of the custom format helper.
+* "input_attribute"; optional name of the attribute which the value is read from.
+* "output_attribute"; optional name of the attribute which the formatted value is written to.
+
 #### Enumeration helpers
 
 Enumeration helpers can be defined to map a value of an event attribute to
@@ -186,10 +215,10 @@ short_message:
 - '{description}'
 ```
 
-enumeration helpers are defined as a set of attributes:
+Enumeration helpers are defined as a set of attributes:
 
-* "input_attribute"; required name of the attribute which the value that needs to be mapped is read from.
-* "output_attribute"; required name of the attribute which the mapped value is written to.
+* "input_attribute"; required name of the attribute which the value is read from.
+* "output_attribute"; required name of the attribute which the formatted value is written to.
 * "default_value"; optional default value if there is no corresponding mapping in "values".
 * "values"; required value mappings, contains key value pairs.
 
@@ -241,10 +270,17 @@ short_message:
 - '{flag_values}'
 ```
 
+Flags helpers are defined as a set of attributes:
+
+* "input_attribute"; required name of the attribute which the value is read from.
+* "output_attribute"; required name of the attribute which the formatted value is written to.
+* "values"; required value mappings, contains key value pairs.
+
 #### Change log
 
 * 20200227 Added support for formatter configuration files.
 * 20200822 Added support for enumeration helpers.
 * 20200904 Added support for flags helpers.
 * 20200916 Removed source types from formatters.
 * 20201220 Added support for boolean helpers.
+* 20201227 Added support for custom helpers.

plaso/formatters/chrome.py

@@ -7,27 +7,31 @@
 from plaso.formatters import manager
 
 
-class ChromePageVisitedFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for Chrome page visited event values."""
+class ChromeHistoryTypedCountFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """Google Chrome history typed count formatter helper."""
 
-  DATA_TYPE = 'chrome:history:page_visited'
+  IDENTIFIER = 'chrome_history_typed_count'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
 
     Args:
       event_values (dict[str, object]): event values.
     """
-    typed_count = event_values.get('typed_count', 0)
-    if typed_count == 0:
-      url_typed_string = '(URL not typed directly)'
-    elif typed_count == 1:
-      url_typed_string = '(URL typed {0:d} time)'
-    else:
-      url_typed_string = '(URL typed {0:d} times)'
+    typed_count = event_values.get('typed_count', None)
+    if typed_count is not None:
+      if typed_count == 0:
+        url_typed_string = '(URL not typed directly)'
+      elif typed_count == 1:
+        url_typed_string = '(URL typed 1 time)'
+      elif typed_count > 1:
+        url_typed_string = '(URL typed {0:d} times)'.format(typed_count)
+      else:
+        url_typed_string = typed_count
 
-    event_values['url_typed_string'] = url_typed_string
+      event_values['url_typed_string'] = url_typed_string
 
 
 manager.FormattersManager.RegisterEventFormatterHelper(
-    ChromePageVisitedFormatter)
+    ChromeHistoryTypedCountFormatterHelper)

plaso/formatters/chrome_preferences.py

@@ -7,19 +7,36 @@
 from plaso.formatters import manager
 
 
-class ChromeContentSettingsExceptionsFormatter(
+class ChromePreferencesPrimaryURLFormatterHelper(
     interface.CustomEventFormatterHelper):
-  """Custom formatter for Chrome content settings exceptions event values."""
+  """Google Chrome preferences primary URL formatter helper."""
 
-  DATA_TYPE = 'chrome:preferences:content_settings:exceptions'
+  IDENTIFIER = 'chrome_preferences_primary_url'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
 
     Args:
       event_values (dict[str, object]): event values.
     """
-    # There is apparently a bug, either in GURL.cc or
+    primary_url = event_values.get('primary_url', None)
+    if primary_url == '':
+      event_values['primary_url'] = 'local file'
+
+
+class ChromePreferencesSecondaryURLFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """Google Chrome preferences secondary URL formatter helper."""
+
+  IDENTIFIER = 'chrome_preferences_secondary_url'
+
+  def FormatEventValues(self, event_values):
+    """Formats event values using the helper.
+
+    Args:
+      event_values (dict[str, object]): event values.
+    """
+    # There appears to be an issue in either GURL.cc or
     # content_settings_pattern.cc where URLs with file:// scheme are stored in
     # the URL as an empty string, which is later detected as being Invalid, and
     # Chrome produces the following example logs:
@@ -29,19 +46,16 @@ def FormatEventValues(self, event_values):
     # More research needed, could be related to https://crbug.com/132659
 
     primary_url = event_values.get('primary_url', None)
-    if primary_url == '':
-      primary_url = 'local file'
-
     secondary_url = event_values.get('secondary_url', None)
     if secondary_url == '':
       secondary_url = 'local file'
 
     if secondary_url in (primary_url, '*'):
       secondary_url = None
 
-    event_values['primary_url'] = primary_url
     event_values['secondary_url'] = secondary_url
 
 
-manager.FormattersManager.RegisterEventFormatterHelper(
-    ChromeContentSettingsExceptionsFormatter)
+manager.FormattersManager.RegisterEventFormatterHelpers([
+    ChromePreferencesPrimaryURLFormatterHelper,
+    ChromePreferencesSecondaryURLFormatterHelper])

plaso/formatters/file_system.py

@@ -7,10 +7,10 @@
 from plaso.formatters import manager
 
 
-class NTFSFileStatEventFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for NTFS file system stat event values."""
+class NTFSFileReferenceFormatterHelper(interface.CustomEventFormatterHelper):
+  """NTFS file reference formatter helper."""
 
-  DATA_TYPE = 'fs:stat:ntfs'
+  IDENTIFIER = 'ntfs_file_reference'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
@@ -23,37 +23,41 @@ def FormatEventValues(self, event_values):
       event_values['file_reference'] = '{0:d}-{1:d}'.format(
           file_reference & 0xffffffffffff, file_reference >> 48)
 
+
+class NTFSParentFileReferenceFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """NTFS parent file reference formatter helper."""
+
+  IDENTIFIER = 'ntfs_parent_file_reference'
+
+  def FormatEventValues(self, event_values):
+    """Formats event values using the helper.
+
+    Args:
+      event_values (dict[str, object]): event values.
+    """
     parent_file_reference = event_values.get('parent_file_reference', None)
     if parent_file_reference:
       event_values['parent_file_reference'] = '{0:d}-{1:d}'.format(
           parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
 
-    path_hints = event_values.get('path_hints', [])
-    if path_hints:
-      event_values['path_hints'] = ';'.join(path_hints)
-
 
-class NTFSUSNChangeEventFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for NTFS USN change event values."""
+class NTFSPathHintsFormatterHelper(interface.CustomEventFormatterHelper):
+  """NTFS path hints formatter helper."""
 
-  DATA_TYPE = 'fs:ntfs:usn_change'
+  IDENTIFIER = 'ntfs_path_hints'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
 
     Args:
       event_values (dict[str, object]): event values.
     """
-    file_reference = event_values.get('file_reference', None)
-    if file_reference:
-      event_values['file_reference'] = '{0:d}-{1:d}'.format(
-          file_reference & 0xffffffffffff, file_reference >> 48)
-
-    parent_file_reference = event_values.get('parent_file_reference', None)
-    if parent_file_reference:
-      event_values['parent_file_reference'] = '{0:d}-{1:d}'.format(
-          parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
+    path_hints = event_values.get('path_hints', None)
+    if path_hints:
+      event_values['path_hints'] = ';'.join(path_hints)
 
 
 manager.FormattersManager.RegisterEventFormatterHelpers([
-    NTFSFileStatEventFormatter, NTFSUSNChangeEventFormatter])
+    NTFSFileReferenceFormatterHelper, NTFSParentFileReferenceFormatterHelper,
+    NTFSPathHintsFormatterHelper])