Added support for custom formatter helpers log2timeline#444

Author: joachimmetz

data/formatters/browser.yaml

@@ -76,6 +76,9 @@ boolean_helpers:
 - input_attribute: 'url_hidden'
   output_attribute: 'url_hidden_string'
   value_if_true: '(URL hidden)'
+custom_helpers:
+- identifier: 'chrome_history_typed_count'
+  output_attribute: 'url_typed_string'
 enumeration_helpers:
 - input_attribute: 'page_transition_type'
   output_attribute: 'page_transition'
@@ -126,6 +129,11 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'chrome:preferences:content_settings:exceptions'
+custom_helpers:
+- identifier: 'chrome_preferences_primary_url'
+  output_attribute: 'primary_url'
+- identifier: 'chrome_preferences_secondary_url'
+  output_attribute: 'secondary_url'
 message:
 - 'Permission {permission}'
 - 'used by {primary_url}'
@@ -253,6 +261,11 @@ short_message: '{title}'
 ---
 type: 'conditional'
 data_type: 'firefox:places:page_visited'
+custom_helpers:
+- identifier: 'firefox_history_typed_count'
+  output_attribute: 'url_typed_string'
+- identifier: 'firefox_history_url_hidden'
+  output_attribute: 'url_hidden_string'
 enumeration_helpers:
 - input_attribute: 'visit_type'
   output_attribute: 'transition_string'
@@ -290,6 +303,9 @@ boolean_helpers:
 - input_attribute: 'recovered'
   output_attribute: 'recovered_string'
   value_if_true: '[Recovered Entry]'
+custom_helpers:
+- identifier: 'msiecf_cached_path'
+  output_attribute: 'cached_file_path'
 message:
 - 'Cached file: {cached_file_path}'
 - 'Cached file size: {cached_file_size}'
@@ -315,6 +331,11 @@ boolean_helpers:
 - input_attribute: 'recovered'
   output_attribute: 'recovered_string'
   value_if_true: '[Recovered Entry]'
+custom_helpers:
+- identifier: 'msiecf_cached_path'
+  output_attribute: 'cached_file_path'
+- identifier: 'msiecf_http_headers'
+  output_attribute: 'http_headers'
 message:
 - 'Location: {url}'
 - 'Number of hits: {number_of_hits}'

data/formatters/generic.yaml

@@ -183,6 +183,11 @@ short_message: '{filename}'
 ---
 type: 'conditional'
 data_type: 'fs:ntfs:usn_change'
+custom_helpers:
+- identifier: 'ntfs_file_reference'
+  output_attribute: 'file_reference'
+- identifier: 'ntfs_parent_file_reference'
+  output_attribute: 'parent_file_reference'
 flags_helpers:
 - input_attribute: 'update_reason_flags'
   output_attribute: 'update_reason'
@@ -245,6 +250,13 @@ boolean_helpers:
 - input_attribute: 'is_allocated'
   output_attribute: 'unallocated'
   value_if_false: 'unallocated'
+custom_helpers:
+- identifier: 'ntfs_file_reference'
+  output_attribute: 'file_reference'
+- identifier: 'ntfs_parent_file_reference'
+  output_attribute: 'parent_file_reference'
+- identifier: 'ntfs_path_hints'
+  output_attribute: 'path_hints'
 enumeration_helpers:
 - input_attribute: 'attribute_type'
   output_attribute: 'attribute_name'

data/formatters/windows.yaml

@@ -161,6 +161,11 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'windows:lnk:link'
+custom_helpers:
+- identifier: 'windows_shortcut_description'
+  output_attribute: 'description'
+- identifier: 'windows_shortcut_linked_path'
+  output_attribute: 'linked_path'
 message:
 - '[{description}]'
 - 'File size: {file_size}'
@@ -224,6 +229,11 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'windows:prefetch:execution'
+custom_helpers:
+- identifier: 'windows_prefetch_path_hints'
+  output_attribute: 'path_hints'
+- identifier: 'windows_prefetch_volumes_string'
+  output_attribute: 'volumes_string'
 message:
 - 'Prefetch'
 - '[{executable}] was executed -'
@@ -330,6 +340,9 @@ short_message:
 ---
 type: 'conditional'
 data_type: 'windows:registry:key_value'
+custom_helpers:
+- identifier: 'windows_registry_values'
+  output_attribute: 'values'
 message:
 - '[{key_path}]'
 - '{values}'
@@ -637,6 +650,9 @@ short_message: '[{key_path}] {entries}'
 ---
 type: 'conditional'
 data_type: 'windows:shell_item:file_entry'
+custom_helpers:
+- identifier: 'shell_item_file_entry_name'
+  output_attribute: 'file_entry_name'
 message:
 - 'Name: {name}'
 - 'Long name: {long_name}'

plaso/formatters/chrome.py

@@ -7,10 +7,11 @@
 from plaso.formatters import manager
 
 
-class ChromePageVisitedFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for Chrome page visited event values."""
+class ChromeHistoryTypedCountFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """Google Chrome history typed count formatter helper."""
 
-  DATA_TYPE = 'chrome:history:page_visited'
+  IDENTIFIER = 'chrome_history_typed_count'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
@@ -30,4 +31,4 @@ def FormatEventValues(self, event_values):
 
 
 manager.FormattersManager.RegisterEventFormatterHelper(
-    ChromePageVisitedFormatter)
+    ChromeHistoryTypedCountFormatterHelper)

plaso/formatters/chrome_preferences.py

@@ -7,19 +7,36 @@
 from plaso.formatters import manager
 
 
-class ChromeContentSettingsExceptionsFormatter(
+class ChromePreferencesPrimaryURLFormatterHelper(
     interface.CustomEventFormatterHelper):
-  """Custom formatter for Chrome content settings exceptions event values."""
+  """Google Chrome preferences primary URL formatter helper."""
 
-  DATA_TYPE = 'chrome:preferences:content_settings:exceptions'
+  IDENTIFIER = 'chrome_preferences_primary_url'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
 
     Args:
       event_values (dict[str, object]): event values.
     """
-    # There is apparently a bug, either in GURL.cc or
+    primary_url = event_values.get('primary_url', None)
+    if primary_url == '':
+      event_values['primary_url'] = 'local file'
+
+
+class ChromePreferencesSecondaryURLFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """Google Chrome preferences secondary URL formatter helper."""
+
+  IDENTIFIER = 'chrome_preferences_secondary_url'
+
+  def FormatEventValues(self, event_values):
+    """Formats event values using the helper.
+
+    Args:
+      event_values (dict[str, object]): event values.
+    """
+    # There appears to be an issue in either GURL.cc or
     # content_settings_pattern.cc where URLs with file:// scheme are stored in
     # the URL as an empty string, which is later detected as being Invalid, and
     # Chrome produces the following example logs:
@@ -29,19 +46,16 @@ def FormatEventValues(self, event_values):
     # More research needed, could be related to https://crbug.com/132659
 
     primary_url = event_values.get('primary_url', None)
-    if primary_url == '':
-      primary_url = 'local file'
-
     secondary_url = event_values.get('secondary_url', None)
     if secondary_url == '':
       secondary_url = 'local file'
 
     if secondary_url in (primary_url, '*'):
       secondary_url = None
 
-    event_values['primary_url'] = primary_url
     event_values['secondary_url'] = secondary_url
 
 
-manager.FormattersManager.RegisterEventFormatterHelper(
-    ChromeContentSettingsExceptionsFormatter)
+manager.FormattersManager.RegisterEventFormatterHelpers([
+    ChromePreferencesPrimaryURLFormatterHelper,
+    ChromePreferencesSecondaryURLFormatterHelper])

plaso/formatters/file_system.py

@@ -7,10 +7,10 @@
 from plaso.formatters import manager
 
 
-class NTFSFileStatEventFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for NTFS file system stat event values."""
+class NTFSFileReferenceFormatterHelper(interface.CustomEventFormatterHelper):
+  """NTFS file reference formatter helper."""
 
-  DATA_TYPE = 'fs:stat:ntfs'
+  IDENTIFIER = 'ntfs_file_reference'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
@@ -23,37 +23,41 @@ def FormatEventValues(self, event_values):
       event_values['file_reference'] = '{0:d}-{1:d}'.format(
           file_reference & 0xffffffffffff, file_reference >> 48)
 
+
+class NTFSParentFileReferenceFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """NTFS parent file reference formatter helper."""
+
+  IDENTIFIER = 'ntfs_parent_file_reference'
+
+  def FormatEventValues(self, event_values):
+    """Formats event values using the helper.
+
+    Args:
+      event_values (dict[str, object]): event values.
+    """
     parent_file_reference = event_values.get('parent_file_reference', None)
     if parent_file_reference:
       event_values['parent_file_reference'] = '{0:d}-{1:d}'.format(
           parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
 
-    path_hints = event_values.get('path_hints', [])
-    if path_hints:
-      event_values['path_hints'] = ';'.join(path_hints)
-
 
-class NTFSUSNChangeEventFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for NTFS USN change event values."""
+class NTFSPathHintsFormatterHelper(interface.CustomEventFormatterHelper):
+  """NTFS path hints formatter helper."""
 
-  DATA_TYPE = 'fs:ntfs:usn_change'
+  IDENTIFIER = 'ntfs_path_hints'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
 
     Args:
       event_values (dict[str, object]): event values.
     """
-    file_reference = event_values.get('file_reference', None)
-    if file_reference:
-      event_values['file_reference'] = '{0:d}-{1:d}'.format(
-          file_reference & 0xffffffffffff, file_reference >> 48)
-
-    parent_file_reference = event_values.get('parent_file_reference', None)
-    if parent_file_reference:
-      event_values['parent_file_reference'] = '{0:d}-{1:d}'.format(
-          parent_file_reference & 0xffffffffffff, parent_file_reference >> 48)
+    path_hints = event_values.get('path_hints', [])
+    if path_hints:
+      event_values['path_hints'] = ';'.join(path_hints)
 
 
 manager.FormattersManager.RegisterEventFormatterHelpers([
-    NTFSFileStatEventFormatter, NTFSUSNChangeEventFormatter])
+    NTFSFileReferenceFormatterHelper, NTFSParentFileReferenceFormatterHelper,
+    NTFSPathHintsFormatterHelper])

plaso/formatters/firefox.py

@@ -7,21 +7,18 @@
 from plaso.formatters import manager
 
 
-class FirefoxPageVisitFormatter(interface.CustomEventFormatterHelper):
-  """Custom formatter for Firefox page visited event values."""
+class FirefoxHistoryTypedCountFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """Mozilla Firefox history typed count formatter helper."""
 
-  DATA_TYPE = 'firefox:places:page_visited'
+  IDENTIFIER = 'firefox_history_typed_count'
 
   def FormatEventValues(self, event_values):
     """Formats event values using the helper.
 
     Args:
       event_values (dict[str, object]): event values.
     """
-    hidden = event_values.get('hidden', None)
-    if hidden == '1':
-      event_values['url_hidden_string'] = '(URL hidden)'
-
     typed = event_values.get('typed', None)
     if typed == '1':
       url_typed_string = '(URL directly typed)'
@@ -31,5 +28,23 @@ def FormatEventValues(self, event_values):
     event_values['url_typed_string'] = url_typed_string
 
 
-manager.FormattersManager.RegisterEventFormatterHelper(
-    FirefoxPageVisitFormatter)
+class FirefoxHistoryURLHiddenFormatterHelper(
+    interface.CustomEventFormatterHelper):
+  """Mozilla Firefox history URL hidden formatter helper."""
+
+  IDENTIFIER = 'firefox_history_url_hidden'
+
+  def FormatEventValues(self, event_values):
+    """Formats event values using the helper.
+
+    Args:
+      event_values (dict[str, object]): event values.
+    """
+    hidden = event_values.get('hidden', None)
+    if hidden == '1':
+      event_values['url_hidden_string'] = '(URL hidden)'
+
+
+manager.FormattersManager.RegisterEventFormatterHelpers([
+    FirefoxHistoryTypedCountFormatterHelper,
+    FirefoxHistoryURLHiddenFormatterHelper])

plaso/formatters/interface.py

@@ -81,6 +81,7 @@ class CustomEventFormatterHelper(EventFormatterHelper):
   """Base class for a helper for custom formatting of event data."""
 
   DATA_TYPE = ''
+  IDENTIFIER = ''
 
   @abc.abstractmethod
   def FormatEventValues(self, event_values):
@@ -195,6 +196,7 @@ class EventFormatter(object):
   event object attribute is defined as {attribute_name}.
 
   Attributes:
+    custom_helpers (list[str]): identifiers of custom event formatter helpers.
     helpers (list[EventFormatterHelper]): event formatter helpers.
   """
 
@@ -208,10 +210,6 @@ class EventFormatter(object):
   FORMAT_STRING = ''
   FORMAT_STRING_SHORT = ''
 
-  # The source short and long strings.
-  SOURCE_SHORT = 'LOG'
-  SOURCE_LONG = ''
-
   # The format string can be defined as:
   # {name}, {name:format}, {name!conversion}, {name!conversion:format}
   _FORMAT_STRING_ATTRIBUTE_NAME_RE = re.compile(
@@ -221,6 +219,7 @@ def __init__(self):
     """Initializes an event formatter object."""
     super(EventFormatter, self).__init__()
     self._format_string_attribute_names = None
+    self.custom_helpers = []
     self.helpers = []
 
   def _FormatMessage(self, format_string, event_values):
@@ -309,6 +308,20 @@ def GetFormatStringAttributeNames(self):
 
     return set(self._format_string_attribute_names)
 
+  # pylint: disable=unused-argument
+  def AddCustomHelper(
+      self, identifier, input_attribute=None, output_attribute=None):
+    """Adds a custom event formatter helper.
+
+    Args:
+      identifier (str): identifier.
+      input_attribute (Optional[str]): name of the attribute that contains
+          the input value.
+      output_attribute (Optional[str]): name of the attribute where the
+          output value should be stored.
+    """
+    self.custom_helpers.append(identifier)
+
   def AddHelper(self, helper):
     """Adds an event formatter helper.