Moved AMCache formatter to configuration log2timeline#444

Author: joachimmetz

data/formatters.yaml

@@ -0,0 +1,41 @@
+# Event formatters.
+
+type: 'conditional'
+data_type: 'windows:registry:amcache'
+message:
+- 'path: {full_path}'
+- 'sha1: {sha1}'
+- 'productname: {productname}'
+- 'companyname: {companyname}'
+- 'fileversion: {fileversion}'
+- 'languagecode: {languagecode}'
+- 'filesize: {filesize}'
+- 'filedescription: {filedescription}'
+- 'linkerts: {linkerts}'
+- 'lastmodifiedts: {lastmodifiedts}'
+- 'createdts: {createdts}'
+- 'programid: {programid}'
+short_message:
+- 'path: {full_path}'
+short_source: 'AMCACHE'
+source: 'Amcache Registry Entry'
+---
+type: 'conditional'
+data_type: 'windows:registry:amcache:programs'
+message:
+- 'name: {name}'
+- 'version: {version}'
+- 'publisher: {publisher}'
+- 'languagecode: {languagecode}'
+- 'entrytype: {entrytype}'
+- 'uninstallkey: {uninstallkey}'
+- 'filepaths: {filepaths}'
+- 'productcode: {productcode}'
+- 'packagecode: {packagecode}'
+- 'msiproductcode: {msiproductcode}'
+- 'msipackagecode: {msipackagecode}'
+- 'files: {files}'
+short_message:
+- 'name: {name}'
+short_source: 'AMCACHEPROGRAM'
+source: 'Amcache Programs Registry Entry'

plaso/cli/extraction_tool.py

@@ -208,7 +208,7 @@ def _ReadParserPresetsFromFile(self):
       parsers_manager.ParsersManager.ReadPresetsFromFile(self._presets_file)
     except errors.MalformedPresetError as exception:
       raise errors.BadConfigOption(
-          'Unable to read presets from file with error: {0!s}'.format(
+          'Unable to read parser presets from file with error: {0!s}'.format(
               exception))
 
   def _SetExtractionParsersAndPlugins(self, configuration, session):

plaso/cli/psort_tool.py

@@ -29,6 +29,7 @@
 from plaso.engine import engine
 from plaso.engine import knowledge_base
 from plaso.filters import event_filter
+from plaso.formatters import manager as formatters_manager
 from plaso.lib import errors
 from plaso.lib import loggers
 from plaso.lib import timelib
@@ -61,6 +62,8 @@ class PsortTool(
       'Application to read, filter and process output from a plaso storage '
       'file.')
 
+  _FORMATTERS_FILE_NAME = 'formatters.yaml'
+
   def __init__(self, input_reader=None, output_writer=None):
     """Initializes the CLI tool object.
 
@@ -79,6 +82,7 @@ def __init__(self, input_reader=None, output_writer=None):
     self._deduplicate_events = True
     self._event_filter_expression = None
     self._event_filter = None
+    self._formatters_file = None
     self._knowledge_base = knowledge_base.KnowledgeBase()
     self._number_of_analysis_reports = 0
     self._preferred_language = 'en-US'
@@ -282,6 +286,26 @@ def _PrintAnalysisReportsDetails(self, storage_reader):
 
       table_view.Write(self._output_writer)
 
+  def _ReadEventFormattersFromFile(self):
+    """Reads the event formatters the formatters.yaml file.
+
+    Raises:
+      BadConfigOption: if the event formatters file cannot be read.
+    """
+    self._formatters_file = os.path.join(
+        self._data_location, self._FORMATTERS_FILE_NAME)
+    if not os.path.isfile(self._formatters_file):
+      raise errors.BadConfigOption(
+          'No such event formatters file: {0:s}.'.format(self._formatters_file))
+
+    try:
+      formatters_manager.FormattersManager.ReadFormattersFromFile(
+          self._formatters_file)
+    except KeyError as exception:
+      raise errors.BadConfigOption(
+          'Unable to read event formatters from file with error: {0!s}'.format(
+              exception))
+
   def AddProcessingOptions(self, argument_group):
     """Adds processing options to the argument group
 
@@ -449,6 +473,8 @@ def ParseOptions(self, options):
     helpers_manager.ArgumentHelperManager.ParseOptions(
         options, self, names=['data_location'])
 
+    self._ReadEventFormattersFromFile()
+
     self._ParseLogFileOptions(options)
 
     self._ParseProcessingOptions(options)

plaso/cli/psteal_tool.py

@@ -23,6 +23,7 @@
 from plaso.engine import engine
 from plaso.engine import knowledge_base
 from plaso.engine import single_process as single_process_engine
+from plaso.formatters import manager as formatters_manager
 from plaso.lib import errors
 from plaso.lib import loggers
 from plaso.multi_processing import psort
@@ -83,6 +84,8 @@ class PstealTool(
       'And that is how you build a timeline using psteal...',
       '']))
 
+  _FORMATTERS_FILE_NAME = 'formatters.yaml'
+
   _SOURCE_TYPES_TO_PREPROCESS = frozenset([
       dfvfs_definitions.SOURCE_TYPE_DIRECTORY,
       dfvfs_definitions.SOURCE_TYPE_STORAGE_MEDIA_DEVICE,
@@ -103,6 +106,7 @@ def __init__(self, input_reader=None, output_writer=None):
     self._command_line_arguments = None
     self._deduplicate_events = True
     self._enable_sigsegv_handler = False
+    self._formatters_file = None
     self._knowledge_base = knowledge_base.KnowledgeBase()
     self._number_of_analysis_reports = 0
     self._number_of_extraction_workers = 0
@@ -172,6 +176,26 @@ def _PrintAnalysisReportsDetails(
 
       table_view.Write(self._output_writer)
 
+  def _ReadEventFormattersFromFile(self):
+    """Reads the event formatters the formatters.yaml file.
+
+    Raises:
+      BadConfigOption: if the event formatters file cannot be read.
+    """
+    self._formatters_file = os.path.join(
+        self._data_location, self._FORMATTERS_FILE_NAME)
+    if not os.path.isfile(self._formatters_file):
+      raise errors.BadConfigOption(
+          'No such event formatters file: {0:s}.'.format(self._formatters_file))
+
+    try:
+      formatters_manager.FormattersManager.ReadFormattersFromFile(
+          self._formatters_file)
+    except KeyError as exception:
+      raise errors.BadConfigOption(
+          'Unable to read event formatters from file with error: {0!s}'.format(
+              exception))
+
   def AnalyzeEvents(self):
     """Analyzes events from a plaso storage file and generate a report.
 
@@ -440,6 +464,7 @@ def ParseOptions(self, options):
         options, self, names=['data_location'])
 
     self._ReadParserPresetsFromFile()
+    self._ReadEventFormattersFromFile()
 
     # The output modules options are dependent on the preferred language
     # and preferred time zone options.

plaso/formatters/__init__.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 """This file contains an import statement for each formatter."""
 
-from plaso.formatters import amcache
 from plaso.formatters import android_app_usage
 from plaso.formatters import android_calls
 from plaso.formatters import android_sms

plaso/formatters/amcache.py

@@ -5,6 +5,7 @@
 
 from plaso.formatters import default
 from plaso.formatters import logger
+from plaso.formatters import yaml_formatters_file
 
 
 class FormattersManager(object):
@@ -13,6 +14,9 @@ class FormattersManager(object):
   _formatter_classes = {}
   _formatter_objects = {}
 
+  # TODO: work-around to prevent the tests re-reading the tests.
+  file_was_read = False
+
   @classmethod
   def DeregisterFormatter(cls, formatter_class):
     """Deregisters a formatter class.
@@ -27,9 +31,8 @@ def DeregisterFormatter(cls, formatter_class):
     """
     formatter_data_type = formatter_class.DATA_TYPE.lower()
     if formatter_data_type not in cls._formatter_classes:
-      raise KeyError(
-          'Formatter class not set for data type: {0:s}.'.format(
-              formatter_class.DATA_TYPE))
+      raise KeyError('Formatter class not set for data type: {0:s}.'.format(
+          formatter_class.DATA_TYPE))
 
     del cls._formatter_classes[formatter_data_type]
 
@@ -55,8 +58,8 @@ def GetFormatterObject(cls, data_type):
         formatter_object = formatter_class()
 
       if not formatter_object:
-        logger.warning(
-            'Using default formatter for data type: {0:s}'.format(data_type))
+        logger.warning('Using default formatter for data type: {0:s}'.format(
+            data_type))
         formatter_object = default.DefaultFormatter()
 
       cls._formatter_objects[data_type] = formatter_object
@@ -95,6 +98,27 @@ def GetSourceStrings(cls, event, event_data):
     formatter_object = cls.GetFormatterObject(event_data.data_type)
     return formatter_object.GetSources(event, event_data)
 
+  @classmethod
+  def ReadFormattersFromFile(cls, path):
+    """Reads formatters from a file.
+
+    Args:
+      path (str): path of file that contains the formatters configuration.
+
+    Raises:
+      KeyError: if formatter class is already set for the corresponding
+          data type.
+    """
+    formatters_file = yaml_formatters_file.YAMLFormattersFile()
+    for formatter in formatters_file.ReadFromFile(path):
+      # TODO: refactor RegisterFormatter to only use formatter objects.
+      cls.RegisterFormatter(formatter)
+
+      data_type = formatter.DATA_TYPE.lower()
+      cls._formatter_objects[data_type] = formatter
+
+    cls.file_was_read = True
+
   @classmethod
   def RegisterFormatter(cls, formatter_class):
     """Registers a formatter class.
@@ -110,9 +134,8 @@ def RegisterFormatter(cls, formatter_class):
     """
     formatter_data_type = formatter_class.DATA_TYPE.lower()
     if formatter_data_type in cls._formatter_classes:
-      raise KeyError((
-          'Formatter class already set for data type: {0:s}.').format(
-              formatter_class.DATA_TYPE))
+      raise KeyError('Formatter class already set for data type: {0:s}.'.format(
+          formatter_class.DATA_TYPE))
 
     cls._formatter_classes[formatter_data_type] = formatter_class
 

plaso/formatters/manager.py

@@ -463,7 +463,7 @@ def ReadPresetsFromFile(cls, path):
     """Reads parser and parser plugin presets from a file.
 
     Args:
-      path (str): path of file that contains the the parser and parser plugin
+      path (str): path of file that contains the parser and parser plugin
           presets configuration.
 
     Raises:

plaso/parsers/manager.py

@@ -36,18 +36,15 @@ def testParse(self):
         'is-f4510.tmp\\idafree50.tmp')
     self.assertEqual(event.full_path, expected_full_path)
 
-    expected_sha1 = '82274eef0911a948f91425f5e5b0e730517fe75e'
-    self.assertEqual(event.sha1, expected_sha1)
+    self.assertEqual(event.sha1, '82274eef0911a948f91425f5e5b0e730517fe75e')
 
     event = events[1148]
 
-    expected_program_name = 'FileInsight - File analysis tool'
-    self.assertEqual(event.name, expected_program_name)
+    self.assertEqual(event.name, 'FileInsight - File analysis tool')
+    self.assertEqual(event.publisher, 'McAfee Inc.')
 
-    expected_publisher = 'McAfee Inc.'
-    self.assertEqual(event.publisher, expected_publisher)
-
-    # TODO: add test for message string
+    expected_message = 'name: FileInsight - File analysis tool'
+    self._TestGetMessageStrings(event, expected_message, expected_message)
 
   @shared_test_lib.skipUnlessHasTestFile(['SYSTEM'])
   def testParseWithSystem(self):

tests/parsers/amcache.py

@@ -3,6 +3,8 @@
 
 from __future__ import unicode_literals
 
+import os
+
 from dfdatetime import posix_time as dfdatetime_posix_time
 
 from dfvfs.lib import definitions as dfvfs_definitions
@@ -175,6 +177,11 @@ def _TestGetMessageStrings(
       expected_message (str): expected message string.
       expected_short_message (str): expected short message string.
     """
+    if not formatters_manager.FormattersManager.file_was_read:
+      formatters_file_path = os.path.join(self._DATA_PATH, 'formatters.yaml')
+      formatters_manager.FormattersManager.ReadFormattersFromFile(
+          formatters_file_path)
+
     formatter_mediator = formatters_mediator.FormatterMediator(
         data_location=self._DATA_PATH)
     message, message_short = (