Use more global paths for system tests (elastic#3591)

This change is the first step to make it possible to execute system tests from different locations. Currently all system test execution relies on the convention that everything is under `tests/system/` and that `nosetests` is executed in this directory. Based on this it is not possible to place system tests in different location, for example inside a metricbeat module. In addition it should simplify the creation of system tests in community beats.

* Unify handling of setUpClass
* Rename SystemTest to Test to be consistent
* Rename `beat_path` variable to `test_binary` because that is what it is
* Expect full template path in template loader
* Use absolute paths for importing other python files
* Remove duplicated code from packetbeat setUp

filebeat/tests/system/filebeat.py

@@ -2,7 +2,7 @@
 import os
 import sys
 
-sys.path.append('../../../libbeat/tests/system')
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system'))
 
 from beat.beat import TestCase
 
@@ -12,6 +12,8 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "filebeat"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+
         super(BaseTest, self).setUpClass()
 
     def get_registry(self):

filebeat/tests/system/test_crawler.py

@@ -577,7 +577,7 @@ def test_encodings(self):
                 "encoding": enc_go
             })
         self.render_config_template(
-            template="filebeat_prospectors.yml.j2",
+            template_name="filebeat_prospectors",
             prospectors=prospectors
         )
 

filebeat/tests/system/test_modules.py

@@ -40,7 +40,7 @@ def test_modules(self):
         # generate a minimal configuration
         cfgfile = os.path.join(self.working_dir, "filebeat.yml")
         self.render_config_template(
-            template="filebeat_modules.yml.j2",
+            template_name="filebeat_modules",
             output=cfgfile,
             index_name=self.index_name,
             elasticsearch_url=self.elasticsearch_url)

filebeat/tests/system/test_prospector.py

@@ -119,7 +119,7 @@ def test_stdin_eof(self):
             close_eof="true",
         )
 
-        args = [self.beat_path,
+        args = [self.test_binary,
                 "-systemTest",
                 "-test.coverprofile",
                 os.path.join(self.working_dir, "coverage.cov"),

generator/beat/{beat}/tests/system/{beat}.py

@@ -1,3 +1,4 @@
+import os
 import sys
 sys.path.append('../../vendor/github.com/elastic/beats/libbeat/tests/system')
 from beat.beat import TestCase
@@ -8,5 +9,5 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "{beat}"
-        self.build_path = "../../build/system-tests/"
-        self.beat_path = "../../{beat}.test"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+        super(BaseTest, self).setUpClass()

heartbeat/tests/system/heartbeat.py

@@ -1,5 +1,7 @@
 import sys
-sys.path.append('../../vendor/github.com/elastic/beats/libbeat/tests/system')
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system'))
+
 from beat.beat import TestCase
 
 
@@ -8,5 +10,4 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "heartbeat"
-        self.build_path = "../../build/system-tests/"
-        self.beat_path = "../../heartbeat.test"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))

libbeat/tests/system/base.py

@@ -1,3 +1,4 @@
+import os
 from beat.beat import TestCase
 
 
@@ -6,5 +7,6 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "mockbeat"
-        self.build_path = "../../build/system-tests/"
-        self.beat_path = "../../libbeat.test"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+        self.test_binary = self.beat_path + "/libbeat.test"
+        super(BaseTest, self).setUpClass()

libbeat/tests/system/beat/beat.py

@@ -102,19 +102,22 @@ class TestCase(unittest.TestCase):
     @classmethod
     def setUpClass(self):
 
-        # Create build path
-        build_dir = "../../build"
-        if 'BUILD_DIR' in os.environ.keys() and os.environ['BUILD_DIR'] != '':
-            build_dir = os.environ['BUILD_DIR']
-        self.build_path = build_dir + "/system-tests/"
+
 
         # Path to test binary
         if not hasattr(self, 'beat_name'):
             self.beat_name = "beat"
 
-        # Path to test binary
         if not hasattr(self, 'beat_path'):
-            self.beat_path = "../../" + self.beat_name + ".test"
+            self.beat_path = "."
+
+        # Path to test binary
+        if not hasattr(self, 'test_binary'):
+            self.test_binary = os.path.abspath(self.beat_path + "/" + self.beat_name + ".test")
+
+        # Create build path
+        build_dir = self.beat_path + "/build"
+        self.build_path = build_dir + "/system-tests/"
 
     def run_beat(self,
                  cmd=None,
@@ -150,7 +153,7 @@ def start_beat(self,
 
         # Init defaults
         if cmd is None:
-            cmd = self.beat_path
+            cmd = self.test_binary
 
         if config is None:
             config = self.beat_name + ".yml"
@@ -175,17 +178,19 @@ def start_beat(self,
         proc.start()
         return proc
 
-    def render_config_template(self, template=None,
+    def render_config_template(self, template_name=None,
                                output=None, **kargs):
 
         # Init defaults
-        if template is None:
-            template = self.beat_name + ".yml.j2"
+        if template_name is None:
+            template_name = self.beat_name
+
+        template_path = "./tests/system/config/" + template_name + ".yml.j2"
 
         if output is None:
             output = self.beat_name + ".yml"
 
-        template = self.template_env.get_template(template)
+        template = self.template_env.get_template(template_path)
 
         kargs["beat"] = self
         output_str = template.render(**kargs)
@@ -242,7 +247,7 @@ def copy_files(self, files, source_dir="files/"):
     def setUp(self):
 
         self.template_env = jinja2.Environment(
-            loader=jinja2.FileSystemLoader("config")
+            loader=jinja2.FileSystemLoader(self.beat_path)
         )
 
         # create working dir
@@ -379,14 +384,18 @@ def all_fields_are_expected(self, objs, expected_fields,
                     raise Exception("Unexpected key '{}' found"
                                     .format(key))
 
-    def load_fields(self, fields_doc="../../_meta/fields.generated.yml"):
+    def load_fields(self, fields_doc=None):
         """
         Returns a list of fields to expect in the output dictionaries
         and a second list that contains the fields that have a
         dictionary type.
 
         Reads these lists from the fields documentation.
         """
+
+        if fields_doc is None:
+            fields_doc = self.beat_path + "/_meta/fields.generated.yml"
+
         def extract_fields(doc_list, name):
             fields = []
             dictfields = []
@@ -414,12 +423,12 @@ def extract_fields(doc_list, name):
 
         # Not all beats have a fields.generated.yml. Fall back to fields.yml
         if not os.path.isfile(fields_doc):
-            fields_doc = "../../_meta/fields.yml"
+            fields_doc = self.beat_path + "/_meta/fields.yml"
 
         # TODO: Make fields_doc path more generic to work with beat-generator
         with open(fields_doc, "r") as f:
-            # TODO: Make this path more generic to work with beat-generator.
-            with open("../../../libbeat/_meta/fields.common.yml") as f2:
+            path = os.path.abspath(os.path.dirname(__file__) + "../../../../_meta/fields.common.yml")
+            with open(path) as f2:
                 content = f2.read()
 
             #content = "fields:\n"

libbeat/tests/system/test_base.py

@@ -33,7 +33,7 @@ def test_invalid_config(self):
         """
         Checks stop on invalid config
         """
-        shutil.copy("../files/invalid.yml",
+        shutil.copy(self.beat_path + "/tests/files/invalid.yml",
                     os.path.join(self.working_dir, "invalid.yml"))
 
         exit_code = self.run_beat(config="invalid.yml")
@@ -68,7 +68,7 @@ def test_config_test(self):
         """
         Checks if -configtest works as expected
         """
-        shutil.copy("../../_meta/config.yml",
+        shutil.copy(self.beat_path + "/_meta/config.yml",
                     os.path.join(self.working_dir, "libbeat.yml"))
         with open(self.working_dir + "/mockbeat.template.json", "w") as f:
             f.write('{"template": true}')
@@ -94,7 +94,7 @@ def test_version(self):
         """
         Checks if version param works
         """
-        args = ["../../libbeat.test"]
+        args = [self.beat_path + "/libbeat.test"]
 
         args.extend(["-version",
                      "-e",

metricbeat/tests/system/metricbeat.py

@@ -1,7 +1,8 @@
 import sys
 import os
 
-sys.path.append('../../../libbeat/tests/system')
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system'))
+
 from beat.beat import TestCase
 
 COMMON_FIELDS = ["@timestamp", "beat", "metricset.name", "metricset.host",
@@ -15,8 +16,8 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "metricbeat"
-        self.build_path = "../../build/system-tests/"
-        self.beat_path = "../../metricbeat.test"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+        super(BaseTest, self).setUpClass()
 
     def assert_fields_are_documented(self, evt):
         """

metricbeat/tests/system/test_system.py

@@ -42,7 +42,7 @@
                          "state", "username", "cgroup"]
 
 
-class SystemTest(metricbeat.BaseTest):
+class Test(metricbeat.BaseTest):
 
     @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os")
     def test_cpu(self):

packetbeat/tests/system/packetbeat.py

@@ -2,10 +2,8 @@
 import sys
 import subprocess
 import json
-import jinja2
-import shutil
 
-sys.path.append('../../../libbeat/tests/system')
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system'))
 
 from beat.beat import TestCase
 from beat.beat import Proc
@@ -22,10 +20,11 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "packetbeat"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
         super(BaseTest, self).setUpClass()
 
     def run_packetbeat(self, pcap,
-                       cmd="../../packetbeat.test",
+                       cmd=None,
                        config="packetbeat.yml",
                        output="packetbeat.log",
                        extra_args=[],
@@ -38,11 +37,14 @@ def run_packetbeat(self, pcap,
         the caller.
         """
 
+        if cmd is None:
+            cmd = self.beat_path + "/packetbeat.test"
+
         args = [cmd]
 
         args.extend([
             "-e",
-            "-I", os.path.join("pcaps", pcap),
+            "-I", os.path.join(self.beat_path + "/tests/system/pcaps", pcap),
             "-c", os.path.join(self.working_dir, config),
             "-t",
             "-systemTest",
@@ -109,24 +111,5 @@ def read_output(self,
 
     def setUp(self):
 
-        self.template_env = jinja2.Environment(
-            loader=jinja2.FileSystemLoader("config")
-        )
-
-        # create working dir
-        self.working_dir = os.path.join(self.build_path + "run", self.id())
-        if os.path.exists(self.working_dir):
-            shutil.rmtree(self.working_dir)
-        os.makedirs(self.working_dir)
-
-        try:
-            # update the last_run link
-            if os.path.islink(self.build_path + "last_run"):
-                os.unlink(self.build_path + "last_run")
-            os.symlink(self.build_path + "run/{}".format(self.id()), self.build_path + "last_run")
-        except:
-            # symlink is best effort and can fail when
-            # running tests in parallel
-            pass
-
         self.expected_fields, self.dict_fields = self.load_fields()
+        super(BaseTest, self).setUp()

winlogbeat/tests/system/winlogbeat.py

@@ -1,3 +1,4 @@
+import os
 import sys
 
 if sys.platform.startswith("win"):
@@ -7,7 +8,8 @@
     import win32security
     import win32evtlogutil
 
-sys.path.append('../../../libbeat/tests/system')
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../../libbeat/tests/system'))
+
 from beat.beat import TestCase
 
 
@@ -16,6 +18,7 @@ class BaseTest(TestCase):
     @classmethod
     def setUpClass(self):
         self.beat_name = "winlogbeat"
+        self.beat_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
         super(BaseTest, self).setUpClass()