Adds clang as second compiler for test cases (#27)

* Added feature to compiler test cases with more than one compiler. Added clang as first examples. Fixed test cases to work with gcc

* Added acceptance tests for clang x64, adjusted Travis scripts (now
runs also unittests), install_cross_compilers installs also clang.

* Skips test for cwe415, which is broken on clang + Ubuntu 16.04

* Added feature to compiler test cases with more than one compiler. Added clang as first examples. Fixed test cases to work with gcc

* Added acceptance tests for clang x64, adjusted Travis scripts (now
runs also unittests), install_cross_compilers installs also clang.

* Skips test for cwe415, which is broken on clang + Ubuntu 16.04

* Added change to CHANGES.md

* Fixed test issue: test for json output was pre-compiler suffix.

.travis_prepare.sh

@@ -1,5 +1,3 @@
-#!/bin/bash
-
 #!/bin/bash
 cd test/artificial_samples/
 ./install_cross_compilers.sh

.travis_run_tests.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
+dune runtest
 pytest

CHANGES.md

@@ -2,6 +2,7 @@ dev
 ====
 
 -   Added more documentation to checks (PR #26)
+-   Added clang as another compiler for test cases, added tests for clang compiled test cases (PR #27)
 -   Fixed check CWE367: use symbols defined in config.json (PR #28)
 -   Refactoring of logging and JSON support via --json (PR #30)
 -   Added file output support via --out (PR #30)

test/acceptance/cwe_checker_testlib.py

@@ -2,37 +2,37 @@
 import subprocess
 
 
-def build_bap_cmd(filename, target, arch):
+def build_bap_cmd(filename, target, arch, compiler):
         if 'travis' in os.environ['USER']:
-                abs_path = os.path.abspath('test/artificial_samples/build/cwe_%s_%s.out' % (filename, arch))
+                abs_path = os.path.abspath('test/artificial_samples/build/cwe_%s_%s_%s.out' % (filename, arch, compiler))
                 cmd = 'docker run --rm -v %s:/tmp/input cwe-checker:latest bap /tmp/input --pass=cwe-checker --cwe-checker-partial=CWE%s  --cwe-checker-config=/home/bap/cwe_checker/src/config.json' % (abs_path, target)
         else:
-                cmd = 'bap test/artificial_samples/build/cwe_%s_%s.out --pass=cwe-checker --cwe-checker-partial=CWE%s --cwe-checker-config=src/config.json' % (filename, arch, target)
+                cmd = 'bap test/artificial_samples/build/cwe_%s_%s_%s.out --pass=cwe-checker --cwe-checker-partial=CWE%s --cwe-checker-config=src/config.json' % (filename, arch, compiler, target)
         return cmd.split()
 
 
-def build_bap_emulation_cmd(filename, target, arch):
+def build_bap_emulation_cmd(filename, target, arch, compiler):
         if 'travis' in os.environ['USER']:
-                abs_path = os.path.abspath('test/artificial_samples/build/cwe_%s_%s.out' % (filename, arch))
+                abs_path = os.path.abspath('test/artificial_samples/build/cwe_%s_%s_%s.out' % (filename, arch, compiler))
                 cmd = 'docker run --rm -v %s:/tmp/input cwe-checker:latest bap /tmp/input --recipe=recipes/emulation' % abs_path
         else:
-                cmd = 'bap test/artificial_samples/build/cwe_%s_%s.out --recipe=recipes/emulation' % (filename, arch)
+                cmd = 'bap test/artificial_samples/build/cwe_%s_%s_%s.out --recipe=recipes/emulation' % (filename, arch, compiler)
         return cmd.split()
 
 
-def execute_and_check_occurence(filename, target, arch, string):
+def execute_and_check_occurence(filename, target, arch, compiler, string):
     occurence = 0
-    bap_cmd = build_bap_cmd(filename, target, arch)
+    bap_cmd = build_bap_cmd(filename, target, arch, compiler)
     output = subprocess.check_output(bap_cmd)
     for l in output.splitlines():
         if string in l:
             occurence += 1
     return occurence
 
 
-def execute_emulation_and_check_occurence(filename, target, arch, string):
+def execute_emulation_and_check_occurence(filename, target, arch, compiler, string):
     occurence = 0
-    bap_cmd = build_bap_emulation_cmd(filename, target, arch)
+    bap_cmd = build_bap_emulation_cmd(filename, target, arch, compiler)
     output = subprocess.check_output(bap_cmd)
     for l in output.splitlines():
         if string in l:

test/acceptance/test_cwe190.py

@@ -8,28 +8,33 @@ def setUp(self):
         self.target = '190'
         self.string = b'Integer Overflow or Wraparound'
 
-    def test_cwe190_01_arm(self):
+    def test_cwe190_01_arm_gcc(self):
         expect_res = 3
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe190_01_x86(self):
+    def test_cwe190_01_x86_gcc(self):
         expect_res = 3
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe190_01_x64(self):
+    def test_cwe190_01_x64_gcc(self):
         expect_res = 3
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'gcc', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe190_01_x64_clang(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'clang', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Depends on proper MIPS support in BAP")
-    def test_cwe190_01_mips(self):
+    def test_cwe190_01_mips_gcc(self):
         expect_res = 3
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe190_01_ppc(self):
+    def test_cwe190_01_ppc_gcc(self):
         expect_res = 3
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', 'gcc', self.string)
         self.assertEqual(res, expect_res)

test/acceptance/test_cwe215.py

@@ -9,27 +9,32 @@ def setUp(self):
         self.filename = '476'
         self.string = b'Information Exposure Through Debug Information'
 
-    def test_cwe215_01_arm(self):
+    def test_cwe215_01_arm_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe215_01_x86(self):
+    def test_cwe215_01_x86_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x86', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe215_01_x64(self):
+    def test_cwe215_01_x64_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x64', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe215_01_ppc(self):
+    def test_cwe215_01_x64_clang(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x64', 'clang', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe215_01_mips(self):
+    def test_cwe215_01_ppc_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'ppc', 'gcc', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe215_01_mips_gcc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)

test/acceptance/test_cwe243.py

@@ -8,55 +8,65 @@ def setUp(self):
         self.target = '243'
         self.string = b'The program utilizes chroot without dropping privileges and/or changing the directory'
 
-    def test_cwe243_01_arm(self):
+    def test_cwe243_01_arm_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe243_01_x86(self):
+    def test_cwe243_01_x86_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe243_01_x64(self):
+    def test_cwe243_01_x64_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe243_01_ppc(self):
+    def test_cwe243_01_x64_clang(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'clang', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe243_01_ppc_gcc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Depends on proper MIPS support in BAP")
-    def test_cwe243_01_mips(self):
+    def test_cwe243_01_mips_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe243_02_arm(self):
+    def test_cwe243_02_arm_gcc(self):
         expect_res = 0
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Investigate and fix this issue")
-    def test_cwe243_02_x86(self):
+    def test_cwe243_02_x86_gcc(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x86', 'gcc', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe243_02_x64_gcc(self):
         expect_res = 0
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x64', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe243_02_x64(self):
+    def test_cwe243_02_x64_clang(self):
         expect_res = 0
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x64', 'clang', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe243_02_ppc(self):
+    def test_cwe243_02_ppc_gcc(self):
         expect_res = 0
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'ppc', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Depends on proper MIPS support in BAP")
-    def test_cwe476_02_mips(self):
+    def test_cwe476_02_mips_gcc(self):
         expect_res = 0
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)

test/acceptance/test_cwe248.py

@@ -8,30 +8,35 @@ def setUp(self):
         self.target = '248'
         self.string = b'Possibly Uncaught Exception'
 
-    def test_cwe248_01_arm(self):
+    def test_cwe248_01_arm_gcc(self):
         expect_res = 2
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Fix CPP compilation issue for x86")
-    def test_cwe248_01_x86(self):
+    def test_cwe248_01_x86_gcc(self):
         expect_res = 2
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe248_01_x64(self):
+    def test_cwe248_01_x64_gcc(self):
         expect_res = 2
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'gcc', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe248_01_x64_clang(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'clang', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Depends on proper MIPS support in BAP")
-    def test_cwe248_01_mips(self):
+    def test_cwe248_01_mips_gcc(self):
         expect_res = 2
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("FIXME")
-    def test_cwe248_01_ppc(self):
+    def test_cwe248_01_ppc_gcc(self):
         expect_res = 2
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', 'gcc', self.string)
         self.assertEqual(res, expect_res)

test/acceptance/test_cwe332.py

@@ -8,28 +8,33 @@ def setUp(self):
         self.target = '332'
         self.string = b'Insufficient Entropy in PRNG'
 
-    def test_cwe332_01_arm(self):
+    def test_cwe332_01_arm_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe332_01_x86(self):
+    def test_cwe332_01_x86_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe332_01_x64(self):
+    def test_cwe332_01_x64_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'gcc', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe332_01_x64_clang(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'clang', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Depends on proper MIPS support in BAP")
-    def test_cwe332_01_mips(self):
+    def test_cwe332_01_mips_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe332_01_ppc(self):
+    def test_cwe332_01_ppc_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', 'gcc', self.string)
         self.assertEqual(res, expect_res)

test/acceptance/test_cwe367.py

@@ -8,28 +8,33 @@ def setUp(self):
         self.target = '367'
         self.string = b'Time-of-check Time-of-use Race Condition'
 
-    def test_cwe367_01_arm(self):
+    def test_cwe367_01_arm_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe367_01_x86(self):
+    def test_cwe367_01_x86_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe367_01_x64(self):
+    def test_cwe367_01_x64_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'gcc', self.string)
+        self.assertEqual(res, expect_res)
+
+    def test_cwe367_01_x64_clang(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', 'clang', self.string)
         self.assertEqual(res, expect_res)
 
     @unittest.skip("Depends on proper MIPS support in BAP")
-    def test_cwe367_01_mips(self):
+    def test_cwe367_01_mips_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', 'gcc', self.string)
         self.assertEqual(res, expect_res)
 
-    def test_cwe367_01_ppc(self):
+    def test_cwe367_01_ppc_gcc(self):
         expect_res = 1
-        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', 'gcc', self.string)
         self.assertEqual(res, expect_res)