core: find and store test files (#1667)

core: fidn and store test files

First go keeping it as simple as possible

Ref: google/oss-fuzz-gen#494

Signed-off-by: David Korczynski <[email protected]>

src/fuzz_introspector/analysis.py

@@ -17,6 +17,7 @@
 import logging
 import multiprocessing
 import os
+import shutil
 
 from typing import (
     Dict,
@@ -979,3 +980,67 @@ def correlate_introspection_functions_to_debug_info(all_functions_json_report,
             else:
                 if_func['function_signature'] = 'N/A'
             if_func['debug_function_info'] = dict()
+
+
+def extract_test_information(report_dict=dict()):
+    """Correlates function data collected by debug information to function
+    data collected by LLVMs module, and uses the correlated data to generate
+    function signatures for each function based on debug information."""
+
+    # Find header files
+    normalized_paths = set()
+    for header_file in report_dict.get('all_files_in_project', []):
+        normalized_paths.add(os.path.normpath(header_file['source_file']))
+
+    directories = set()
+
+    # All directories added
+    for path in normalized_paths:
+        if path.startswith('/usr/'):
+            continue
+        directories.add('/'.join(path.split('/')[:-1]))
+
+    all_files_in_subtree = set()
+    for dir in directories:
+        for root, dirs, files in os.walk(dir):
+            for f in files:
+                all_files_in_subtree.add(os.path.join(root, f))
+
+    all_directories = set()
+    for file in all_files_in_subtree:
+        assembled_dir = '/'
+        for dd2 in file.split('/'):
+            assembled_dir += dd2
+            if os.path.isdir(assembled_dir):
+                all_directories.add(assembled_dir)
+            assembled_dir += '/'
+
+    inspirations = ["sample", "test"]
+    all_inspiration_dirs = set()
+    for directory in all_directories:
+        if any(ins in directory for ins in inspirations):
+            all_inspiration_dirs.add(directory)
+
+    # Get all .c files
+    test_extensions = ['.cc', '.cpp', '.cxx', '.c++', '.c']
+    all_test_files = set()
+    for dir in all_inspiration_dirs:
+        for root, dirs, files in os.walk(dir):
+            for f in files:
+                if not any(f.endswith(ext) for ext in test_extensions):
+                    continue
+                # Absolute path
+                absolute_path = os.path.join(root, f)
+                with open(absolute_path, 'r') as file_fp:
+                    if 'LLVMFuzzerTestOneInput' in file_fp.read():
+                        continue
+                all_test_files.add(absolute_path)
+
+    logger.info("All test files")
+    for test_file in all_test_files:
+        logger.info(test_file)
+        dst = constants.SAVED_SOURCE_FOLDER + '/' + test_file
+        os.makedirs(os.path.dirname(dst), exist_ok=True)
+        shutil.copy(test_file, dst)
+
+    return all_test_files

src/fuzz_introspector/constants.py

@@ -22,6 +22,7 @@
 ALL_FUNCTIONS_JSON = 'all-fuzz-introspector-functions.json'
 ALL_JVM_CONSTRUCTOR_JSON = 'all-fuzz-introspector-jvm-constructor.json'
 BRANCH_BLOCKERS_FILE = 'branch-blockers.json'
+TEST_FILES_JSON = 'test-files.json'
 
 HTML_REPORT = "fuzz_report.html"
 

src/fuzz_introspector/html_report.py

@@ -769,6 +769,11 @@ def create_html_report(introspection_proj: analysis.IntrospectionProject,
         all_functions_json_report, introspection_proj.debug_all_functions,
         proj_profile.target_lang, introspection_proj.debug_report)
 
+    all_test_files = analysis.extract_test_information(
+        introspection_proj.debug_report)
+    with open(constants.TEST_FILES_JSON, 'w') as test_file_fd:
+        test_file_fd.write(json.dumps(list(all_test_files)))
+
     # Write various stats and all-functions data to summary.json
     proj_profile.write_stats_to_summary_file()