CM-23340 - Skip on collect node_modules folder for SCA npm (#119)

* added is_file_relevant for sca

* added tests

* emptied test data files

* fixed according to windows test failures

* naming refactoring

* added is_file_relevant_for_sca_scan tests

* removed old tests

* add type hints; remove unused imports

---------

Co-authored-by: Ilya Siamionau <[email protected]>

Author: PelegCycode

cycode/cli/code_scanner.py

@@ -730,6 +730,14 @@ def _get_package_name(detection) -> str:
     return f'{package_name}@{package_version}'
 
 
+def _is_file_relevant_for_sca_scan(filename: str) -> bool:
+    if any([sca_excluded_path in filename for sca_excluded_path in SCA_EXCLUDED_PATHS]):
+        logger.debug("file is irrelevant because it is from node_modules's inner path, %s",
+                     {'filename': filename})
+        return False
+    return True
+
+
 def _is_relevant_file_to_scan(scan_type: str, filename: str) -> bool:
     if _is_subpath_of_cycode_configuration_folder(filename):
         logger.debug("file is irrelevant because it is in cycode configuration directory, %s",
@@ -755,6 +763,10 @@ def _is_relevant_file_to_scan(scan_type: str, filename: str) -> bool:
         logger.debug("file is irrelevant because its exceeded max size limit, %s",
                      {'filename': filename})
         return False
+
+    if scan_type == SCA_SCAN_TYPE and not _is_file_relevant_for_sca_scan(filename):
+        return False
+
     return True
 
 
@@ -814,8 +826,8 @@ def _does_document_exceed_max_size_limit(content: str) -> bool:
 
 def _is_subpath_of_cycode_configuration_folder(filename: str) -> bool:
     return is_sub_path(configuration_manager.global_config_file_manager.get_config_directory_path(), filename) \
-           or is_sub_path(configuration_manager.local_config_file_manager.get_config_directory_path(), filename) \
-           or filename.endswith(ConfigFileManager.get_config_file_route())
+        or is_sub_path(configuration_manager.local_config_file_manager.get_config_directory_path(), filename) \
+        or filename.endswith(ConfigFileManager.get_config_file_route())
 
 
 def _handle_exception(context: click.Context, e: Exception):

cycode/cli/consts.py

@@ -30,6 +30,10 @@
     'pipfile', 'pipfile.lock', 'requirements.txt', 'setup.py'
 ]
 
+SCA_EXCLUDED_PATHS = [
+    'node_modules'
+]
+
 PROJECT_FILES_BY_ECOSYSTEM_MAP = {
     "crates": ["Cargo.lock", "Cargo.toml"],
     "composer": ["composer.json", "composer.lock"],

tests/cli/test_code_scanner.py

@@ -1,10 +1,12 @@
+import os
+
 import click
 import pytest
 from click import ClickException
 from git import InvalidGitRepositoryError
 from requests import Response
 
-from cycode.cli.code_scanner import _handle_exception  # noqa
+from cycode.cli.code_scanner import _handle_exception, _is_file_relevant_for_sca_scan, exclude_irrelevant_files  # noqa
 from cycode.cli.exceptions import custom_exceptions
 
 
@@ -58,3 +60,14 @@ def mock_secho(msg, *_, **__):
     with ctx:
         with pytest.raises(ClickException):
             _handle_exception(ctx, ValueError('test'))
+
+
+def test_is_file_relevant_for_sca_scan():
+    path = os.path.join('some_package', 'node_modules', 'package.json')
+    assert _is_file_relevant_for_sca_scan(path) is False
+    path = os.path.join('some_package', 'node_modules', 'package.lock')
+    assert _is_file_relevant_for_sca_scan(path) is False
+    path = os.path.join('some_package', 'package.json')
+    assert _is_file_relevant_for_sca_scan(path) is True
+    path = os.path.join('some_package', 'package.lock')
+    assert _is_file_relevant_for_sca_scan(path) is True