CM-48734 - Update file filtering for all scan types (#313)

Author: MarshalX

cycode/cli/consts.py

@@ -14,38 +14,40 @@
 SCA_SCAN_TYPE = 'sca'
 SAST_SCAN_TYPE = 'sast'
 
-IAC_SCAN_SUPPORTED_FILES = ('.tf', '.tf.json', '.json', '.yaml', '.yml', 'dockerfile')
+IAC_SCAN_SUPPORTED_FILE_EXTENSIONS = ('.tf', '.tf.json', '.json', '.yaml', '.yml', '.dockerfile', '.containerfile')
+IAC_SCAN_SUPPORTED_FILE_PREFIXES = ('dockerfile', 'containerfile')
 
 SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE = (
-    '.7z',
+    '.DS_Store',
     '.bmp',
-    '.bz2',
-    '.dmg',
-    '.exe',
     '.gif',
-    '.gz',
     '.ico',
-    '.jar',
-    '.jpg',
-    '.jpeg',
-    '.png',
-    '.rar',
-    '.realm',
-    '.s7z',
-    '.svg',
-    '.tar',
     '.tif',
     '.tiff',
     '.webp',
-    '.zi',
+    '.mp3',
+    '.mp4',
+    '.mkv',
+    '.avi',
+    '.mov',
+    '.mpg',
+    '.mpeg',
+    '.wav',
+    '.vob',
+    '.aac',
+    '.flac',
+    '.ogg',
+    '.mka',
+    '.wma',
+    '.wmv',
+    '.psd',
+    '.ai',
+    '.model',
     '.lock',
     '.css',
-    '.less',
-    '.dll',
-    '.enc',
-    '.deb',
-    '.obj',
-    '.model',
+    '.pdf',
+    '.odt',
+    '.iso',
 )
 
 SCA_CONFIGURATION_SCAN_SUPPORTED_FILES = (  # keep in lowercase
@@ -55,11 +57,18 @@
     'composer.lock',
     'go.sum',
     'go.mod',
+    'go.mod.graph',
     'gopkg.lock',
     'pom.xml',
+    'bom.json',
+    'bcde.mvndeps',
     'build.gradle',
+    '.gradle',
     'gradle.lockfile',
     'build.gradle.kts',
+    '.gradle.kts',
+    '.properties',
+    '.kt',  # config KT files
     'package.json',
     'package-lock.json',
     'yarn.lock',
@@ -69,9 +78,10 @@
     'packages.lock.json',
     'nuget.config',
     '.csproj',
+    '.vbproj',
     'gemfile',
     'gemfile.lock',
-    'build.sbt',
+    '.sbt',
     'build.scala',
     'build.sbt.lock',
     'pyproject.toml',
@@ -84,14 +94,36 @@
     'mix.lock',
     'package.swift',
     'package.resolved',
+    'pubspec.yaml',
+    'pubspec.lock',
+    'conanfile.py',
+    'conanfile.txt',
+    'maven_install.json',
+    'conan.lock',
 )
 
-SCA_EXCLUDED_PATHS = ('node_modules',)
+SCA_EXCLUDED_PATHS = (
+    'node_modules',
+    'venv',
+    '.venv',
+    '__pycache__',
+    '.pytest_cache',
+    '.tox',
+    '.mvn',
+    '.gradle',
+    '.npm',
+    '.yarn',
+    '.bundle',
+    '.bloop',
+    '.build',
+    '.dart_tool',
+    '.pub',
+)
 
 PROJECT_FILES_BY_ECOSYSTEM_MAP = {
     'crates': ['Cargo.lock', 'Cargo.toml'],
     'composer': ['composer.json', 'composer.lock'],
-    'go': ['go.sum', 'go.mod', 'Gopkg.lock'],
+    'go': ['go.sum', 'go.mod', 'go.mod.graph', 'Gopkg.lock'],
     'maven_pom': ['pom.xml'],
     'maven_gradle': ['build.gradle', 'build.gradle.kts', 'gradle.lockfile'],
     'npm': ['package.json', 'package-lock.json', 'yarn.lock', 'npm-shrinkwrap.json', '.npmrc'],
@@ -104,6 +136,8 @@
     'pypi_setup': ['setup.py'],
     'hex': ['mix.exs', 'mix.lock'],
     'swift_pm': ['Package.swift', 'Package.resolved'],
+    'dart': ['pubspec.yaml', 'pubspec.lock'],
+    'conan': ['conanfile.py', 'conanfile.txt', 'conan.lock'],
 }
 
 COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE]

cycode/cli/files_collector/excluder.py

@@ -51,8 +51,11 @@ def _is_file_relevant_for_sca_scan(filename: str) -> bool:
 
 class Excluder:
     def __init__(self) -> None:
+        self._scannable_prefixes: dict[str, tuple[str, ...]] = {
+            consts.IAC_SCAN_TYPE: consts.IAC_SCAN_SUPPORTED_FILE_PREFIXES,
+        }
         self._scannable_extensions: dict[str, tuple[str, ...]] = {
-            consts.IAC_SCAN_TYPE: consts.IAC_SCAN_SUPPORTED_FILES,
+            consts.IAC_SCAN_TYPE: consts.IAC_SCAN_SUPPORTED_FILE_EXTENSIONS,
             consts.SCA_SCAN_TYPE: consts.SCA_CONFIGURATION_SCAN_SUPPORTED_FILES,
         }
         self._non_scannable_extensions: dict[str, tuple[str, ...]] = {
@@ -74,6 +77,10 @@ def _is_file_extension_supported(self, scan_type: str, filename: str) -> bool:
         if non_scannable_extensions:
             return not filename.endswith(non_scannable_extensions)
 
+        scannable_prefixes = self._scannable_prefixes.get(scan_type)
+        if scannable_prefixes:
+            return filename.startswith(scannable_prefixes)
+
         return True
 
     def _is_relevant_file_to_scan_common(self, scan_type: str, filename: str) -> bool: