CM-48075 - Update pre-commit hook to work with compact output (#307)

Author: MarshalX

.pre-commit-hooks.yaml

@@ -3,10 +3,10 @@
   language: python
   language_version: python3
   entry: cycode
-  args: [ '--no-progress-meter', 'scan', '--scan-type', 'secret', 'pre-commit' ]
+  args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'secret', 'pre-commit' ]
 - id: cycode-sca
   name: Cycode SCA pre-commit defender
   language: python
   language_version: python3
   entry: cycode
-  args: [ '--no-progress-meter', 'scan', '--scan-type', 'sca', 'pre-commit' ]
+  args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sca', 'pre-commit' ]

README.md

@@ -221,26 +221,26 @@ Perform the following steps to install the pre-commit hook:
     ```yaml
     repos:
       - repo: https://github.com/cycodehq/cycode-cli
-        rev: v2.3.0
+        rev: v3.0.0
         hooks:
           - id: cycode
             stages:
-              - commit
+              - pre-commit
     ```
 
 4. Modify the created file for your specific needs. Use hook ID `cycode` to enable scan for Secrets. Use hook ID `cycode-sca` to enable SCA scan. If you want to enable both, use this configuration:
 
     ```yaml
     repos:
       - repo: https://github.com/cycodehq/cycode-cli
-        rev: v2.3.0
+        rev: v3.0.0
         hooks:
           - id: cycode
             stages:
-              - commit
+              - pre-commit
           - id: cycode-sca
             stages:
-              - commit
+              - pre-commit
     ```
 
 5. Install Cycode’s hook:

cycode/cli/apps/scan/code_scanner.py

@@ -48,15 +48,17 @@
 logger = get_logger('Code Scanner')
 
 
-def scan_sca_pre_commit(ctx: typer.Context) -> None:
+def scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None:
     scan_type = ctx.obj['scan_type']
     scan_parameters = get_scan_parameters(ctx)
     git_head_documents, pre_committed_documents = get_pre_commit_modified_documents(
-        ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES
+        progress_bar=ctx.obj['progress_bar'],
+        progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES,
+        repo_path=repo_path,
     )
     git_head_documents = exclude_irrelevant_documents_to_scan(scan_type, git_head_documents)
     pre_committed_documents = exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents)
-    sca_code_scanner.perform_pre_hook_range_scan_actions(git_head_documents, pre_committed_documents)
+    sca_code_scanner.perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents)
     scan_commit_range_documents(
         ctx,
         git_head_documents,
@@ -269,14 +271,13 @@ def scan_commit_range(
         commit_id = commit.hexsha
         commit_ids_to_scan.append(commit_id)
         parent = commit.parents[0] if commit.parents else git_proxy.get_null_tree()
-        diff = commit.diff(parent, create_patch=True, R=True)
+        diff_index = commit.diff(parent, create_patch=True, R=True)
         commit_documents_to_scan = []
-        for blob in diff:
-            blob_path = get_path_by_os(os.path.join(path, get_diff_file_path(blob)))
+        for diff in diff_index:
             commit_documents_to_scan.append(
                 Document(
-                    path=blob_path,
-                    content=blob.diff.decode('UTF-8', errors='replace'),
+                    path=get_path_by_os(get_diff_file_path(diff)),
+                    content=diff.diff.decode('UTF-8', errors='replace'),
                     is_git_diff_format=True,
                     unique_id=commit_id,
                 )
@@ -413,10 +414,10 @@ def scan_commit_range_documents(
     _report_scan_status(
         cycode_client,
         scan_type,
-        local_scan_result.scan_id,
+        scan_id,
         scan_completed,
-        local_scan_result.relevant_detections_count,
-        local_scan_result.detections_count,
+        relevant_detections_count,
+        detections_count,
         len(to_documents_to_scan),
         zip_file_size,
         scan_command_type,
@@ -658,7 +659,11 @@ def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = N
     scan_parameters['paths'] = paths
 
     if len(paths) != 1:
-        # ignore remote url if multiple paths are provided
+        logger.debug('Multiple paths provided, going to ignore remote url')
+        return scan_parameters
+
+    if not os.path.isdir(paths[0]):
+        logger.debug('Path is not a directory, going to ignore remote url')
         return scan_parameters
 
     remote_url = try_get_git_remote_url(paths[0])

cycode/cli/apps/scan/pre_commit/pre_commit_command.py

@@ -27,14 +27,16 @@ def pre_commit_command(
 
     scan_type = ctx.obj['scan_type']
 
+    repo_path = os.getcwd()  # change locally for easy testing
+
     progress_bar = ctx.obj['progress_bar']
     progress_bar.start()
 
     if scan_type == consts.SCA_SCAN_TYPE:
-        scan_sca_pre_commit(ctx)
+        scan_sca_pre_commit(ctx, repo_path)
         return
 
-    diff_files = git_proxy.get_repo(os.getcwd()).index.diff('HEAD', create_patch=True, R=True)
+    diff_files = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
 
     progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files))
 

cycode/cli/apps/scan/repository/repository_command.py

@@ -1,4 +1,3 @@
-import os
 from pathlib import Path
 from typing import Annotated, Optional
 
@@ -44,16 +43,16 @@ def repository_command(
         progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries))
 
         documents_to_scan = []
-        for file in file_entries:
+        for blob in file_entries:
             # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only
             progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES)
 
-            absolute_path = get_path_by_os(os.path.join(path, file.path))
-            file_path = file.path if monitor else absolute_path
+            absolute_path = get_path_by_os(blob.abspath)
+            file_path = get_path_by_os(blob.path) if monitor else absolute_path
             documents_to_scan.append(
                 Document(
                     file_path,
-                    file.data_stream.read().decode('UTF-8', errors='replace'),
+                    blob.data_stream.read().decode('UTF-8', errors='replace'),
                     absolute_path=absolute_path,
                 )
             )

cycode/cli/cli_types.py

@@ -3,42 +3,50 @@
 from cycode.cli import consts
 
 
-class OutputTypeOption(str, Enum):
+class StrEnum(str, Enum):
+    def __str__(self) -> str:
+        return self.value
+
+
+class OutputTypeOption(StrEnum):
     RICH = 'rich'
     TEXT = 'text'
     JSON = 'json'
     TABLE = 'table'
 
 
-class ExportTypeOption(str, Enum):
+class ExportTypeOption(StrEnum):
     JSON = 'json'
     HTML = 'html'
     SVG = 'svg'
 
 
-class ScanTypeOption(str, Enum):
+class ScanTypeOption(StrEnum):
     SECRET = consts.SECRET_SCAN_TYPE
     SCA = consts.SCA_SCAN_TYPE
     IAC = consts.IAC_SCAN_TYPE
     SAST = consts.SAST_SCAN_TYPE
 
+    def __str__(self) -> str:
+        return self.value
+
 
-class ScaScanTypeOption(str, Enum):
+class ScaScanTypeOption(StrEnum):
     PACKAGE_VULNERABILITIES = 'package-vulnerabilities'
     LICENSE_COMPLIANCE = 'license-compliance'
 
 
-class SbomFormatOption(str, Enum):
+class SbomFormatOption(StrEnum):
     SPDX_2_2 = 'spdx-2.2'
     SPDX_2_3 = 'spdx-2.3'
     CYCLONEDX_1_4 = 'cyclonedx-1.4'
 
 
-class SbomOutputFormatOption(str, Enum):
+class SbomOutputFormatOption(StrEnum):
     JSON = 'json'
 
 
-class SeverityOption(str, Enum):
+class SeverityOption(StrEnum):
     INFO = 'info'
     LOW = 'low'
     MEDIUM = 'medium'

cycode/cli/files_collector/repository_documents.py

@@ -3,7 +3,7 @@
 from typing import TYPE_CHECKING, Optional, Union
 
 from cycode.cli import consts
-from cycode.cli.files_collector.sca import sca_code_scanner
+from cycode.cli.files_collector.sca.sca_code_scanner import get_file_content_from_commit_diff
 from cycode.cli.models import Document
 from cycode.cli.utils.git_proxy import git_proxy
 from cycode.cli.utils.path_utils import get_file_content, get_path_by_os
@@ -38,30 +38,36 @@ def parse_commit_range(commit_range: str, path: str) -> tuple[str, str]:
     return from_commit_rev, to_commit_rev
 
 
-def get_diff_file_path(file: 'Diff') -> Optional[str]:
-    return file.b_path if file.b_path else file.a_path
+def get_diff_file_path(file: 'Diff', relative: bool = False) -> Optional[str]:
+    if relative:
+        # relative to the repository root
+        return file.b_path if file.b_path else file.a_path
+
+    if file.b_blob:
+        return file.b_blob.abspath
+    return file.a_blob.abspath
 
 
 def get_diff_file_content(file: 'Diff') -> str:
     return file.diff.decode('UTF-8', errors='replace')
 
 
 def get_pre_commit_modified_documents(
-    progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection'
+    progress_bar: 'BaseProgressBar',
+    progress_bar_section: 'ProgressBarSection',
+    repo_path: str,
 ) -> tuple[list[Document], list[Document]]:
     git_head_documents = []
     pre_committed_documents = []
 
-    repo = git_proxy.get_repo(os.getcwd())
-    diff_files = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
-    progress_bar.set_section_length(progress_bar_section, len(diff_files))
-    for file in diff_files:
+    repo = git_proxy.get_repo(repo_path)
+    diff_index = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True)
+    progress_bar.set_section_length(progress_bar_section, len(diff_index))
+    for diff in diff_index:
         progress_bar.update(progress_bar_section)
 
-        diff_file_path = get_diff_file_path(file)
-        file_path = get_path_by_os(diff_file_path)
-
-        file_content = sca_code_scanner.get_file_content_from_commit(repo, consts.GIT_HEAD_COMMIT_REV, diff_file_path)
+        file_path = get_path_by_os(get_diff_file_path(diff))
+        file_content = get_file_content_from_commit_diff(repo, consts.GIT_HEAD_COMMIT_REV, diff)
         if file_content is not None:
             git_head_documents.append(Document(file_path, file_content))
 
@@ -92,14 +98,13 @@ def get_commit_range_modified_documents(
     for blob in modified_files_diff:
         progress_bar.update(progress_bar_section)
 
-        diff_file_path = get_diff_file_path(blob)
-        file_path = get_path_by_os(diff_file_path)
+        file_path = get_path_by_os(get_diff_file_path(blob))
 
-        file_content = sca_code_scanner.get_file_content_from_commit(repo, from_commit_rev, diff_file_path)
+        file_content = get_file_content_from_commit_diff(repo, from_commit_rev, blob)
         if file_content is not None:
             from_commit_documents.append(Document(file_path, file_content))
 
-        file_content = sca_code_scanner.get_file_content_from_commit(repo, to_commit_rev, diff_file_path)
+        file_content = get_file_content_from_commit_diff(repo, to_commit_rev, blob)
         if file_content is not None:
             to_commit_documents.append(Document(file_path, file_content))
 

cycode/cli/files_collector/sca/sca_code_scanner.py

@@ -1,4 +1,3 @@
-import os
 from typing import TYPE_CHECKING, Optional
 
 import typer
@@ -18,7 +17,7 @@
 from cycode.logger import get_logger
 
 if TYPE_CHECKING:
-    from git import Repo
+    from git import Diff, Repo
 
 BUILD_DEP_TREE_TIMEOUT = 180
 
@@ -39,9 +38,9 @@ def perform_pre_commit_range_scan_actions(
 
 
 def perform_pre_hook_range_scan_actions(
-    git_head_documents: list[Document], pre_committed_documents: list[Document]
+    repo_path: str, git_head_documents: list[Document], pre_committed_documents: list[Document]
 ) -> None:
-    repo = git_proxy.get_repo(os.getcwd())
+    repo = git_proxy.get_repo(repo_path)
     add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV)
     add_ecosystem_related_files_if_exists(pre_committed_documents)
 
@@ -69,7 +68,7 @@ def get_doc_ecosystem_related_project_files(
         file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file)
         if not is_project_file_exists_in_documents(documents, file_to_search):
             if repo:
-                file_content = get_file_content_from_commit(repo, commit_rev, file_to_search)
+                file_content = get_file_content_from_commit_path(repo, commit_rev, file_to_search)
             else:
                 file_content = get_file_content(file_to_search)
 
@@ -151,13 +150,20 @@ def get_manifest_file_path(document: Document, is_monitor_action: bool, project_
     return join_paths(project_path, document.path) if is_monitor_action else document.path
 
 
-def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> Optional[str]:
+def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]:
     try:
         return repo.git.show(f'{commit}:{file_path}')
     except git_proxy.get_git_command_error():
         return None
 
 
+def get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]:
+    from cycode.cli.files_collector.repository_documents import get_diff_file_path
+
+    file_path = get_diff_file_path(diff, relative=True)
+    return get_file_content_from_commit_path(repo, commit, file_path)
+
+
 def perform_pre_scan_documents_actions(
     ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False
 ) -> None:

cycode/cli/printers/console_printer.py

@@ -28,9 +28,8 @@ class ConsolePrinter:
         'text': TextPrinter,
         'json': JsonPrinter,
         'table': TablePrinter,
-        # overrides
+        # overrides:
         'table_sca': ScaTablePrinter,
-        'text_sca': ScaTablePrinter,
     }
 
     def __init__(