diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 42467e02..4101ded8 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -27,10 +27,10 @@ jobs: run: | git checkout ${{ steps.latest_tag.outputs.LATEST_TAG }} - - name: Set up Python 3.8 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached_poetry diff --git a/.github/workflows/pre_release.yml b/.github/workflows/pre_release.yml index 7aca89c1..9b665d73 100644 --- a/.github/workflows/pre_release.yml +++ b/.github/workflows/pre_release.yml @@ -32,10 +32,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40913767..aacbae5e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,10 +31,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.8 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 3a91d0f3..575abfd0 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -21,9 +21,9 @@ jobs: uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0b5ddb58..a62a01b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,9 +26,9 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Load cached Poetry setup id: cached-poetry diff --git a/.github/workflows/tests_full.yml b/.github/workflows/tests_full.yml index 985a3d36..a9ddd4f6 100644 --- a/.github/workflows/tests_full.yml +++ b/.github/workflows/tests_full.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ macos-latest, ubuntu-latest, windows-latest ] - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12", "3.13" ] + python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] runs-on: ${{matrix.os}} diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 40e7a614..ab69bf3f 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -3,10 +3,10 @@ language: python language_version: python3 entry: cycode - args: [ '--no-progress-meter', 'scan', '--scan-type', 'secret', 'pre_commit' ] + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'secret', 'pre-commit' ] - id: cycode-sca name: Cycode SCA pre-commit defender language: python language_version: python3 entry: cycode - args: [ '--no-progress-meter', 'scan', '--scan-type', 'sca', 'pre_commit' ] + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sca', 'pre-commit' ] diff --git a/CODEOWNERS b/CODEOWNERS index 32a2011c..aba89cba 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @MarshalX @MichalBor @MaorDavidzon @artem-fedorov @elsapet @gotbadger @cfabianski +* @MarshalX @elsapet @gotbadger @cfabianski diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a95c8c28..857a27cd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,14 +4,14 @@ ## How to contribute to Cycode CLI -The minimum version of Python that we support is 3.8. +The minimum version of Python that we support is 3.9. We recommend using this version for local development. But it’s fine to use a higher version without using new features from these versions. The project is under Poetry project management. To deal with it, you should install it on your system: -Install Poetry (feel free to use Brew, etc): +Install Poetry (feel free to use Brew, etc.): ```shell curl -sSL https://install.python-poetry.org | python - -y @@ -70,6 +70,8 @@ poetry run ruff format . Many rules support auto-fixing. You can run it with the `--fix` flag. +Plugin for JB IDEs with auto formatting on save is available [here](https://plugins.jetbrains.com/plugin/20574-ruff). + ### Branching and versioning We use the `main` branch as the main one. diff --git a/README.md b/README.md index 189d69f2..13e23a6f 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ This guide walks you through both installation and usage. 1. [Options](#options) 1. [Severity Threshold](#severity-option) 2. [Monitor](#monitor-option) - 3. [Report](#report-option) + 3. [Cycode Report](#cycode-report-option) 4. [Package Vulnerabilities](#package-vulnerabilities-option) 5. [License Compliance](#license-compliance-option) 6. [Lock Restore](#lock-restore-option) @@ -54,7 +54,7 @@ This guide walks you through both installation and usage. # Prerequisites -- The Cycode CLI application requires Python version 3.8 or later. +- The Cycode CLI application requires Python version 3.9 or later. - Use the [`cycode auth` command](#using-the-auth-command) to authenticate to Cycode with the CLI - Alternatively, you can get a Cycode Client ID and Client Secret Key by following the steps detailed in the [Service Account Token](https://docs.cycode.com/docs/en/service-accounts) and [Personal Access Token](https://docs.cycode.com/v1/docs/managing-personal-access-tokens) pages, which contain details on getting these values. @@ -208,7 +208,7 @@ Cycode’s pre-commit hook can be set up within your local repository so that th Perform the following steps to install the pre-commit hook: -1. Install the pre-commit framework (Python 3.8 or higher must be installed): +1. Install the pre-commit framework (Python 3.9 or higher must be installed): ```bash pip3 install pre-commit @@ -221,11 +221,11 @@ Perform the following steps to install the pre-commit hook: ```yaml repos: - repo: https://github.com/cycodehq/cycode-cli - rev: v2.3.0 + rev: v3.0.0 hooks: - id: cycode stages: - - commit + - pre-commit ``` 4. Modify the created file for your specific needs. Use hook ID `cycode` to enable scan for Secrets. Use hook ID `cycode-sca` to enable SCA scan. If you want to enable both, use this configuration: @@ -233,14 +233,14 @@ Perform the following steps to install the pre-commit hook: ```yaml repos: - repo: https://github.com/cycodehq/cycode-cli - rev: v2.3.0 + rev: v3.0.0 hooks: - id: cycode stages: - - commit + - pre-commit - id: cycode-sca stages: - - commit + - pre-commit ``` 5. Install Cycode’s hook: @@ -281,8 +281,8 @@ The following are the options and commands available with the Cycode CLI applica | [auth](#using-the-auth-command) | Authenticate your machine to associate the CLI with your Cycode account. | | [configure](#using-the-configure-command) | Initial command to configure your CLI client authentication. | | [ignore](#ignoring-scan-results) | Ignores a specific value, path or rule ID. | -| [scan](#running-a-scan) | Scan the content for Secrets/IaC/SCA/SAST violations. You`ll need to specify which scan type to perform: commit_history/path/repository/etc. | -| [report](#report-command) | Generate report. You`ll need to specify which report type to perform. | +| [scan](#running-a-scan) | Scan the content for Secrets/IaC/SCA/SAST violations. You`ll need to specify which scan type to perform: commit-history/path/repository/etc. | +| [report](#report-command) | Generate report. You`ll need to specify which report type to perform as SBOM. | | status | Show the CLI status and exit. | # Scan Command @@ -294,24 +294,23 @@ The Cycode CLI application offers several types of scans so that you can choose | Option | Description | |------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`), the default is `secret`. | -| `--secret TEXT` | Specify a Cycode client secret for this specific scan execution. | +| `--client-secret TEXT` | Specify a Cycode client secret for this specific scan execution. | | `--client-id TEXT` | Specify a Cycode client ID for this specific scan execution. | | `--show-secret BOOLEAN` | Show secrets in plain text. See [Show/Hide Secrets](#showhide-secrets) section for more details. | | `--soft-fail BOOLEAN` | Run scan without failing, always return a non-error status code. See [Soft Fail](#soft-fail) section for more details. | | `--severity-threshold [INFO\|LOW\|MEDIUM\|HIGH\|CRITICAL]` | Show only violations at the specified level or higher. | | `--sca-scan` | Specify the SCA scan you wish to execute (`package-vulnerabilities`/`license-compliance`). The default is both. | | `--monitor` | When specified, the scan results will be recorded in the knowledge graph. Please note that when working in `monitor` mode, the knowledge graph will not be updated as a result of SCM events (Push, Repo creation). (Supported for SCA scan type only). | -| `--report` | When specified, a violations report will be generated. A URL link to the report will be printed as an output to the command execution. | +| `--cycode-report` | When specified, displays a link to the scan report in the Cycode platform in the console output. | | `--no-restore` | When specified, Cycode will not run restore command. Will scan direct dependencies ONLY! | -| `--sync` | Run scan synchronously (the default is asynchronous). | | `--gradle-all-sub-projects` | When specified, Cycode will run gradle restore command for all sub projects. Should run from root project directory ONLY! | | `--help` | Show options for given command. | | Command | Description | |----------------------------------------|-----------------------------------------------------------------| -| [commit_history](#commit-history-scan) | Scan all the commits history in this git repository | +| [commit-history](#commit-history-scan) | Scan all the commits history in this git repository | | [path](#path-scan) | Scan the files in the path supplied in the command | -| [pre_commit](#pre-commit-scan) | Use this command to scan the content that was not committed yet | +| [pre-commit](#pre-commit-scan) | Use this command to scan the content that was not committed yet | | [repository](#repository-scan) | Scan git repository including its history | ### Options @@ -340,18 +339,15 @@ When using this option, the scan results from this scan will appear in the knowl > [!WARNING] > You must be an `owner` or an `admin` in Cycode to view the knowledge graph page. -#### Report Option +#### Cycode Report Option -> [!NOTE] -> This option is not available to IaC scans. - -To push scan results tied to the [SCA policies](https://docs.cycode.com/docs/sca-policies) found in the Repository scan to Cycode, add the argument `--report` to the scan command. +For every scan performed using the Cycode CLI, a report is automatically generated and its results are sent to Cycode. These results are tied to the relevant policies (e.g., [SCA policies](https://docs.cycode.com/docs/sca-policies) for Repository scans) within the Cycode platform. -`cycode scan -t sca --report repository ~/home/git/codebase` +To have the direct URL to this Cycode report printed in your CLI output after the scan completes, add the argument `--cycode-report` to your scan command. -In the same way, you can push scan results of Secrets and SAST scans to Cycode by adding the `--report` option to the scan command. +`cycode scan --cycode-report repository ~/home/git/codebase` -When using this option, the scan results from this scan will appear in the On-Demand Scans section of Cycode. To get to this page, click the link that appears after the printed results: +All scan results from the CLI will appear in the CLI Logs section of Cycode. If you included the `--cycode-report` flag in your command, a direct link to the specific report will be displayed in your terminal following the scan results. > [!WARNING] > You must be an `owner` or an `admin` in Cycode to view this page. @@ -467,25 +463,25 @@ A commit history scan is limited to a local repository’s previous commits, foc To execute a commit history scan, execute the following: -`cycode scan commit_history {{path}}` +`cycode scan commit-history {{path}}` For example, consider a scenario in which you want to scan the commit history for a repository stored in `~/home/git/codebase`. You could then execute the following: -`cycode scan commit_history ~/home/git/codebase` +`cycode scan commit-history ~/home/git/codebase` The following options are available for use with this command: | Option | Description | |---------------------------|----------------------------------------------------------------------------------------------------------| -| `-r, --commit_range TEXT` | Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1) | +| `-r, --commit-range TEXT` | Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1) | #### Commit Range Option -The commit history scan, by default, examines the repository’s entire commit history, all the way back to the initial commit. You can instead limit the scan to a specific commit range by adding the argument `--commit_range` (`-r`) followed by the name you specify. +The commit history scan, by default, examines the repository’s entire commit history, all the way back to the initial commit. You can instead limit the scan to a specific commit range by adding the argument `--commit-range` (`-r`) followed by the name you specify. Consider the previous example. If you wanted to scan only specific commits in your repository, you could execute the following: -`cycode scan commit_history -r {{from-commit-id}}...{{to-commit-id}} ~/home/git/codebase` +`cycode scan commit-history -r {{from-commit-id}}...{{to-commit-id}} ~/home/git/codebase` ### Pre-Commit Scan @@ -824,7 +820,7 @@ The following commands are available for use with this command: | Command | Description | |------------------|-----------------------------------------------------------------| | `path` | Generate SBOM report for provided path in the command | -| `repository_url` | Generate SBOM report for provided repository URI in the command | +| `repository-url` | Generate SBOM report for provided repository URI in the command | ### Repository diff --git a/cycode/__main__.py b/cycode/__main__.py new file mode 100644 index 00000000..7ad8ef7e --- /dev/null +++ b/cycode/__main__.py @@ -0,0 +1,4 @@ +from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.main import app + +app(prog_name=PROGRAM_NAME) diff --git a/cycode/cli/app.py b/cycode/cli/app.py new file mode 100644 index 00000000..6fd4f70d --- /dev/null +++ b/cycode/cli/app.py @@ -0,0 +1,156 @@ +import logging +from typing import Annotated, Optional + +import typer +from typer import rich_utils +from typer._completion_classes import completion_init +from typer.completion import install_callback, show_callback + +from cycode import __version__ +from cycode.cli.apps import ai_remediation, auth, configure, ignore, report, scan, status +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.consts import CLI_CONTEXT_SETTINGS +from cycode.cli.printers import ConsolePrinter +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar +from cycode.cli.utils.sentry import add_breadcrumb, init_sentry +from cycode.cli.utils.version_checker import version_checker +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.models import UserAgentOptionScheme +from cycode.logger import set_logging_level + +# By default, it uses dim style which is hard to read with the combination of color from RICH_HELP +rich_utils.STYLE_ERRORS_SUGGESTION = 'bold' +# By default, it uses blue color which is too dark for some terminals +rich_utils.RICH_HELP = "Try [cyan]'{command_path} {help_option}'[/] for help." + +completion_init() # DO NOT TOUCH; this is required for the completion to work properly + +_cycode_cli_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md' +_cycode_cli_epilog = f'[bold]Documentation:[/] [link={_cycode_cli_docs}]{_cycode_cli_docs}[/link]' + +app = typer.Typer( + pretty_exceptions_show_locals=False, + pretty_exceptions_short=True, + context_settings=CLI_CONTEXT_SETTINGS, + epilog=_cycode_cli_epilog, + rich_markup_mode='rich', + no_args_is_help=True, + add_completion=False, # we add it manually to control the rich help panel +) + +app.add_typer(ai_remediation.app) +app.add_typer(auth.app) +app.add_typer(configure.app) +app.add_typer(ignore.app) +app.add_typer(report.app) +app.add_typer(scan.app) +app.add_typer(status.app) + + +def check_latest_version_on_close(ctx: typer.Context) -> None: + output = ctx.obj.get('output') + # don't print anything if the output is JSON + if output == OutputTypeOption.JSON: + return + + # we always want to check the latest version for "version" and "status" commands + should_use_cache = ctx.invoked_subcommand not in {'version', 'status'} + version_checker.check_and_notify_update(current_version=__version__, use_cache=should_use_cache) + + +def export_if_needed_on_close(ctx: typer.Context) -> None: + scan_finalized = ctx.obj.get('scan_finalized') + printer = ctx.obj.get('console_printer') + if scan_finalized and printer.is_recording: + printer.export() + + +_AUTH_RICH_HELP_PANEL = 'Authentication options' +_COMPLETION_RICH_HELP_PANEL = 'Completion options' + + +@app.callback() +def app_callback( + ctx: typer.Context, + verbose: Annotated[bool, typer.Option('--verbose', '-v', help='Show detailed logs.')] = False, + no_progress_meter: Annotated[ + bool, typer.Option('--no-progress-meter', help='Do not show the progress meter.') + ] = False, + no_update_notifier: Annotated[ + bool, typer.Option('--no-update-notifier', help='Do not check CLI for updates.') + ] = False, + output: Annotated[ + OutputTypeOption, typer.Option('--output', '-o', case_sensitive=False, help='Specify the output type.') + ] = OutputTypeOption.RICH, + user_agent: Annotated[ + Optional[str], + typer.Option(hidden=True, help='Characteristic JSON object that lets servers identify the application.'), + ] = None, + client_secret: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode client secret for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, + ), + ] = None, + client_id: Annotated[ + Optional[str], + typer.Option( + help='Specify a Cycode client ID for this specific scan execution.', + rich_help_panel=_AUTH_RICH_HELP_PANEL, + ), + ] = None, + _: Annotated[ + Optional[bool], + typer.Option( + '--install-completion', + callback=install_callback, + is_eager=True, + expose_value=False, + help='Install completion for the current shell.', + rich_help_panel=_COMPLETION_RICH_HELP_PANEL, + ), + ] = False, + __: Annotated[ + Optional[bool], + typer.Option( + '--show-completion', + callback=show_callback, + is_eager=True, + expose_value=False, + help='Show completion for the current shell, to copy it or customize the installation.', + rich_help_panel=_COMPLETION_RICH_HELP_PANEL, + ), + ] = False, +) -> None: + """[bold cyan]Cycode CLI - Command Line Interface for Cycode.[/]""" + init_sentry() + add_breadcrumb('cycode') + + ctx.ensure_object(dict) + configuration_manager = ConfigurationManager() + + verbose = verbose or configuration_manager.get_verbose_flag() + ctx.obj['verbose'] = verbose + if verbose: + set_logging_level(logging.DEBUG) + + ctx.obj['output'] = output + if output == OutputTypeOption.JSON: + no_progress_meter = True + + ctx.obj['client_id'] = client_id + ctx.obj['client_secret'] = client_secret + + ctx.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) + + ctx.obj['console_printer'] = ConsolePrinter(ctx) + ctx.call_on_close(lambda: export_if_needed_on_close(ctx)) + + if user_agent: + user_agent_option = UserAgentOptionScheme().loads(user_agent) + CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) + + if not no_update_notifier: + ctx.call_on_close(lambda: check_latest_version_on_close(ctx)) diff --git a/cycode/cli/commands/__init__.py b/cycode/cli/apps/__init__.py similarity index 100% rename from cycode/cli/commands/__init__.py rename to cycode/cli/apps/__init__.py diff --git a/cycode/cli/apps/ai_remediation/__init__.py b/cycode/cli/apps/ai_remediation/__init__.py new file mode 100644 index 00000000..00d0c7c5 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/__init__.py @@ -0,0 +1,20 @@ +import typer + +from cycode.cli.apps.ai_remediation.ai_remediation_command import ai_remediation_command + +app = typer.Typer() + +_ai_remediation_epilog = ( + 'Note: AI remediation suggestions are generated automatically and should be reviewed before applying.' +) + +app.command( + name='ai-remediation', + short_help='Get AI remediation (INTERNAL).', + epilog=_ai_remediation_epilog, + hidden=True, + no_args_is_help=True, +)(ai_remediation_command) + +# backward compatibility +app.command(hidden=True, name='ai_remediation')(ai_remediation_command) diff --git a/cycode/cli/apps/ai_remediation/ai_remediation_command.py b/cycode/cli/apps/ai_remediation/ai_remediation_command.py new file mode 100644 index 00000000..ab2eca5e --- /dev/null +++ b/cycode/cli/apps/ai_remediation/ai_remediation_command.py @@ -0,0 +1,39 @@ +from typing import Annotated +from uuid import UUID + +import typer + +from cycode.cli.apps.ai_remediation.apply_fix import apply_fix +from cycode.cli.apps.ai_remediation.print_remediation import print_remediation +from cycode.cli.exceptions.handle_ai_remediation_errors import handle_ai_remediation_exception +from cycode.cli.utils.get_api_client import get_scan_cycode_client + + +def ai_remediation_command( + ctx: typer.Context, + detection_id: Annotated[UUID, typer.Argument(help='Detection ID to get remediation for', show_default=False)], + fix: Annotated[ + bool, typer.Option('--fix', help='Apply fixes to resolve violations. Note: fix could be not available.') + ] = False, +) -> None: + """:robot: [bold cyan]Get AI-powered remediation for security issues.[/] + + This command provides AI-generated remediation guidance for detected security issues. + + Example usage: + * `cycode ai-remediation `: View remediation guidance + * `cycode ai-remediation --fix`: Apply suggested fixes + """ + client = get_scan_cycode_client(ctx) + + try: + remediation_markdown = client.get_ai_remediation(detection_id) + fix_diff = client.get_ai_remediation(detection_id, fix=True) + is_fix_available = bool(fix_diff) # exclude empty string, None, etc. + + if fix: + apply_fix(ctx, fix_diff, is_fix_available) + else: + print_remediation(ctx, remediation_markdown, is_fix_available) + except Exception as err: + handle_ai_remediation_exception(ctx, err) diff --git a/cycode/cli/apps/ai_remediation/apply_fix.py b/cycode/cli/apps/ai_remediation/apply_fix.py new file mode 100644 index 00000000..bd840411 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/apply_fix.py @@ -0,0 +1,24 @@ +import os + +import typer +from patch_ng import fromstring + +from cycode.cli.models import CliResult + + +def apply_fix(ctx: typer.Context, diff: str, is_fix_available: bool) -> None: + printer = ctx.obj.get('console_printer') + if not is_fix_available: + printer.print_result(CliResult(success=False, message='Fix is not available for this violation')) + return + + patch = fromstring(diff.encode('UTF-8')) + if patch is False: + printer.print_result(CliResult(success=False, message='Failed to parse fix diff')) + return + + is_fix_applied = patch.apply(root=os.getcwd(), strip=0) + if is_fix_applied: + printer.print_result(CliResult(success=True, message='Fix applied successfully')) + else: + printer.print_result(CliResult(success=False, message='Failed to apply fix')) diff --git a/cycode/cli/apps/ai_remediation/print_remediation.py b/cycode/cli/apps/ai_remediation/print_remediation.py new file mode 100644 index 00000000..92272b76 --- /dev/null +++ b/cycode/cli/apps/ai_remediation/print_remediation.py @@ -0,0 +1,14 @@ +import typer +from rich.markdown import Markdown + +from cycode.cli.console import console +from cycode.cli.models import CliResult + + +def print_remediation(ctx: typer.Context, remediation_markdown: str, is_fix_available: bool) -> None: + printer = ctx.obj.get('console_printer') + if printer.is_json_printer: + data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} + printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) + else: # text or table + console.print(Markdown(remediation_markdown)) diff --git a/cycode/cli/apps/auth/__init__.py b/cycode/cli/apps/auth/__init__.py new file mode 100644 index 00000000..f487e1bf --- /dev/null +++ b/cycode/cli/apps/auth/__init__.py @@ -0,0 +1,9 @@ +import typer + +from cycode.cli.apps.auth.auth_command import auth_command + +_auth_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-auth-command' +_auth_command_epilog = f'[bold]Documentation:[/] [link={_auth_command_docs}]{_auth_command_docs}[/link]' + +app = typer.Typer(no_args_is_help=False) +app.command(name='auth', epilog=_auth_command_epilog, short_help='Authenticate your machine with Cycode.')(auth_command) diff --git a/cycode/cli/apps/auth/auth_command.py b/cycode/cli/apps/auth/auth_command.py new file mode 100644 index 00000000..817e0213 --- /dev/null +++ b/cycode/cli/apps/auth/auth_command.py @@ -0,0 +1,31 @@ +import typer + +from cycode.cli.apps.auth.auth_manager import AuthManager +from cycode.cli.exceptions.handle_auth_errors import handle_auth_exception +from cycode.cli.logger import logger +from cycode.cli.models import CliResult +from cycode.cli.utils.sentry import add_breadcrumb + + +def auth_command(ctx: typer.Context) -> None: + """:key: [bold cyan]Authenticate your machine with Cycode.[/] + + This command handles authentication with Cycode's security platform. + + Example usage: + * `cycode auth`: Start interactive authentication + * `cycode auth --help`: View authentication options + """ + add_breadcrumb('auth') + printer = ctx.obj.get('console_printer') + + try: + logger.debug('Starting authentication process') + + auth_manager = AuthManager() + auth_manager.authenticate() + + result = CliResult(success=True, message='Successfully logged into cycode') + printer.print_result(result) + except Exception as err: + handle_auth_exception(ctx, err) diff --git a/cycode/cli/commands/auth_common.py b/cycode/cli/apps/auth/auth_common.py similarity index 60% rename from cycode/cli/commands/auth_common.py rename to cycode/cli/apps/auth/auth_common.py index bf8d5d41..96fec4cf 100644 --- a/cycode/cli/commands/auth_common.py +++ b/cycode/cli/apps/auth/auth_common.py @@ -1,21 +1,22 @@ -from typing import NamedTuple, Optional - -import click +from typing import TYPE_CHECKING, Optional +from cycode.cli.apps.auth.models import AuthInfo from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError -from cycode.cli.printers import ConsolePrinter from cycode.cli.user_settings.credentials_manager import CredentialsManager from cycode.cli.utils.jwt_utils import get_user_and_tenant_ids_from_access_token from cycode.cyclient.cycode_token_based_client import CycodeTokenBasedClient +if TYPE_CHECKING: + from typer import Context + -class AuthInfo(NamedTuple): - user_id: str - tenant_id: str +def get_authorization_info(ctx: 'Context') -> Optional[AuthInfo]: + printer = ctx.obj.get('console_printer') + client_id, client_secret = ctx.obj.get('client_id'), ctx.obj.get('client_secret') + if not client_id or not client_secret: + client_id, client_secret = CredentialsManager().get_credentials() -def get_authorization_info(context: Optional[click.Context] = None) -> Optional[AuthInfo]: - client_id, client_secret = CredentialsManager().get_credentials() if not client_id or not client_secret: return None @@ -27,7 +28,7 @@ def get_authorization_info(context: Optional[click.Context] = None) -> Optional[ user_id, tenant_id = get_user_and_tenant_ids_from_access_token(access_token) return AuthInfo(user_id=user_id, tenant_id=tenant_id) except (RequestHttpError, HttpUnauthorizedError): - if context: - ConsolePrinter(context).print_exception() + if ctx: + printer.print_exception() return None diff --git a/cycode/cli/commands/auth/auth_manager.py b/cycode/cli/apps/auth/auth_manager.py similarity index 83% rename from cycode/cli/commands/auth/auth_manager.py rename to cycode/cli/apps/auth/auth_manager.py index ab621842..56a480e4 100644 --- a/cycode/cli/commands/auth/auth_manager.py +++ b/cycode/cli/apps/auth/auth_manager.py @@ -1,21 +1,22 @@ import time import webbrowser -from typing import TYPE_CHECKING, Tuple - -from requests import Request +from typing import TYPE_CHECKING from cycode.cli.exceptions.custom_exceptions import AuthProcessError from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.user_settings.credentials_manager import CredentialsManager from cycode.cli.utils.string_utils import generate_random_string, hash_string_to_sha256 -from cycode.cyclient import logger from cycode.cyclient.auth_client import AuthClient from cycode.cyclient.models import ApiTokenGenerationPollingResponse +from cycode.logger import get_logger if TYPE_CHECKING: from cycode.cyclient.models import ApiToken +logger = get_logger('Auth Manager') + + class AuthManager: CODE_VERIFIER_LENGTH = 101 POLLING_WAIT_INTERVAL_IN_SECONDS = 3 @@ -50,7 +51,7 @@ def start_session(self, code_challenge: str) -> str: return auth_session.session_id def redirect_to_login_page(self, code_challenge: str, session_id: str) -> None: - login_url = self._build_login_url(code_challenge, session_id) + login_url = self.auth_client.build_login_url(code_challenge, session_id) webbrowser.open(login_url) def get_api_token(self, session_id: str, code_verifier: str) -> 'ApiToken': @@ -72,20 +73,12 @@ def get_api_token_polling(self, session_id: str, code_verifier: str) -> 'ApiToke raise AuthProcessError('Error while obtaining API token') time.sleep(self.POLLING_WAIT_INTERVAL_IN_SECONDS) - raise AuthProcessError('session expired') + raise AuthProcessError('Timeout while obtaining API token (session expired)') def save_api_token(self, api_token: 'ApiToken') -> None: self.credentials_manager.update_credentials(api_token.client_id, api_token.secret) - def _build_login_url(self, code_challenge: str, session_id: str) -> str: - app_url = self.configuration_manager.get_cycode_app_url() - login_url = f'{app_url}/account/sign-in' - query_params = {'source': 'cycode_cli', 'code_challenge': code_challenge, 'session_id': session_id} - # TODO(MarshalX). Use auth_client instead and don't depend on "requests" lib here - request = Request(url=login_url, params=query_params) - return request.prepare().url - - def _generate_pkce_code_pair(self) -> Tuple[str, str]: + def _generate_pkce_code_pair(self) -> tuple[str, str]: code_verifier = generate_random_string(self.CODE_VERIFIER_LENGTH) code_challenge = hash_string_to_sha256(code_verifier) return code_challenge, code_verifier diff --git a/cycode/cli/apps/auth/models.py b/cycode/cli/apps/auth/models.py new file mode 100644 index 00000000..4b41dd3e --- /dev/null +++ b/cycode/cli/apps/auth/models.py @@ -0,0 +1,6 @@ +from typing import NamedTuple + + +class AuthInfo(NamedTuple): + user_id: str + tenant_id: str diff --git a/cycode/cli/apps/configure/__init__.py b/cycode/cli/apps/configure/__init__.py new file mode 100644 index 00000000..4944a3e3 --- /dev/null +++ b/cycode/cli/apps/configure/__init__.py @@ -0,0 +1,14 @@ +import typer + +from cycode.cli.apps.configure.configure_command import configure_command + +_configure_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#using-the-configure-command' +_configure_command_epilog = f'[bold]Documentation:[/] [link={_configure_command_docs}]{_configure_command_docs}[/link]' + + +app = typer.Typer(no_args_is_help=True) +app.command( + name='configure', + epilog=_configure_command_epilog, + short_help='Initial command to configure your CLI client authentication.', +)(configure_command) diff --git a/cycode/cli/apps/configure/configure_command.py b/cycode/cli/apps/configure/configure_command.py new file mode 100644 index 00000000..348e3ccb --- /dev/null +++ b/cycode/cli/apps/configure/configure_command.py @@ -0,0 +1,69 @@ +from typing import Optional + +from cycode.cli.apps.configure.consts import CONFIGURATION_MANAGER, CREDENTIALS_MANAGER +from cycode.cli.apps.configure.messages import get_credentials_update_result_message, get_urls_update_result_message +from cycode.cli.apps.configure.prompts import ( + get_api_url_input, + get_app_url_input, + get_client_id_input, + get_client_secret_input, +) +from cycode.cli.console import console +from cycode.cli.utils.sentry import add_breadcrumb + + +def _should_update_value( + old_value: Optional[str], + new_value: Optional[str], +) -> bool: + if not new_value: + return False + + return old_value != new_value + + +def configure_command() -> None: + """:gear: [bold cyan]Configure Cycode CLI settings.[/] + + This command allows you to configure various aspects of the Cycode CLI. + + Configuration options: + * API URL: The base URL for Cycode's API (for on-premise or EU installations) + * APP URL: The base URL for Cycode's web application (for on-premise or EU installations) + * Client ID: Your Cycode client ID for authentication + * Client Secret: Your Cycode client secret for authentication + + Example usage: + * `cycode configure`: Start interactive configuration + * `cycode configure --help`: View configuration options + """ + add_breadcrumb('configure') + + global_config_manager = CONFIGURATION_MANAGER.global_config_file_manager + + current_api_url = global_config_manager.get_api_url() + current_app_url = global_config_manager.get_app_url() + api_url = get_api_url_input(current_api_url) + app_url = get_app_url_input(current_app_url) + + config_updated = False + if _should_update_value(current_api_url, api_url): + global_config_manager.update_api_base_url(api_url) + config_updated = True + if _should_update_value(current_app_url, app_url): + global_config_manager.update_app_base_url(app_url) + config_updated = True + + current_client_id, current_client_secret = CREDENTIALS_MANAGER.get_credentials_from_file() + client_id = get_client_id_input(current_client_id) + client_secret = get_client_secret_input(current_client_secret) + + credentials_updated = False + if _should_update_value(current_client_id, client_id) or _should_update_value(current_client_secret, client_secret): + credentials_updated = True + CREDENTIALS_MANAGER.update_credentials(client_id, client_secret) + + if config_updated: + console.print(get_urls_update_result_message()) + if credentials_updated: + console.print(get_credentials_update_result_message()) diff --git a/cycode/cli/apps/configure/consts.py b/cycode/cli/apps/configure/consts.py new file mode 100644 index 00000000..15c9b7a5 --- /dev/null +++ b/cycode/cli/apps/configure/consts.py @@ -0,0 +1,19 @@ +from cycode.cli import config, consts +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.user_settings.credentials_manager import CredentialsManager + +URLS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured Cycode URLs! Saved to: {filename}' +URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( + 'Note that the URLs (APP and API) that already exist in environment variables ' + f'({consts.CYCODE_API_URL_ENV_VAR_NAME} and {consts.CYCODE_APP_URL_ENV_VAR_NAME}) ' + 'take precedent over these URLs; either update or remove the environment variables.' +) +CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured CLI credentials! Saved to: {filename}' +CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( + 'Note that the credentials that already exist in environment variables ' + f'({config.CYCODE_CLIENT_ID_ENV_VAR_NAME} and {config.CYCODE_CLIENT_SECRET_ENV_VAR_NAME}) ' + 'take precedent over these credentials; either update or remove the environment variables.' +) + +CREDENTIALS_MANAGER = CredentialsManager() +CONFIGURATION_MANAGER = ConfigurationManager() diff --git a/cycode/cli/apps/configure/messages.py b/cycode/cli/apps/configure/messages.py new file mode 100644 index 00000000..36ce807b --- /dev/null +++ b/cycode/cli/apps/configure/messages.py @@ -0,0 +1,37 @@ +from cycode.cli.apps.configure.consts import ( + CONFIGURATION_MANAGER, + CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE, + CREDENTIALS_MANAGER, + CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE, + URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE, + URLS_UPDATED_SUCCESSFULLY_MESSAGE, +) + + +def _are_credentials_exist_in_environment_variables() -> bool: + client_id, client_secret = CREDENTIALS_MANAGER.get_credentials_from_environment_variables() + return any([client_id, client_secret]) + + +def get_credentials_update_result_message() -> str: + success_message = CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE.format(filename=CREDENTIALS_MANAGER.get_filename()) + if _are_credentials_exist_in_environment_variables(): + return f'{success_message}. {CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' + + return success_message + + +def _are_urls_exist_in_environment_variables() -> bool: + api_url = CONFIGURATION_MANAGER.get_api_url_from_environment_variables() + app_url = CONFIGURATION_MANAGER.get_app_url_from_environment_variables() + return any([api_url, app_url]) + + +def get_urls_update_result_message() -> str: + success_message = URLS_UPDATED_SUCCESSFULLY_MESSAGE.format( + filename=CONFIGURATION_MANAGER.global_config_file_manager.get_filename() + ) + if _are_urls_exist_in_environment_variables(): + return f'{success_message}. {URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' + + return success_message diff --git a/cycode/cli/apps/configure/prompts.py b/cycode/cli/apps/configure/prompts.py new file mode 100644 index 00000000..3025688d --- /dev/null +++ b/cycode/cli/apps/configure/prompts.py @@ -0,0 +1,48 @@ +from typing import Optional + +import typer + +from cycode.cli import consts +from cycode.cli.utils.string_utils import obfuscate_text + + +def get_client_id_input(current_client_id: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode Client ID' + + prompt_suffix = ' []: ' + if current_client_id: + prompt_suffix = f' [{obfuscate_text(current_client_id)}]: ' + + new_client_id = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_client_id or current_client_id + + +def get_client_secret_input(current_client_secret: Optional[str]) -> Optional[str]: + prompt_text = 'Cycode Client Secret' + + prompt_suffix = ' []: ' + if current_client_secret: + prompt_suffix = f' [{obfuscate_text(current_client_secret)}]: ' + + new_client_secret = typer.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) + return new_client_secret or current_client_secret + + +def get_app_url_input(current_app_url: Optional[str]) -> str: + prompt_text = 'Cycode APP URL' + + default = consts.DEFAULT_CYCODE_APP_URL + if current_app_url: + default = current_app_url + + return typer.prompt(text=prompt_text, default=default, type=str) + + +def get_api_url_input(current_api_url: Optional[str]) -> str: + prompt_text = 'Cycode API URL' + + default = consts.DEFAULT_CYCODE_API_URL + if current_api_url: + default = current_api_url + + return typer.prompt(text=prompt_text, default=default, type=str) diff --git a/cycode/cli/apps/ignore/__init__.py b/cycode/cli/apps/ignore/__init__.py new file mode 100644 index 00000000..e6573b69 --- /dev/null +++ b/cycode/cli/apps/ignore/__init__.py @@ -0,0 +1,6 @@ +import typer + +from cycode.cli.apps.ignore.ignore_command import ignore_command + +app = typer.Typer(no_args_is_help=True) +app.command(name='ignore', short_help='Ignores a specific value, path or rule ID.')(ignore_command) diff --git a/cycode/cli/apps/ignore/ignore_command.py b/cycode/cli/apps/ignore/ignore_command.py new file mode 100644 index 00000000..1183114a --- /dev/null +++ b/cycode/cli/apps/ignore/ignore_command.py @@ -0,0 +1,161 @@ +import re +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli import consts +from cycode.cli.cli_types import ScanTypeOption +from cycode.cli.config import configuration_manager +from cycode.cli.logger import logger +from cycode.cli.utils.path_utils import get_absolute_path, is_path_exists +from cycode.cli.utils.sentry import add_breadcrumb +from cycode.cli.utils.string_utils import hash_string_to_sha256 + +_FILTER_BY_RICH_HELP_PANEL = 'Filter options' +_SECRETS_FILTER_BY_RICH_HELP_PANEL = 'Secrets filter options' +_SCA_FILTER_BY_RICH_HELP_PANEL = 'SCA filter options' + + +def _is_package_pattern_valid(package: str) -> bool: + return re.search('^[^@]+@[^@]+$', package) is not None + + +def ignore_command( # noqa: C901 + by_path: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific file or directory while scanning.', + show_default=False, + rich_help_panel=_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_rule: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific Secrets rule ID or IaC rule ID.', + show_default=False, + rich_help_panel=_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_value: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific value.', + show_default=False, + rich_help_panel=_SECRETS_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_sha: Annotated[ + Optional[str], + typer.Option( + help='Ignore a specific SHA512 representation of a string.', + show_default=False, + rich_help_panel=_SECRETS_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_package: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific package version. Expected pattern: [cyan]name@version[/].', + show_default=False, + rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + by_cve: Annotated[ + Optional[str], + typer.Option( + help='Ignore scanning a specific CVE. Expected pattern: [cyan]CVE-YYYY-NNN[/].', + show_default=False, + rich_help_panel=_SCA_FILTER_BY_RICH_HELP_PANEL, + ), + ] = None, + scan_type: Annotated[ + ScanTypeOption, + typer.Option( + '--scan-type', + '-t', + help='Specify the type of scan you wish to execute.', + case_sensitive=False, + ), + ] = ScanTypeOption.SECRET, + is_global: Annotated[ + bool, typer.Option('--global', '-g', help='Add an ignore rule to the global CLI config.') + ] = False, +) -> None: + """:no_entry: [bold cyan]Ignore specific findings or paths in scans.[/] + + This command allows you to exclude specific items from Cycode scans, including: + * Paths: Exclude specific files or directories + * Rules: Ignore specific security rules + * Values: Exclude specific sensitive values + * Packages: Ignore specific package versions + * CVEs: Exclude specific vulnerabilities + + Example usage: + * `cycode ignore --by-path .env`: Ignore the tests directory + * `cycode ignore --by-rule GUID`: Ignore rule with the specified GUID + * `cycode ignore --by-package lodash@4.17.21`: Ignore lodash version 4.17.21 + """ + add_breadcrumb('ignore') + + all_by_values = [by_value, by_sha, by_path, by_rule, by_package, by_cve] + if all(by is None for by in all_by_values): + raise click.ClickException('Ignore by type is missing') + if len([by for by in all_by_values if by is not None]) != 1: + raise click.ClickException('You must specify only one ignore by type') + + if any(by is not None for by in [by_value, by_sha]) and scan_type != consts.SECRET_SCAN_TYPE: + raise click.ClickException('This exclude is supported only for Secret scan type') + if (by_cve or by_package) and scan_type != consts.SCA_SCAN_TYPE: + raise click.ClickException('This exclude is supported only for SCA scan type') + + # only one of the by values must be set + # at least one of the by values must be set + exclusion_type = exclusion_value = None + + if by_value: + exclusion_type = consts.EXCLUSIONS_BY_VALUE_SECTION_NAME + exclusion_value = hash_string_to_sha256(by_value) + + if by_sha: + exclusion_type = consts.EXCLUSIONS_BY_SHA_SECTION_NAME + exclusion_value = by_sha + + if by_path: + absolute_path = get_absolute_path(by_path) + if not is_path_exists(absolute_path): + raise click.ClickException('The provided path to ignore by does not exist') + + exclusion_type = consts.EXCLUSIONS_BY_PATH_SECTION_NAME + exclusion_value = get_absolute_path(absolute_path) + + if by_rule: + exclusion_type = consts.EXCLUSIONS_BY_RULE_SECTION_NAME + exclusion_value = by_rule + + if by_package: + if not _is_package_pattern_valid(by_package): + raise click.ClickException('wrong package pattern. should be name@version.') + + exclusion_type = consts.EXCLUSIONS_BY_PACKAGE_SECTION_NAME + exclusion_value = by_package + + if by_cve: + exclusion_type = consts.EXCLUSIONS_BY_CVE_SECTION_NAME + exclusion_value = by_cve + + if not exclusion_type or not exclusion_value: + # should never happen + raise click.ClickException('Invalid ignore by type') + + configuration_scope = 'global' if is_global else 'local' + logger.debug( + 'Adding ignore rule, %s', + { + 'configuration_scope': configuration_scope, + 'exclusion_type': exclusion_type, + 'exclusion_value': exclusion_value, + }, + ) + configuration_manager.add_exclusion(configuration_scope, str(scan_type), exclusion_type, exclusion_value) diff --git a/cycode/cli/apps/report/__init__.py b/cycode/cli/apps/report/__init__.py new file mode 100644 index 00000000..751157a4 --- /dev/null +++ b/cycode/cli/apps/report/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.report import sbom +from cycode.cli.apps.report.report_command import report_command + +app = typer.Typer(name='report', no_args_is_help=True) +app.callback(short_help='Generate report. You`ll need to specify which report type to perform as SBOM.')(report_command) +app.add_typer(sbom.app) diff --git a/cycode/cli/apps/report/report_command.py b/cycode/cli/apps/report/report_command.py new file mode 100644 index 00000000..75debb33 --- /dev/null +++ b/cycode/cli/apps/report/report_command.py @@ -0,0 +1,15 @@ +import typer + +from cycode.cli.utils.progress_bar import SBOM_REPORT_PROGRESS_BAR_SECTIONS, get_progress_bar +from cycode.cli.utils.sentry import add_breadcrumb + + +def report_command(ctx: typer.Context) -> int: + """:bar_chart: [bold cyan]Generate security reports.[/] + + Example usage: + * `cycode report sbom`: Generate SBOM report + """ + add_breadcrumb('report') + ctx.obj['progress_bar'] = get_progress_bar(hidden=False, sections=SBOM_REPORT_PROGRESS_BAR_SECTIONS) + return 1 diff --git a/cycode/cli/apps/report/sbom/__init__.py b/cycode/cli/apps/report/sbom/__init__.py new file mode 100644 index 00000000..77d081e8 --- /dev/null +++ b/cycode/cli/apps/report/sbom/__init__.py @@ -0,0 +1,15 @@ +import typer + +from cycode.cli.apps.report.sbom.path.path_command import path_command +from cycode.cli.apps.report.sbom.repository_url.repository_url_command import repository_url_command +from cycode.cli.apps.report.sbom.sbom_command import sbom_command + +app = typer.Typer(name='sbom') +app.callback(short_help='Generate SBOM report for remote repository by url or local directory by path.')(sbom_command) +app.command(name='path', short_help='Generate SBOM report for provided path in the command.')(path_command) +app.command(name='repository-url', short_help='Generate SBOM report for provided repository URI in the command.')( + repository_url_command +) + +# backward compatibility +app.command(hidden=True, name='repository_url')(repository_url_command) diff --git a/cycode/cli/commands/report/sbom/common.py b/cycode/cli/apps/report/sbom/common.py similarity index 95% rename from cycode/cli/commands/report/sbom/common.py rename to cycode/cli/apps/report/sbom/common.py index 6ea843f5..067a9fa6 100644 --- a/cycode/cli/commands/report/sbom/common.py +++ b/cycode/cli/apps/report/sbom/common.py @@ -4,11 +4,11 @@ from typing import TYPE_CHECKING, Optional from cycode.cli import consts -from cycode.cli.commands.report.sbom.sbom_report_file import SbomReportFile +from cycode.cli.apps.report.sbom.sbom_report_file import SbomReportFile from cycode.cli.config import configuration_manager from cycode.cli.exceptions.custom_exceptions import ReportAsyncError +from cycode.cli.logger import logger from cycode.cli.utils.progress_bar import SbomReportProgressBarSection -from cycode.cyclient import logger from cycode.cyclient.models import ReportExecutionSchema if TYPE_CHECKING: @@ -30,7 +30,7 @@ def _poll_report_execution_until_completed( report_execution = client.get_report_execution(report_execution_id) report_label = report_execution.error_message or report_execution.status_message - progress_bar.update_label(report_label) + progress_bar.update_right_side_label(report_label) if report_execution.status == consts.REPORT_STATUS_COMPLETED: return report_execution diff --git a/cycode/cli/commands/ai_remediation/__init__.py b/cycode/cli/apps/report/sbom/path/__init__.py similarity index 100% rename from cycode/cli/commands/ai_remediation/__init__.py rename to cycode/cli/apps/report/sbom/path/__init__.py diff --git a/cycode/cli/commands/report/sbom/path/path_command.py b/cycode/cli/apps/report/sbom/path/path_command.py similarity index 73% rename from cycode/cli/commands/report/sbom/path/path_command.py rename to cycode/cli/apps/report/sbom/path/path_command.py index c52bc611..9741aa73 100644 --- a/cycode/cli/commands/report/sbom/path/path_command.py +++ b/cycode/cli/apps/report/sbom/path/path_command.py @@ -1,30 +1,35 @@ import time +from pathlib import Path +from typing import Annotated -import click +import typer from cycode.cli import consts -from cycode.cli.commands.report.sbom.common import create_sbom_report, send_report_feedback +from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception from cycode.cli.files_collector.path_documents import get_relevant_documents from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.get_api_client import get_report_cycode_client from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Generate SBOM report for provided path in the command.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.pass_context -def path_command(context: click.Context, path: str) -> None: +def path_command( + ctx: typer.Context, + path: Annotated[ + Path, + typer.Argument(exists=True, resolve_path=True, help='Path to generate SBOM report for.', show_default=False), + ], +) -> None: add_breadcrumb('path') - client = get_report_cycode_client() - report_parameters = context.obj['report_parameters'] + client = get_report_cycode_client(ctx) + report_parameters = ctx.obj['report_parameters'] output_format = report_parameters.output_format - output_file = context.obj['output_file'] + output_file = ctx.obj['output_file'] - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() start_scan_time = time.time() @@ -32,11 +37,11 @@ def path_command(context: click.Context, path: str) -> None: try: documents = get_relevant_documents( - progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (path,) + progress_bar, SbomReportProgressBarSection.PREPARE_LOCAL_FILES, consts.SCA_SCAN_TYPE, (str(path),) ) # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document. # unhardcode usage of context in perform_pre_scan_documents_actions - perform_pre_scan_documents_actions(context, consts.SCA_SCAN_TYPE, documents) + perform_pre_scan_documents_actions(ctx, consts.SCA_SCAN_TYPE, documents) zipped_documents = zip_documents(consts.SCA_SCAN_TYPE, documents) report_execution = client.request_sbom_report_execution(report_parameters, zip_file=zipped_documents) @@ -66,4 +71,4 @@ def path_command(context: click.Context, path: str) -> None: error_message=str(e), ) - handle_report_exception(context, e) + handle_report_exception(ctx, e) diff --git a/cycode/cli/commands/auth/__init__.py b/cycode/cli/apps/report/sbom/repository_url/__init__.py similarity index 100% rename from cycode/cli/commands/auth/__init__.py rename to cycode/cli/apps/report/sbom/repository_url/__init__.py diff --git a/cycode/cli/commands/report/sbom/repository_url/repository_url_command.py b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py similarity index 71% rename from cycode/cli/commands/report/sbom/repository_url/repository_url_command.py rename to cycode/cli/apps/report/sbom/repository_url/repository_url_command.py index 189fd961..9e2f4885 100644 --- a/cycode/cli/commands/report/sbom/repository_url/repository_url_command.py +++ b/cycode/cli/apps/report/sbom/repository_url/repository_url_command.py @@ -1,27 +1,28 @@ import time +from typing import Annotated -import click +import typer -from cycode.cli.commands.report.sbom.common import create_sbom_report, send_report_feedback +from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.get_api_client import get_report_cycode_client from cycode.cli.utils.progress_bar import SbomReportProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Generate SBOM report for provided repository URI in the command.') -@click.argument('uri', nargs=1, type=str, required=True) -@click.pass_context -def repository_url_command(context: click.Context, uri: str) -> None: +def repository_url_command( + ctx: typer.Context, + uri: Annotated[str, typer.Argument(help='Repository URL to generate SBOM report for.', show_default=False)], +) -> None: add_breadcrumb('repository_url') - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() progress_bar.set_section_length(SbomReportProgressBarSection.PREPARE_LOCAL_FILES) - client = get_report_cycode_client() - report_parameters = context.obj['report_parameters'] - output_file = context.obj['output_file'] + client = get_report_cycode_client(ctx) + report_parameters = ctx.obj['report_parameters'] + output_file = ctx.obj['output_file'] output_format = report_parameters.output_format start_scan_time = time.time() @@ -56,4 +57,4 @@ def repository_url_command(context: click.Context, uri: str) -> None: repository_uri=uri, ) - handle_report_exception(context, e) + handle_report_exception(ctx, e) diff --git a/cycode/cli/apps/report/sbom/sbom_command.py b/cycode/cli/apps/report/sbom/sbom_command.py new file mode 100644 index 00000000..06126dd0 --- /dev/null +++ b/cycode/cli/apps/report/sbom/sbom_command.py @@ -0,0 +1,72 @@ +from pathlib import Path +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli.cli_types import SbomFormatOption, SbomOutputFormatOption +from cycode.cli.utils.sentry import add_breadcrumb +from cycode.cyclient.report_client import ReportParameters + +_OUTPUT_RICH_HELP_PANEL = 'Output options' + + +def sbom_command( + ctx: typer.Context, + sbom_format: Annotated[ + SbomFormatOption, + typer.Option( + '--format', + '-f', + help='SBOM format.', + case_sensitive=False, + show_default=False, + ), + ], + output_format: Annotated[ + SbomOutputFormatOption, + typer.Option( + '--output-format', + '-o', + help='Specify the output file format.', + rich_help_panel=_OUTPUT_RICH_HELP_PANEL, + ), + ] = SbomOutputFormatOption.JSON, + output_file: Annotated[ + Optional[Path], + typer.Option( + help='Output file.', + show_default='Autogenerated filename saved to the current directory', + dir_okay=False, + writable=True, + rich_help_panel=_OUTPUT_RICH_HELP_PANEL, + ), + ] = None, + include_vulnerabilities: Annotated[ + bool, typer.Option('--include-vulnerabilities', help='Include vulnerabilities.', show_default=False) + ] = False, + include_dev_dependencies: Annotated[ + bool, typer.Option('--include-dev-dependencies', help='Include dev dependencies.', show_default=False) + ] = False, +) -> int: + """Generate SBOM report.""" + add_breadcrumb('sbom') + + sbom_format_parts = sbom_format.split('-') + if len(sbom_format_parts) != 2: + raise click.ClickException('Invalid SBOM format.') + + sbom_format, sbom_format_version = sbom_format_parts + + report_parameters = ReportParameters( + entity_type='SbomCli', + sbom_report_type=sbom_format, + sbom_version=sbom_format_version, + output_format=output_format, + include_vulnerabilities=include_vulnerabilities, + include_dev_dependencies=include_dev_dependencies, + ) + ctx.obj['report_parameters'] = report_parameters + ctx.obj['output_file'] = output_file + + return 1 diff --git a/cycode/cli/commands/report/sbom/sbom_report_file.py b/cycode/cli/apps/report/sbom/sbom_report_file.py similarity index 88% rename from cycode/cli/commands/report/sbom/sbom_report_file.py rename to cycode/cli/apps/report/sbom/sbom_report_file.py index 4d58f89f..f3178b44 100644 --- a/cycode/cli/commands/report/sbom/sbom_report_file.py +++ b/cycode/cli/apps/report/sbom/sbom_report_file.py @@ -3,7 +3,9 @@ import re from typing import Optional -import click +import typer + +from cycode.cli.console import console class SbomReportFile: @@ -21,14 +23,14 @@ def is_exists(self) -> bool: return self._file_path.exists() def _prompt_overwrite(self) -> bool: - return click.confirm(f'File {self._file_path} already exists. Save with a different filename?', default=True) + return typer.confirm(f'File {self._file_path} already exists. Save with a different filename?', default=True) def _write(self, content: str) -> None: with open(self._file_path, 'w', encoding='UTF-8') as f: f.write(content) def _notify_about_saved_file(self) -> None: - click.echo(f'Report saved to {self._file_path}') + console.print(f'Report saved to {self._file_path}') def _find_and_set_unique_filename(self) -> None: attempt_no = 0 diff --git a/cycode/cli/apps/scan/__init__.py b/cycode/cli/apps/scan/__init__.py new file mode 100644 index 00000000..b4d8ab79 --- /dev/null +++ b/cycode/cli/apps/scan/__init__.py @@ -0,0 +1,41 @@ +import typer + +from cycode.cli.apps.scan.commit_history.commit_history_command import commit_history_command +from cycode.cli.apps.scan.path.path_command import path_command +from cycode.cli.apps.scan.pre_commit.pre_commit_command import pre_commit_command +from cycode.cli.apps.scan.pre_receive.pre_receive_command import pre_receive_command +from cycode.cli.apps.scan.repository.repository_command import repository_command +from cycode.cli.apps.scan.scan_command import scan_command, scan_command_result_callback + +app = typer.Typer(name='scan', no_args_is_help=True) + +_scan_command_docs = 'https://github.com/cycodehq/cycode-cli/blob/main/README.md#scan-command' +_scan_command_epilog = f'[bold]Documentation:[/] [link={_scan_command_docs}]{_scan_command_docs}[/link]' + +app.callback( + short_help='Scan the content for Secrets, IaC, SCA, and SAST violations.', + result_callback=scan_command_result_callback, + epilog=_scan_command_epilog, +)(scan_command) + +app.command(name='path', short_help='Scan the files in the paths provided in the command.')(path_command) +app.command(name='repository', short_help='Scan the Git repository included files.')(repository_command) +app.command(name='commit-history', short_help='Scan all the commits history in this Git repository.')( + commit_history_command +) +app.command( + name='pre-commit', + short_help='Use this command in pre-commit hook to scan any content that was not committed yet.', + rich_help_panel='Automation commands', +)(pre_commit_command) +app.command( + name='pre-receive', + short_help='Use this command in pre-receive hook ' + 'to scan commits on the server side before pushing them to the repository.', + rich_help_panel='Automation commands', +)(pre_receive_command) + +# backward compatibility +app.command(hidden=True, name='commit_history')(commit_history_command) +app.command(hidden=True, name='pre_commit')(pre_commit_command) +app.command(hidden=True, name='pre_receive')(pre_receive_command) diff --git a/cycode/cli/commands/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py similarity index 77% rename from cycode/cli/commands/scan/code_scanner.py rename to cycode/cli/apps/scan/code_scanner.py index 4091118f..e7dff93f 100644 --- a/cycode/cli/commands/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -3,13 +3,16 @@ import sys import time from platform import platform -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Callable, Optional from uuid import UUID, uuid4 import click +import typer from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption from cycode.cli.config import configuration_manager +from cycode.cli.console import console from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan @@ -24,8 +27,7 @@ from cycode.cli.files_collector.sca import sca_code_scanner from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents -from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult, Severity -from cycode.cli.printers import ConsolePrinter +from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult from cycode.cli.utils import scan_utils from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_path_by_os @@ -33,9 +35,8 @@ from cycode.cli.utils.scan_batch import run_parallel_batched_scan from cycode.cli.utils.scan_utils import set_issue_detected from cycode.cli.utils.shell_executor import shell -from cycode.cyclient import logger -from cycode.cyclient.config import set_logging_level from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult +from cycode.logger import get_logger, set_logging_level if TYPE_CHECKING: from cycode.cyclient.models import ScanDetailsResponse @@ -44,17 +45,22 @@ start_scan_time = time.time() -def scan_sca_pre_commit(context: click.Context) -> None: - scan_type = context.obj['scan_type'] - scan_parameters = get_scan_parameters(context) +logger = get_logger('Code Scanner') + + +def scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None: + scan_type = ctx.obj['scan_type'] + scan_parameters = get_scan_parameters(ctx) git_head_documents, pre_committed_documents = get_pre_commit_modified_documents( - context.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES + progress_bar=ctx.obj['progress_bar'], + progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path=repo_path, ) git_head_documents = exclude_irrelevant_documents_to_scan(scan_type, git_head_documents) pre_committed_documents = exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents) - sca_code_scanner.perform_pre_hook_range_scan_actions(git_head_documents, pre_committed_documents) + sca_code_scanner.perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents) scan_commit_range_documents( - context, + ctx, git_head_documents, pre_committed_documents, scan_parameters, @@ -62,11 +68,11 @@ def scan_sca_pre_commit(context: click.Context) -> None: ) -def scan_sca_commit_range(context: click.Context, path: str, commit_range: str) -> None: - scan_type = context.obj['scan_type'] - progress_bar = context.obj['progress_bar'] +def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> None: + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] - scan_parameters = get_scan_parameters(context, (path,)) + scan_parameters = get_scan_parameters(ctx, (path,)) from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path) from_commit_documents, to_commit_documents = get_commit_range_modified_documents( progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev @@ -77,43 +83,50 @@ def scan_sca_commit_range(context: click.Context, path: str, commit_range: str) path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev ) - scan_commit_range_documents(context, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) + scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) -def scan_disk_files(context: click.Context, paths: Tuple[str]) -> None: - scan_type = context.obj['scan_type'] - progress_bar = context.obj['progress_bar'] +def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None: + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] try: documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths) - perform_pre_scan_documents_actions(context, scan_type, documents) - scan_documents(context, documents, get_scan_parameters(context, paths)) + perform_pre_scan_documents_actions(ctx, scan_type, documents) + scan_documents(ctx, documents, get_scan_parameters(ctx, paths)) except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) + +def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: list[LocalScanResult]) -> None: + set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) -def set_issue_detected_by_scan_results(context: click.Context, scan_results: List[LocalScanResult]) -> None: - set_issue_detected(context, any(scan_result.issue_detected for scan_result in scan_results)) +def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: bool) -> bool: + """Decide whether to use sync flow or async flow for the scan. -def _should_use_scan_service(scan_type: str, scan_parameters: dict) -> bool: - return scan_type == consts.SECRET_SCAN_TYPE and scan_parameters.get('report') is True + Note: + Passing `--sync` option does not mean that sync flow will be used in all cases. + The logic: + - for IAC scan, sync flow is always used + - for SAST scan, sync flow is not supported + - for SCA and Secrets scan, sync flow is supported only for path/repository scan -def _should_use_sync_flow( - command_scan_type: str, scan_type: str, sync_option: bool, scan_parameters: Optional[dict] = None -) -> bool: - if not sync_option: + """ + if not sync_option and scan_type != consts.IAC_SCAN_TYPE: return False if command_scan_type not in {'path', 'repository'}: - raise ValueError(f'Sync flow is not available for "{command_scan_type}" command type. Remove --sync option.') + return False - if scan_type is consts.SAST_SCAN_TYPE: - raise ValueError('Sync scan is not available for SAST scan type.') + if scan_type == consts.IAC_SCAN_TYPE: + # sync in the only available flow for IAC scan; we do not use detector directly anymore + return True - if scan_parameters.get('report') is True: - raise ValueError('You can not use sync flow with report option. Either remove "report" or "sync" option.') + if scan_type is consts.SAST_SCAN_TYPE: # noqa: SIM103 + # SAST does not support sync flow + return False return True @@ -150,23 +163,22 @@ def _enrich_scan_result_with_data_from_detection_rules( def _get_scan_documents_thread_func( - context: click.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict -) -> Callable[[List[Document]], Tuple[str, CliError, LocalScanResult]]: - cycode_client = context.obj['client'] - scan_type = context.obj['scan_type'] - severity_threshold = context.obj['severity_threshold'] - sync_option = context.obj['sync'] - command_scan_type = context.info_name - - def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, LocalScanResult]: + ctx: typer.Context, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict +) -> Callable[[list[Document]], tuple[str, CliError, LocalScanResult]]: + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + sync_option = ctx.obj['sync'] + command_scan_type = ctx.info_name + + def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, LocalScanResult]: local_scan_result = error = error_message = None detections_count = relevant_detections_count = zip_file_size = 0 scan_id = str(_generate_unique_id()) scan_completed = False - should_use_scan_service = _should_use_scan_service(scan_type, scan_parameters) - should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option, scan_parameters) + should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option) try: logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)}) @@ -176,11 +188,9 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local cycode_client, zipped_documents, scan_type, - scan_id, is_git_diff, is_commit_range, scan_parameters, - should_use_scan_service, should_use_sync_flow, ) @@ -192,7 +202,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local scan_completed = True except Exception as e: - error = handle_scan_exception(context, e, return_exception=True) + error = handle_scan_exception(ctx, e, return_exception=True) error_message = str(e) if local_scan_result: @@ -220,7 +230,6 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local zip_file_size, command_scan_type, error_message, - should_use_scan_service or should_use_sync_flow, # sync flow implies scan service ) return scan_id, error, local_scan_result @@ -229,18 +238,18 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local def scan_commit_range( - context: click.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None + ctx: typer.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None ) -> None: - scan_type = context.obj['scan_type'] + scan_type = ctx.obj['scan_type'] - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: raise click.ClickException(f'Commit range scanning for {str.upper(scan_type)} is not supported') if scan_type == consts.SCA_SCAN_TYPE: - return scan_sca_commit_range(context, path, commit_range) + return scan_sca_commit_range(ctx, path, commit_range) documents_to_scan = [] commit_ids_to_scan = [] @@ -262,14 +271,13 @@ def scan_commit_range( commit_id = commit.hexsha commit_ids_to_scan.append(commit_id) parent = commit.parents[0] if commit.parents else git_proxy.get_null_tree() - diff = commit.diff(parent, create_patch=True, R=True) + diff_index = commit.diff(parent, create_patch=True, R=True) commit_documents_to_scan = [] - for blob in diff: - blob_path = get_path_by_os(os.path.join(path, get_diff_file_path(blob))) + for diff in diff_index: commit_documents_to_scan.append( Document( - path=blob_path, - content=blob.diff.decode('UTF-8', errors='replace'), + path=get_path_by_os(get_diff_file_path(diff)), + content=diff.diff.decode('UTF-8', errors='replace'), is_git_diff_format=True, unique_id=commit_id, ) @@ -285,25 +293,24 @@ def scan_commit_range( logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) logger.debug('Starting to scan commit range (it may take a few minutes)') - scan_documents( - context, documents_to_scan, get_scan_parameters(context, (path,)), is_git_diff=True, is_commit_range=True - ) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (path,)), is_git_diff=True, is_commit_range=True) return None def scan_documents( - context: click.Context, - documents_to_scan: List[Document], + ctx: typer.Context, + documents_to_scan: list[Document], scan_parameters: dict, is_git_diff: bool = False, is_commit_range: bool = False, ) -> None: - scan_type = context.obj['scan_type'] - progress_bar = context.obj['progress_bar'] + scan_type = ctx.obj['scan_type'] + progress_bar = ctx.obj['progress_bar'] + printer = ctx.obj.get('console_printer') if not documents_to_scan: progress_bar.stop() - ConsolePrinter(context).print_error( + printer.print_error( CliError( code='no_relevant_files', message='Error: The scan could not be completed - relevant files to scan are not found. ' @@ -312,38 +319,35 @@ def scan_documents( ) return - scan_batch_thread_func = _get_scan_documents_thread_func(context, is_git_diff, is_commit_range, scan_parameters) + scan_batch_thread_func = _get_scan_documents_thread_func(ctx, is_git_diff, is_commit_range, scan_parameters) errors, local_scan_results = run_parallel_batched_scan( scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar ) - aggregation_report_url = _try_get_aggregation_report_url_if_needed( - scan_parameters, context.obj['client'], scan_type - ) - _set_aggregation_report_url(context, aggregation_report_url) + aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, ctx.obj['client'], scan_type) + _set_aggregation_report_url(ctx, aggregation_report_url) progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) progress_bar.stop() - set_issue_detected_by_scan_results(context, local_scan_results) - print_results(context, local_scan_results, errors) + set_issue_detected_by_scan_results(ctx, local_scan_results) + print_results(ctx, local_scan_results, errors) def scan_commit_range_documents( - context: click.Context, - from_documents_to_scan: List[Document], - to_documents_to_scan: List[Document], + ctx: typer.Context, + from_documents_to_scan: list[Document], + to_documents_to_scan: list[Document], scan_parameters: Optional[dict] = None, timeout: Optional[int] = None, ) -> None: - """Used by SCA only""" - - cycode_client = context.obj['client'] - scan_type = context.obj['scan_type'] - severity_threshold = context.obj['severity_threshold'] - scan_command_type = context.info_name - progress_bar = context.obj['progress_bar'] + """In use by SCA only.""" + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + scan_command_type = ctx.info_name + progress_bar = ctx.obj['progress_bar'] local_scan_result = error_message = None scan_completed = False @@ -377,17 +381,17 @@ def scan_commit_range_documents( local_scan_result = create_local_scan_result( scan_result, to_documents_to_scan, scan_command_type, scan_type, severity_threshold ) - set_issue_detected_by_scan_results(context, [local_scan_result]) + set_issue_detected_by_scan_results(ctx, [local_scan_result]) progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) progress_bar.stop() # errors will be handled with try-except block; printing will not occur on errors - print_results(context, [local_scan_result]) + print_results(ctx, [local_scan_result]) scan_completed = True except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) error_message = str(e) zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size @@ -410,10 +414,10 @@ def scan_commit_range_documents( _report_scan_status( cycode_client, scan_type, - local_scan_result.scan_id, + scan_id, scan_completed, - local_scan_result.relevant_detections_count, - local_scan_result.detections_count, + relevant_detections_count, + detections_count, len(to_documents_to_scan), zip_file_size, scan_command_type, @@ -421,13 +425,13 @@ def scan_commit_range_documents( ) -def should_scan_documents(from_documents_to_scan: List[Document], to_documents_to_scan: List[Document]) -> bool: +def should_scan_documents(from_documents_to_scan: list[Document], to_documents_to_scan: list[Document]) -> bool: return len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0 def create_local_scan_result( scan_result: ZippedFileScanResult, - documents_to_scan: List[Document], + documents_to_scan: list[Document], command_scan_type: str, scan_type: str, severity_threshold: str, @@ -456,24 +460,16 @@ def perform_scan( cycode_client: 'ScanClient', zipped_documents: 'InMemoryZip', scan_type: str, - scan_id: str, is_git_diff: bool, is_commit_range: bool, scan_parameters: dict, - should_use_scan_service: bool = False, should_use_sync_flow: bool = False, ) -> ZippedFileScanResult: if should_use_sync_flow: # it does not support commit range scans; should_use_sync_flow handles it return perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff) - if scan_type in (consts.SCA_SCAN_TYPE, consts.SAST_SCAN_TYPE) or should_use_scan_service: - return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) - - if is_commit_range: - return cycode_client.commit_range_zipped_file_scan(scan_type, zipped_documents, scan_id) - - return cycode_client.zipped_file_scan(scan_type, zipped_documents, scan_id, scan_parameters, is_git_diff) + return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) def perform_scan_async( @@ -573,15 +569,16 @@ def print_debug_scan_details(scan_details_response: 'ScanDetailsResponse') -> No def print_results( - context: click.Context, local_scan_results: List[LocalScanResult], errors: Optional[Dict[str, 'CliError']] = None + ctx: typer.Context, local_scan_results: list[LocalScanResult], errors: Optional[dict[str, 'CliError']] = None ) -> None: - printer = ConsolePrinter(context) + printer = ctx.obj.get('console_printer') + printer.update_ctx(ctx) printer.print_scan_results(local_scan_results, errors) def get_document_detections( - scan_result: ZippedFileScanResult, documents_to_scan: List[Document] -) -> List[DocumentDetections]: + scan_result: ZippedFileScanResult, documents_to_scan: list[Document] +) -> list[DocumentDetections]: logger.debug('Getting document detections') document_detections = [] @@ -600,11 +597,11 @@ def get_document_detections( def exclude_irrelevant_document_detections( - document_detections_list: List[DocumentDetections], + document_detections_list: list[DocumentDetections], scan_type: str, command_scan_type: str, severity_threshold: str, -) -> List[DocumentDetections]: +) -> list[DocumentDetections]: relevant_document_detections_list = [] for document_detections in document_detections_list: relevant_detections = exclude_irrelevant_detections( @@ -619,8 +616,7 @@ def exclude_irrelevant_document_detections( def parse_pre_receive_input() -> str: - """ - Parsing input to pushed branch update details + """Parse input to pushed branch update details. Example input: old_value new_value refname @@ -629,7 +625,7 @@ def parse_pre_receive_input() -> str: 973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch 59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop - :return: first branch update details (input's first line) + :return: First branch update details (input's first line) """ # FIXME(MarshalX): this blocks main thread forever if called outside of pre-receive hook pre_receive_input = sys.stdin.read().strip() @@ -643,19 +639,19 @@ def parse_pre_receive_input() -> str: return pre_receive_input.splitlines()[0] -def _get_default_scan_parameters(context: click.Context) -> dict: +def _get_default_scan_parameters(ctx: typer.Context) -> dict: return { - 'monitor': context.obj.get('monitor'), - 'report': context.obj.get('report'), - 'package_vulnerabilities': context.obj.get('package-vulnerabilities'), - 'license_compliance': context.obj.get('license-compliance'), - 'command_type': context.info_name, + 'monitor': ctx.obj.get('monitor'), + 'report': ctx.obj.get('report'), + 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), + 'license_compliance': ctx.obj.get('license-compliance'), + 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility 'aggregation_id': str(_generate_unique_id()), } -def get_scan_parameters(context: click.Context, paths: Optional[Tuple[str]] = None) -> dict: - scan_parameters = _get_default_scan_parameters(context) +def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = None) -> dict: + scan_parameters = _get_default_scan_parameters(ctx) if not paths: return scan_parameters @@ -663,7 +659,11 @@ def get_scan_parameters(context: click.Context, paths: Optional[Tuple[str]] = No scan_parameters['paths'] = paths if len(paths) != 1: - # ignore remote url if multiple paths are provided + logger.debug('Multiple paths provided, going to ignore remote url') + return scan_parameters + + if not os.path.isdir(paths[0]): + logger.debug('Path is not a directory, going to ignore remote url') return scan_parameters remote_url = try_get_git_remote_url(paths[0]) @@ -672,7 +672,7 @@ def get_scan_parameters(context: click.Context, paths: Optional[Tuple[str]] = No if remote_url: # TODO(MarshalX): remove hardcode in context - context.obj['remote_url'] = remote_url + ctx.obj['remote_url'] = remote_url scan_parameters['remote_url'] = remote_url return scan_parameters @@ -683,13 +683,13 @@ def try_get_git_remote_url(path: str) -> Optional[str]: remote_url = git_proxy.get_repo(path).remotes[0].config_reader.get('url') logger.debug('Found Git remote URL, %s', {'remote_url': remote_url, 'path': path}) return remote_url - except Exception as e: - logger.debug('Failed to get Git remote URL', exc_info=e) + except Exception: + logger.debug('Failed to get Git remote URL. Probably not a Git repository') return None def _get_plastic_repository_name(path: str) -> Optional[str]: - """Gets the name of the Plastic repository from the current working directory. + """Get the name of the Plastic repository from the current working directory. The command to execute is: cm status --header --machinereadable --fieldseparator=":::" @@ -697,7 +697,6 @@ def _get_plastic_repository_name(path: str) -> Optional[str]: Example of status header in machine-readable format: STATUS:::0:::Project/RepoName:::OrgName@ServerInfo """ - try: command = [ 'cm', @@ -707,7 +706,9 @@ def _get_plastic_repository_name(path: str) -> Optional[str]: f'--fieldseparator={consts.PLASTIC_VCS_DATA_SEPARATOR}', ] - status = shell(command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path) + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path, silent_exc_info=True + ) if not status: logger.debug('Failed to get Plastic repository name (command failed)') return None @@ -718,13 +719,13 @@ def _get_plastic_repository_name(path: str) -> Optional[str]: return None return status_parts[2].strip() - except Exception as e: - logger.debug('Failed to get Plastic repository name', exc_info=e) + except Exception: + logger.debug('Failed to get Plastic repository name. Probably not a Plastic repository') return None -def _get_plastic_repository_list(working_dir: Optional[str] = None) -> Dict[str, str]: - """Gets the list of Plastic repositories and their GUIDs. +def _get_plastic_repository_list(working_dir: Optional[str] = None) -> dict[str, str]: + """Get the list of Plastic repositories and their GUIDs. The command to execute is: cm repo list --format="{repname}:::{repguid}" @@ -734,13 +735,14 @@ def _get_plastic_repository_list(working_dir: Optional[str] = None) -> Dict[str, Each line represents an individual repository. """ - repo_name_to_guid = {} try: command = ['cm', 'repo', 'ls', f'--format={{repname}}{consts.PLASTIC_VCS_DATA_SEPARATOR}{{repguid}}'] - status = shell(command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir) + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir, silent_exc_info=True + ) if not status: logger.debug('Failed to get Plastic repository list (command failed)') return repo_name_to_guid @@ -776,14 +778,14 @@ def try_to_get_plastic_remote_url(path: str) -> Optional[str]: def exclude_irrelevant_detections( - detections: List[Detection], scan_type: str, command_scan_type: str, severity_threshold: str -) -> List[Detection]: + detections: list[Detection], scan_type: str, command_scan_type: str, severity_threshold: str +) -> list[Detection]: relevant_detections = _exclude_detections_by_exclusions_configuration(detections, scan_type) relevant_detections = _exclude_detections_by_scan_type(relevant_detections, scan_type, command_scan_type) return _exclude_detections_by_severity(relevant_detections, severity_threshold) -def _exclude_detections_by_severity(detections: List[Detection], severity_threshold: str) -> List[Detection]: +def _exclude_detections_by_severity(detections: list[Detection], severity_threshold: str) -> list[Detection]: relevant_detections = [] for detection in detections: severity = detection.severity @@ -800,8 +802,8 @@ def _exclude_detections_by_severity(detections: List[Detection], severity_thresh def _exclude_detections_by_scan_type( - detections: List[Detection], scan_type: str, command_scan_type: str -) -> List[Detection]: + detections: list[Detection], scan_type: str, command_scan_type: str +) -> list[Detection]: if command_scan_type == consts.PRE_COMMIT_COMMAND_SCAN_TYPE: return exclude_detections_in_deleted_lines(detections) @@ -816,16 +818,16 @@ def _exclude_detections_by_scan_type( return detections -def exclude_detections_in_deleted_lines(detections: List[Detection]) -> List[Detection]: +def exclude_detections_in_deleted_lines(detections: list[Detection]) -> list[Detection]: return [detection for detection in detections if detection.detection_details.get('line_type') != 'Removed'] -def _exclude_detections_by_exclusions_configuration(detections: List[Detection], scan_type: str) -> List[Detection]: +def _exclude_detections_by_exclusions_configuration(detections: list[Detection], scan_type: str) -> list[Detection]: exclusions = configuration_manager.get_exclusions_by_scan_type(scan_type) return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)] -def _should_exclude_detection(detection: Detection, exclusions: Dict) -> bool: +def _should_exclude_detection(detection: Detection, exclusions: dict) -> bool: # FIXME(MarshalX): what the difference between by_value and by_sha? exclusions_by_value = exclusions.get(consts.EXCLUSIONS_BY_VALUE_SECTION_NAME, []) if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value): @@ -867,7 +869,7 @@ def _should_exclude_detection(detection: Detection, exclusions: Dict) -> bool: return False -def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: List[str]) -> bool: +def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: list[str]) -> bool: detection_sha = detection.detection_details.get('sha512') return detection_sha in exclusions @@ -891,7 +893,7 @@ def _get_cve_identifier(detection: Detection) -> Optional[str]: def _get_document_by_file_name( - documents: List[Document], file_name: str, unique_id: Optional[str] = None + documents: list[Document], file_name: str, unique_id: Optional[str] = None ) -> Optional[Document]: for document in documents: if _normalize_file_path(document.path) == _normalize_file_path(file_name) and document.unique_id == unique_id: @@ -911,7 +913,6 @@ def _report_scan_status( zip_size: int, command_scan_type: str, error_message: Optional[str], - should_use_scan_service: bool = False, ) -> None: try: end_scan_time = time.time() @@ -928,19 +929,22 @@ def _report_scan_status( 'scan_type': scan_type, } - cycode_client.report_scan_status(scan_type, scan_id, scan_status, should_use_scan_service) + cycode_client.report_scan_status(scan_type, scan_id, scan_status) except Exception as e: logger.debug('Failed to report scan status', exc_info=e) def _generate_unique_id() -> UUID: + if 'PYTEST_TEST_UNIQUE_ID' in os.environ: + return UUID(os.environ['PYTEST_TEST_UNIQUE_ID']) + return uuid4() def _does_severity_match_severity_threshold(severity: str, severity_threshold: str) -> bool: - detection_severity_value = Severity.try_get_value(severity) - severity_threshold_value = Severity.try_get_value(severity_threshold) - if detection_severity_value is None or severity_threshold_value is None: + detection_severity_value = SeverityOption.get_member_weight(severity) + severity_threshold_value = SeverityOption.get_member_weight(severity_threshold) + if detection_severity_value < 0 or severity_threshold_value < 0: return True return detection_severity_value >= severity_threshold_value @@ -956,13 +960,13 @@ def _get_scan_result( if not scan_details.detections_count: return init_default_scan_result(scan_id) - scan_raw_detections = cycode_client.get_scan_raw_detections(scan_type, scan_id) + scan_raw_detections = cycode_client.get_scan_raw_detections(scan_id) return ZippedFileScanResult( did_detect=True, detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), scan_id=scan_id, - report_url=_try_get_any_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters), + report_url=_try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), ) @@ -974,39 +978,8 @@ def init_default_scan_result(scan_id: str) -> ZippedFileScanResult: ) -def _try_get_any_report_url_if_needed( - cycode_client: 'ScanClient', - scan_id: str, - scan_type: str, - scan_parameters: dict, -) -> Optional[str]: - """Tries to get aggregation report URL if needed, otherwise tries to get report URL.""" - aggregation_report_url = None - if scan_parameters: - _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters) - aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type) - - if aggregation_report_url: - return aggregation_report_url - - return _try_get_report_url_if_needed(cycode_client, scan_id, scan_type, scan_parameters) - - -def _try_get_report_url_if_needed( - cycode_client: 'ScanClient', scan_id: str, scan_type: str, scan_parameters: dict -) -> Optional[str]: - if not scan_parameters.get('report', False): - return None - - try: - report_url_response = cycode_client.get_scan_report_url(scan_id, scan_type) - return report_url_response.report_url - except Exception as e: - logger.debug('Failed to get report URL', exc_info=e) - - -def _set_aggregation_report_url(context: click.Context, aggregation_report_url: Optional[str] = None) -> None: - context.obj['aggregation_report_url'] = aggregation_report_url +def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: + ctx.obj['aggregation_report_url'] = aggregation_report_url def _try_get_aggregation_report_url_if_needed( @@ -1026,10 +999,11 @@ def _try_get_aggregation_report_url_if_needed( logger.debug('Failed to get aggregation report url: %s', str(e)) -def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: List[dict]) -> List[DetectionsPerFile]: - """Converts list of detections (async flow) to list of DetectionsPerFile objects (sync flow). +def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: list[dict]) -> list[DetectionsPerFile]: + """Convert a list of detections (async flow) to list of DetectionsPerFile objects (sync flow). Args: + scan_type: Type of the scan. raw_detections: List of detections as is returned from the server. Note: @@ -1038,6 +1012,7 @@ def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: List[ Note: Aggregation is performed by file name and commit ID (if available) + """ detections_per_files = {} for raw_detection in raw_detections: @@ -1079,7 +1054,7 @@ def _get_secret_file_name_from_detection(raw_detection: dict) -> str: return os.path.join(file_path, file_name) -def _does_reach_to_max_commits_to_scan_limit(commit_ids: List[str], max_commits_count: Optional[int]) -> bool: +def _does_reach_to_max_commits_to_scan_limit(commit_ids: list[str], max_commits_count: Optional[int]) -> bool: if max_commits_count is None: return False @@ -1094,13 +1069,13 @@ def _normalize_file_path(path: str) -> str: return path -def perform_post_pre_receive_scan_actions(context: click.Context) -> None: - if scan_utils.is_scan_failed(context): - click.echo(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) +def perform_post_pre_receive_scan_actions(ctx: typer.Context) -> None: + if scan_utils.is_scan_failed(ctx): + console.print(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) -def enable_verbose_mode(context: click.Context) -> None: - context.obj['verbose'] = True +def enable_verbose_mode(ctx: typer.Context) -> None: + ctx.obj['verbose'] = True set_logging_level(logging.DEBUG) diff --git a/cycode/cli/commands/configure/__init__.py b/cycode/cli/apps/scan/commit_history/__init__.py similarity index 100% rename from cycode/cli/commands/configure/__init__.py rename to cycode/cli/apps/scan/commit_history/__init__.py diff --git a/cycode/cli/apps/scan/commit_history/commit_history_command.py b/cycode/cli/apps/scan/commit_history/commit_history_command.py new file mode 100644 index 00000000..fc1ef23f --- /dev/null +++ b/cycode/cli/apps/scan/commit_history/commit_history_command.py @@ -0,0 +1,33 @@ +from pathlib import Path +from typing import Annotated + +import typer + +from cycode.cli.apps.scan.code_scanner import scan_commit_range +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.logger import logger +from cycode.cli.utils.sentry import add_breadcrumb + + +def commit_history_command( + ctx: typer.Context, + path: Annotated[ + Path, typer.Argument(exists=True, resolve_path=True, help='Path to Git repository to scan', show_default=False) + ], + commit_range: Annotated[ + str, + typer.Option( + '--commit-range', + '-r', + help='Scan a commit range in this Git repository (example: HEAD~1)', + show_default='cycode scans all commit history', + ), + ] = '--all', +) -> None: + try: + add_breadcrumb('commit_history') + + logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) + scan_commit_range(ctx, path=str(path), commit_range=commit_range) + except Exception as e: + handle_scan_exception(ctx, e) diff --git a/cycode/cli/commands/ignore/__init__.py b/cycode/cli/apps/scan/path/__init__.py similarity index 100% rename from cycode/cli/commands/ignore/__init__.py rename to cycode/cli/apps/scan/path/__init__.py diff --git a/cycode/cli/apps/scan/path/path_command.py b/cycode/cli/apps/scan/path/path_command.py new file mode 100644 index 00000000..3ee87350 --- /dev/null +++ b/cycode/cli/apps/scan/path/path_command.py @@ -0,0 +1,25 @@ +from pathlib import Path +from typing import Annotated + +import typer + +from cycode.cli.apps.scan.code_scanner import scan_disk_files +from cycode.cli.logger import logger +from cycode.cli.utils.sentry import add_breadcrumb + + +def path_command( + ctx: typer.Context, + paths: Annotated[ + list[Path], typer.Argument(exists=True, resolve_path=True, help='Paths to scan', show_default=False) + ], +) -> None: + add_breadcrumb('path') + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + logger.debug('Starting path scan process, %s', {'paths': paths}) + + tuple_paths = tuple(str(path) for path in paths) + scan_disk_files(ctx, tuple_paths) diff --git a/cycode/cli/commands/report/__init__.py b/cycode/cli/apps/scan/pre_commit/__init__.py similarity index 100% rename from cycode/cli/commands/report/__init__.py rename to cycode/cli/apps/scan/pre_commit/__init__.py diff --git a/cycode/cli/commands/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py similarity index 56% rename from cycode/cli/commands/scan/pre_commit/pre_commit_command.py rename to cycode/cli/apps/scan/pre_commit/pre_commit_command.py index e71f2772..40e6a8c1 100644 --- a/cycode/cli/commands/scan/pre_commit/pre_commit_command.py +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -1,40 +1,42 @@ import os -from typing import List +from typing import Annotated, Optional -import click +import typer from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import get_scan_parameters, scan_documents, scan_sca_pre_commit +from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents, scan_sca_pre_commit from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.repository_documents import ( get_diff_file_content, get_diff_file_path, ) from cycode.cli.models import Document -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import ( get_path_by_os, ) from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Use this command to scan any content that was not committed yet.') -@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_commit_command(context: click.Context, ignored_args: List[str]) -> None: +def pre_commit_command( + ctx: typer.Context, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: add_breadcrumb('pre_commit') - scan_type = context.obj['scan_type'] + scan_type = ctx.obj['scan_type'] - progress_bar = context.obj['progress_bar'] + repo_path = os.getcwd() # change locally for easy testing + + progress_bar = ctx.obj['progress_bar'] progress_bar.start() if scan_type == consts.SCA_SCAN_TYPE: - scan_sca_pre_commit(context) + scan_sca_pre_commit(ctx, repo_path) return - diff_files = git_proxy.get_repo(os.getcwd()).index.diff('HEAD', create_patch=True, R=True) + diff_files = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files)) @@ -44,4 +46,4 @@ def pre_commit_command(context: click.Context, ignored_args: List[str]) -> None: documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))) documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - scan_documents(context, documents_to_scan, get_scan_parameters(context), is_git_diff=True) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True) diff --git a/cycode/cli/commands/report/sbom/__init__.py b/cycode/cli/apps/scan/pre_receive/__init__.py similarity index 100% rename from cycode/cli/commands/report/sbom/__init__.py rename to cycode/cli/apps/scan/pre_receive/__init__.py diff --git a/cycode/cli/commands/scan/pre_receive/pre_receive_command.py b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py similarity index 72% rename from cycode/cli/commands/scan/pre_receive/pre_receive_command.py rename to cycode/cli/apps/scan/pre_receive/pre_receive_command.py index 3ad59bad..eb4f1420 100644 --- a/cycode/cli/commands/scan/pre_receive/pre_receive_command.py +++ b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py @@ -1,10 +1,11 @@ import os -from typing import List +from typing import Annotated, Optional import click +import typer from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import ( +from cycode.cli.apps.scan.code_scanner import ( enable_verbose_mode, is_verbose_mode_requested_in_pre_receive_scan, parse_pre_receive_input, @@ -17,19 +18,19 @@ from cycode.cli.files_collector.repository_documents import ( calculate_pre_receive_commit_range, ) -from cycode.cli.sentry import add_breadcrumb +from cycode.cli.logger import logger +from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.task_timer import TimeoutAfter -from cycode.cyclient import logger -@click.command(short_help='Use this command to scan commits on the server side before pushing them to the repository.') -@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None: +def pre_receive_command( + ctx: typer.Context, + _: Annotated[Optional[list[str]], typer.Argument(help='Ignored arguments', hidden=True)] = None, +) -> None: try: add_breadcrumb('pre_receive') - scan_type = context.obj['scan_type'] + scan_type = ctx.obj['scan_type'] if scan_type != consts.SECRET_SCAN_TYPE: raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') @@ -41,10 +42,10 @@ def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None return if is_verbose_mode_requested_in_pre_receive_scan(): - enable_verbose_mode(context) + enable_verbose_mode(ctx) logger.debug('Verbose mode enabled: all log levels will be displayed.') - command_scan_type = context.info_name + command_scan_type = ctx.info_name timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) with TimeoutAfter(timeout): if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: @@ -60,7 +61,7 @@ def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None return max_commits_to_scan = configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type) - scan_commit_range(context, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) - perform_post_pre_receive_scan_actions(context) + scan_commit_range(ctx, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) + perform_post_pre_receive_scan_actions(ctx) except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) diff --git a/cycode/cli/commands/report/sbom/path/__init__.py b/cycode/cli/apps/scan/repository/__init__.py similarity index 100% rename from cycode/cli/commands/report/sbom/path/__init__.py rename to cycode/cli/apps/scan/repository/__init__.py diff --git a/cycode/cli/commands/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py similarity index 57% rename from cycode/cli/commands/scan/repository/repository_command.py rename to cycode/cli/apps/scan/repository/repository_command.py index b0a0effb..c96ca577 100644 --- a/cycode/cli/commands/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -1,68 +1,67 @@ -import os +from pathlib import Path +from typing import Annotated, Optional import click +import typer from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import get_scan_parameters, scan_documents +from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions +from cycode.cli.logger import logger from cycode.cli.models import Document -from cycode.cli.sentry import add_breadcrumb from cycode.cli.utils.path_utils import get_path_by_os from cycode.cli.utils.progress_bar import ScanProgressBarSection -from cycode.cyclient import logger +from cycode.cli.utils.sentry import add_breadcrumb -@click.command(short_help='Scan the git repository including its history.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.option( - '--branch', - '-b', - default=None, - help='Branch to scan, if not set scanning the default branch', - type=str, - required=False, -) -@click.pass_context -def repository_command(context: click.Context, path: str, branch: str) -> None: +def repository_command( + ctx: typer.Context, + path: Annotated[ + Path, typer.Argument(exists=True, resolve_path=True, help='Path to Git repository to scan.', show_default=False) + ], + branch: Annotated[ + Optional[str], typer.Option('--branch', '-b', help='Branch to scan.', show_default='default branch') + ] = None, +) -> None: try: add_breadcrumb('repository') logger.debug('Starting repository scan process, %s', {'path': path, 'branch': branch}) - scan_type = context.obj['scan_type'] - monitor = context.obj.get('monitor') + scan_type = ctx.obj['scan_type'] + monitor = ctx.obj.get('monitor') if monitor and scan_type != consts.SCA_SCAN_TYPE: raise click.ClickException('Monitor flag is currently supported for SCA scan type only') - progress_bar = context.obj['progress_bar'] + progress_bar = ctx.obj['progress_bar'] progress_bar.start() - file_entries = list(get_git_repository_tree_file_entries(path, branch)) + file_entries = list(get_git_repository_tree_file_entries(str(path), branch)) progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries)) documents_to_scan = [] - for file in file_entries: + for blob in file_entries: # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - absolute_path = get_path_by_os(os.path.join(path, file.path)) - file_path = file.path if monitor else absolute_path + absolute_path = get_path_by_os(blob.abspath) + file_path = get_path_by_os(blob.path) if monitor else absolute_path documents_to_scan.append( Document( file_path, - file.data_stream.read().decode('UTF-8', errors='replace'), + blob.data_stream.read().decode('UTF-8', errors='replace'), absolute_path=absolute_path, ) ) documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - perform_pre_scan_documents_actions(context, scan_type, documents_to_scan) + perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan) logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) - scan_documents(context, documents_to_scan, get_scan_parameters(context, (path,))) + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (str(path),))) except Exception as e: - handle_scan_exception(context, e) + handle_scan_exception(ctx, e) diff --git a/cycode/cli/commands/report/sbom/repository_url/__init__.py b/cycode/cli/apps/scan/scan_ci/__init__.py similarity index 100% rename from cycode/cli/commands/report/sbom/repository_url/__init__.py rename to cycode/cli/apps/scan/scan_ci/__init__.py diff --git a/cycode/cli/commands/scan/scan_ci/ci_integrations.py b/cycode/cli/apps/scan/scan_ci/ci_integrations.py similarity index 88% rename from cycode/cli/commands/scan/scan_ci/ci_integrations.py rename to cycode/cli/apps/scan/scan_ci/ci_integrations.py index f2869b2f..3cb617a9 100644 --- a/cycode/cli/commands/scan/scan_ci/ci_integrations.py +++ b/cycode/cli/apps/scan/scan_ci/ci_integrations.py @@ -2,6 +2,8 @@ import click +from cycode.cli.console import console + def github_action_range() -> str: before_sha = os.getenv('BEFORE_SHA') @@ -11,7 +13,7 @@ def github_action_range() -> str: head_sha = os.getenv('GITHUB_SHA') ref = os.getenv('GITHUB_REF') - click.echo(f'{before_sha}, {push_base_sha}, {pr_base_sha}, {default_branch}, {head_sha}, {ref}') + console.print(f'{before_sha}, {push_base_sha}, {pr_base_sha}, {default_branch}, {head_sha}, {ref}') if before_sha and before_sha != NO_COMMITS: return f'{before_sha}...' @@ -26,7 +28,7 @@ def circleci_range() -> str: before_sha = os.getenv('BEFORE_SHA') current_sha = os.getenv('CURRENT_SHA') commit_range = f'{before_sha}...{current_sha}' - click.echo(f'commit range: {commit_range}') + console.print(f'commit range: {commit_range}') if not commit_range.startswith('...'): return commit_range diff --git a/cycode/cli/apps/scan/scan_ci/scan_ci_command.py b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py new file mode 100644 index 00000000..cbfebb72 --- /dev/null +++ b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py @@ -0,0 +1,20 @@ +import os + +import click +import typer + +from cycode.cli.apps.scan.code_scanner import scan_commit_range +from cycode.cli.apps.scan.scan_ci.ci_integrations import get_commit_range +from cycode.cli.utils.sentry import add_breadcrumb + +# This command is not finished yet. It is not used in the codebase. + + +@click.command( + short_help='Execute scan in a CI environment which relies on the ' + 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' +) +@click.pass_context +def scan_ci_command(ctx: typer.Context) -> None: + add_breadcrumb('ci') + scan_commit_range(ctx, path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py new file mode 100644 index 00000000..a2ffb550 --- /dev/null +++ b/cycode/cli/apps/scan/scan_command.py @@ -0,0 +1,178 @@ +from pathlib import Path +from typing import Annotated, Optional + +import click +import typer + +from cycode.cli.cli_types import ExportTypeOption, ScanTypeOption, ScaScanTypeOption, SeverityOption +from cycode.cli.consts import ( + ISSUE_DETECTED_STATUS_CODE, + NO_ISSUES_STATUS_CODE, +) +from cycode.cli.utils import scan_utils +from cycode.cli.utils.get_api_client import get_scan_cycode_client +from cycode.cli.utils.sentry import add_breadcrumb + +_EXPORT_RICH_HELP_PANEL = 'Export options' +_SCA_RICH_HELP_PANEL = 'SCA options' +_SECRET_RICH_HELP_PANEL = 'Secret options' + + +def scan_command( + ctx: typer.Context, + scan_type: Annotated[ + ScanTypeOption, + typer.Option( + '--scan-type', + '-t', + help='Specify the type of scan you wish to execute.', + case_sensitive=False, + ), + ] = ScanTypeOption.SECRET, + soft_fail: Annotated[ + bool, typer.Option('--soft-fail', help='Run the scan without failing; always return a non-error status code.') + ] = False, + severity_threshold: Annotated[ + SeverityOption, + typer.Option( + help='Show violations only for the specified level or higher.', + case_sensitive=False, + ), + ] = SeverityOption.INFO, + sync: Annotated[ + bool, + typer.Option( + '--sync', help='Run scan synchronously (INTERNAL FOR IDEs).', show_default='asynchronously', hidden=True + ), + ] = False, + report: Annotated[ + bool, + typer.Option( + '--cycode-report', + help='When specified, displays a link to the scan report in the Cycode platform in the console output.', + ), + ] = False, + show_secret: Annotated[ + bool, typer.Option('--show-secret', help='Show Secrets in plain text.', rich_help_panel=_SECRET_RICH_HELP_PANEL) + ] = False, + sca_scan: Annotated[ + list[ScaScanTypeOption], + typer.Option( + help='Specify the type of SCA scan you wish to execute.', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), + ] = (ScaScanTypeOption.PACKAGE_VULNERABILITIES, ScaScanTypeOption.LICENSE_COMPLIANCE), + monitor: Annotated[ + bool, + typer.Option( + '--monitor', + help='When specified, the scan results are recorded in the Discovery module.', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), + ] = False, + no_restore: Annotated[ + bool, + typer.Option( + '--no-restore', + help='When specified, Cycode will not run restore command. Will scan direct dependencies [b]only[/]!', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), + ] = False, + gradle_all_sub_projects: Annotated[ + bool, + typer.Option( + '--gradle-all-sub-projects', + help='When specified, Cycode will run gradle restore command for all sub projects. ' + 'Should run from root project directory [b]only[/]!', + rich_help_panel=_SCA_RICH_HELP_PANEL, + ), + ] = False, + export_type: Annotated[ + ExportTypeOption, + typer.Option( + '--export-type', + case_sensitive=False, + help='Specify the export type. ' + 'HTML and SVG will export terminal output and rely on --output option. ' + 'JSON always exports JSON.', + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, + export_file: Annotated[ + Optional[Path], + typer.Option( + '--export-file', + help='Export file. Path to the file where the export will be saved.', + dir_okay=False, + writable=True, + rich_help_panel=_EXPORT_RICH_HELP_PANEL, + ), + ] = None, +) -> None: + """:mag: [bold cyan]Scan code for vulnerabilities (Secrets, IaC, SCA, SAST).[/] + + This command scans your code for various types of security issues, including: + * [yellow]Secrets:[/] Hardcoded credentials and sensitive information. + * [dodger_blue1]Infrastructure as Code (IaC):[/] Misconfigurations in Terraform, CloudFormation, etc. + * [green]Software Composition Analysis (SCA):[/] Vulnerabilities and license issues in dependencies. + * [magenta]Static Application Security Testing (SAST):[/] Code quality and security flaws. + + Example usage: + * `cycode scan path `: Scan a specific local directory or file. + * `cycode scan repository `: Scan Git related files in a local Git repository. + * `cycode scan commit-history `: Scan the commit history of a local Git repository. + + """ + add_breadcrumb('scan') + + if export_file and export_type is None: + raise typer.BadParameter( + 'Export type must be specified when --export-file is provided.', + param_hint='--export-type', + ) + if export_type and export_file is None: + raise typer.BadParameter( + 'Export file must be specified when --export-type is provided.', + param_hint='--export-file', + ) + + ctx.obj['show_secret'] = show_secret + ctx.obj['soft_fail'] = soft_fail + ctx.obj['client'] = get_scan_cycode_client(ctx) + ctx.obj['scan_type'] = scan_type + ctx.obj['sync'] = sync + ctx.obj['severity_threshold'] = severity_threshold + ctx.obj['monitor'] = monitor + ctx.obj['report'] = report + + if export_type and export_file: + console_printer = ctx.obj['console_printer'] + console_printer.enable_recording(export_type, export_file) + + _ = no_restore, gradle_all_sub_projects # they are actually used; via ctx.params + + _sca_scan_to_context(ctx, sca_scan) + + +def _sca_scan_to_context(ctx: typer.Context, sca_scan_user_selected: list[str]) -> None: + for sca_scan_option_selected in sca_scan_user_selected: + ctx.obj[sca_scan_option_selected] = True + + +@click.pass_context +def scan_command_result_callback(ctx: click.Context, *_, **__) -> None: + add_breadcrumb('scan_finalized') + ctx.obj['scan_finalized'] = True + + progress_bar = ctx.obj.get('progress_bar') + if progress_bar: + progress_bar.stop() + + if ctx.obj['soft_fail']: + raise typer.Exit(0) + + exit_code = NO_ISSUES_STATUS_CODE + if scan_utils.is_scan_failed(ctx): + exit_code = ISSUE_DETECTED_STATUS_CODE + + raise typer.Exit(exit_code) diff --git a/cycode/cli/apps/status/__init__.py b/cycode/cli/apps/status/__init__.py new file mode 100644 index 00000000..1161b2e6 --- /dev/null +++ b/cycode/cli/apps/status/__init__.py @@ -0,0 +1,8 @@ +import typer + +from cycode.cli.apps.status.status_command import status_command +from cycode.cli.apps.status.version_command import version_command + +app = typer.Typer(no_args_is_help=True) +app.command(name='status', short_help='Show the CLI status and exit.')(status_command) +app.command(name='version', hidden=True, short_help='Alias to status command.')(version_command) diff --git a/cycode/cli/apps/status/get_cli_status.py b/cycode/cli/apps/status/get_cli_status.py new file mode 100644 index 00000000..0cf6e8fd --- /dev/null +++ b/cycode/cli/apps/status/get_cli_status.py @@ -0,0 +1,49 @@ +import platform +from typing import TYPE_CHECKING + +from cycode import __version__ +from cycode.cli.apps.auth.auth_common import get_authorization_info +from cycode.cli.apps.status.models import CliStatus, CliSupportedModulesStatus +from cycode.cli.consts import PROGRAM_NAME +from cycode.cli.logger import logger +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.get_api_client import get_scan_cycode_client + +if TYPE_CHECKING: + from typer import Context + + +def get_cli_status(ctx: 'Context') -> CliStatus: + configuration_manager = ConfigurationManager() + + auth_info = get_authorization_info(ctx) + is_authenticated = auth_info is not None + + supported_modules_status = CliSupportedModulesStatus() + if is_authenticated: + try: + client = get_scan_cycode_client(ctx) + supported_modules_preferences = client.get_supported_modules_preferences() + + supported_modules_status.secret_scanning = supported_modules_preferences.secret_scanning + supported_modules_status.sca_scanning = supported_modules_preferences.sca_scanning + supported_modules_status.iac_scanning = supported_modules_preferences.iac_scanning + supported_modules_status.sast_scanning = supported_modules_preferences.sast_scanning + supported_modules_status.ai_large_language_model = supported_modules_preferences.ai_large_language_model + except Exception as e: + logger.debug('Failed to get supported modules preferences', exc_info=e) + + return CliStatus( + program=PROGRAM_NAME, + version=__version__, + os=platform.system(), + arch=platform.machine(), + python_version=platform.python_version(), + installation_id=configuration_manager.get_or_create_installation_id(), + app_url=configuration_manager.get_cycode_app_url(), + api_url=configuration_manager.get_cycode_api_url(), + is_authenticated=is_authenticated, + user_id=auth_info.user_id if auth_info else None, + tenant_id=auth_info.tenant_id if auth_info else None, + supported_modules=supported_modules_status, + ) diff --git a/cycode/cli/apps/status/models.py b/cycode/cli/apps/status/models.py new file mode 100644 index 00000000..82b9751a --- /dev/null +++ b/cycode/cli/apps/status/models.py @@ -0,0 +1,61 @@ +import json +from dataclasses import asdict, dataclass + + +class CliStatusBase: + def as_dict(self) -> dict[str, any]: + return asdict(self) + + def _get_text_message_part(self, key: str, value: any, intent: int = 0) -> str: + message_parts = [] + + intent_prefix = ' ' * intent * 2 + human_readable_key = key.replace('_', ' ').capitalize() + + if isinstance(value, dict): + message_parts.append(f'{intent_prefix}{human_readable_key}:') + for sub_key, sub_value in value.items(): + message_parts.append(self._get_text_message_part(sub_key, sub_value, intent=intent + 1)) + elif isinstance(value, (list, set, tuple)): + message_parts.append(f'{intent_prefix}{human_readable_key}:') + for index, sub_value in enumerate(value): + message_parts.append(self._get_text_message_part(f'#{index}', sub_value, intent=intent + 1)) + else: + message_parts.append(f'{intent_prefix}{human_readable_key}: {value}') + + return '\n'.join(message_parts) + + def as_text(self) -> str: + message_parts = [] + for key, value in self.as_dict().items(): + message_parts.append(self._get_text_message_part(key, value)) + + return '\n'.join(message_parts) + + def as_json(self) -> str: + return json.dumps(self.as_dict()) + + +@dataclass +class CliSupportedModulesStatus(CliStatusBase): + secret_scanning: bool = False + sca_scanning: bool = False + iac_scanning: bool = False + sast_scanning: bool = False + ai_large_language_model: bool = False + + +@dataclass +class CliStatus(CliStatusBase): + program: str + version: str + os: str + arch: str + python_version: str + installation_id: str + app_url: str + api_url: str + is_authenticated: bool + user_id: str = None + tenant_id: str = None + supported_modules: CliSupportedModulesStatus = None diff --git a/cycode/cli/apps/status/status_command.py b/cycode/cli/apps/status/status_command.py new file mode 100644 index 00000000..4654ef20 --- /dev/null +++ b/cycode/cli/apps/status/status_command.py @@ -0,0 +1,31 @@ +import typer + +from cycode.cli.apps.status.get_cli_status import get_cli_status +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.console import console + + +def status_command(ctx: typer.Context) -> None: + """:information_source: [bold cyan]Show Cycode CLI status and configuration.[/] + + This command displays the current status and configuration of the Cycode CLI, including: + * Authentication status: Whether you're logged in + * Version information: Current CLI version + * Configuration: Current API endpoints and settings + * System information: Operating system and environment details + + Output formats: + * Text: Human-readable format (default) + * JSON: Machine-readable format + + Example usage: + * `cycode status`: Show status in text format + * `cycode -o json status`: Show status in JSON format + """ + output = ctx.obj['output'] + + cli_status = get_cli_status(ctx) + if output == OutputTypeOption.JSON: + console.print_json(cli_status.as_json()) + else: + console.print(cli_status.as_text()) diff --git a/cycode/cli/apps/status/version_command.py b/cycode/cli/apps/status/version_command.py new file mode 100644 index 00000000..ef117fc7 --- /dev/null +++ b/cycode/cli/apps/status/version_command.py @@ -0,0 +1,10 @@ +import typer + +from cycode.cli.apps.status.status_command import status_command +from cycode.cli.console import console + + +def version_command(ctx: typer.Context) -> None: + console.print('[b yellow]This command is deprecated. Please use the "status" command instead.[/]') + console.line() + status_command(ctx) diff --git a/cycode/cli/cli_types.py b/cycode/cli/cli_types.py new file mode 100644 index 00000000..c2fa12a2 --- /dev/null +++ b/cycode/cli/cli_types.py @@ -0,0 +1,107 @@ +from enum import Enum + +from cycode.cli import consts + + +class StrEnum(str, Enum): + def __str__(self) -> str: + return self.value + + +class OutputTypeOption(StrEnum): + RICH = 'rich' + TEXT = 'text' + JSON = 'json' + TABLE = 'table' + + +class ExportTypeOption(StrEnum): + JSON = 'json' + HTML = 'html' + SVG = 'svg' + + +class ScanTypeOption(StrEnum): + SECRET = consts.SECRET_SCAN_TYPE + SCA = consts.SCA_SCAN_TYPE + IAC = consts.IAC_SCAN_TYPE + SAST = consts.SAST_SCAN_TYPE + + def __str__(self) -> str: + return self.value + + +class ScaScanTypeOption(StrEnum): + PACKAGE_VULNERABILITIES = 'package-vulnerabilities' + LICENSE_COMPLIANCE = 'license-compliance' + + +class SbomFormatOption(StrEnum): + SPDX_2_2 = 'spdx-2.2' + SPDX_2_3 = 'spdx-2.3' + CYCLONEDX_1_4 = 'cyclonedx-1.4' + + +class SbomOutputFormatOption(StrEnum): + JSON = 'json' + + +class SeverityOption(StrEnum): + INFO = 'info' + LOW = 'low' + MEDIUM = 'medium' + HIGH = 'high' + CRITICAL = 'critical' + + @classmethod + def _missing_(cls, value: str) -> str: + value = value.lower() + for member in cls: + if member.lower() == value: + return member + + return cls.INFO # fallback to INFO if no match is found + + @staticmethod + def get_member_weight(name: str) -> int: + return _SEVERITY_WEIGHTS.get(name.lower(), _SEVERITY_DEFAULT_WEIGHT) + + @staticmethod + def get_member_color(name: str) -> str: + return _SEVERITY_COLORS.get(name.lower(), _SEVERITY_DEFAULT_COLOR) + + @staticmethod + def get_member_emoji(name: str) -> str: + return _SEVERITY_EMOJIS.get(name.lower(), _SEVERITY_DEFAULT_EMOJI) + + def __rich__(self) -> str: + color = self.get_member_color(self.value) + return f'[{color}]{self.value.upper()}[/]' + + +_SEVERITY_DEFAULT_WEIGHT = -1 +_SEVERITY_WEIGHTS = { + SeverityOption.INFO.value: 0, + SeverityOption.LOW.value: 1, + SeverityOption.MEDIUM.value: 2, + SeverityOption.HIGH.value: 3, + SeverityOption.CRITICAL.value: 4, +} + +_SEVERITY_DEFAULT_COLOR = 'white' +_SEVERITY_COLORS = { + SeverityOption.INFO.value: 'deep_sky_blue1', + SeverityOption.LOW.value: 'gold1', + SeverityOption.MEDIUM.value: 'dark_orange', + SeverityOption.HIGH.value: 'red1', + SeverityOption.CRITICAL.value: 'red3', +} + +_SEVERITY_DEFAULT_EMOJI = ':white_circle:' +_SEVERITY_EMOJIS = { + SeverityOption.INFO.value: ':blue_circle:', + SeverityOption.LOW.value: ':yellow_circle:', + SeverityOption.MEDIUM.value: ':orange_circle:', + SeverityOption.HIGH.value: ':red_circle:', + SeverityOption.CRITICAL.value: ':exclamation_mark:', # double_exclamation_mark is not red +} diff --git a/cycode/cli/commands/ai_remediation/ai_remediation_command.py b/cycode/cli/commands/ai_remediation/ai_remediation_command.py deleted file mode 100644 index 608fc9f4..00000000 --- a/cycode/cli/commands/ai_remediation/ai_remediation_command.py +++ /dev/null @@ -1,67 +0,0 @@ -import os - -import click -from patch_ng import fromstring -from rich.console import Console -from rich.markdown import Markdown - -from cycode.cli.exceptions.handle_ai_remediation_errors import handle_ai_remediation_exception -from cycode.cli.models import CliResult -from cycode.cli.printers import ConsolePrinter -from cycode.cli.utils.get_api_client import get_scan_cycode_client - - -def _echo_remediation(context: click.Context, remediation_markdown: str, is_fix_available: bool) -> None: - printer = ConsolePrinter(context) - if printer.is_json_printer: - data = {'remediation': remediation_markdown, 'is_fix_available': is_fix_available} - printer.print_result(CliResult(success=True, message='Remediation fetched successfully', data=data)) - else: # text or table - Console().print(Markdown(remediation_markdown)) - - -def _apply_fix(context: click.Context, diff: str, is_fix_available: bool) -> None: - printer = ConsolePrinter(context) - if not is_fix_available: - printer.print_result(CliResult(success=False, message='Fix is not available for this violation')) - return - - patch = fromstring(diff.encode('UTF-8')) - if patch is False: - printer.print_result(CliResult(success=False, message='Failed to parse fix diff')) - return - - is_fix_applied = patch.apply(root=os.getcwd(), strip=0) - if is_fix_applied: - printer.print_result(CliResult(success=True, message='Fix applied successfully')) - else: - printer.print_result(CliResult(success=False, message='Failed to apply fix')) - - -@click.command(short_help='Get AI remediation (INTERNAL).', hidden=True) -@click.argument('detection_id', nargs=1, type=click.UUID, required=True) -@click.option( - '--fix', - is_flag=True, - default=False, - help='Apply fixes to resolve violations. Fix is not available for all violations.', - type=click.BOOL, - required=False, -) -@click.pass_context -def ai_remediation_command(context: click.Context, detection_id: str, fix: bool) -> None: - client = get_scan_cycode_client() - - try: - remediation_markdown = client.get_ai_remediation(detection_id) - fix_diff = client.get_ai_remediation(detection_id, fix=True) - is_fix_available = bool(fix_diff) # exclude empty string, None, etc. - - if fix: - _apply_fix(context, fix_diff, is_fix_available) - else: - _echo_remediation(context, remediation_markdown, is_fix_available) - except Exception as err: - handle_ai_remediation_exception(context, err) - - context.exit() diff --git a/cycode/cli/commands/auth/auth_command.py b/cycode/cli/commands/auth/auth_command.py deleted file mode 100644 index 0862db2b..00000000 --- a/cycode/cli/commands/auth/auth_command.py +++ /dev/null @@ -1,82 +0,0 @@ -import click - -from cycode.cli.commands.auth.auth_manager import AuthManager -from cycode.cli.commands.auth_common import get_authorization_info -from cycode.cli.exceptions.custom_exceptions import ( - KNOWN_USER_FRIENDLY_REQUEST_ERRORS, - AuthProcessError, -) -from cycode.cli.models import CliError, CliErrors, CliResult -from cycode.cli.printers import ConsolePrinter -from cycode.cli.sentry import add_breadcrumb, capture_exception -from cycode.cyclient import logger - - -@click.group( - invoke_without_command=True, short_help='Authenticate your machine to associate the CLI with your Cycode account.' -) -@click.pass_context -def auth_command(context: click.Context) -> None: - """Authenticates your machine.""" - add_breadcrumb('auth') - - if context.invoked_subcommand is not None: - # if it is a subcommand, do nothing - return - - try: - logger.debug('Starting authentication process') - - auth_manager = AuthManager() - auth_manager.authenticate() - - result = CliResult(success=True, message='Successfully logged into cycode') - ConsolePrinter(context).print_result(result) - except Exception as e: - _handle_exception(context, e) - - -@auth_command.command( - name='check', short_help='Checks that your machine is associating the CLI with your Cycode account.' -) -@click.pass_context -def authorization_check(context: click.Context) -> None: - """Validates that your Cycode account has permission to work with the CLI.""" - add_breadcrumb('check') - - printer = ConsolePrinter(context) - auth_info = get_authorization_info(context) - if auth_info is None: - printer.print_result(CliResult(success=False, message='Cycode authentication failed')) - return - - printer.print_result( - CliResult( - success=True, - message='Cycode authentication verified', - data={'user_id': auth_info.user_id, 'tenant_id': auth_info.tenant_id}, - ) - ) - - -def _handle_exception(context: click.Context, e: Exception) -> None: - ConsolePrinter(context).print_exception() - - errors: CliErrors = { - **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, - AuthProcessError: CliError( - code='auth_error', message='Authentication failed. Please try again later using the command `cycode auth`' - ), - } - - error = errors.get(type(e)) - if error: - ConsolePrinter(context).print_error(error) - return - - if isinstance(e, click.ClickException): - raise e - - capture_exception(e) - - raise click.ClickException(str(e)) diff --git a/cycode/cli/commands/configure/configure_command.py b/cycode/cli/commands/configure/configure_command.py deleted file mode 100644 index 8f76d159..00000000 --- a/cycode/cli/commands/configure/configure_command.py +++ /dev/null @@ -1,140 +0,0 @@ -from typing import Optional - -import click - -from cycode.cli import config, consts -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.user_settings.credentials_manager import CredentialsManager -from cycode.cli.utils.string_utils import obfuscate_text - -_URLS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured Cycode URLs! Saved to: {filename}' -_URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( - 'Note that the URLs (APP and API) that already exist in environment variables ' - f'({consts.CYCODE_API_URL_ENV_VAR_NAME} and {consts.CYCODE_APP_URL_ENV_VAR_NAME}) ' - 'take precedent over these URLs; either update or remove the environment variables.' -) -_CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE = 'Successfully configured CLI credentials! Saved to: {filename}' -_CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE = ( - 'Note that the credentials that already exist in environment variables ' - f'({config.CYCODE_CLIENT_ID_ENV_VAR_NAME} and {config.CYCODE_CLIENT_SECRET_ENV_VAR_NAME}) ' - 'take precedent over these credentials; either update or remove the environment variables.' -) -_CREDENTIALS_MANAGER = CredentialsManager() -_CONFIGURATION_MANAGER = ConfigurationManager() - - -@click.command(short_help='Initial command to configure your CLI client authentication.') -def configure_command() -> None: - """Configure your CLI client authentication manually.""" - add_breadcrumb('configure') - - global_config_manager = _CONFIGURATION_MANAGER.global_config_file_manager - - current_api_url = global_config_manager.get_api_url() - current_app_url = global_config_manager.get_app_url() - api_url = _get_api_url_input(current_api_url) - app_url = _get_app_url_input(current_app_url) - - config_updated = False - if _should_update_value(current_api_url, api_url): - global_config_manager.update_api_base_url(api_url) - config_updated = True - if _should_update_value(current_app_url, app_url): - global_config_manager.update_app_base_url(app_url) - config_updated = True - - current_client_id, current_client_secret = _CREDENTIALS_MANAGER.get_credentials_from_file() - client_id = _get_client_id_input(current_client_id) - client_secret = _get_client_secret_input(current_client_secret) - - credentials_updated = False - if _should_update_value(current_client_id, client_id) or _should_update_value(current_client_secret, client_secret): - credentials_updated = True - _CREDENTIALS_MANAGER.update_credentials(client_id, client_secret) - - if config_updated: - click.echo(_get_urls_update_result_message()) - if credentials_updated: - click.echo(_get_credentials_update_result_message()) - - -def _get_client_id_input(current_client_id: Optional[str]) -> Optional[str]: - prompt_text = 'Cycode Client ID' - - prompt_suffix = ' []: ' - if current_client_id: - prompt_suffix = f' [{obfuscate_text(current_client_id)}]: ' - - new_client_id = click.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) - return new_client_id or current_client_id - - -def _get_client_secret_input(current_client_secret: Optional[str]) -> Optional[str]: - prompt_text = 'Cycode Client Secret' - - prompt_suffix = ' []: ' - if current_client_secret: - prompt_suffix = f' [{obfuscate_text(current_client_secret)}]: ' - - new_client_secret = click.prompt(text=prompt_text, prompt_suffix=prompt_suffix, default='', show_default=False) - return new_client_secret or current_client_secret - - -def _get_app_url_input(current_app_url: Optional[str]) -> str: - prompt_text = 'Cycode APP URL' - - default = consts.DEFAULT_CYCODE_APP_URL - if current_app_url: - default = current_app_url - - return click.prompt(text=prompt_text, default=default, type=click.STRING) - - -def _get_api_url_input(current_api_url: Optional[str]) -> str: - prompt_text = 'Cycode API URL' - - default = consts.DEFAULT_CYCODE_API_URL - if current_api_url: - default = current_api_url - - return click.prompt(text=prompt_text, default=default, type=click.STRING) - - -def _get_credentials_update_result_message() -> str: - success_message = _CREDENTIALS_UPDATED_SUCCESSFULLY_MESSAGE.format(filename=_CREDENTIALS_MANAGER.get_filename()) - if _are_credentials_exist_in_environment_variables(): - return f'{success_message}. {_CREDENTIALS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' - - return success_message - - -def _are_credentials_exist_in_environment_variables() -> bool: - client_id, client_secret = _CREDENTIALS_MANAGER.get_credentials_from_environment_variables() - return any([client_id, client_secret]) - - -def _get_urls_update_result_message() -> str: - success_message = _URLS_UPDATED_SUCCESSFULLY_MESSAGE.format( - filename=_CONFIGURATION_MANAGER.global_config_file_manager.get_filename() - ) - if _are_urls_exist_in_environment_variables(): - return f'{success_message}. {_URLS_ARE_SET_IN_ENVIRONMENT_VARIABLES_MESSAGE}' - - return success_message - - -def _are_urls_exist_in_environment_variables() -> bool: - api_url = _CONFIGURATION_MANAGER.get_api_url_from_environment_variables() - app_url = _CONFIGURATION_MANAGER.get_app_url_from_environment_variables() - return any([api_url, app_url]) - - -def _should_update_value( - old_value: Optional[str], - new_value: Optional[str], -) -> bool: - if not new_value: - return False - - return old_value != new_value diff --git a/cycode/cli/commands/ignore/ignore_command.py b/cycode/cli/commands/ignore/ignore_command.py deleted file mode 100644 index b94c5612..00000000 --- a/cycode/cli/commands/ignore/ignore_command.py +++ /dev/null @@ -1,141 +0,0 @@ -import re -from typing import Optional - -import click - -from cycode.cli import consts -from cycode.cli.config import config, configuration_manager -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.utils.path_utils import get_absolute_path, is_path_exists -from cycode.cli.utils.string_utils import hash_string_to_sha256 -from cycode.cyclient import logger - - -def _is_package_pattern_valid(package: str) -> bool: - return re.search('^[^@]+@[^@]+$', package) is not None - - -@click.command(short_help='Ignores a specific value, path or rule ID.') -@click.option( - '--by-value', type=click.STRING, required=False, help='Ignore a specific value while scanning for Secrets.' -) -@click.option( - '--by-sha', - type=click.STRING, - required=False, - help='Ignore a specific SHA512 representation of a string while scanning for Secrets.', -) -@click.option( - '--by-path', - type=click.STRING, - required=False, - help='Avoid scanning a specific path. You`ll need to specify the scan type.', -) -@click.option( - '--by-rule', - type=click.STRING, - required=False, - help='Ignore scanning a specific secret rule ID or IaC rule ID. You`ll to specify the scan type.', -) -@click.option( - '--by-package', - type=click.STRING, - required=False, - help='Ignore scanning a specific package version while running an SCA scan. Expected pattern: name@version.', -) -@click.option( - '--by-cve', - type=click.STRING, - required=False, - help='Ignore scanning a specific CVE while running an SCA scan. Expected pattern: CVE-YYYY-NNN.', -) -@click.option( - '--scan-type', - '-t', - default=consts.SECRET_SCAN_TYPE, - help='Specify the type of scan you wish to execute (the default is Secrets).', - type=click.Choice(config['scans']['supported_scans']), - required=False, -) -@click.option( - '--global', - '-g', - 'is_global', - is_flag=True, - default=False, - required=False, - help='Add an ignore rule to the global CLI config.', -) -def ignore_command( # noqa: C901 - by_value: Optional[str], - by_sha: Optional[str], - by_path: Optional[str], - by_rule: Optional[str], - by_package: Optional[str], - by_cve: Optional[str], - scan_type: str = consts.SECRET_SCAN_TYPE, - is_global: bool = False, -) -> None: - """Ignores a specific value, path or rule ID.""" - add_breadcrumb('ignore') - - all_by_values = [by_value, by_sha, by_path, by_rule, by_package, by_cve] - if all(by is None for by in all_by_values): - raise click.ClickException('Ignore by type is missing') - if len([by for by in all_by_values if by is not None]) != 1: - raise click.ClickException('You must specify only one ignore by type') - - if any(by is not None for by in [by_value, by_sha]) and scan_type != consts.SECRET_SCAN_TYPE: - raise click.ClickException('This exclude is supported only for Secret scan type') - if (by_cve or by_package) and scan_type != consts.SCA_SCAN_TYPE: - raise click.ClickException('This exclude is supported only for SCA scan type') - - # only one of the by values must be set - # at least one of the by values must be set - exclusion_type = exclusion_value = None - - if by_value: - exclusion_type = consts.EXCLUSIONS_BY_VALUE_SECTION_NAME - exclusion_value = hash_string_to_sha256(by_value) - - if by_sha: - exclusion_type = consts.EXCLUSIONS_BY_SHA_SECTION_NAME - exclusion_value = by_sha - - if by_path: - absolute_path = get_absolute_path(by_path) - if not is_path_exists(absolute_path): - raise click.ClickException('The provided path to ignore by does not exist') - - exclusion_type = consts.EXCLUSIONS_BY_PATH_SECTION_NAME - exclusion_value = get_absolute_path(absolute_path) - - if by_rule: - exclusion_type = consts.EXCLUSIONS_BY_RULE_SECTION_NAME - exclusion_value = by_rule - - if by_package: - if not _is_package_pattern_valid(by_package): - raise click.ClickException('wrong package pattern. should be name@version.') - - exclusion_type = consts.EXCLUSIONS_BY_PACKAGE_SECTION_NAME - exclusion_value = by_package - - if by_cve: - exclusion_type = consts.EXCLUSIONS_BY_CVE_SECTION_NAME - exclusion_value = by_cve - - if not exclusion_type or not exclusion_value: - # should never happen - raise click.ClickException('Invalid ignore by type') - - configuration_scope = 'global' if is_global else 'local' - logger.debug( - 'Adding ignore rule, %s', - { - 'configuration_scope': configuration_scope, - 'exclusion_type': exclusion_type, - 'exclusion_value': exclusion_value, - }, - ) - configuration_manager.add_exclusion(configuration_scope, scan_type, exclusion_type, exclusion_value) diff --git a/cycode/cli/commands/main_cli.py b/cycode/cli/commands/main_cli.py deleted file mode 100644 index 59b8625f..00000000 --- a/cycode/cli/commands/main_cli.py +++ /dev/null @@ -1,117 +0,0 @@ -import logging -from typing import Optional - -import click - -from cycode import __version__ -from cycode.cli.commands.ai_remediation.ai_remediation_command import ai_remediation_command -from cycode.cli.commands.auth.auth_command import auth_command -from cycode.cli.commands.configure.configure_command import configure_command -from cycode.cli.commands.ignore.ignore_command import ignore_command -from cycode.cli.commands.report.report_command import report_command -from cycode.cli.commands.scan.scan_command import scan_command -from cycode.cli.commands.status.status_command import status_command -from cycode.cli.commands.version.version_checker import version_checker -from cycode.cli.commands.version.version_command import version_command -from cycode.cli.consts import ( - CLI_CONTEXT_SETTINGS, -) -from cycode.cli.sentry import add_breadcrumb, init_sentry -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar -from cycode.cyclient.config import set_logging_level -from cycode.cyclient.cycode_client_base import CycodeClientBase -from cycode.cyclient.models import UserAgentOptionScheme - - -@click.group( - commands={ - 'scan': scan_command, - 'report': report_command, - 'configure': configure_command, - 'ignore': ignore_command, - 'auth': auth_command, - 'version': version_command, - 'status': status_command, - 'ai_remediation': ai_remediation_command, - }, - context_settings=CLI_CONTEXT_SETTINGS, -) -@click.option( - '--verbose', - '-v', - is_flag=True, - default=False, - help='Show detailed logs.', -) -@click.option( - '--no-progress-meter', - is_flag=True, - default=False, - help='Do not show the progress meter.', -) -@click.option( - '--no-update-notifier', - is_flag=True, - default=False, - help='Do not check CLI for updates.', -) -@click.option( - '--output', - '-o', - default='text', - help='Specify the output type (the default is text).', - type=click.Choice(['text', 'json', 'table']), -) -@click.option( - '--user-agent', - default=None, - help='Characteristic JSON object that lets servers identify the application.', - type=str, -) -@click.pass_context -def main_cli( - context: click.Context, - verbose: bool, - no_progress_meter: bool, - no_update_notifier: bool, - output: str, - user_agent: Optional[str], -) -> None: - init_sentry() - add_breadcrumb('cycode') - - context.ensure_object(dict) - configuration_manager = ConfigurationManager() - - verbose = verbose or configuration_manager.get_verbose_flag() - context.obj['verbose'] = verbose - if verbose: - set_logging_level(logging.DEBUG) - - context.obj['output'] = output - if output == 'json': - no_progress_meter = True - - context.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) - - if user_agent: - user_agent_option = UserAgentOptionScheme().loads(user_agent) - CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) - - if not no_update_notifier: - context.call_on_close(lambda: check_latest_version_on_close()) - - -@click.pass_context -def check_latest_version_on_close(context: click.Context) -> None: - output = context.obj.get('output') - # don't print anything if the output is JSON - if output == 'json': - return - - # we always want to check the latest version for "version" and "status" commands - should_use_cache = context.invoked_subcommand not in {'version', 'status'} - version_checker.check_and_notify_update( - current_version=__version__, use_color=context.color, use_cache=should_use_cache - ) diff --git a/cycode/cli/commands/report/report_command.py b/cycode/cli/commands/report/report_command.py deleted file mode 100644 index 9e92a64f..00000000 --- a/cycode/cli/commands/report/report_command.py +++ /dev/null @@ -1,21 +0,0 @@ -import click - -from cycode.cli.commands.report.sbom.sbom_command import sbom_command -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.utils.progress_bar import SBOM_REPORT_PROGRESS_BAR_SECTIONS, get_progress_bar - - -@click.group( - commands={ - 'sbom': sbom_command, - }, - short_help='Generate report. You`ll need to specify which report type to perform.', -) -@click.pass_context -def report_command( - context: click.Context, -) -> int: - """Generate report.""" - add_breadcrumb('report') - context.obj['progress_bar'] = get_progress_bar(hidden=False, sections=SBOM_REPORT_PROGRESS_BAR_SECTIONS) - return 1 diff --git a/cycode/cli/commands/report/sbom/sbom_command.py b/cycode/cli/commands/report/sbom/sbom_command.py deleted file mode 100644 index a938fd90..00000000 --- a/cycode/cli/commands/report/sbom/sbom_command.py +++ /dev/null @@ -1,87 +0,0 @@ -import pathlib -from typing import Optional - -import click - -from cycode.cli.commands.report.sbom.path.path_command import path_command -from cycode.cli.commands.report.sbom.repository_url.repository_url_command import repository_url_command -from cycode.cli.config import config -from cycode.cli.sentry import add_breadcrumb -from cycode.cyclient.report_client import ReportParameters - - -@click.group( - commands={ - 'path': path_command, - 'repository_url': repository_url_command, - }, - short_help='Generate SBOM report for remote repository by url or local directory by path.', -) -@click.option( - '--format', - '-f', - help='SBOM format.', - type=click.Choice(config['scans']['supported_sbom_formats']), - required=True, -) -@click.option( - '--output-format', - '-o', - default='json', - help='Specify the output file format (the default is json).', - type=click.Choice(['json']), - required=False, -) -@click.option( - '--output-file', - help='Output file (the default is autogenerated filename saved to the current directory).', - default=None, - type=click.Path(resolve_path=True, writable=True, path_type=pathlib.Path), - required=False, -) -@click.option( - '--include-vulnerabilities', - is_flag=True, - default=False, - help='Include vulnerabilities.', - type=bool, - required=False, -) -@click.option( - '--include-dev-dependencies', - is_flag=True, - default=False, - help='Include dev dependencies.', - type=bool, - required=False, -) -@click.pass_context -def sbom_command( - context: click.Context, - format: str, - output_format: Optional[str], - output_file: Optional[pathlib.Path], - include_vulnerabilities: bool, - include_dev_dependencies: bool, -) -> int: - """Generate SBOM report.""" - add_breadcrumb('sbom') - - sbom_format_parts = format.split('-') - if len(sbom_format_parts) != 2: - raise click.ClickException('Invalid SBOM format.') - - sbom_format, sbom_format_version = sbom_format_parts - - report_parameters = ReportParameters( - entity_type='SbomCli', - sbom_report_type=sbom_format, - sbom_version=sbom_format_version, - output_format=output_format, - include_vulnerabilities=include_vulnerabilities, - include_dev_dependencies=include_dev_dependencies, - ) - context.obj['report_parameters'] = report_parameters - context.obj['output_file'] = output_file - - return 1 diff --git a/cycode/cli/commands/scan/commit_history/__init__.py b/cycode/cli/commands/scan/commit_history/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/commit_history/commit_history_command.py b/cycode/cli/commands/scan/commit_history/commit_history_command.py deleted file mode 100644 index bfb57c29..00000000 --- a/cycode/cli/commands/scan/commit_history/commit_history_command.py +++ /dev/null @@ -1,27 +0,0 @@ -import click - -from cycode.cli.commands.scan.code_scanner import scan_commit_range -from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception -from cycode.cli.sentry import add_breadcrumb -from cycode.cyclient import logger - - -@click.command(short_help='Scan all the commits history in this git repository.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.option( - '--commit_range', - '-r', - help='Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1)', - type=click.STRING, - default='--all', - required=False, -) -@click.pass_context -def commit_history_command(context: click.Context, path: str, commit_range: str) -> None: - try: - add_breadcrumb('commit_history') - - logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) - scan_commit_range(context, path=path, commit_range=commit_range) - except Exception as e: - handle_scan_exception(context, e) diff --git a/cycode/cli/commands/scan/path/__init__.py b/cycode/cli/commands/scan/path/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/path/path_command.py b/cycode/cli/commands/scan/path/path_command.py deleted file mode 100644 index ec62b224..00000000 --- a/cycode/cli/commands/scan/path/path_command.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Tuple - -import click - -from cycode.cli.commands.scan.code_scanner import scan_disk_files -from cycode.cli.sentry import add_breadcrumb -from cycode.cyclient import logger - - -@click.command(short_help='Scan the files in the path provided in the command.') -@click.argument('paths', nargs=-1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.pass_context -def path_command(context: click.Context, paths: Tuple[str]) -> None: - add_breadcrumb('path') - - progress_bar = context.obj['progress_bar'] - progress_bar.start() - - logger.debug('Starting path scan process, %s', {'paths': paths}) - scan_disk_files(context, paths) diff --git a/cycode/cli/commands/scan/pre_commit/__init__.py b/cycode/cli/commands/scan/pre_commit/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/pre_receive/__init__.py b/cycode/cli/commands/scan/pre_receive/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/repository/__init__.py b/cycode/cli/commands/scan/repository/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/scan_ci/__init__.py b/cycode/cli/commands/scan/scan_ci/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/scan/scan_ci/scan_ci_command.py b/cycode/cli/commands/scan/scan_ci/scan_ci_command.py deleted file mode 100644 index 6d4fbd36..00000000 --- a/cycode/cli/commands/scan/scan_ci/scan_ci_command.py +++ /dev/null @@ -1,19 +0,0 @@ -import os - -import click - -from cycode.cli.commands.scan.code_scanner import scan_commit_range -from cycode.cli.commands.scan.scan_ci.ci_integrations import get_commit_range -from cycode.cli.sentry import add_breadcrumb - -# This command is not finished yet. It is not used in the codebase. - - -@click.command( - short_help='Execute scan in a CI environment which relies on the ' - 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' -) -@click.pass_context -def scan_ci_command(context: click.Context) -> None: - add_breadcrumb('ci') - scan_commit_range(context, path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py deleted file mode 100644 index 95259f4a..00000000 --- a/cycode/cli/commands/scan/scan_command.py +++ /dev/null @@ -1,187 +0,0 @@ -import sys -from typing import List - -import click - -from cycode.cli import consts -from cycode.cli.commands.scan.commit_history.commit_history_command import commit_history_command -from cycode.cli.commands.scan.path.path_command import path_command -from cycode.cli.commands.scan.pre_commit.pre_commit_command import pre_commit_command -from cycode.cli.commands.scan.pre_receive.pre_receive_command import pre_receive_command -from cycode.cli.commands.scan.repository.repository_command import repository_command -from cycode.cli.config import config -from cycode.cli.consts import ( - ISSUE_DETECTED_STATUS_CODE, - NO_ISSUES_STATUS_CODE, - SCA_GRADLE_ALL_SUB_PROJECTS_FLAG, - SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, -) -from cycode.cli.models import Severity -from cycode.cli.sentry import add_breadcrumb -from cycode.cli.utils import scan_utils -from cycode.cli.utils.get_api_client import get_scan_cycode_client - - -@click.group( - commands={ - 'repository': repository_command, - 'commit_history': commit_history_command, - 'path': path_command, - 'pre_commit': pre_commit_command, - 'pre_receive': pre_receive_command, - }, - short_help='Scan the content for Secrets/IaC/SCA/SAST violations. ' - 'You`ll need to specify which scan type to perform: commit_history/path/repository/etc.', -) -@click.option( - '--scan-type', - '-t', - default=consts.SECRET_SCAN_TYPE, - help='Specify the type of scan you wish to execute (the default is Secrets).', - type=click.Choice(config['scans']['supported_scans']), -) -@click.option( - '--secret', - default=None, - help='Specify a Cycode client secret for this specific scan execution.', - type=str, - required=False, -) -@click.option( - '--client-id', - default=None, - help='Specify a Cycode client ID for this specific scan execution.', - type=str, - required=False, -) -@click.option( - '--show-secret', is_flag=True, default=False, help='Show Secrets in plain text.', type=bool, required=False -) -@click.option( - '--soft-fail', - is_flag=True, - default=False, - help='Run the scan without failing; always return a non-error status code.', - type=bool, - required=False, -) -@click.option( - '--severity-threshold', - default=Severity.INFO.name, - help='Show violations only for the specified level or higher.', - type=click.Choice([e.name for e in Severity]), - required=False, -) -@click.option( - '--sca-scan', - default=None, - help='Specify the type of SCA scan you wish to execute (the default is both).', - multiple=True, - type=click.Choice(config['scans']['supported_sca_scans']), -) -@click.option( - '--monitor', - is_flag=True, - default=False, - help='Used for SCA scan types only; when specified, the scan results are recorded in the Discovery module.', - type=bool, - required=False, -) -@click.option( - '--report', - is_flag=True, - default=False, - help='When specified, generates a violations report. A link to the report will be displayed in the console output.', - type=bool, - required=False, -) -@click.option( - f'--{SCA_SKIP_RESTORE_DEPENDENCIES_FLAG}', - is_flag=True, - default=False, - help='When specified, Cycode will not run restore command. Will scan direct dependencies ONLY!', - type=bool, - required=False, -) -@click.option( - '--sync', - is_flag=True, - default=False, - help='Run scan synchronously (the default is asynchronous).', - type=bool, - required=False, -) -@click.option( - f'--{SCA_GRADLE_ALL_SUB_PROJECTS_FLAG}', - is_flag=True, - default=False, - help='When specified, Cycode will run gradle restore command for all sub projects. ' - 'Should run from root project directory ONLY!', - type=bool, - required=False, -) -@click.pass_context -def scan_command( - context: click.Context, - scan_type: str, - secret: str, - client_id: str, - show_secret: bool, - soft_fail: bool, - severity_threshold: str, - sca_scan: List[str], - monitor: bool, - report: bool, - no_restore: bool, - sync: bool, - gradle_all_sub_projects: bool, -) -> int: - """Scans for Secrets, IaC, SCA or SAST violations.""" - add_breadcrumb('scan') - - if show_secret: - context.obj['show_secret'] = show_secret - else: - context.obj['show_secret'] = config['result_printer']['default']['show_secret'] - - if soft_fail: - context.obj['soft_fail'] = soft_fail - else: - context.obj['soft_fail'] = config['soft_fail'] - - context.obj['client'] = get_scan_cycode_client(client_id, secret, not context.obj['show_secret']) - context.obj['scan_type'] = scan_type - context.obj['sync'] = sync - context.obj['severity_threshold'] = severity_threshold - context.obj['monitor'] = monitor - context.obj['report'] = report - context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore - context.obj[SCA_GRADLE_ALL_SUB_PROJECTS_FLAG] = gradle_all_sub_projects - - _sca_scan_to_context(context, sca_scan) - - return 1 - - -def _sca_scan_to_context(context: click.Context, sca_scan_user_selected: List[str]) -> None: - for sca_scan_option_selected in sca_scan_user_selected: - context.obj[sca_scan_option_selected] = True - - -@scan_command.result_callback() -@click.pass_context -def finalize(context: click.Context, *_, **__) -> None: - add_breadcrumb('scan_finalize') - - progress_bar = context.obj.get('progress_bar') - if progress_bar: - progress_bar.stop() - - if context.obj['soft_fail']: - sys.exit(0) - - exit_code = NO_ISSUES_STATUS_CODE - if scan_utils.is_scan_failed(context): - exit_code = ISSUE_DETECTED_STATUS_CODE - - sys.exit(exit_code) diff --git a/cycode/cli/commands/status/__init__.py b/cycode/cli/commands/status/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/status/status_command.py b/cycode/cli/commands/status/status_command.py deleted file mode 100644 index f5d9aec3..00000000 --- a/cycode/cli/commands/status/status_command.py +++ /dev/null @@ -1,122 +0,0 @@ -import dataclasses -import json -import platform -from typing import Dict - -import click - -from cycode import __version__ -from cycode.cli.commands.auth_common import get_authorization_info -from cycode.cli.consts import PROGRAM_NAME -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils.get_api_client import get_scan_cycode_client -from cycode.cyclient import logger - - -class CliStatusBase: - def as_dict(self) -> Dict[str, any]: - return dataclasses.asdict(self) - - def _get_text_message_part(self, key: str, value: any, intent: int = 0) -> str: - message_parts = [] - - intent_prefix = ' ' * intent * 2 - human_readable_key = key.replace('_', ' ').capitalize() - - if isinstance(value, dict): - message_parts.append(f'{intent_prefix}{human_readable_key}:') - for sub_key, sub_value in value.items(): - message_parts.append(self._get_text_message_part(sub_key, sub_value, intent=intent + 1)) - elif isinstance(value, (list, set, tuple)): - message_parts.append(f'{intent_prefix}{human_readable_key}:') - for index, sub_value in enumerate(value): - message_parts.append(self._get_text_message_part(f'#{index}', sub_value, intent=intent + 1)) - else: - message_parts.append(f'{intent_prefix}{human_readable_key}: {value}') - - return '\n'.join(message_parts) - - def as_text(self) -> str: - message_parts = [] - for key, value in self.as_dict().items(): - message_parts.append(self._get_text_message_part(key, value)) - - return '\n'.join(message_parts) - - def as_json(self) -> str: - return json.dumps(self.as_dict()) - - -@dataclasses.dataclass -class CliSupportedModulesStatus(CliStatusBase): - secret_scanning: bool = False - sca_scanning: bool = False - iac_scanning: bool = False - sast_scanning: bool = False - ai_large_language_model: bool = False - - -@dataclasses.dataclass -class CliStatus(CliStatusBase): - program: str - version: str - os: str - arch: str - python_version: str - installation_id: str - app_url: str - api_url: str - is_authenticated: bool - user_id: str = None - tenant_id: str = None - supported_modules: CliSupportedModulesStatus = None - - -def get_cli_status() -> CliStatus: - configuration_manager = ConfigurationManager() - - auth_info = get_authorization_info() - is_authenticated = auth_info is not None - - supported_modules_status = CliSupportedModulesStatus() - if is_authenticated: - try: - client = get_scan_cycode_client() - supported_modules_preferences = client.get_supported_modules_preferences() - - supported_modules_status.secret_scanning = supported_modules_preferences.secret_scanning - supported_modules_status.sca_scanning = supported_modules_preferences.sca_scanning - supported_modules_status.iac_scanning = supported_modules_preferences.iac_scanning - supported_modules_status.sast_scanning = supported_modules_preferences.sast_scanning - supported_modules_status.ai_large_language_model = supported_modules_preferences.ai_large_language_model - except Exception as e: - logger.debug('Failed to get supported modules preferences', exc_info=e) - - return CliStatus( - program=PROGRAM_NAME, - version=__version__, - os=platform.system(), - arch=platform.machine(), - python_version=platform.python_version(), - installation_id=configuration_manager.get_or_create_installation_id(), - app_url=configuration_manager.get_cycode_app_url(), - api_url=configuration_manager.get_cycode_api_url(), - is_authenticated=is_authenticated, - user_id=auth_info.user_id if auth_info else None, - tenant_id=auth_info.tenant_id if auth_info else None, - supported_modules=supported_modules_status, - ) - - -@click.command(short_help='Show the CLI status and exit.') -@click.pass_context -def status_command(context: click.Context) -> None: - output = context.obj['output'] - - status = get_cli_status() - message = status.as_text() - if output == 'json': - message = status.as_json() - - click.echo(message, color=context.color) - context.exit() diff --git a/cycode/cli/commands/version/__init__.py b/cycode/cli/commands/version/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/cycode/cli/commands/version/version_command.py b/cycode/cli/commands/version/version_command.py deleted file mode 100644 index 107aedbc..00000000 --- a/cycode/cli/commands/version/version_command.py +++ /dev/null @@ -1,22 +0,0 @@ -import json - -import click - -from cycode import __version__ -from cycode.cli.consts import PROGRAM_NAME - - -@click.command(short_help='Show the CLI version and exit. Use `cycode status` instead.', deprecated=True) -@click.pass_context -def version_command(context: click.Context) -> None: - output = context.obj['output'] - - prog = PROGRAM_NAME - ver = __version__ - - message = f'{prog}, version {ver}' - if output == 'json': - message = json.dumps({'name': prog, 'version': ver}) - - click.echo(message, color=context.color) - context.exit() diff --git a/cycode/cli/config.py b/cycode/cli/config.py index 71f354ad..73491546 100644 --- a/cycode/cli/config.py +++ b/cycode/cli/config.py @@ -1,13 +1,7 @@ -import os - from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils.yaml_utils import read_file -relative_path = os.path.dirname(__file__) -config_file_path = os.path.join(relative_path, 'config.yaml') -config = read_file(config_file_path) configuration_manager = ConfigurationManager() # env vars CYCODE_CLIENT_ID_ENV_VAR_NAME = 'CYCODE_CLIENT_ID' -CYCODE_CLIENT_SECRET_ENV_VAR_NAME = 'CYCODE_CLIENT_SECRET' # noqa: S105 +CYCODE_CLIENT_SECRET_ENV_VAR_NAME = 'CYCODE_CLIENT_SECRET' diff --git a/cycode/cli/config.yaml b/cycode/cli/config.yaml deleted file mode 100644 index 875f37c1..00000000 --- a/cycode/cli/config.yaml +++ /dev/null @@ -1,25 +0,0 @@ -soft_fail: False -scans: - supported_scans: - - secret - - iac - - sca - - sast - supported_sca_scans: - - package-vulnerabilities - - license-compliance - supported_sbom_formats: - - spdx-2.2 - - spdx-2.3 - - cyclonedx-1.4 -result_printer: - default: - lines_to_display: 3 - show_secret: False - secret: - pre_receive: - lines_to_display: 1 - show_secret: False - commit_history: - lines_to_display: 1 - show_secret: False \ No newline at end of file diff --git a/cycode/cli/console.py b/cycode/cli/console.py new file mode 100644 index 00000000..5d78fc36 --- /dev/null +++ b/cycode/cli/console.py @@ -0,0 +1,69 @@ +import os +from typing import TYPE_CHECKING, Optional + +from rich.console import Console, RenderResult +from rich.markdown import Heading, Markdown +from rich.text import Text + +if TYPE_CHECKING: + from rich.console import ConsoleOptions + +console_out = Console() +console_err = Console(stderr=True) + +console = console_out # alias + + +def is_dark_console() -> Optional[bool]: + """Detect if the console is dark or light. + + This function checks the environment variables and terminal type to determine if the console is dark or light. + + Used approaches: + 1. Check the `LC_DARK_BG` environment variable. + 2. Check the `COLORFGBG` environment variable for background color. + + And it still could be wrong in some cases. + + TODO(MarshalX): migrate to https://github.com/dalance/termbg when someone will implement it for Python. + """ + dark = None + + dark_bg = os.environ.get('LC_DARK_BG') + if dark_bg is not None: + return dark_bg != '0' + + # If BG color in {0, 1, 2, 3, 4, 5, 6, 8} then dark, else light. + try: + color = os.environ.get('COLORFGBG') + *_, bg = color.split(';') + bg = int(bg) + dark = bool(0 <= bg <= 6 or bg == 8) + except Exception: # noqa: S110 + pass + + return dark + + +_SYNTAX_HIGHLIGHT_DARK_THEME = 'monokai' +_SYNTAX_HIGHLIGHT_LIGHT_THEME = 'default' + +# when we could not detect it, use dark theme as most terminals are dark +_SYNTAX_HIGHLIGHT_THEME = _SYNTAX_HIGHLIGHT_LIGHT_THEME if is_dark_console() is False else _SYNTAX_HIGHLIGHT_DARK_THEME + + +class CycodeHeading(Heading): + """Custom Rich Heading for Markdown. + + Changes: + - remove justify to 'center' + - remove the box for h1 + """ + + def __rich_console__(self, console: 'Console', options: 'ConsoleOptions') -> RenderResult: + if self.tag == 'h2': + yield Text('') + yield self.text + + +Markdown.elements['heading_open'] = CycodeHeading diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index 003218d6..286f1f95 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -1,20 +1,20 @@ PROGRAM_NAME = 'cycode' APP_NAME = 'CycodeCLI' -CLI_CONTEXT_SETTINGS = { - 'terminal_width': 10**9, - 'max_content_width': 10**9, -} +CLI_CONTEXT_SETTINGS = {'terminal_width': 10**9, 'max_content_width': 10**9, 'help_option_names': ['-h', '--help']} -PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre_commit' -PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre_receive' -COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit_history' +PRE_COMMIT_COMMAND_SCAN_TYPE = 'pre-commit' +PRE_COMMIT_COMMAND_SCAN_TYPE_OLD = 'pre_commit' +PRE_RECEIVE_COMMAND_SCAN_TYPE = 'pre-receive' +PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD = 'pre_receive' +COMMIT_HISTORY_COMMAND_SCAN_TYPE = 'commit-history' +COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD = 'commit_history' -SECRET_SCAN_TYPE = 'secret' # noqa: S105 -INFRA_CONFIGURATION_SCAN_TYPE = 'iac' +SECRET_SCAN_TYPE = 'secret' +IAC_SCAN_TYPE = 'iac' SCA_SCAN_TYPE = 'sca' SAST_SCAN_TYPE = 'sast' -INFRA_CONFIGURATION_SCAN_SUPPORTED_FILES = ('.tf', '.tf.json', '.json', '.yaml', '.yml', 'dockerfile') +IAC_SCAN_SUPPORTED_FILES = ('.tf', '.tf.json', '.json', '.yaml', '.yml', 'dockerfile') SECRET_SCAN_FILE_EXTENSIONS_TO_IGNORE = ( '.7z', @@ -108,7 +108,12 @@ COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE] -COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [PRE_RECEIVE_COMMAND_SCAN_TYPE, COMMIT_HISTORY_COMMAND_SCAN_TYPE] +COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [ + PRE_RECEIVE_COMMAND_SCAN_TYPE, + PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD, + COMMIT_HISTORY_COMMAND_SCAN_TYPE, + COMMIT_HISTORY_COMMAND_SCAN_TYPE_OLD, +] DEFAULT_CYCODE_DOMAIN = 'cycode.com' DEFAULT_CYCODE_API_URL = f'https://api.{DEFAULT_CYCODE_DOMAIN}' diff --git a/cycode/cli/exceptions/custom_exceptions.py b/cycode/cli/exceptions/custom_exceptions.py index 40abed63..59c0f693 100644 --- a/cycode/cli/exceptions/custom_exceptions.py +++ b/cycode/cli/exceptions/custom_exceptions.py @@ -4,13 +4,17 @@ class CycodeError(Exception): - """Base class for all custom exceptions""" + """Base class for all custom exceptions.""" + + def __str__(self) -> str: + class_name = self.__class__.__name__ + return f'{class_name} error occurred.' class RequestError(CycodeError): ... -class RequestTimeout(RequestError): ... +class RequestTimeoutError(RequestError): ... class RequestConnectionError(RequestError): ... @@ -27,10 +31,7 @@ def __init__(self, status_code: int, error_message: str, response: Response) -> super().__init__(self.error_message) def __str__(self) -> str: - return ( - f'error occurred during the request. status code: {self.status_code}, error message: ' - f'{self.error_message}' - ) + return f'HTTP error occurred during the request (code {self.status_code}). Message: {self.error_message}' class ScanAsyncError(CycodeError): @@ -39,7 +40,7 @@ def __init__(self, error_message: str) -> None: super().__init__(self.error_message) def __str__(self) -> str: - return f'error occurred during the scan. error message: {self.error_message}' + return f'Async scan error occurred during the scan. Message: {self.error_message}' class ReportAsyncError(CycodeError): @@ -54,7 +55,7 @@ def __init__(self, error_message: str, response: Response) -> None: super().__init__(self.error_message) def __str__(self) -> str: - return 'Http Unauthorized Error' + return f'HTTP unauthorized error occurred during the request. Message: {self.error_message}' class ZipTooLargeError(CycodeError): @@ -72,7 +73,7 @@ def __init__(self, error_message: str) -> None: super().__init__() def __str__(self) -> str: - return f'Something went wrong during the authentication process, error message: {self.error_message}' + return f'Something went wrong during the authentication process. Message: {self.error_message}' class TfplanKeyError(CycodeError): @@ -90,7 +91,7 @@ def __str__(self) -> str: code='cycode_error', message='Cycode was unable to complete this scan. Please try again by executing the `cycode scan` command', ), - RequestTimeout: CliError( + RequestTimeoutError: CliError( soft_fail=True, code='timeout_error', message='The request timed out. Please try again by executing the `cycode scan` command', @@ -106,6 +107,6 @@ def __str__(self) -> str: code='ssl_error', message='An SSL error occurred when trying to connect to the Cycode API. ' 'If you use an on-premises installation or a proxy that intercepts SSL traffic ' - 'you should use the CURL_CA_BUNDLE environment variable to specify path to a valid .pem or similar.', + 'you should use the CURL_CA_BUNDLE environment variable to specify path to a valid .pem or similar', ), } diff --git a/cycode/cli/exceptions/handle_ai_remediation_errors.py b/cycode/cli/exceptions/handle_ai_remediation_errors.py index ba46cbf7..961acd62 100644 --- a/cycode/cli/exceptions/handle_ai_remediation_errors.py +++ b/cycode/cli/exceptions/handle_ai_remediation_errors.py @@ -1,14 +1,14 @@ -import click +import typer -from cycode.cli.exceptions.common import handle_errors from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS, RequestHttpError +from cycode.cli.exceptions.handle_errors import handle_errors from cycode.cli.models import CliError, CliErrors class AiRemediationNotFoundError(Exception): ... -def handle_ai_remediation_exception(context: click.Context, err: Exception) -> None: +def handle_ai_remediation_exception(ctx: typer.Context, err: Exception) -> None: if isinstance(err, RequestHttpError) and err.status_code == 404: err = AiRemediationNotFoundError() @@ -19,4 +19,4 @@ def handle_ai_remediation_exception(context: click.Context, err: Exception) -> N message='The AI remediation was not found. Please try different detection ID', ), } - handle_errors(context, err, errors) + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_auth_errors.py b/cycode/cli/exceptions/handle_auth_errors.py new file mode 100644 index 00000000..72e18c88 --- /dev/null +++ b/cycode/cli/exceptions/handle_auth_errors.py @@ -0,0 +1,18 @@ +import typer + +from cycode.cli.exceptions.custom_exceptions import ( + KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AuthProcessError, +) +from cycode.cli.exceptions.handle_errors import handle_errors +from cycode.cli.models import CliError, CliErrors + + +def handle_auth_exception(ctx: typer.Context, err: Exception) -> None: + errors: CliErrors = { + **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, + AuthProcessError: CliError( + code='auth_error', message='Authentication failed. Please try again later using the command `cycode auth`' + ), + } + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/common.py b/cycode/cli/exceptions/handle_errors.py similarity index 52% rename from cycode/cli/exceptions/common.py rename to cycode/cli/exceptions/handle_errors.py index 51433af7..8d230902 100644 --- a/cycode/cli/exceptions/common.py +++ b/cycode/cli/exceptions/handle_errors.py @@ -1,27 +1,28 @@ from typing import Optional import click +import typer from cycode.cli.models import CliError, CliErrors -from cycode.cli.printers import ConsolePrinter -from cycode.cli.sentry import capture_exception +from cycode.cli.utils.sentry import capture_exception def handle_errors( - context: click.Context, err: BaseException, cli_errors: CliErrors, *, return_exception: bool = False + ctx: typer.Context, err: BaseException, cli_errors: CliErrors, *, return_exception: bool = False ) -> Optional['CliError']: - ConsolePrinter(context).print_exception(err) + printer = ctx.obj.get('console_printer') + printer.print_exception(err) if type(err) in cli_errors: - error = cli_errors[type(err)] + error = cli_errors[type(err)].enrich(additional_message=str(err)) if error.soft_fail is True: - context.obj['soft_fail'] = True + ctx.obj['soft_fail'] = True if return_exception: return error - ConsolePrinter(context).print_error(error) + printer.print_error(error) return None if isinstance(err, click.ClickException): @@ -33,5 +34,5 @@ def handle_errors( if return_exception: return unknown_error - ConsolePrinter(context).print_error(unknown_error) - exit(1) + printer.print_error(unknown_error) + raise typer.Exit(1) diff --git a/cycode/cli/exceptions/handle_report_sbom_errors.py b/cycode/cli/exceptions/handle_report_sbom_errors.py index 70cf6277..22707c8c 100644 --- a/cycode/cli/exceptions/handle_report_sbom_errors.py +++ b/cycode/cli/exceptions/handle_report_sbom_errors.py @@ -1,12 +1,12 @@ -import click +import typer from cycode.cli.exceptions import custom_exceptions -from cycode.cli.exceptions.common import handle_errors from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS +from cycode.cli.exceptions.handle_errors import handle_errors from cycode.cli.models import CliError, CliErrors -def handle_report_exception(context: click.Context, err: Exception) -> None: +def handle_report_exception(ctx: typer.Context, err: Exception) -> None: errors: CliErrors = { **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, custom_exceptions.ScanAsyncError: CliError( @@ -20,4 +20,4 @@ def handle_report_exception(context: click.Context, err: Exception) -> None: 'Please try again by executing the `cycode report` command', ), } - handle_errors(context, err, errors) + handle_errors(ctx, err, errors) diff --git a/cycode/cli/exceptions/handle_scan_errors.py b/cycode/cli/exceptions/handle_scan_errors.py index 550e6879..229e0f02 100644 --- a/cycode/cli/exceptions/handle_scan_errors.py +++ b/cycode/cli/exceptions/handle_scan_errors.py @@ -1,26 +1,23 @@ from typing import Optional -import click +import typer from cycode.cli.exceptions import custom_exceptions -from cycode.cli.exceptions.common import handle_errors from cycode.cli.exceptions.custom_exceptions import KNOWN_USER_FRIENDLY_REQUEST_ERRORS +from cycode.cli.exceptions.handle_errors import handle_errors from cycode.cli.models import CliError, CliErrors from cycode.cli.utils.git_proxy import git_proxy -def handle_scan_exception( - context: click.Context, err: Exception, *, return_exception: bool = False -) -> Optional[CliError]: - context.obj['did_fail'] = True +def handle_scan_exception(ctx: typer.Context, err: Exception, *, return_exception: bool = False) -> Optional[CliError]: + ctx.obj['did_fail'] = True errors: CliErrors = { **KNOWN_USER_FRIENDLY_REQUEST_ERRORS, custom_exceptions.ScanAsyncError: CliError( soft_fail=True, code='scan_error', - message='Cycode was unable to complete this scan. ' - 'Please try again by executing the `cycode scan` command', + message='Cycode was unable to complete this scan. Please try again by executing the `cycode scan` command', ), custom_exceptions.ZipTooLargeError: CliError( soft_fail=True, @@ -40,9 +37,9 @@ def handle_scan_exception( git_proxy.get_invalid_git_repository_error(): CliError( soft_fail=False, code='invalid_git_error', - message='The path you supplied does not correlate to a git repository. ' + message='The path you supplied does not correlate to a Git repository. ' 'If you still wish to scan this path, use: `cycode scan path `', ), } - return handle_errors(context, err, errors, return_exception=return_exception) + return handle_errors(ctx, err, errors, return_exception=return_exception) diff --git a/cycode/cli/files_collector/excluder.py b/cycode/cli/files_collector/excluder.py index b8cb7920..9ef5e3d6 100644 --- a/cycode/cli/files_collector/excluder.py +++ b/cycode/cli/files_collector/excluder.py @@ -1,20 +1,23 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from cycode.cli import consts from cycode.cli.config import configuration_manager from cycode.cli.user_settings.config_file_manager import ConfigFileManager from cycode.cli.utils.path_utils import get_file_size, is_binary_file, is_sub_path from cycode.cli.utils.string_utils import get_content_size, is_binary_content -from cycode.cyclient import logger +from cycode.logger import get_logger if TYPE_CHECKING: from cycode.cli.models import Document from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection +logger = get_logger('File Excluder') + + def exclude_irrelevant_files( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, filenames: List[str] -) -> List[str]: + progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, filenames: list[str] +) -> list[str]: relevant_files = [] for filename in filenames: progress_bar.update(progress_bar_section) @@ -26,7 +29,7 @@ def exclude_irrelevant_files( return relevant_files -def exclude_irrelevant_documents_to_scan(scan_type: str, documents_to_scan: List['Document']) -> List['Document']: +def exclude_irrelevant_documents_to_scan(scan_type: str, documents_to_scan: list['Document']) -> list['Document']: logger.debug('Excluding irrelevant documents to scan') relevant_documents = [] @@ -149,8 +152,8 @@ def _is_relevant_document_to_scan(scan_type: str, filename: str, content: str) - def _is_file_extension_supported(scan_type: str, filename: str) -> bool: filename = filename.lower() - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - return filename.endswith(consts.INFRA_CONFIGURATION_SCAN_SUPPORTED_FILES) + if scan_type == consts.IAC_SCAN_TYPE: + return filename.endswith(consts.IAC_SCAN_SUPPORTED_FILES) if scan_type == consts.SCA_SCAN_TYPE: return filename.endswith(consts.SCA_CONFIGURATION_SCAN_SUPPORTED_FILES) diff --git a/cycode/cli/files_collector/iac/tf_content_generator.py b/cycode/cli/files_collector/iac/tf_content_generator.py index 57ebb4b1..63be9e47 100644 --- a/cycode/cli/files_collector/iac/tf_content_generator.py +++ b/cycode/cli/files_collector/iac/tf_content_generator.py @@ -1,6 +1,5 @@ import json import time -from typing import List from cycode.cli import consts from cycode.cli.exceptions.custom_exceptions import TfplanKeyError @@ -17,7 +16,7 @@ def generate_tfplan_document_name(path: str) -> str: def is_iac(scan_type: str) -> bool: - return scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE + return scan_type == consts.IAC_SCAN_TYPE def is_tfplan_file(file: str, content: str) -> bool: @@ -34,7 +33,7 @@ def generate_tf_content_from_tfplan(filename: str, tfplan: str) -> str: return _generate_tf_content(planned_resources) -def _generate_tf_content(resource_changes: List[ResourceChange]) -> str: +def _generate_tf_content(resource_changes: list[ResourceChange]) -> str: tf_content = '' for resource_change in resource_changes: if not any(item in resource_change.actions for item in ACTIONS_TO_OMIT_RESOURCE): @@ -62,9 +61,9 @@ def _get_resource_name(resource_change: ResourceChange) -> str: return '.'.join(valid_parts) -def _extract_resources(tfplan: str, filename: str) -> List[ResourceChange]: +def _extract_resources(tfplan: str, filename: str) -> list[ResourceChange]: tfplan_json = load_json(tfplan) - resources: List[ResourceChange] = [] + resources: list[ResourceChange] = [] try: resource_changes = tfplan_json['resource_changes'] for resource_change in resource_changes: diff --git a/cycode/cli/files_collector/models/in_memory_zip.py b/cycode/cli/files_collector/models/in_memory_zip.py index a0700f6b..8f58b12b 100644 --- a/cycode/cli/files_collector/models/in_memory_zip.py +++ b/cycode/cli/files_collector/models/in_memory_zip.py @@ -10,7 +10,7 @@ from pathlib import Path -class InMemoryZip(object): +class InMemoryZip: def __init__(self) -> None: self.configuration_manager = ConfigurationManager() diff --git a/cycode/cli/files_collector/path_documents.py b/cycode/cli/files_collector/path_documents.py index 14f88888..e0f06312 100644 --- a/cycode/cli/files_collector/path_documents.py +++ b/cycode/cli/files_collector/path_documents.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING from cycode.cli.files_collector.excluder import exclude_irrelevant_files from cycode.cli.files_collector.iac.tf_content_generator import ( @@ -9,16 +9,16 @@ is_tfplan_file, ) from cycode.cli.files_collector.walk_ignore import walk_ignore +from cycode.cli.logger import logger from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_absolute_path, get_file_content -from cycode.cyclient import logger if TYPE_CHECKING: from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection -def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns: bool = True) -> List[str]: - files: List[str] = [] +def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns: bool = True) -> list[str]: + files: list[str] = [] walk_func = walk_ignore if walk_with_ignore_patterns else os.walk for root, _, filenames in walk_func(path): @@ -28,7 +28,7 @@ def _get_all_existing_files_in_directory(path: str, *, walk_with_ignore_patterns return files -def _get_relevant_files_in_path(path: str) -> List[str]: +def _get_relevant_files_in_path(path: str) -> list[str]: absolute_path = get_absolute_path(path) if not os.path.isfile(absolute_path) and not os.path.isdir(absolute_path): @@ -42,8 +42,8 @@ def _get_relevant_files_in_path(path: str) -> List[str]: def _get_relevant_files( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: Tuple[str] -) -> List[str]: + progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, paths: tuple[str, ...] +) -> list[str]: all_files_to_scan = [] for path in paths: all_files_to_scan.extend(_get_relevant_files_in_path(path)) @@ -89,13 +89,13 @@ def get_relevant_documents( progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection', scan_type: str, - paths: Tuple[str], + paths: tuple[str, ...], *, is_git_diff: bool = False, -) -> List[Document]: +) -> list[Document]: relevant_files = _get_relevant_files(progress_bar, progress_bar_section, scan_type, paths) - documents: List[Document] = [] + documents: list[Document] = [] for file in relevant_files: progress_bar.update(progress_bar_section) diff --git a/cycode/cli/files_collector/repository_documents.py b/cycode/cli/files_collector/repository_documents.py index df49aa95..379346f8 100644 --- a/cycode/cli/files_collector/repository_documents.py +++ b/cycode/cli/files_collector/repository_documents.py @@ -1,8 +1,9 @@ import os -from typing import TYPE_CHECKING, Iterator, List, Optional, Tuple, Union +from collections.abc import Iterator +from typing import TYPE_CHECKING, Optional, Union from cycode.cli import consts -from cycode.cli.files_collector.sca import sca_code_scanner +from cycode.cli.files_collector.sca.sca_code_scanner import get_file_content_from_commit_diff from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_file_content, get_path_by_os @@ -25,7 +26,7 @@ def get_git_repository_tree_file_entries( return git_proxy.get_repo(path).tree(branch).traverse(predicate=should_process_git_object) -def parse_commit_range(commit_range: str, path: str) -> Tuple[str, str]: +def parse_commit_range(commit_range: str, path: str) -> tuple[str, str]: from_commit_rev = None to_commit_rev = None @@ -37,8 +38,14 @@ def parse_commit_range(commit_range: str, path: str) -> Tuple[str, str]: return from_commit_rev, to_commit_rev -def get_diff_file_path(file: 'Diff') -> Optional[str]: - return file.b_path if file.b_path else file.a_path +def get_diff_file_path(file: 'Diff', relative: bool = False) -> Optional[str]: + if relative: + # relative to the repository root + return file.b_path if file.b_path else file.a_path + + if file.b_blob: + return file.b_blob.abspath + return file.a_blob.abspath def get_diff_file_content(file: 'Diff') -> str: @@ -46,21 +53,21 @@ def get_diff_file_content(file: 'Diff') -> str: def get_pre_commit_modified_documents( - progress_bar: 'BaseProgressBar', progress_bar_section: 'ProgressBarSection' -) -> Tuple[List[Document], List[Document]]: + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + repo_path: str, +) -> tuple[list[Document], list[Document]]: git_head_documents = [] pre_committed_documents = [] - repo = git_proxy.get_repo(os.getcwd()) - diff_files = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) - progress_bar.set_section_length(progress_bar_section, len(diff_files)) - for file in diff_files: + repo = git_proxy.get_repo(repo_path) + diff_index = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) + progress_bar.set_section_length(progress_bar_section, len(diff_index)) + for diff in diff_index: progress_bar.update(progress_bar_section) - diff_file_path = get_diff_file_path(file) - file_path = get_path_by_os(diff_file_path) - - file_content = sca_code_scanner.get_file_content_from_commit(repo, consts.GIT_HEAD_COMMIT_REV, diff_file_path) + file_path = get_path_by_os(get_diff_file_path(diff)) + file_content = get_file_content_from_commit_diff(repo, consts.GIT_HEAD_COMMIT_REV, diff) if file_content is not None: git_head_documents.append(Document(file_path, file_content)) @@ -77,7 +84,7 @@ def get_commit_range_modified_documents( path: str, from_commit_rev: str, to_commit_rev: str, -) -> Tuple[List[Document], List[Document]]: +) -> tuple[list[Document], list[Document]]: from_commit_documents = [] to_commit_documents = [] @@ -91,14 +98,13 @@ def get_commit_range_modified_documents( for blob in modified_files_diff: progress_bar.update(progress_bar_section) - diff_file_path = get_diff_file_path(blob) - file_path = get_path_by_os(diff_file_path) + file_path = get_path_by_os(get_diff_file_path(blob)) - file_content = sca_code_scanner.get_file_content_from_commit(repo, from_commit_rev, diff_file_path) + file_content = get_file_content_from_commit_diff(repo, from_commit_rev, blob) if file_content is not None: from_commit_documents.append(Document(file_path, file_content)) - file_content = sca_code_scanner.get_file_content_from_commit(repo, to_commit_rev, diff_file_path) + file_content = get_file_content_from_commit_diff(repo, to_commit_rev, blob) if file_content is not None: to_commit_documents.append(Document(file_path, file_content)) diff --git a/cycode/cli/files_collector/sca/base_restore_dependencies.py b/cycode/cli/files_collector/sca/base_restore_dependencies.py index 81caea1d..ea8a0bb7 100644 --- a/cycode/cli/files_collector/sca/base_restore_dependencies.py +++ b/cycode/cli/files_collector/sca/base_restore_dependencies.py @@ -1,12 +1,12 @@ +import os from abc import ABC, abstractmethod -from typing import List, Optional +from typing import Optional -import click +import typer from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths from cycode.cli.utils.shell_executor import shell -from cycode.cyclient import logger def build_dep_tree_path(path: str, generated_file_name: str) -> str: @@ -14,38 +14,35 @@ def build_dep_tree_path(path: str, generated_file_name: str) -> str: def execute_commands( - commands: List[List[str]], - file_name: str, - command_timeout: int, - dependencies_file_name: Optional[str] = None, + commands: list[list[str]], + timeout: int, + output_file_path: Optional[str] = None, working_directory: Optional[str] = None, ) -> Optional[str]: try: - all_dependencies = [] + outputs = [] - # Run all commands and collect outputs for command in commands: - dependencies = shell(command=command, timeout=command_timeout, working_directory=working_directory) - all_dependencies.append(dependencies) # Collect each command's output + command_output = shell(command=command, timeout=timeout, working_directory=working_directory) + if command_output: + outputs.append(command_output) - dependencies = '\n'.join(all_dependencies) + joined_output = '\n'.join(outputs) - # Write all collected outputs to the file if dependencies_file_name is provided - if dependencies_file_name: - with open(dependencies_file_name, 'w') as output_file: # Open once in 'w' mode to start fresh - output_file.writelines(dependencies) - except Exception as e: - logger.debug('Failed to restore dependencies via shell command, %s', {'filename': file_name}, exc_info=e) + if output_file_path: + with open(output_file_path, 'w', encoding='UTF-8') as output_file: + output_file.writelines(joined_output) + except Exception: return None - return dependencies + return joined_output class BaseRestoreDependencies(ABC): def __init__( - self, context: click.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool = False + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, create_output_file_manually: bool = False ) -> None: - self.context = context + self.ctx = ctx self.is_git_diff = is_git_diff self.command_timeout = command_timeout self.create_output_file_manually = create_output_file_manually @@ -55,9 +52,7 @@ def restore(self, document: Document) -> Optional[Document]: def get_manifest_file_path(self, document: Document) -> str: return ( - join_paths(get_path_from_context(self.context), document.path) - if self.context.obj.get('monitor') - else document.path + join_paths(get_path_from_context(self.ctx), document.path) if self.ctx.obj.get('monitor') else document.path ) def try_restore_dependencies(self, document: Document) -> Optional[Document]: @@ -66,34 +61,32 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: relative_restore_file_path = build_dep_tree_path(document.path, self.get_lock_file_name()) working_directory_path = self.get_working_directory(document) - if self.verify_restore_file_already_exist(restore_file_path): - restore_file_content = get_file_content(restore_file_path) - else: - output_file_path = restore_file_path if self.create_output_file_manually else None - execute_commands( + if not self.verify_restore_file_already_exist(restore_file_path): + output = execute_commands( self.get_commands(manifest_file_path), - manifest_file_path, self.command_timeout, - output_file_path, - working_directory_path, + output_file_path=restore_file_path if self.create_output_file_manually else None, + working_directory=working_directory_path, ) - restore_file_content = get_file_content(restore_file_path) + if output is None: # one of the commands failed + return None + restore_file_content = get_file_content(restore_file_path) return Document(relative_restore_file_path, restore_file_content, self.is_git_diff) def get_working_directory(self, document: Document) -> Optional[str]: return None - @abstractmethod - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - pass + @staticmethod + def verify_restore_file_already_exist(restore_file_path: str) -> bool: + return os.path.isfile(restore_file_path) @abstractmethod def is_project(self, document: Document) -> bool: pass @abstractmethod - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: pass @abstractmethod diff --git a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py index 1986b3a2..6eb48a76 100644 --- a/cycode/cli/files_collector/sca/go/restore_go_dependencies.py +++ b/cycode/cli/files_collector/sca/go/restore_go_dependencies.py @@ -1,10 +1,10 @@ -import logging import os -from typing import List, Optional +from typing import Optional -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies +from cycode.cli.logger import logger from cycode.cli.models import Document GO_PROJECT_FILE_EXTENSIONS = ['.mod', '.sum'] @@ -14,15 +14,15 @@ class RestoreGoDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) def try_restore_dependencies(self, document: Document) -> Optional[Document]: manifest_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_FILE_NAME) lock_exists = os.path.isfile(self.get_working_directory(document) + os.sep + BUILD_GO_LOCK_FILE_NAME) if not manifest_exists or not lock_exists: - logging.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found') + logger.info('No manifest go.mod file found' if not manifest_exists else 'No manifest go.sum file found') manifest_files_exists = manifest_exists & lock_exists @@ -34,7 +34,7 @@ def try_restore_dependencies(self, document: Document) -> Optional[Document]: def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in GO_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [ ['go', 'list', '-m', '-json', 'all'], ['echo', '------------------------------------------------------'], @@ -44,8 +44,5 @@ def get_commands(self, manifest_file_path: str) -> List[List[str]]: def get_lock_file_name(self) -> str: return GO_RESTORE_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py index 85dc9e20..777ae727 100644 --- a/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_gradle_dependencies.py @@ -1,10 +1,9 @@ import os import re -from typing import List, Optional, Set +from typing import Optional -import click +import typer -from cycode.cli.consts import SCA_GRADLE_ALL_SUB_PROJECTS_FLAG from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_path_from_context @@ -20,20 +19,20 @@ class RestoreGradleDependencies(BaseRestoreDependencies): def __init__( - self, context: click.Context, is_git_diff: bool, command_timeout: int, projects: Optional[Set[str]] = None + self, ctx: typer.Context, is_git_diff: bool, command_timeout: int, projects: Optional[set[str]] = None ) -> None: - super().__init__(context, is_git_diff, command_timeout, create_output_file_manually=True) + super().__init__(ctx, is_git_diff, command_timeout, create_output_file_manually=True) if projects is None: projects = set() self.projects = self.get_all_projects() if self.is_gradle_sub_projects() else projects def is_gradle_sub_projects(self) -> bool: - return self.context.obj.get(SCA_GRADLE_ALL_SUB_PROJECTS_FLAG) + return self.ctx.params.get('gradle-all-sub-projects', False) def is_project(self, document: Document) -> bool: return document.path.endswith(BUILD_GRADLE_FILE_NAME) or document.path.endswith(BUILD_GRADLE_KTS_FILE_NAME) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return ( self.get_commands_for_sub_projects(manifest_file_path) if self.is_gradle_sub_projects() @@ -43,24 +42,21 @@ def get_commands(self, manifest_file_path: str) -> List[List[str]]: def get_lock_file_name(self) -> str: return BUILD_GRADLE_DEP_TREE_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: - return get_path_from_context(self.context) if self.is_gradle_sub_projects() else None + return get_path_from_context(self.ctx) if self.is_gradle_sub_projects() else None - def get_all_projects(self) -> Set[str]: - projects_output = shell( + def get_all_projects(self) -> set[str]: + output = shell( command=BUILD_GRADLE_ALL_PROJECTS_COMMAND, timeout=BUILD_GRADLE_ALL_PROJECTS_TIMEOUT, - working_directory=get_path_from_context(self.context), + working_directory=get_path_from_context(self.ctx), ) + if not output: + return set() - projects = re.findall(ALL_PROJECTS_REGEX, projects_output) - - return set(projects) + return set(re.findall(ALL_PROJECTS_REGEX, output)) - def get_commands_for_sub_projects(self, manifest_file_path: str) -> List[List[str]]: + def get_commands_for_sub_projects(self, manifest_file_path: str) -> list[list[str]]: project_name = os.path.basename(os.path.dirname(manifest_file_path)) project_name = f':{project_name}' return ( diff --git a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py index a44a27e0..b9a2b1ed 100644 --- a/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py +++ b/cycode/cli/files_collector/sca/maven/restore_maven_dependencies.py @@ -1,8 +1,7 @@ -import os from os import path -from typing import List, Optional +from typing import Optional -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import ( BaseRestoreDependencies, @@ -18,21 +17,18 @@ class RestoreMavenDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return path.basename(document.path).split('/')[-1] == BUILD_MAVEN_FILE_NAME - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['mvn', 'org.cyclonedx:cyclonedx-maven-plugin:2.7.4:makeAggregateBom', '-f', manifest_file_path]] def get_lock_file_name(self) -> str: return join_paths('target', MAVEN_CYCLONE_DEP_TREE_FILE_NAME) - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def try_restore_dependencies(self, document: Document) -> Optional[Document]: restore_dependencies_document = super().try_restore_dependencies(document) manifest_file_path = self.get_manifest_file_path(document) @@ -51,8 +47,8 @@ def restore_from_secondary_command( self, document: Document, manifest_file_path: str, restore_dependencies_document: Optional[Document] ) -> Optional[Document]: # TODO(MarshalX): does it even work? Ignored restore_dependencies_document arg - secondary_restore_command = create_secondary_restore_command(manifest_file_path) - backup_restore_content = execute_commands(secondary_restore_command, manifest_file_path, self.command_timeout) + secondary_restore_command = create_secondary_restore_commands(manifest_file_path) + backup_restore_content = execute_commands(secondary_restore_command, self.command_timeout) restore_dependencies_document = Document( build_dep_tree_path(document.path, MAVEN_DEP_TREE_FILE_NAME), backup_restore_content, self.is_git_diff ) @@ -64,13 +60,15 @@ def restore_from_secondary_command( return restore_dependencies -def create_secondary_restore_command(manifest_file_path: str) -> List[str]: +def create_secondary_restore_commands(manifest_file_path: str) -> list[list[str]]: return [ - 'mvn', - 'dependency:tree', - '-B', - '-DoutputType=text', - '-f', - manifest_file_path, - f'-DoutputFile={MAVEN_DEP_TREE_FILE_NAME}', + [ + 'mvn', + 'dependency:tree', + '-B', + '-DoutputType=text', + '-f', + manifest_file_path, + f'-DoutputFile={MAVEN_DEP_TREE_FILE_NAME}', + ] ] diff --git a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py index c3026938..2563612f 100644 --- a/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py +++ b/cycode/cli/files_collector/sca/npm/restore_npm_dependencies.py @@ -1,7 +1,6 @@ import os -from typing import List -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -12,13 +11,13 @@ class RestoreNpmDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NPM_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [ [ 'npm', @@ -34,8 +33,6 @@ def get_commands(self, manifest_file_path: str) -> List[List[str]]: def get_lock_file_name(self) -> str: return NPM_LOCK_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - - def prepare_manifest_file_path_for_command(self, manifest_file_path: str) -> str: + @staticmethod + def prepare_manifest_file_path_for_command(manifest_file_path: str) -> str: return manifest_file_path.replace(os.sep + NPM_MANIFEST_FILE_NAME, '') diff --git a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py index 0e2ed83d..3035e206 100644 --- a/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py +++ b/cycode/cli/files_collector/sca/nuget/restore_nuget_dependencies.py @@ -1,7 +1,4 @@ -import os -from typing import List - -import click +import typer from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -11,17 +8,14 @@ class RestoreNugetDependencies(BaseRestoreDependencies): - def __init__(self, context: click.Context, is_git_diff: bool, command_timeout: int) -> None: - super().__init__(context, is_git_diff, command_timeout) + def __init__(self, ctx: typer.Context, is_git_diff: bool, command_timeout: int) -> None: + super().__init__(ctx, is_git_diff, command_timeout) def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in NUGET_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['dotnet', 'restore', manifest_file_path, '--use-lock-file', '--verbosity', 'quiet']] def get_lock_file_name(self) -> str: return NUGET_LOCK_FILE_NAME - - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) diff --git a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py index 3dfc4a16..8c256f27 100644 --- a/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py +++ b/cycode/cli/files_collector/sca/ruby/restore_ruby_dependencies.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Optional from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -12,14 +12,11 @@ class RestoreRubyDependencies(BaseRestoreDependencies): def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in RUBY_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['bundle', '--quiet']] def get_lock_file_name(self) -> str: return RUBY_LOCK_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py index b8e1c41b..26a88646 100644 --- a/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py +++ b/cycode/cli/files_collector/sca/sbt/restore_sbt_dependencies.py @@ -1,5 +1,5 @@ import os -from typing import List, Optional +from typing import Optional from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.models import Document @@ -12,14 +12,11 @@ class RestoreSbtDependencies(BaseRestoreDependencies): def is_project(self, document: Document) -> bool: return any(document.path.endswith(ext) for ext in SBT_PROJECT_FILE_EXTENSIONS) - def get_commands(self, manifest_file_path: str) -> List[List[str]]: + def get_commands(self, manifest_file_path: str) -> list[list[str]]: return [['sbt', 'dependencyLockWrite', '--verbose']] def get_lock_file_name(self) -> str: return SBT_LOCK_FILE_NAME - def verify_restore_file_already_exist(self, restore_file_path: str) -> bool: - return os.path.isfile(restore_file_path) - def get_working_directory(self, document: Document) -> Optional[str]: return os.path.dirname(document.absolute_path) diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_code_scanner.py index ca6908b6..b9988122 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_code_scanner.py @@ -1,7 +1,6 @@ -import os -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional -import click +import typer from cycode.cli import consts from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies @@ -15,19 +14,22 @@ from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy from cycode.cli.utils.path_utils import get_file_content, get_file_dir, get_path_from_context, join_paths -from cycode.cyclient import logger +from cycode.logger import get_logger if TYPE_CHECKING: - from git import Repo + from git import Diff, Repo BUILD_DEP_TREE_TIMEOUT = 180 +logger = get_logger('SCA Code Scanner') + + def perform_pre_commit_range_scan_actions( path: str, - from_commit_documents: List[Document], + from_commit_documents: list[Document], from_commit_rev: str, - to_commit_documents: List[Document], + to_commit_documents: list[Document], to_commit_rev: str, ) -> None: repo = git_proxy.get_repo(path) @@ -36,17 +38,17 @@ def perform_pre_commit_range_scan_actions( def perform_pre_hook_range_scan_actions( - git_head_documents: List[Document], pre_committed_documents: List[Document] + repo_path: str, git_head_documents: list[Document], pre_committed_documents: list[Document] ) -> None: - repo = git_proxy.get_repo(os.getcwd()) + repo = git_proxy.get_repo(repo_path) add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) add_ecosystem_related_files_if_exists(pre_committed_documents) def add_ecosystem_related_files_if_exists( - documents: List[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None + documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None ) -> None: - documents_to_add: List[Document] = [] + documents_to_add: list[Document] = [] for doc in documents: ecosystem = get_project_file_ecosystem(doc) if ecosystem is None: @@ -59,14 +61,14 @@ def add_ecosystem_related_files_if_exists( def get_doc_ecosystem_related_project_files( - doc: Document, documents: List[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] -) -> List[Document]: - documents_to_add: List[Document] = [] + doc: Document, documents: list[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] +) -> list[Document]: + documents_to_add: list[Document] = [] for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file) if not is_project_file_exists_in_documents(documents, file_to_search): if repo: - file_content = get_file_content_from_commit(repo, commit_rev, file_to_search) + file_content = get_file_content_from_commit_path(repo, commit_rev, file_to_search) else: file_content = get_file_content(file_to_search) @@ -76,7 +78,7 @@ def get_doc_ecosystem_related_project_files( return documents_to_add -def is_project_file_exists_in_documents(documents: List[Document], file: str) -> bool: +def is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool: return any(doc for doc in documents if file == doc.path) @@ -89,8 +91,8 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]: def try_restore_dependencies( - context: click.Context, - documents_to_add: Dict[str, Document], + ctx: typer.Context, + documents_to_add: dict[str, Document], restore_dependencies: 'BaseRestoreDependencies', document: Document, ) -> None: @@ -106,8 +108,8 @@ def try_restore_dependencies( logger.warning('Error occurred while trying to generate dependencies tree, %s', {'filename': document.path}) restore_dependencies_document.content = '' else: - is_monitor_action = context.obj.get('monitor', False) - project_path = get_path_from_context(context) + is_monitor_action = ctx.obj.get('monitor', False) + project_path = get_path_from_context(ctx) manifest_file_path = get_manifest_file_path(document, is_monitor_action, project_path) logger.debug('Succeeded to generate dependencies tree on path: %s', manifest_file_path) @@ -119,27 +121,28 @@ def try_restore_dependencies( def add_dependencies_tree_document( - context: click.Context, documents_to_scan: List[Document], is_git_diff: bool = False + ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False ) -> None: - documents_to_add: Dict[str, Document] = {} - restore_dependencies_list = restore_handlers(context, is_git_diff) + documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan} + restore_dependencies_list = restore_handlers(ctx, is_git_diff) for restore_dependencies in restore_dependencies_list: for document in documents_to_scan: - try_restore_dependencies(context, documents_to_add, restore_dependencies, document) + try_restore_dependencies(ctx, documents_to_add, restore_dependencies, document) - documents_to_scan.extend(list(documents_to_add.values())) + # mutate original list using slice assignment + documents_to_scan[:] = list(documents_to_add.values()) -def restore_handlers(context: click.Context, is_git_diff: bool) -> List[BaseRestoreDependencies]: +def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]: return [ - RestoreGradleDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreMavenDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreSbtDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreGoDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreNugetDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreNpmDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), - RestoreRubyDependencies(context, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreGradleDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreMavenDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreSbtDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreGoDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreNugetDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreNpmDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), + RestoreRubyDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), ] @@ -147,16 +150,24 @@ def get_manifest_file_path(document: Document, is_monitor_action: bool, project_ return join_paths(project_path, document.path) if is_monitor_action else document.path -def get_file_content_from_commit(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: +def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: try: return repo.git.show(f'{commit}:{file_path}') except git_proxy.get_git_command_error(): return None +def get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]: + from cycode.cli.files_collector.repository_documents import get_diff_file_path + + file_path = get_diff_file_path(diff, relative=True) + return get_file_content_from_commit_path(repo, commit, file_path) + + def perform_pre_scan_documents_actions( - context: click.Context, scan_type: str, documents_to_scan: List[Document], is_git_diff: bool = False + ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False ) -> None: - if scan_type == consts.SCA_SCAN_TYPE and not context.obj.get(consts.SCA_SKIP_RESTORE_DEPENDENCIES_FLAG): + no_restore = ctx.params.get('no-restore', False) + if scan_type == consts.SCA_SCAN_TYPE and not no_restore: logger.debug('Perform pre-scan document add_dependencies_tree_document action') - add_dependencies_tree_document(context, documents_to_scan, is_git_diff) + add_dependencies_tree_document(ctx, documents_to_scan, is_git_diff) diff --git a/cycode/cli/files_collector/walk_ignore.py b/cycode/cli/files_collector/walk_ignore.py index 93286c87..35855ff4 100644 --- a/cycode/cli/files_collector/walk_ignore.py +++ b/cycode/cli/files_collector/walk_ignore.py @@ -1,8 +1,8 @@ import os -from typing import Generator, Iterable, List, Tuple +from collections.abc import Generator, Iterable +from cycode.cli.logger import logger from cycode.cli.utils.ignore_utils import IgnoreFilterManager -from cycode.cyclient import logger _SUPPORTED_IGNORE_PATTERN_FILES = { # oneday we will bring .cycodeignore or something like that '.gitignore', @@ -22,7 +22,7 @@ def _walk_to_top(path: str) -> Iterable[str]: yield path # Include the top-level directory -def _collect_top_level_ignore_files(path: str) -> List[str]: +def _collect_top_level_ignore_files(path: str) -> list[str]: ignore_files = [] top_paths = reversed(list(_walk_to_top(path))) # we must reverse it to make top levels more prioritized for dir_path in top_paths: @@ -34,7 +34,7 @@ def _collect_top_level_ignore_files(path: str) -> List[str]: return ignore_files -def walk_ignore(path: str) -> Generator[Tuple[str, List[str], List[str]], None, None]: +def walk_ignore(path: str) -> Generator[tuple[str, list[str], list[str]], None, None]: ignore_filter_manager = IgnoreFilterManager.build( path=path, global_ignore_file_paths=_collect_top_level_ignore_files(path), diff --git a/cycode/cli/files_collector/zip_documents.py b/cycode/cli/files_collector/zip_documents.py index 9547f7fb..770121fa 100644 --- a/cycode/cli/files_collector/zip_documents.py +++ b/cycode/cli/files_collector/zip_documents.py @@ -1,12 +1,14 @@ import timeit from pathlib import Path -from typing import List, Optional +from typing import Optional from cycode.cli import consts from cycode.cli.exceptions import custom_exceptions from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.models import Document -from cycode.cyclient import logger +from cycode.logger import get_logger + +logger = get_logger('ZIP') def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None: @@ -15,7 +17,7 @@ def _validate_zip_file_size(scan_type: str, zip_file_size: int) -> None: raise custom_exceptions.ZipTooLargeError(max_size_limit) -def zip_documents(scan_type: str, documents: List[Document], zip_file: Optional[InMemoryZip] = None) -> InMemoryZip: +def zip_documents(scan_type: str, documents: list[Document], zip_file: Optional[InMemoryZip] = None) -> InMemoryZip: if zip_file is None: zip_file = InMemoryZip() @@ -25,7 +27,7 @@ def zip_documents(scan_type: str, documents: List[Document], zip_file: Optional[ _validate_zip_file_size(scan_type, zip_file.size) logger.debug( - 'Adding file to ZIP, %s', + 'Adding file, %s', {'index': index, 'filename': document.path, 'unique_id': document.unique_id}, ) zip_file.append(document.path, document.unique_id, document.content) @@ -34,11 +36,14 @@ def zip_documents(scan_type: str, documents: List[Document], zip_file: Optional[ end_zip_creation_time = timeit.default_timer() zip_creation_time = int(end_zip_creation_time - start_zip_creation_time) - logger.debug('Finished to create ZIP file, %s', {'zip_creation_time': zip_creation_time}) + logger.debug( + 'Finished to create file, %s', + {'zip_creation_time': zip_creation_time, 'zip_size': zip_file.size, 'documents_count': len(documents)}, + ) if zip_file.configuration_manager.get_debug_flag(): zip_file_path = Path.joinpath(Path.cwd(), f'{scan_type}_scan_{end_zip_creation_time}.zip') - logger.debug('Writing ZIP file to disk, %s', {'zip_file_path': zip_file_path}) + logger.debug('Writing file to disk, %s', {'zip_file_path': zip_file_path}) zip_file.write_on_disk(zip_file_path) return zip_file diff --git a/cycode/cli/logger.py b/cycode/cli/logger.py new file mode 100644 index 00000000..46748bff --- /dev/null +++ b/cycode/cli/logger.py @@ -0,0 +1,3 @@ +from cycode.logger import get_logger + +logger = get_logger('CLI') diff --git a/cycode/cli/main.py b/cycode/cli/main.py index dd2d1fa7..c6a857a4 100644 --- a/cycode/cli/main.py +++ b/cycode/cli/main.py @@ -1,11 +1,10 @@ from multiprocessing import freeze_support -from cycode.cli.commands.main_cli import main_cli +from cycode.cli.app import app -if __name__ == '__main__': - # DO NOT REMOVE OR MOVE THIS LINE - # this is required to support multiprocessing in executables files packaged with PyInstaller - # see https://pyinstaller.org/en/latest/common-issues-and-pitfalls.html#multi-processing - freeze_support() +# DO NOT REMOVE OR MOVE THIS LINE +# this is required to support multiprocessing in executables files packaged with PyInstaller +# see https://pyinstaller.org/en/latest/common-issues-and-pitfalls.html#multi-processing +freeze_support() - main_cli() +app() diff --git a/cycode/cli/models.py b/cycode/cli/models.py index 25b2347f..3c59eeee 100644 --- a/cycode/cli/models.py +++ b/cycode/cli/models.py @@ -1,6 +1,5 @@ from dataclasses import dataclass -from enum import Enum -from typing import Dict, List, NamedTuple, Optional, Type +from typing import NamedTuple, Optional from cycode.cyclient.models import Detection @@ -21,44 +20,16 @@ def __init__( self.absolute_path = absolute_path def __repr__(self) -> str: - return 'path:{0}, content:{1}'.format(self.path, self.content) + return f'path:{self.path}, content:{self.content}' class DocumentDetections: - def __init__(self, document: Document, detections: List[Detection]) -> None: + def __init__(self, document: Document, detections: list[Detection]) -> None: self.document = document self.detections = detections def __repr__(self) -> str: - return 'document:{0}, detections:{1}'.format(self.document, self.detections) - - -SEVERITY_UNKNOWN_WEIGHT = -2 - - -class Severity(Enum): - INFO = -1 - LOW = 0 - MEDIUM = 1 - MODERATE = 1 # noqa: PIE796. TODO(MarshalX): rework. should not be Enum - HIGH = 2 - CRITICAL = 3 - - @staticmethod - def try_get_value(name: str) -> Optional[int]: - name = name.upper() - if name not in Severity.__members__: - return None - - return Severity[name].value - - @staticmethod - def get_member_weight(name: str) -> int: - weight = Severity.try_get_value(name) - if weight is None: # unknown severity - return SEVERITY_UNKNOWN_WEIGHT - - return weight + return f'document:{self.document}, detections:{self.detections}' class CliError(NamedTuple): @@ -66,20 +37,24 @@ class CliError(NamedTuple): message: str soft_fail: bool = False + def enrich(self, additional_message: str) -> 'CliError': + message = f'{self.message} ({additional_message})' + return CliError(self.code, message, self.soft_fail) + -CliErrors = Dict[Type[BaseException], CliError] +CliErrors = dict[type[BaseException], CliError] class CliResult(NamedTuple): success: bool message: str - data: Optional[Dict[str, any]] = None + data: Optional[dict[str, any]] = None class LocalScanResult(NamedTuple): scan_id: str report_url: Optional[str] - document_detections: List[DocumentDetections] + document_detections: list[DocumentDetections] issue_detected: bool detections_count: int relevant_detections_count: int @@ -91,8 +66,8 @@ class ResourceChange: resource_type: str name: str index: Optional[int] - actions: List[str] - values: Dict[str, str] + actions: list[str] + values: dict[str, str] def __repr__(self) -> str: return f'resource_type: {self.resource_type}, name: {self.name}' diff --git a/cycode/cli/printers/console_printer.py b/cycode/cli/printers/console_printer.py index 1f70836c..50d48fd7 100644 --- a/cycode/cli/printers/console_printer.py +++ b/cycode/cli/printers/console_printer.py @@ -1,74 +1,159 @@ -from typing import TYPE_CHECKING, ClassVar, Dict, List, Optional, Type +import io +from typing import TYPE_CHECKING, ClassVar, Optional -import click +import typer +from rich.console import Console +from cycode.cli import consts +from cycode.cli.cli_types import ExportTypeOption +from cycode.cli.console import console, console_err from cycode.cli.exceptions.custom_exceptions import CycodeError from cycode.cli.models import CliError, CliResult from cycode.cli.printers.json_printer import JsonPrinter +from cycode.cli.printers.rich_printer import RichPrinter from cycode.cli.printers.tables.sca_table_printer import ScaTablePrinter from cycode.cli.printers.tables.table_printer import TablePrinter from cycode.cli.printers.text_printer import TextPrinter if TYPE_CHECKING: + from pathlib import Path + from cycode.cli.models import LocalScanResult from cycode.cli.printers.tables.table_printer_base import PrinterBase class ConsolePrinter: - _AVAILABLE_PRINTERS: ClassVar[Dict[str, Type['PrinterBase']]] = { + _AVAILABLE_PRINTERS: ClassVar[dict[str, type['PrinterBase']]] = { + 'rich': RichPrinter, 'text': TextPrinter, 'json': JsonPrinter, 'table': TablePrinter, - # overrides + # overrides: 'table_sca': ScaTablePrinter, - 'text_sca': ScaTablePrinter, } - def __init__(self, context: click.Context) -> None: - self.context = context - self.scan_type = self.context.obj.get('scan_type') - self.output_type = self.context.obj.get('output') - self.aggregation_report_url = self.context.obj.get('aggregation_report_url') - self._printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) - if self._printer_class is None: - raise CycodeError(f'"{self.output_type}" output type is not supported.') - - def print_scan_results( + def __init__( self, - local_scan_results: List['LocalScanResult'], - errors: Optional[Dict[str, 'CliError']] = None, + ctx: typer.Context, + console_override: Optional['Console'] = None, + console_err_override: Optional['Console'] = None, + output_type_override: Optional[str] = None, ) -> None: - printer = self._get_scan_printer() - printer.print_scan_results(local_scan_results, errors) + self.ctx = ctx + self.console = console_override or console + self.console_err = console_err_override or console_err + self.output_type = output_type_override or self.ctx.obj.get('output') + + self.export_type: Optional[str] = None + self.export_file: Optional[Path] = None + self.console_record: Optional[ConsolePrinter] = None + + @property + def scan_type(self) -> str: + return self.ctx.obj.get('scan_type') + + @property + def aggregation_report_url(self) -> str: + return self.ctx.obj.get('aggregation_report_url') - def _get_scan_printer(self) -> 'PrinterBase': - printer_class = self._printer_class + @property + def printer(self) -> 'PrinterBase': + printer_class = self._AVAILABLE_PRINTERS.get(self.output_type) composite_printer = self._AVAILABLE_PRINTERS.get(f'{self.output_type}_{self.scan_type}') if composite_printer: printer_class = composite_printer - return printer_class(self.context) + if not printer_class: + raise CycodeError(f'"{self.output_type}" output type is not supported.') + + return printer_class(self.ctx, self.console, self.console_err) + + def update_ctx(self, ctx: 'typer.Context') -> None: + self.ctx = ctx + + def enable_recording(self, export_type: str, export_file: 'Path') -> None: + if self.console_record is None: + self.export_file = export_file + self.export_type = export_type + + self.console_record = ConsolePrinter( + self.ctx, + console_override=Console(record=True, file=io.StringIO()), + console_err_override=Console(stderr=True, record=True, file=io.StringIO()), + output_type_override='json' if self.export_type == 'json' else self.output_type, + ) + + def print_scan_results( + self, + local_scan_results: list['LocalScanResult'], + errors: Optional[dict[str, 'CliError']] = None, + ) -> None: + if self.console_record: + self.console_record.print_scan_results(local_scan_results, errors) + self.printer.print_scan_results(local_scan_results, errors) def print_result(self, result: CliResult) -> None: - self._printer_class(self.context).print_result(result) + if self.console_record: + self.console_record.print_result(result) + self.printer.print_result(result) def print_error(self, error: CliError) -> None: - self._printer_class(self.context).print_error(error) + if self.console_record: + self.console_record.print_error(error) + self.printer.print_error(error) def print_exception(self, e: Optional[BaseException] = None, force_print: bool = False) -> None: """Print traceback message in stderr if verbose mode is set.""" - if force_print or self.context.obj.get('verbose', False): - self._printer_class(self.context).print_exception(e) + if force_print or self.ctx.obj.get('verbose', False): + if self.console_record: + self.console_record.print_exception(e) + self.printer.print_exception(e) + + def export(self) -> None: + if self.console_record is None: + raise CycodeError('Console recording was not enabled. Cannot export.') + + export_file = self.export_file + if not export_file.suffix: + # resolve file extension based on the export type if not provided in the file name + export_file = export_file.with_suffix(f'.{self.export_type.lower()}') + + export_file = str(export_file) + if self.export_type is ExportTypeOption.HTML: + self.console_record.console.save_html(export_file) + elif self.export_type is ExportTypeOption.SVG: + self.console_record.console.save_svg(export_file, title=consts.APP_NAME) + elif self.export_type is ExportTypeOption.JSON: + with open(export_file, 'w', encoding='UTF-8') as f: + self.console_record.console.file.seek(0) + f.write(self.console_record.console.file.read()) + else: + raise CycodeError(f'Export type "{self.export_type}" is not supported.') + + export_format_msg = f'{self.export_type.upper()} format' + if self.export_type in {ExportTypeOption.HTML, ExportTypeOption.SVG}: + export_format_msg += f' with {self.output_type.upper()} output type' + + clickable_path = f'[link=file://{self.export_file}]{self.export_file}[/link]' + self.console.print(f'[b green]Cycode CLI output exported to {clickable_path} in {export_format_msg}[/]') + + @property + def is_recording(self) -> bool: + return self.console_record is not None @property def is_json_printer(self) -> bool: - return self._printer_class == JsonPrinter + return isinstance(self.printer, JsonPrinter) @property def is_table_printer(self) -> bool: - return self._printer_class == TablePrinter + return isinstance(self.printer, TablePrinter) @property def is_text_printer(self) -> bool: - return self._printer_class == TextPrinter + return isinstance(self.printer, TextPrinter) + + @property + def is_rich_printer(self) -> bool: + return isinstance(self.printer, RichPrinter) diff --git a/cycode/cli/printers/json_printer.py b/cycode/cli/printers/json_printer.py index b682b8c7..acb7912f 100644 --- a/cycode/cli/printers/json_printer.py +++ b/cycode/cli/printers/json_printer.py @@ -1,7 +1,5 @@ import json -from typing import TYPE_CHECKING, Dict, List, Optional - -import click +from typing import TYPE_CHECKING, Optional from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase @@ -15,20 +13,20 @@ class JsonPrinter(PrinterBase): def print_result(self, result: CliResult) -> None: result = {'result': result.success, 'message': result.message, 'data': result.data} - click.echo(self.get_data_json(result)) + self.console.print_json(self.get_data_json(result)) def print_error(self, error: CliError) -> None: result = {'error': error.code, 'message': error.message} - click.echo(self.get_data_json(result)) + self.console.print_json(self.get_data_json(result)) def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: scan_ids = [] report_urls = [] detections = [] - aggregation_report_url = self.context.obj.get('aggregation_report_url') + aggregation_report_url = self.ctx.obj.get('aggregation_report_url') if aggregation_report_url: report_urls.append(aggregation_report_url) @@ -47,13 +45,12 @@ def print_scan_results( # FIXME(MarshalX): we don't care about scan IDs in JSON output due to clumsy JSON root structure inlined_errors = [err._asdict() for err in errors.values()] - click.echo(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) + self.console.print_json(self._get_json_scan_result(scan_ids, detections_dict, report_urls, inlined_errors)) def _get_json_scan_result( - self, scan_ids: List[str], detections: dict, report_urls: List[str], errors: List[dict] + self, scan_ids: list[str], detections: dict, report_urls: list[str], errors: list[dict] ) -> str: result = { - 'scan_id': 'DEPRECATED', # backward compatibility 'scan_ids': scan_ids, 'detections': detections, 'report_urls': report_urls, @@ -65,4 +62,4 @@ def _get_json_scan_result( @staticmethod def get_data_json(data: dict) -> str: # ensure_ascii is disabled for symbols like "`". Eg: `cycode scan` - return json.dumps(data, indent=4, ensure_ascii=False) + return json.dumps(data, ensure_ascii=False) diff --git a/cycode/cli/printers/printer_base.py b/cycode/cli/printers/printer_base.py index fa5bf435..69596e2a 100644 --- a/cycode/cli/printers/printer_base.py +++ b/cycode/cli/printers/printer_base.py @@ -1,27 +1,57 @@ -import traceback +import sys from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Dict, List, Optional +from collections import defaultdict +from typing import TYPE_CHECKING, Optional -import click +import typer +from cycode.cli.cli_types import SeverityOption from cycode.cli.models import CliError, CliResult from cycode.cyclient.headers import get_correlation_id if TYPE_CHECKING: + from rich.console import Console + from cycode.cli.models import LocalScanResult +from rich.traceback import Traceback as RichTraceback + + class PrinterBase(ABC): - RED_COLOR_NAME = 'red' - WHITE_COLOR_NAME = 'white' - GREEN_COLOR_NAME = 'green' + NO_DETECTIONS_MESSAGE = ( + '[b green]Good job! No issues were found!!! :clapping_hands::clapping_hands::clapping_hands:[/]' + ) + FAILED_SCAN_MESSAGE = ( + '[b red]Unfortunately, Cycode was unable to complete the full scan. ' + 'Please note that not all results may be available:[/]' + ) + + def __init__( + self, + ctx: typer.Context, + console: 'Console', + console_err: 'Console', + ) -> None: + self.ctx = ctx + self.console = console + self.console_err = console_err + + @property + def scan_type(self) -> str: + return self.ctx.obj.get('scan_type') - def __init__(self, context: click.Context) -> None: - self.context = context + @property + def command_scan_type(self) -> str: + return self.ctx.info_name + + @property + def show_secret(self) -> bool: + return self.ctx.obj.get('show_secret', False) @abstractmethod def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: pass @@ -38,15 +68,48 @@ def print_exception(self, e: Optional[BaseException] = None) -> None: Note: Called only when the verbose flag is set. + """ - if e is None: - # gets the most recent exception caught by an except clause - message = f'Error: {traceback.format_exc()}' - else: - traceback_message = ''.join(traceback.format_exception(None, e, e.__traceback__)) - message = f'Error: {traceback_message}' + rich_traceback = ( + RichTraceback.from_exception(type(e), e, e.__traceback__) + if e + else RichTraceback.from_exception(*sys.exc_info()) + ) + rich_traceback.show_locals = False + self.console_err.print(rich_traceback) + + self.console_err.print(f'[red]Correlation ID:[/] {get_correlation_id()}') + + def print_scan_results_summary(self, local_scan_results: list['LocalScanResult']) -> None: + """Print a summary of scan results based on severity levels. + + Args: + local_scan_results (List['LocalScanResult']): A list of local scan results containing detections. + + The summary includes the count of detections for each severity level + and is displayed in the console in a formatted string. + + """ + detections_count = 0 + severity_counts = defaultdict(int) + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + for detection in document_detections.detections: + if detection.severity: + detections_count += 1 + severity_counts[SeverityOption(detection.severity)] += 1 + + self.console.line() + self.console.print(f'[bold]Cycode found {detections_count} violations[/]', end=': ') + + # Example of string: CRITICAL - 6 | HIGH - 0 | MEDIUM - 14 | LOW - 0 | INFO - 0 + for index, severity in enumerate(reversed(SeverityOption), start=1): + end = ' | ' + if index == len(SeverityOption): + end = '\n' - click.secho(message, err=True, fg=self.RED_COLOR_NAME) + self.console.print( + SeverityOption.get_member_emoji(severity), severity, '-', severity_counts[severity], end=end + ) - correlation_message = f'Correlation ID: {get_correlation_id()}' - click.secho(correlation_message, err=True, fg=self.RED_COLOR_NAME) + self.console.line() diff --git a/cycode/cli/printers/rich_printer.py b/cycode/cli/printers/rich_printer.py new file mode 100644 index 00000000..7ee0f853 --- /dev/null +++ b/cycode/cli/printers/rich_printer.py @@ -0,0 +1,177 @@ +from typing import TYPE_CHECKING, Optional + +from rich.console import Group +from rich.panel import Panel +from rich.table import Table +from rich.text import Text + +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.printers.text_printer import TextPrinter +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax +from cycode.cli.printers.utils.detection_data import ( + get_detection_clickable_cwe_cve, + get_detection_file_path, + get_detection_title, +) +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result +from cycode.cli.printers.utils.rich_helpers import get_columns_in_1_to_3_ratio, get_markdown_panel, get_panel + +if TYPE_CHECKING: + from cycode.cli.models import CliError, Detection, Document, LocalScanResult + + +class RichPrinter(TextPrinter): + MAX_PATH_LENGTH = 60 + + def print_scan_results( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + if not errors and all(result.issue_detected == 0 for result in local_scan_results): + self.console.print(self.NO_DETECTIONS_MESSAGE) + return + + detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) + detections_count = len(detections) + for detection_number, (detection, document) in enumerate(detections, start=1): + self._print_violation_card( + document, + detection, + detection_number, + detections_count, + ) + + self.print_scan_results_summary(local_scan_results) + self.print_report_urls_and_errors(local_scan_results, errors) + + def _get_details_table(self, detection: 'Detection') -> Table: + details_table = Table(show_header=False, box=None, padding=(0, 1)) + + details_table.add_column('Key', style='dim') + details_table.add_column('Value', style='', overflow='fold') + + severity = detection.severity if detection.severity else 'N/A' + severity_icon = SeverityOption.get_member_emoji(severity.lower()) + details_table.add_row('Severity', f'{severity_icon} {SeverityOption(severity).__rich__()}') + + path = str(get_detection_file_path(self.scan_type, detection)) + shorten_path = f'...{path[-self.MAX_PATH_LENGTH :]}' if len(path) > self.MAX_PATH_LENGTH else path + details_table.add_row('In file', f'[link=file://{path}]{shorten_path}[/]') + + self._add_scan_related_rows(details_table, detection) + + details_table.add_row('Rule ID', detection.detection_rule_id) + + return details_table + + def _add_scan_related_rows(self, details_table: Table, detection: 'Detection') -> None: + scan_type_details_handlers = { + consts.SECRET_SCAN_TYPE: self.__add_secret_scan_related_rows, + consts.SCA_SCAN_TYPE: self.__add_sca_scan_related_rows, + consts.IAC_SCAN_TYPE: self.__add_iac_scan_related_rows, + consts.SAST_SCAN_TYPE: self.__add_sast_scan_related_rows, + } + + if self.scan_type not in scan_type_details_handlers: + raise ValueError(f'Unknown scan type: {self.scan_type}') + + scan_enricher_function = scan_type_details_handlers[self.scan_type] + scan_enricher_function(details_table, detection) + + @staticmethod + def __add_secret_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('Secret SHA', detection.detection_details.get('sha512')) + + @staticmethod + def __add_sca_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + detection_details = detection.detection_details + + details_table.add_row('CVEs', get_detection_clickable_cwe_cve(consts.SCA_SCAN_TYPE, detection)) + details_table.add_row('Package', detection_details.get('package_name')) + details_table.add_row('Version', detection_details.get('package_version')) + + if detection.has_alert: + patched_version = detection_details['alert'].get('patched_version') + details_table.add_row('First patched version', patched_version or 'Not fixed') + + dependency_path = detection_details.get('dependency_paths') + details_table.add_row('Dependency path', dependency_path or 'N/A') + + if not detection.has_alert: + details_table.add_row('License', detection_details.get('license')) + + @staticmethod + def __add_iac_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('IaC Provider', detection.detection_details.get('infra_provider')) + + @staticmethod + def __add_sast_scan_related_rows(details_table: Table, detection: 'Detection') -> None: + details_table.add_row('CWE', get_detection_clickable_cwe_cve(consts.SAST_SCAN_TYPE, detection)) + details_table.add_row('Subcategory', detection.detection_details.get('category')) + details_table.add_row('Language', ', '.join(detection.detection_details.get('languages', []))) + + engine_id_to_display_name = { + '5db84696-88dc-11ec-a8a3-0242ac120002': 'Semgrep OSS (Orchestrated by Cycode)', + '560a0abd-d7da-4e6d-a3f1-0ed74895295c': 'Bearer (Powered by Cycode)', + } + engine_id = detection.detection_details.get('external_scanner_id') + details_table.add_row('Security Tool', engine_id_to_display_name.get(engine_id, 'N/A')) + + def _print_violation_card( + self, document: 'Document', detection: 'Detection', detection_number: int, detections_count: int + ) -> None: + details_table = self._get_details_table(detection) + details_panel = get_panel( + details_table, + title=':mag: Details', + ) + + code_snippet_panel = get_panel( + get_code_snippet_syntax( + self.scan_type, + self.command_scan_type, + detection, + document, + obfuscate=not self.show_secret, + lines_to_display_before=3, + lines_to_display_after=3, + ), + title=':computer: Code Snippet', + ) + + if detection.has_alert: + summary = detection.detection_details['alert'].get('description') + else: + summary = detection.detection_details.get('description') or detection.message + + summary_panel = None + if summary: + summary_panel = get_markdown_panel( + summary, + title=':memo: Summary', + ) + + custom_guidelines_panel = None + custom_guidelines = detection.detection_details.get('custom_remediation_guidelines') + if custom_guidelines: + custom_guidelines_panel = get_markdown_panel( + custom_guidelines, + title=':office: Company Guidelines', + ) + + navigation = Text(f'Violation {detection_number} of {detections_count}', style='dim', justify='right') + + renderables = [navigation, get_columns_in_1_to_3_ratio(details_panel, code_snippet_panel)] + if summary_panel: + renderables.append(summary_panel) + if custom_guidelines_panel: + renderables.append(custom_guidelines_panel) + + violation_card_panel = Panel( + Group(*renderables), + title=get_detection_title(self.scan_type, detection), + border_style=SeverityOption.get_member_color(detection.severity), + title_align='center', + ) + + self.console.print(violation_card_panel) diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index e92b2be7..1bf358c8 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -1,13 +1,14 @@ from collections import defaultdict -from typing import TYPE_CHECKING, Dict, List - -import click +from typing import TYPE_CHECKING +from cycode.cli.cli_types import SeverityOption from cycode.cli.consts import LICENSE_COMPLIANCE_POLICY_ID, PACKAGE_VULNERABILITY_POLICY_ID -from cycode.cli.models import SEVERITY_UNKNOWN_WEIGHT, Detection, Severity +from cycode.cli.models import Detection from cycode.cli.printers.tables.table import Table -from cycode.cli.printers.tables.table_models import ColumnInfoBuilder, ColumnWidths +from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils import is_git_diff_based_scan +from cycode.cli.printers.utils.detection_ordering.sca_ordering import sort_and_group_detections from cycode.cli.utils.string_utils import shortcut_dependency_paths if TYPE_CHECKING: @@ -18,42 +19,32 @@ # Building must have strict order. Represents the order of the columns in the table (from left to right) SEVERITY_COLUMN = column_builder.build(name='Severity') REPOSITORY_COLUMN = column_builder.build(name='Repository') -CODE_PROJECT_COLUMN = column_builder.build(name='Code Project') # File path to manifest file -ECOSYSTEM_COLUMN = column_builder.build(name='Ecosystem') -PACKAGE_COLUMN = column_builder.build(name='Package') -CVE_COLUMNS = column_builder.build(name='CVE') +CODE_PROJECT_COLUMN = column_builder.build(name='Code Project', highlight=False) # File path to the manifest file +ECOSYSTEM_COLUMN = column_builder.build(name='Ecosystem', highlight=False) +PACKAGE_COLUMN = column_builder.build(name='Package', highlight=False) +CVE_COLUMNS = column_builder.build(name='CVE', highlight=False) DEPENDENCY_PATHS_COLUMN = column_builder.build(name='Dependency Paths') UPGRADE_COLUMN = column_builder.build(name='Upgrade') -LICENSE_COLUMN = column_builder.build(name='License') +LICENSE_COLUMN = column_builder.build(name='License', highlight=False) DIRECT_DEPENDENCY_COLUMN = column_builder.build(name='Direct Dependency') DEVELOPMENT_DEPENDENCY_COLUMN = column_builder.build(name='Development Dependency') -COLUMN_WIDTHS_CONFIG: ColumnWidths = { - REPOSITORY_COLUMN: 2, - CODE_PROJECT_COLUMN: 2, - PACKAGE_COLUMN: 3, - CVE_COLUMNS: 5, - UPGRADE_COLUMN: 3, - LICENSE_COLUMN: 2, -} - class ScaTablePrinter(TablePrinterBase): - def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: - aggregation_report_url = self.context.obj.get('aggregation_report_url') + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: detections_per_policy_id = self._extract_detections_per_policy_id(local_scan_results) for policy_id, detections in detections_per_policy_id.items(): table = self._get_table(policy_id) - table.set_cols_width(COLUMN_WIDTHS_CONFIG) - for detection in self._sort_and_group_detections(detections): + resulting_detections, group_separator_indexes = sort_and_group_detections(detections) + for detection in resulting_detections: self._enrich_table_with_values(table, detection) + table.set_group_separator_indexes(group_separator_indexes) + self._print_summary_issues(len(detections), self._get_title(policy_id)) self._print_table(table) - self._print_report_urls(local_scan_results, aggregation_report_url) - @staticmethod def _get_title(policy_id: str) -> str: if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: @@ -63,73 +54,25 @@ def _get_title(policy_id: str) -> str: return 'Unknown' - @staticmethod - def __group_by(detections: List[Detection], details_field_name: str) -> Dict[str, List[Detection]]: - grouped = defaultdict(list) - for detection in detections: - grouped[detection.detection_details.get(details_field_name)].append(detection) - return grouped - - @staticmethod - def __severity_sort_key(detection: Detection) -> int: - if detection.severity: - return Severity.get_member_weight(detection.severity) - - return SEVERITY_UNKNOWN_WEIGHT - - def _sort_detections_by_severity(self, detections: List[Detection]) -> List[Detection]: - return sorted(detections, key=self.__severity_sort_key, reverse=True) - - @staticmethod - def __package_sort_key(detection: Detection) -> int: - return detection.detection_details.get('package_name') - - def _sort_detections_by_package(self, detections: List[Detection]) -> List[Detection]: - return sorted(detections, key=self.__package_sort_key) - - def _sort_and_group_detections(self, detections: List[Detection]) -> List[Detection]: - """Sort detections by severity and group by repository, code project and package name. - - Note: - Code Project is path to manifest file. - - Grouping by code projects also groups by ecosystem. - Because manifest files are unique per ecosystem. - """ - result = [] - - # we sort detections by package name to make persist output order - sorted_detections = self._sort_detections_by_package(detections) - - grouped_by_repository = self.__group_by(sorted_detections, 'repository_name') - for repository_group in grouped_by_repository.values(): - grouped_by_code_project = self.__group_by(repository_group, 'file_name') - for code_project_group in grouped_by_code_project.values(): - grouped_by_package = self.__group_by(code_project_group, 'package_name') - for package_group in grouped_by_package.values(): - result.extend(self._sort_detections_by_severity(package_group)) - - return result - def _get_table(self, policy_id: str) -> Table: table = Table() if policy_id == PACKAGE_VULNERABILITY_POLICY_ID: - table.add(SEVERITY_COLUMN) - table.add(CVE_COLUMNS) - table.add(UPGRADE_COLUMN) + table.add_column(CVE_COLUMNS) + table.add_column(UPGRADE_COLUMN) elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: - table.add(LICENSE_COLUMN) + table.add_column(LICENSE_COLUMN) - if self._is_git_repository(): - table.add(REPOSITORY_COLUMN) + if is_git_diff_based_scan(self.scan_type, self.command_scan_type): + table.add_column(REPOSITORY_COLUMN) - table.add(CODE_PROJECT_COLUMN) - table.add(ECOSYSTEM_COLUMN) - table.add(PACKAGE_COLUMN) - table.add(DIRECT_DEPENDENCY_COLUMN) - table.add(DEVELOPMENT_DEPENDENCY_COLUMN) - table.add(DEPENDENCY_PATHS_COLUMN) + table.add_column(SEVERITY_COLUMN) + table.add_column(CODE_PROJECT_COLUMN) + table.add_column(ECOSYSTEM_COLUMN) + table.add_column(PACKAGE_COLUMN) + table.add_column(DIRECT_DEPENDENCY_COLUMN) + table.add_column(DEVELOPMENT_DEPENDENCY_COLUMN) + table.add_column(DEPENDENCY_PATHS_COLUMN) return table @@ -137,38 +80,53 @@ def _get_table(self, policy_id: str) -> Table: def _enrich_table_with_values(table: Table, detection: Detection) -> None: detection_details = detection.detection_details - table.set(SEVERITY_COLUMN, detection.severity) - table.set(REPOSITORY_COLUMN, detection_details.get('repository_name')) - - table.set(CODE_PROJECT_COLUMN, detection_details.get('file_name')) - table.set(ECOSYSTEM_COLUMN, detection_details.get('ecosystem')) - table.set(PACKAGE_COLUMN, detection_details.get('package_name')) - table.set(DIRECT_DEPENDENCY_COLUMN, detection_details.get('is_direct_dependency_str')) - table.set(DEVELOPMENT_DEPENDENCY_COLUMN, detection_details.get('is_dev_dependency_str')) + if detection.severity: + table.add_cell(SEVERITY_COLUMN, SeverityOption(detection.severity)) + else: + table.add_cell(SEVERITY_COLUMN, 'N/A') + + table.add_cell(REPOSITORY_COLUMN, detection_details.get('repository_name')) + table.add_file_path_cell(CODE_PROJECT_COLUMN, detection_details.get('file_name')) + table.add_cell(ECOSYSTEM_COLUMN, detection_details.get('ecosystem')) + table.add_cell(PACKAGE_COLUMN, detection_details.get('package_name')) + + dependency_bool_to_color = { + True: 'green', + False: 'red', + } # by default, not colored (None) + table.add_cell( + column=DIRECT_DEPENDENCY_COLUMN, + value=detection_details.get('is_direct_dependency_str'), + color=dependency_bool_to_color.get(detection_details.get('is_direct_dependency')), + ) + table.add_cell( + column=DEVELOPMENT_DEPENDENCY_COLUMN, + value=detection_details.get('is_dev_dependency_str'), + color=dependency_bool_to_color.get(detection_details.get('is_dev_dependency')), + ) dependency_paths = 'N/A' dependency_paths_raw = detection_details.get('dependency_paths') if dependency_paths_raw: dependency_paths = shortcut_dependency_paths(dependency_paths_raw) - table.set(DEPENDENCY_PATHS_COLUMN, dependency_paths) + table.add_cell(DEPENDENCY_PATHS_COLUMN, dependency_paths) upgrade = '' alert = detection_details.get('alert') if alert and alert.get('first_patched_version'): upgrade = f'{alert.get("vulnerable_requirements")} -> {alert.get("first_patched_version")}' - table.set(UPGRADE_COLUMN, upgrade) + table.add_cell(UPGRADE_COLUMN, upgrade) - table.set(CVE_COLUMNS, detection_details.get('vulnerability_id')) - table.set(LICENSE_COLUMN, detection_details.get('license')) + table.add_cell(CVE_COLUMNS, detection_details.get('vulnerability_id')) + table.add_cell(LICENSE_COLUMN, detection_details.get('license')) - @staticmethod - def _print_summary_issues(detections_count: int, title: str) -> None: - click.echo(f'⛔ Found {detections_count} issues of type: {click.style(title, bold=True)}') + def _print_summary_issues(self, detections_count: int, title: str) -> None: + self.console.print(f'[bold]Cycode found {detections_count} violations of type: [cyan]{title}[/]') @staticmethod def _extract_detections_per_policy_id( - local_scan_results: List['LocalScanResult'], - ) -> Dict[str, List[Detection]]: + local_scan_results: list['LocalScanResult'], + ) -> dict[str, list[Detection]]: detections_to_policy_id = defaultdict(list) for local_scan_result in local_scan_results: diff --git a/cycode/cli/printers/tables/table.py b/cycode/cli/printers/tables/table.py index 2017b9c8..61e143ca 100644 --- a/cycode/cli/printers/tables/table.py +++ b/cycode/cli/printers/tables/table.py @@ -1,62 +1,64 @@ -from typing import TYPE_CHECKING, Dict, List, Optional +import urllib.parse +from typing import TYPE_CHECKING, Optional -from texttable import Texttable +from rich.markup import escape +from rich.table import Table as RichTable if TYPE_CHECKING: - from cycode.cli.printers.tables.table_models import ColumnInfo, ColumnWidths + from cycode.cli.printers.tables.table_models import ColumnInfo class Table: """Helper class to manage columns and their values in the right order and only if the column should be presented.""" - def __init__(self, column_infos: Optional[List['ColumnInfo']] = None) -> None: - self._column_widths = None + def __init__(self, column_infos: Optional[list['ColumnInfo']] = None) -> None: + self._group_separator_indexes: set[int] = set() - self._columns: Dict['ColumnInfo', List[str]] = {} + self._columns: dict[ColumnInfo, list[str]] = {} if column_infos: - self._columns: Dict['ColumnInfo', List[str]] = {columns: [] for columns in column_infos} + self._columns = {columns: [] for columns in column_infos} - def add(self, column: 'ColumnInfo') -> None: + def add_column(self, column: 'ColumnInfo') -> None: self._columns[column] = [] - def set(self, column: 'ColumnInfo', value: str) -> None: + def _add_cell_no_error(self, column: 'ColumnInfo', value: str) -> None: # we push values only for existing columns what were added before if column in self._columns: self._columns[column].append(value) - def _get_ordered_columns(self) -> List['ColumnInfo']: + def add_cell(self, column: 'ColumnInfo', value: str, color: Optional[str] = None) -> None: + if color: + value = f'[{color}]{value}[/]' + + self._add_cell_no_error(column, value) + + def add_file_path_cell(self, column: 'ColumnInfo', path: str) -> None: + encoded_path = urllib.parse.quote(path) + escaped_path = escape(encoded_path) + self._add_cell_no_error(column, f'[link file://{escaped_path}]{path}') + + def set_group_separator_indexes(self, group_separator_indexes: set[int]) -> None: + self._group_separator_indexes = group_separator_indexes + + def _get_ordered_columns(self) -> list['ColumnInfo']: # we are sorting columns by index to make sure that columns will be printed in the right order return sorted(self._columns, key=lambda column_info: column_info.index) - def get_columns_info(self) -> List['ColumnInfo']: + def get_columns_info(self) -> list['ColumnInfo']: return self._get_ordered_columns() - def get_headers(self) -> List[str]: - return [header.name for header in self._get_ordered_columns()] - - def get_rows(self) -> List[str]: + def get_rows(self) -> list[str]: column_values = [self._columns[column_info] for column_info in self._get_ordered_columns()] return list(zip(*column_values)) - def set_cols_width(self, column_widths: 'ColumnWidths') -> None: - header_width_size = [] - for header in self.get_columns_info(): - width_multiplier = 1 - if header in column_widths: - width_multiplier = column_widths[header] - - header_width_size.append(len(header.name) * width_multiplier) - - self._column_widths = header_width_size - - def get_table(self, max_width: int = 80) -> Texttable: - table = Texttable(max_width) - table.header(self.get_headers()) + def get_table(self) -> 'RichTable': + table = RichTable(expand=True, highlight=True) - for row in self.get_rows(): - table.add_row(row) + for column in self.get_columns_info(): + extra_args = column.column_opts if column.column_opts else {} + table.add_column(header=column.name, overflow='fold', **extra_args) - if self._column_widths: - table.set_cols_width(self._column_widths) + for index, raw in enumerate(self.get_rows()): + table.add_row(*raw, end_section=index in self._group_separator_indexes) return table diff --git a/cycode/cli/printers/tables/table_models.py b/cycode/cli/printers/tables/table_models.py index c162a8ce..58e41aaa 100644 --- a/cycode/cli/printers/tables/table_models.py +++ b/cycode/cli/printers/tables/table_models.py @@ -1,12 +1,12 @@ -from typing import Dict, NamedTuple +from typing import NamedTuple, Optional class ColumnInfoBuilder: def __init__(self) -> None: self._index = 0 - def build(self, name: str) -> 'ColumnInfo': - column_info = ColumnInfo(name, self._index) + def build(self, name: str, **column_opts) -> 'ColumnInfo': + column_info = ColumnInfo(name, self._index, column_opts) self._index += 1 return column_info @@ -14,7 +14,12 @@ def build(self, name: str) -> 'ColumnInfo': class ColumnInfo(NamedTuple): name: str index: int # Represents the order of the columns, starting from the left + column_opts: Optional[dict] = None + def __hash__(self) -> int: + return hash((self.name, self.index)) -ColumnWidths = Dict[ColumnInfo, int] -ColumnWidthsConfig = Dict[str, ColumnWidths] + def __eq__(self, other: object) -> bool: + if not isinstance(other, ColumnInfo): + return NotImplemented + return (self.name, self.index) == (other.name, other.index) diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index f2153e56..6fc85a1b 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -1,12 +1,13 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING -import click - -from cycode.cli.consts import INFRA_CONFIGURATION_SCAN_TYPE, SAST_SCAN_TYPE, SECRET_SCAN_TYPE +from cycode.cli.cli_types import SeverityOption +from cycode.cli.consts import SECRET_SCAN_TYPE from cycode.cli.models import Detection, Document from cycode.cli.printers.tables.table import Table -from cycode.cli.printers.tables.table_models import ColumnInfoBuilder, ColumnWidthsConfig +from cycode.cli.printers.tables.table_models import ColumnInfoBuilder from cycode.cli.printers.tables.table_printer_base import TablePrinterBase +from cycode.cli.printers.utils import is_git_diff_based_scan +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text if TYPE_CHECKING: @@ -15,73 +16,45 @@ column_builder = ColumnInfoBuilder() # Building must have strict order. Represents the order of the columns in the table (from left to right) +SEVERITY_COLUMN = column_builder.build(name='Severity') ISSUE_TYPE_COLUMN = column_builder.build(name='Issue Type') -RULE_ID_COLUMN = column_builder.build(name='Rule ID') -FILE_PATH_COLUMN = column_builder.build(name='File Path') +FILE_PATH_COLUMN = column_builder.build(name='File Path', highlight=False) +LINE_NUMBER_COLUMN = column_builder.build(name='Line') +COLUMN_NUMBER_COLUMN = column_builder.build(name='Column') +VIOLATION_COLUMN = column_builder.build(name='Violation', highlight=False) +VIOLATION_LENGTH_COLUMN = column_builder.build(name='Length') SECRET_SHA_COLUMN = column_builder.build(name='Secret SHA') COMMIT_SHA_COLUMN = column_builder.build(name='Commit SHA') -LINE_NUMBER_COLUMN = column_builder.build(name='Line Number') -COLUMN_NUMBER_COLUMN = column_builder.build(name='Column Number') -VIOLATION_LENGTH_COLUMN = column_builder.build(name='Violation Length') -VIOLATION_COLUMN = column_builder.build(name='Violation') -SCAN_ID_COLUMN = column_builder.build(name='Scan ID') - -COLUMN_WIDTHS_CONFIG: ColumnWidthsConfig = { - SECRET_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 2, - RULE_ID_COLUMN: 2, - FILE_PATH_COLUMN: 2, - SECRET_SHA_COLUMN: 2, - VIOLATION_COLUMN: 2, - SCAN_ID_COLUMN: 2, - }, - INFRA_CONFIGURATION_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 4, - RULE_ID_COLUMN: 3, - FILE_PATH_COLUMN: 3, - SCAN_ID_COLUMN: 2, - }, - SAST_SCAN_TYPE: { - ISSUE_TYPE_COLUMN: 7, - RULE_ID_COLUMN: 2, - FILE_PATH_COLUMN: 3, - SCAN_ID_COLUMN: 2, - }, -} class TablePrinter(TablePrinterBase): - def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: table = self._get_table() - if self.scan_type in COLUMN_WIDTHS_CONFIG: - table.set_cols_width(COLUMN_WIDTHS_CONFIG[self.scan_type]) - for local_scan_result in local_scan_results: - for document_detections in local_scan_result.document_detections: - for detection in document_detections.detections: - table.set(SCAN_ID_COLUMN, local_scan_result.scan_id) - self._enrich_table_with_values(table, detection, document_detections.document) + detections, group_separator_indexes = sort_and_group_detections_from_scan_result(local_scan_results) + for detection, document in detections: + self._enrich_table_with_values(table, detection, document) + + table.set_group_separator_indexes(group_separator_indexes) self._print_table(table) - self._print_report_urls(local_scan_results, self.context.obj.get('aggregation_report_url')) def _get_table(self) -> Table: table = Table() - table.add(ISSUE_TYPE_COLUMN) - table.add(RULE_ID_COLUMN) - table.add(FILE_PATH_COLUMN) - table.add(LINE_NUMBER_COLUMN) - table.add(COLUMN_NUMBER_COLUMN) - table.add(SCAN_ID_COLUMN) + table.add_column(SEVERITY_COLUMN) + table.add_column(ISSUE_TYPE_COLUMN) + table.add_column(FILE_PATH_COLUMN) + table.add_column(LINE_NUMBER_COLUMN) + table.add_column(COLUMN_NUMBER_COLUMN) - if self._is_git_repository(): - table.add(COMMIT_SHA_COLUMN) + if is_git_diff_based_scan(self.scan_type, self.command_scan_type): + table.add_column(COMMIT_SHA_COLUMN) if self.scan_type == SECRET_SCAN_TYPE: - table.add(SECRET_SHA_COLUMN) - table.add(VIOLATION_LENGTH_COLUMN) - table.add(VIOLATION_COLUMN) + table.add_column(SECRET_SHA_COLUMN) + table.add_column(VIOLATION_LENGTH_COLUMN) + table.add_column(VIOLATION_COLUMN) return table @@ -96,11 +69,11 @@ def _enrich_table_with_detection_summary_values( if self.scan_type == SECRET_SCAN_TYPE: issue_type = detection.type - table.set(ISSUE_TYPE_COLUMN, issue_type) - table.set(RULE_ID_COLUMN, detection.detection_rule_id) - table.set(FILE_PATH_COLUMN, click.format_filename(document.path)) - table.set(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) - table.set(COMMIT_SHA_COLUMN, detection.detection_details.get('commit_id', '')) + table.add_cell(SEVERITY_COLUMN, SeverityOption(detection.severity)) + table.add_cell(ISSUE_TYPE_COLUMN, issue_type) + table.add_file_path_cell(FILE_PATH_COLUMN, document.path) + table.add_cell(SECRET_SHA_COLUMN, detection.detection_details.get('sha512', '')) + table.add_cell(COMMIT_SHA_COLUMN, detection.detection_details.get('commit_id', '')) def _enrich_table_with_detection_code_segment_values( self, table: Table, detection: Detection, document: Document @@ -123,7 +96,7 @@ def _enrich_table_with_detection_code_segment_values( if not self.show_secret: violation = obfuscate_text(violation) - table.set(LINE_NUMBER_COLUMN, str(detection_line)) - table.set(COLUMN_NUMBER_COLUMN, str(detection_column)) - table.set(VIOLATION_LENGTH_COLUMN, f'{violation_length} chars') - table.set(VIOLATION_COLUMN, violation) + table.add_cell(LINE_NUMBER_COLUMN, str(detection_line)) + table.add_cell(COLUMN_NUMBER_COLUMN, str(detection_column)) + table.add_cell(VIOLATION_LENGTH_COLUMN, f'{violation_length} chars') + table.add_cell(VIOLATION_COLUMN, violation) diff --git a/cycode/cli/printers/tables/table_printer_base.py b/cycode/cli/printers/tables/table_printer_base.py index be41454f..8cb4cbda 100644 --- a/cycode/cli/printers/tables/table_printer_base.py +++ b/cycode/cli/printers/tables/table_printer_base.py @@ -1,7 +1,5 @@ import abc -from typing import TYPE_CHECKING, Dict, List, Optional - -import click +from typing import TYPE_CHECKING, Optional from cycode.cli.models import CliError, CliResult from cycode.cli.printers.printer_base import PrinterBase @@ -13,62 +11,32 @@ class TablePrinterBase(PrinterBase, abc.ABC): - def __init__(self, context: click.Context) -> None: - super().__init__(context) - self.scan_type: str = context.obj.get('scan_type') - self.show_secret: bool = context.obj.get('show_secret', False) + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.text_printer = TextPrinter(self.ctx, self.console, self.console_err) def print_result(self, result: CliResult) -> None: - TextPrinter(self.context).print_result(result) + self.text_printer.print_result(result) def print_error(self, error: CliError) -> None: - TextPrinter(self.context).print_error(error) + self.text_printer.print_error(error) def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - click.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) + self.console.print(self.NO_DETECTIONS_MESSAGE) return self._print_results(local_scan_results) - if not errors: - return - - click.secho( - 'Unfortunately, Cycode was unable to complete the full scan. ' - 'Please note that not all results may be available:', - fg='red', - ) - for scan_id, error in errors.items(): - click.echo(f'- {scan_id}: ', nl=False) - self.print_error(error) - - def _is_git_repository(self) -> bool: - return self.context.obj.get('remote_url') is not None + self.print_scan_results_summary(local_scan_results) + self.text_printer.print_report_urls_and_errors(local_scan_results, errors) @abc.abstractmethod - def _print_results(self, local_scan_results: List['LocalScanResult']) -> None: + def _print_results(self, local_scan_results: list['LocalScanResult']) -> None: raise NotImplementedError - @staticmethod - def _print_table(table: 'Table') -> None: + def _print_table(self, table: 'Table') -> None: if table.get_rows(): - click.echo(table.get_table().draw()) - - @staticmethod - def _print_report_urls( - local_scan_results: List['LocalScanResult'], - aggregation_report_url: Optional[str] = None, - ) -> None: - report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] - if not report_urls and not aggregation_report_url: - return - if aggregation_report_url: - click.echo(f'Report URL: {aggregation_report_url}') - return - - click.echo('Report URLs:') - for report_url in report_urls: - click.echo(f'- {report_url}') + self.console.print(table.get_table()) diff --git a/cycode/cli/printers/text_printer.py b/cycode/cli/printers/text_printer.py index 0b503207..51da53c5 100644 --- a/cycode/cli/printers/text_printer.py +++ b/cycode/cli/printers/text_printer.py @@ -1,261 +1,137 @@ -import math -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional -import click - -from cycode.cli.config import config -from cycode.cli.consts import COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES, SECRET_SCAN_TYPE -from cycode.cli.models import CliError, CliResult, Detection, Document, DocumentDetections +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.models import CliError, CliResult, Document from cycode.cli.printers.printer_base import PrinterBase -from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text +from cycode.cli.printers.utils.code_snippet_syntax import get_code_snippet_syntax, get_detection_line +from cycode.cli.printers.utils.detection_data import get_detection_title +from cycode.cli.printers.utils.detection_ordering.common_ordering import sort_and_group_detections_from_scan_result if TYPE_CHECKING: - from cycode.cli.models import LocalScanResult + from cycode.cli.models import Detection, LocalScanResult class TextPrinter(PrinterBase): - def __init__(self, context: click.Context) -> None: - super().__init__(context) - self.scan_type: str = context.obj.get('scan_type') - self.command_scan_type: str = context.info_name - self.show_secret: bool = context.obj.get('show_secret', False) - def print_result(self, result: CliResult) -> None: - color = None + color = 'default' if not result.success: - color = self.RED_COLOR_NAME + color = 'red' - click.secho(result.message, fg=color) + self.console.print(result.message, style=color) if not result.data: return - click.secho('\nAdditional data:', fg=color) + self.console.print('\nAdditional data:', style=color) for name, value in result.data.items(): - click.secho(f'- {name}: {value}', fg=color) + self.console.print(f'- {name}: {value}', style=color) def print_error(self, error: CliError) -> None: - click.secho(error.message, fg=self.RED_COLOR_NAME) + self.console.print(f'[red]Error: {error.message}[/]', highlight=False) def print_scan_results( - self, local_scan_results: List['LocalScanResult'], errors: Optional[Dict[str, 'CliError']] = None + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None ) -> None: if not errors and all(result.issue_detected == 0 for result in local_scan_results): - click.secho('Good job! No issues were found!!! 👏👏👏', fg=self.GREEN_COLOR_NAME) + self.console.print(self.NO_DETECTIONS_MESSAGE) return - for local_scan_result in local_scan_results: - for document_detections in local_scan_result.document_detections: - self._print_document_detections(document_detections, local_scan_result.scan_id) + detections, _ = sort_and_group_detections_from_scan_result(local_scan_results) + for detection, document in detections: + self.__print_document_detection(document, detection) - report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] + self.print_scan_results_summary(local_scan_results) + self.print_report_urls_and_errors(local_scan_results, errors) - self._print_report_urls(report_urls, self.context.obj.get('aggregation_report_url')) - if not errors: - return + def __print_document_detection(self, document: 'Document', detection: 'Detection') -> None: + self.__print_detection_summary(detection, document.path) + self.__print_detection_code_segment(detection, document) + self._print_new_line() - click.secho( - 'Unfortunately, Cycode was unable to complete the full scan. ' - 'Please note that not all results may be available:', - fg='red', - ) - for scan_id, error in errors.items(): - click.echo(f'- {scan_id}: ', nl=False) - self.print_error(error) + def _print_new_line(self) -> None: + self.console.line() - def _print_document_detections(self, document_detections: DocumentDetections, scan_id: str) -> None: - document = document_detections.document - lines_to_display = self._get_lines_to_display_count() - for detection in document_detections.detections: - self._print_detection_summary(detection, document.path, scan_id) - self._print_detection_code_segment(detection, document, lines_to_display) + def __print_detection_summary(self, detection: 'Detection', document_path: str) -> None: + title = get_detection_title(self.scan_type, detection) - def _print_detection_summary(self, detection: Detection, document_path: str, scan_id: str) -> None: - detection_name = detection.type if self.scan_type == SECRET_SCAN_TYPE else detection.message - detection_name_styled = click.style(detection_name, fg='bright_red', bold=True) + severity = SeverityOption(detection.severity) if detection.severity else 'N/A' + severity_icon = SeverityOption.get_member_emoji(detection.severity) if detection.severity else '' - detection_sha = detection.detection_details.get('sha512') - detection_sha_message = f'\nSecret SHA: {detection_sha}' if detection_sha else '' + line_no = get_detection_line(self.scan_type, detection) + 1 + clickable_document_path = f'[u]{document_path}:{line_no}[/]' - scan_id_message = f'\nScan ID: {scan_id}' detection_commit_id = detection.detection_details.get('commit_id') detection_commit_id_message = f'\nCommit SHA: {detection_commit_id}' if detection_commit_id else '' - company_guidelines = detection.detection_details.get('custom_remediation_guidelines') - company_guidelines_message = f'\nCompany Guideline: {company_guidelines}' if company_guidelines else '' - - click.echo( - f'⛔ ' - f'Found issue of type: {detection_name_styled} ' - f'(rule ID: {detection.detection_rule_id}) in file: {click.format_filename(document_path)} ' - f'{detection_sha_message}' - f'{scan_id_message}' - f'{detection_commit_id_message}' - f'{company_guidelines_message}' - f' ⛔' + self.console.print( + severity_icon, + severity, + f'violation: [b bright_red]{title}[/]{detection_commit_id_message}\n', + *self.__get_intermediate_summary_lines(detection), + f'[dodger_blue1]File: {clickable_document_path}[/]', ) - def _print_detection_code_segment(self, detection: Detection, document: Document, code_segment_size: int) -> None: - if self._is_git_diff_based_scan(): - self._print_detection_from_git_diff(detection, document) - return - - self._print_detection_from_file(detection, document, code_segment_size) + def __get_intermediate_summary_lines(self, detection: 'Detection') -> list[str]: + intermediate_summary_lines = [] - @staticmethod - def _print_report_urls(report_urls: List[str], aggregation_report_url: Optional[str] = None) -> None: - if not report_urls and not aggregation_report_url: - return - if aggregation_report_url: - click.echo(f'Report URL: {aggregation_report_url}') - return + if self.scan_type == consts.SCA_SCAN_TYPE: + intermediate_summary_lines.extend(self.__get_sca_related_summary_lines(detection)) - click.echo('Report URLs:') - for report_url in report_urls: - click.echo(f'- {report_url}') + return intermediate_summary_lines @staticmethod - def _get_code_segment_start_line(detection_line: int, code_segment_size: int) -> int: - start_line = detection_line - math.ceil(code_segment_size / 2) - return 0 if start_line < 0 else start_line - - def _print_line_of_code_segment( - self, - document: Document, - line: str, - line_number: int, - detection_position_in_line: int, - violation_length: int, - is_detection_line: bool, - ) -> None: - if is_detection_line: - self._print_detection_line(document, line, line_number, detection_position_in_line, violation_length) - else: - self._print_line(document, line, line_number) - - def _print_detection_line( - self, document: Document, line: str, line_number: int, detection_position_in_line: int, violation_length: int - ) -> None: - detection_line = self._get_detection_line_style( - line, document.is_git_diff_format, detection_position_in_line, violation_length - ) - - click.echo(f'{self._get_line_number_style(line_number)} {detection_line}') + def __get_sca_related_summary_lines(detection: 'Detection') -> list[str]: + summary_lines = [] - def _print_line(self, document: Document, line: str, line_number: int) -> None: - line_no = self._get_line_number_style(line_number) - line = self._get_line_style(line, document.is_git_diff_format) + if detection.has_alert: + patched_version = detection.detection_details['alert'].get('first_patched_version') + patched_version = patched_version or 'Not fixed' - click.echo(f'{line_no} {line}') - - def _get_detection_line_style(self, line: str, is_git_diff: bool, start_position: int, length: int) -> str: - line_color = self._get_line_color(line, is_git_diff) - if self.scan_type != SECRET_SCAN_TYPE or start_position < 0 or length < 0: - return self._get_line_style(line, is_git_diff, line_color) - - violation = line[start_position : start_position + length] - if not self.show_secret: - violation = obfuscate_text(violation) + summary_lines.append(f'First patched version: [cyan]{patched_version}[/]\n') + else: + package_license = detection.detection_details.get('license', 'N/A') + summary_lines.append(f'License: [cyan]{package_license}[/]\n') - line_to_violation = line[0:start_position] - line_from_violation = line[start_position + length :] + return summary_lines - return ( - f'{self._get_line_style(line_to_violation, is_git_diff, line_color)}' - f'{self._get_line_style(violation, is_git_diff, line_color, underline=True)}' - f'{self._get_line_style(line_from_violation, is_git_diff, line_color)}' + def __print_detection_code_segment(self, detection: 'Detection', document: Document) -> None: + self.console.print( + get_code_snippet_syntax( + self.scan_type, + self.command_scan_type, + detection, + document, + obfuscate=not self.show_secret, + ) ) - def _get_line_style( - self, line: str, is_git_diff: bool, color: Optional[str] = None, underline: bool = False - ) -> str: - if color is None: - color = self._get_line_color(line, is_git_diff) - - return click.style(line, fg=color, bold=False, underline=underline) - - def _get_line_color(self, line: str, is_git_diff: bool) -> str: - if not is_git_diff: - return self.WHITE_COLOR_NAME - - if line.startswith('+'): - return self.GREEN_COLOR_NAME - - if line.startswith('-'): - return self.RED_COLOR_NAME - - return self.WHITE_COLOR_NAME + def print_report_urls_and_errors( + self, local_scan_results: list['LocalScanResult'], errors: Optional[dict[str, 'CliError']] = None + ) -> None: + report_urls = [scan_result.report_url for scan_result in local_scan_results if scan_result.report_url] - def _get_line_number_style(self, line_number: int) -> str: - return ( - f'{click.style(str(line_number), fg=self.WHITE_COLOR_NAME, bold=False)} ' - f'{click.style("|", fg=self.RED_COLOR_NAME, bold=False)}' - ) + self.print_report_urls(report_urls, self.ctx.obj.get('aggregation_report_url')) + if not errors: + return - def _get_lines_to_display_count(self) -> int: - result_printer_configuration = config.get('result_printer') - lines_to_display_of_scan = ( - result_printer_configuration.get(self.scan_type, {}).get(self.command_scan_type, {}).get('lines_to_display') - ) - if lines_to_display_of_scan: - return lines_to_display_of_scan + self.console.print(self.FAILED_SCAN_MESSAGE) + for scan_id, error in errors.items(): + self.console.print(f'- {scan_id}: ', end='') + self.print_error(error) - return result_printer_configuration.get('default').get('lines_to_display') + def print_report_urls(self, report_urls: list[str], aggregation_report_url: Optional[str] = None) -> None: + if not report_urls and not aggregation_report_url: + return - def _print_detection_from_file(self, detection: Detection, document: Document, code_segment_size: int) -> None: - detection_details = detection.detection_details - detection_line = ( - detection_details.get('line', -1) - if self.scan_type == SECRET_SCAN_TYPE - else detection_details.get('line_in_file', -1) - ) - detection_position = detection_details.get('start_position', -1) - violation_length = detection_details.get('length', -1) - - file_content = document.content - file_lines = file_content.splitlines() - start_line = self._get_code_segment_start_line(detection_line, code_segment_size) - detection_position_in_line = get_position_in_line(file_content, detection_position) - - click.echo() - for i in range(code_segment_size): - current_line_index = start_line + i - if current_line_index >= len(file_lines): - break - - current_line = file_lines[current_line_index] - is_detection_line = current_line_index == detection_line - self._print_line_of_code_segment( - document, - current_line, - current_line_index + 1, - detection_position_in_line, - violation_length, - is_detection_line, - ) - click.echo() - - def _print_detection_from_git_diff(self, detection: Detection, document: Document) -> None: - detection_details = detection.detection_details - detection_line_number = detection_details.get('line', -1) - detection_line_number_in_original_file = detection_details.get('line_in_file', -1) - detection_position = detection_details.get('start_position', -1) - violation_length = detection_details.get('length', -1) - - git_diff_content = document.content - git_diff_lines = git_diff_content.splitlines() - detection_line = git_diff_lines[detection_line_number] - detection_position_in_line = get_position_in_line(git_diff_content, detection_position) - - click.echo() - self._print_detection_line( - document, - detection_line, - detection_line_number_in_original_file, - detection_position_in_line, - violation_length, - ) - click.echo() + # Prioritize aggregation report URL; if report urls is only one, use it instead + single_url = report_urls[0] if len(report_urls) == 1 else None + single_url = aggregation_report_url or single_url + if single_url: + self.console.print(f'[b]Report URL:[/] {single_url}') + return - def _is_git_diff_based_scan(self) -> bool: - return self.command_scan_type in COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES and self.scan_type == SECRET_SCAN_TYPE + # If there are multiple report URLs, print them all + self.console.print('[b]Report URLs:[/]') + for report_url in report_urls: + self.console.print(f'- {report_url}') diff --git a/cycode/cli/printers/utils/__init__.py b/cycode/cli/printers/utils/__init__.py new file mode 100644 index 00000000..e1676c35 --- /dev/null +++ b/cycode/cli/printers/utils/__init__.py @@ -0,0 +1,8 @@ +from cycode.cli import consts + + +def is_git_diff_based_scan(scan_type: str, command_scan_type: str) -> bool: + return ( + command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES + and scan_type in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES + ) diff --git a/cycode/cli/printers/utils/code_snippet_syntax.py b/cycode/cli/printers/utils/code_snippet_syntax.py new file mode 100644 index 00000000..12501544 --- /dev/null +++ b/cycode/cli/printers/utils/code_snippet_syntax.py @@ -0,0 +1,117 @@ +from typing import TYPE_CHECKING + +from rich.syntax import Syntax + +from cycode.cli import consts +from cycode.cli.console import _SYNTAX_HIGHLIGHT_THEME +from cycode.cli.printers.utils import is_git_diff_based_scan +from cycode.cli.utils.string_utils import get_position_in_line, obfuscate_text + +if TYPE_CHECKING: + from cycode.cli.models import Document + from cycode.cyclient.models import Detection + + +def _get_code_segment_start_line(detection_line: int, lines_to_display_before: int) -> int: + start_line = detection_line - lines_to_display_before + return 0 if start_line < 0 else start_line + + +def get_detection_line(scan_type: str, detection: 'Detection') -> int: + return ( + detection.detection_details.get('line', -1) + if scan_type == consts.SECRET_SCAN_TYPE + else detection.detection_details.get('line_in_file', -1) - 1 + ) + + +def _get_code_snippet_syntax_from_file( + scan_type: str, + detection: 'Detection', + document: 'Document', + lines_to_display_before: int, + lines_to_display_after: int, + obfuscate: bool, +) -> Syntax: + detection_details = detection.detection_details + detection_line = get_detection_line(scan_type, detection) + start_line_index = _get_code_segment_start_line(detection_line, lines_to_display_before) + detection_position = get_position_in_line(document.content, detection_details.get('start_position', -1)) + violation_length = detection_details.get('length', -1) + + code_lines_to_render = [] + document_content_lines = document.content.splitlines() + total_lines_to_display = lines_to_display_before + 1 + lines_to_display_after + + for line_index in range(total_lines_to_display): + current_line_index = start_line_index + line_index + if current_line_index >= len(document_content_lines): + break + + line_content = document_content_lines[current_line_index] + + line_with_detection = current_line_index == detection_line + if scan_type == consts.SECRET_SCAN_TYPE and line_with_detection and obfuscate: + violation = line_content[detection_position : detection_position + violation_length] + code_lines_to_render.append(line_content.replace(violation, obfuscate_text(violation))) + else: + code_lines_to_render.append(line_content) + + code_to_render = '\n'.join(code_lines_to_render) + return Syntax( + theme=_SYNTAX_HIGHLIGHT_THEME, + code=code_to_render, + lexer=Syntax.guess_lexer(document.path, code=code_to_render), + line_numbers=True, + word_wrap=True, + dedent=True, + tab_size=2, + start_line=start_line_index + 1, + highlight_lines={ + detection_line + 1, + }, + ) + + +def _get_code_snippet_syntax_from_git_diff( + scan_type: str, detection: 'Detection', document: 'Document', obfuscate: bool +) -> Syntax: + detection_details = detection.detection_details + detection_line = get_detection_line(scan_type, detection) + detection_position = detection_details.get('start_position', -1) + violation_length = detection_details.get('length', -1) + + line_content = document.content.splitlines()[detection_line] + detection_position_in_line = get_position_in_line(document.content, detection_position) + if scan_type == consts.SECRET_SCAN_TYPE and obfuscate: + violation = line_content[detection_position_in_line : detection_position_in_line + violation_length] + line_content = line_content.replace(violation, obfuscate_text(violation)) + + return Syntax( + theme=_SYNTAX_HIGHLIGHT_THEME, + code=line_content, + lexer='diff', + line_numbers=True, + start_line=detection_line, + dedent=True, + tab_size=2, + highlight_lines={detection_line + 1}, + ) + + +def get_code_snippet_syntax( + scan_type: str, + command_scan_type: str, + detection: 'Detection', + document: 'Document', + lines_to_display_before: int = 1, + lines_to_display_after: int = 1, + obfuscate: bool = True, +) -> Syntax: + if is_git_diff_based_scan(scan_type, command_scan_type): + # it will return syntax with just one line + return _get_code_snippet_syntax_from_git_diff(scan_type, detection, document, obfuscate) + + return _get_code_snippet_syntax_from_file( + scan_type, detection, document, lines_to_display_before, lines_to_display_after, obfuscate + ) diff --git a/cycode/cli/printers/utils/detection_data.py b/cycode/cli/printers/utils/detection_data.py new file mode 100644 index 00000000..37bee310 --- /dev/null +++ b/cycode/cli/printers/utils/detection_data.py @@ -0,0 +1,108 @@ +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +from cycode.cli import consts + +if TYPE_CHECKING: + from cycode.cyclient.models import Detection + + +def get_cwe_cve_link(cwe_cve: Optional[str]) -> Optional[str]: + if not cwe_cve: + return None + + if cwe_cve.startswith('GHSA'): + return f'https://github.com/advisories/{cwe_cve}' + + if cwe_cve.startswith('CWE'): + # string example: 'CWE-532: Insertion of Sensitive Information into Log File' + parts = cwe_cve.split('-') + if len(parts) < 1: + return None + + number = '' + for char in parts[1]: + if char.isdigit(): + number += char + else: + break + + return f'https://cwe.mitre.org/data/definitions/{number}' + + if cwe_cve.startswith('CVE'): + return f'https://cve.mitre.org/cgi-bin/cvename.cgi?name={cwe_cve}' + + return None + + +def clear_cwe_name(cwe: str) -> str: + """Clear CWE. + + Intput: CWE-532: Insertion of Sensitive Information into Log File + Output: CWE-532 + """ + if cwe.startswith('CWE'): + return cwe.split(':')[0] + + return cwe + + +def get_detection_clickable_cwe_cve(scan_type: str, detection: 'Detection') -> str: + def link(url: str, name: str) -> str: + return f'[link={url}]{clear_cwe_name(name)}[/]' + + if scan_type == consts.SCA_SCAN_TYPE: + cve = detection.detection_details.get('vulnerability_id') + return link(get_cwe_cve_link(cve), cve) if cve else '' + if scan_type == consts.SAST_SCAN_TYPE: + renderables = [] + for cwe in detection.detection_details.get('cwe', []): + cwe and renderables.append(link(get_cwe_cve_link(cwe), cwe)) + return ', '.join(renderables) + + return '' + + +def get_detection_cwe_cve(scan_type: str, detection: 'Detection') -> Optional[str]: + if scan_type == consts.SCA_SCAN_TYPE: + return detection.detection_details.get('vulnerability_id') + if scan_type == consts.SAST_SCAN_TYPE: + cwes = detection.detection_details.get('cwe') # actually it is List[str] + if not cwes: + return None + + return ' | '.join(cwes) + + return None + + +def get_detection_title(scan_type: str, detection: 'Detection') -> str: + title = detection.message + if scan_type == consts.SAST_SCAN_TYPE: + title = detection.detection_details['policy_display_name'] + elif scan_type == consts.SECRET_SCAN_TYPE: + title = f'Hardcoded {detection.type} is used' + + is_sca_package_vulnerability = scan_type == consts.SCA_SCAN_TYPE and detection.has_alert + if is_sca_package_vulnerability: + title = detection.detection_details['alert'].get('summary', 'N/A') + + cwe_cve = get_detection_cwe_cve(scan_type, detection) + return f'[{cwe_cve}] {title}' if cwe_cve else title + + +def get_detection_file_path(scan_type: str, detection: 'Detection') -> Path: + if scan_type == consts.SECRET_SCAN_TYPE: + folder_path = detection.detection_details.get('file_path', '') + file_name = detection.detection_details.get('file_name', '') + return Path.joinpath(Path(folder_path), Path(file_name)) + if scan_type == consts.SAST_SCAN_TYPE: + file_path = detection.detection_details.get('file_path', '') + + # fix the absolute path...BE returns string which does not start with / + if not file_path.startswith('/'): + file_path = f'/{file_path}' + + return Path(file_path) + + return Path(detection.detection_details.get('file_name', '')) diff --git a/cycode/cli/commands/scan/__init__.py b/cycode/cli/printers/utils/detection_ordering/__init__.py similarity index 100% rename from cycode/cli/commands/scan/__init__.py rename to cycode/cli/printers/utils/detection_ordering/__init__.py diff --git a/cycode/cli/printers/utils/detection_ordering/common_ordering.py b/cycode/cli/printers/utils/detection_ordering/common_ordering.py new file mode 100644 index 00000000..c4b431ef --- /dev/null +++ b/cycode/cli/printers/utils/detection_ordering/common_ordering.py @@ -0,0 +1,57 @@ +from typing import TYPE_CHECKING + +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cli.models import Document, LocalScanResult + from cycode.cyclient.models import Detection + + +GroupedDetections = tuple[list[tuple['Detection', 'Document']], set[int]] + + +def __severity_sort_key(detection_with_document: tuple['Detection', 'Document']) -> int: + detection, _ = detection_with_document + severity = detection.severity if detection.severity else '' + return SeverityOption.get_member_weight(severity) + + +def _sort_detections_by_severity( + detections_with_documents: list[tuple['Detection', 'Document']], +) -> list[tuple['Detection', 'Document']]: + return sorted(detections_with_documents, key=__severity_sort_key, reverse=True) + + +def __file_path_sort_key(detection_with_document: tuple['Detection', 'Document']) -> str: + _, document = detection_with_document + return document.path + + +def _sort_detections_by_file_path( + detections_with_documents: list[tuple['Detection', 'Document']], +) -> list[tuple['Detection', 'Document']]: + return sorted(detections_with_documents, key=__file_path_sort_key) + + +def sort_and_group_detections( + detections_with_documents: list[tuple['Detection', 'Document']], +) -> GroupedDetections: + """Sort detections by severity. We do not have grouping here (don't find the best one yet).""" + group_separator_indexes = set() + + # we sort detections by file path to make persist output order + sorted_by_path_detections = _sort_detections_by_file_path(detections_with_documents) + sorted_by_severity = _sort_detections_by_severity(sorted_by_path_detections) + + return sorted_by_severity, group_separator_indexes + + +def sort_and_group_detections_from_scan_result(local_scan_results: list['LocalScanResult']) -> GroupedDetections: + detections_with_documents = [] + for local_scan_result in local_scan_results: + for document_detections in local_scan_result.document_detections: + detections_with_documents.extend( + [(detection, document_detections.document) for detection in document_detections.detections] + ) + + return sort_and_group_detections(detections_with_documents) diff --git a/cycode/cli/printers/utils/detection_ordering/sca_ordering.py b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py new file mode 100644 index 00000000..a8be3430 --- /dev/null +++ b/cycode/cli/printers/utils/detection_ordering/sca_ordering.py @@ -0,0 +1,59 @@ +from collections import defaultdict +from typing import TYPE_CHECKING + +from cycode.cli.cli_types import SeverityOption + +if TYPE_CHECKING: + from cycode.cyclient.models import Detection + + +def __group_by(detections: list['Detection'], details_field_name: str) -> dict[str, list['Detection']]: + grouped = defaultdict(list) + for detection in detections: + grouped[detection.detection_details.get(details_field_name)].append(detection) + return grouped + + +def __severity_sort_key(detection: 'Detection') -> int: + severity = detection.severity if detection.severity else 'unknown' + return SeverityOption.get_member_weight(severity) + + +def _sort_detections_by_severity(detections: list['Detection']) -> list['Detection']: + return sorted(detections, key=__severity_sort_key, reverse=True) + + +def __package_sort_key(detection: 'Detection') -> int: + return detection.detection_details.get('package_name') + + +def _sort_detections_by_package(detections: list['Detection']) -> list['Detection']: + return sorted(detections, key=__package_sort_key) + + +def sort_and_group_detections(detections: list['Detection']) -> tuple[list['Detection'], set[int]]: + """Sort detections by severity and group by repository, code project and package name. + + Note: + Code Project is path to the manifest file. + + Grouping by code projects also groups by ecosystem. + Because manifest files are unique per ecosystem. + + """ + resulting_detections = [] + group_separator_indexes = set() + + # we sort detections by package name to make persist output order + sorted_detections = _sort_detections_by_package(detections) + + grouped_by_repository = __group_by(sorted_detections, 'repository_name') + for repository_group in grouped_by_repository.values(): + grouped_by_code_project = __group_by(repository_group, 'file_name') + for code_project_group in grouped_by_code_project.values(): + grouped_by_package = __group_by(code_project_group, 'package_name') + for package_group in grouped_by_package.values(): + group_separator_indexes.add(len(resulting_detections) - 1) # indexing starts from 0 + resulting_detections.extend(_sort_detections_by_severity(package_group)) + + return resulting_detections, group_separator_indexes diff --git a/cycode/cli/printers/utils/rich_helpers.py b/cycode/cli/printers/utils/rich_helpers.py new file mode 100644 index 00000000..52d2a0f2 --- /dev/null +++ b/cycode/cli/printers/utils/rich_helpers.py @@ -0,0 +1,37 @@ +from typing import TYPE_CHECKING + +from rich.columns import Columns +from rich.markdown import Markdown +from rich.panel import Panel + +from cycode.cli.console import console + +if TYPE_CHECKING: + from rich.console import RenderableType + + +def get_panel(renderable: 'RenderableType', title: str) -> Panel: + return Panel( + renderable, + title=title, + title_align='left', + border_style='dim', + ) + + +def get_markdown_panel(markdown_text: str, title: str) -> Panel: + return get_panel( + Markdown(markdown_text.strip()), + title=title, + ) + + +def get_columns_in_1_to_3_ratio(left: 'Panel', right: 'Panel', panel_border_offset: int = 5) -> Columns: + terminal_width = console.width + one_third_width = terminal_width // 3 + two_thirds_width = terminal_width - one_third_width - panel_border_offset + + left.width = one_third_width + right.width = two_thirds_width + + return Columns([left, right]) diff --git a/cycode/cli/user_settings/base_file_manager.py b/cycode/cli/user_settings/base_file_manager.py index b7be273d..4f07f11c 100644 --- a/cycode/cli/user_settings/base_file_manager.py +++ b/cycode/cli/user_settings/base_file_manager.py @@ -1,18 +1,19 @@ import os from abc import ABC, abstractmethod -from typing import Any, Dict, Hashable +from collections.abc import Hashable +from typing import Any -from cycode.cli.utils.yaml_utils import read_file, update_file +from cycode.cli.utils.yaml_utils import read_yaml_file, update_yaml_file class BaseFileManager(ABC): @abstractmethod def get_filename(self) -> str: ... - def read_file(self) -> Dict[Hashable, Any]: - return read_file(self.get_filename()) + def read_file(self) -> dict[Hashable, Any]: + return read_yaml_file(self.get_filename()) - def write_content_to_file(self, content: Dict[Hashable, Any]) -> None: + def write_content_to_file(self, content: dict[Hashable, Any]) -> None: filename = self.get_filename() os.makedirs(os.path.dirname(filename), exist_ok=True) - update_file(filename, content) + update_yaml_file(filename, content) diff --git a/cycode/cli/user_settings/config_file_manager.py b/cycode/cli/user_settings/config_file_manager.py index e4e5e6b1..5b029e39 100644 --- a/cycode/cli/user_settings/config_file_manager.py +++ b/cycode/cli/user_settings/config_file_manager.py @@ -1,5 +1,6 @@ import os -from typing import TYPE_CHECKING, Any, Dict, Hashable, List, Optional, Union +from collections.abc import Hashable +from typing import TYPE_CHECKING, Any, Optional, Union from cycode.cli.consts import CYCODE_CONFIGURATION_DIRECTORY from cycode.cli.user_settings.base_file_manager import BaseFileManager @@ -37,7 +38,7 @@ def get_app_url(self) -> Optional[Any]: def get_verbose_flag(self) -> Optional[Any]: return self._get_value_from_environment_section(self.VERBOSE_FIELD_NAME) - def get_exclusions_by_scan_type(self, scan_type: str) -> Dict[Hashable, Any]: + def get_exclusions_by_scan_type(self, scan_type: str) -> dict[Hashable, Any]: exclusions_section = self._get_section(self.EXCLUSIONS_SECTION_NAME) return exclusions_section.get(scan_type, {}) @@ -87,7 +88,7 @@ def get_filename(self) -> str: def get_config_file_route() -> str: return os.path.join(ConfigFileManager.CYCODE_HIDDEN_DIRECTORY, ConfigFileManager.FILE_NAME) - def _get_exclusions_by_exclusion_type(self, scan_type: str, exclusion_type: str) -> List[Any]: + def _get_exclusions_by_exclusion_type(self, scan_type: str, exclusion_type: str) -> list[Any]: scan_type_exclusions = self.get_exclusions_by_scan_type(scan_type) return scan_type_exclusions.get(exclusion_type, []) @@ -95,7 +96,7 @@ def _get_value_from_environment_section(self, field_name: str) -> Optional[Any]: environment_section = self._get_section(self.ENVIRONMENT_SECTION_NAME) return environment_section.get(field_name) - def _get_scan_configuration_by_scan_type(self, command_scan_type: str) -> Dict[Hashable, Any]: + def _get_scan_configuration_by_scan_type(self, command_scan_type: str) -> dict[Hashable, Any]: scan_section = self._get_section(self.SCAN_SECTION_NAME) return scan_section.get(command_scan_type, {}) @@ -103,6 +104,6 @@ def _get_value_from_command_scan_type_configuration(self, command_scan_type: str command_scan_type_configuration = self._get_scan_configuration_by_scan_type(command_scan_type) return command_scan_type_configuration.get(field_name) - def _get_section(self, section_name: str) -> Dict[Hashable, Any]: + def _get_section(self, section_name: str) -> dict[Hashable, Any]: file_content = self.read_file() return file_content.get(section_name, {}) diff --git a/cycode/cli/user_settings/configuration_manager.py b/cycode/cli/user_settings/configuration_manager.py index f8d67c42..3b83f1c9 100644 --- a/cycode/cli/user_settings/configuration_manager.py +++ b/cycode/cli/user_settings/configuration_manager.py @@ -1,7 +1,7 @@ import os -from functools import lru_cache +from functools import cache from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Optional from uuid import uuid4 from cycode.cli import consts @@ -69,8 +69,8 @@ def get_verbose_flag_from_environment_variables(self) -> bool: value = self._get_value_from_environment_variables(consts.VERBOSE_ENV_VAR_NAME, '') return value.lower() in ('true', '1') - @lru_cache(maxsize=None) # noqa: B019 - def get_exclusions_by_scan_type(self, scan_type: str) -> Dict: + @cache # noqa: B019 + def get_exclusions_by_scan_type(self, scan_type: str) -> dict: local_exclusions = self.local_config_file_manager.get_exclusions_by_scan_type(scan_type) global_exclusions = self.global_config_file_manager.get_exclusions_by_scan_type(scan_type) return self._merge_exclusions(local_exclusions, global_exclusions) @@ -80,7 +80,7 @@ def add_exclusion(self, scope: str, scan_type: str, exclusion_type: str, value: config_file_manager.add_exclusion(scan_type, exclusion_type, value) @staticmethod - def _merge_exclusions(local_exclusions: Dict, global_exclusions: Dict) -> Dict: + def _merge_exclusions(local_exclusions: dict, global_exclusions: dict) -> dict: keys = set(list(local_exclusions.keys()) + list(global_exclusions.keys())) return {key: local_exclusions.get(key, []) + global_exclusions.get(key, []) for key in keys} diff --git a/cycode/cli/user_settings/credentials_manager.py b/cycode/cli/user_settings/credentials_manager.py index ad380e8a..7af43569 100644 --- a/cycode/cli/user_settings/credentials_manager.py +++ b/cycode/cli/user_settings/credentials_manager.py @@ -1,11 +1,11 @@ import os from pathlib import Path -from typing import Optional, Tuple +from typing import Optional from cycode.cli.config import CYCODE_CLIENT_ID_ENV_VAR_NAME, CYCODE_CLIENT_SECRET_ENV_VAR_NAME -from cycode.cli.sentry import setup_scope_from_access_token from cycode.cli.user_settings.base_file_manager import BaseFileManager from cycode.cli.user_settings.jwt_creator import JwtCreator +from cycode.cli.utils.sentry import setup_scope_from_access_token class CredentialsManager(BaseFileManager): @@ -19,7 +19,7 @@ class CredentialsManager(BaseFileManager): ACCESS_TOKEN_EXPIRES_IN_FIELD_NAME: str = 'cycode_access_token_expires_in' ACCESS_TOKEN_CREATOR_FIELD_NAME: str = 'cycode_access_token_creator' - def get_credentials(self) -> Tuple[str, str]: + def get_credentials(self) -> tuple[str, str]: client_id, client_secret = self.get_credentials_from_environment_variables() if client_id is not None and client_secret is not None: return client_id, client_secret @@ -27,12 +27,12 @@ def get_credentials(self) -> Tuple[str, str]: return self.get_credentials_from_file() @staticmethod - def get_credentials_from_environment_variables() -> Tuple[str, str]: + def get_credentials_from_environment_variables() -> tuple[str, str]: client_id = os.getenv(CYCODE_CLIENT_ID_ENV_VAR_NAME) client_secret = os.getenv(CYCODE_CLIENT_SECRET_ENV_VAR_NAME) return client_id, client_secret - def get_credentials_from_file(self) -> Tuple[Optional[str], Optional[str]]: + def get_credentials_from_file(self) -> tuple[Optional[str], Optional[str]]: file_content = self.read_file() client_id = file_content.get(self.CLIENT_ID_FIELD_NAME) client_secret = file_content.get(self.CLIENT_SECRET_FIELD_NAME) @@ -42,7 +42,7 @@ def update_credentials(self, client_id: str, client_secret: str) -> None: file_content_to_update = {self.CLIENT_ID_FIELD_NAME: client_id, self.CLIENT_SECRET_FIELD_NAME: client_secret} self.write_content_to_file(file_content_to_update) - def get_access_token(self) -> Tuple[Optional[str], Optional[float], Optional[JwtCreator]]: + def get_access_token(self) -> tuple[Optional[str], Optional[float], Optional[JwtCreator]]: file_content = self.read_file() access_token = file_content.get(self.ACCESS_TOKEN_FIELD_NAME) diff --git a/cycode/cli/utils/enum_utils.py b/cycode/cli/utils/enum_utils.py index 6ea9ef72..3280a5bb 100644 --- a/cycode/cli/utils/enum_utils.py +++ b/cycode/cli/utils/enum_utils.py @@ -1,8 +1,7 @@ from enum import Enum -from typing import List class AutoCountEnum(Enum): @staticmethod - def _generate_next_value_(name: str, start: int, count: int, last_values: List[int]) -> int: + def _generate_next_value_(name: str, start: int, count: int, last_values: list[int]) -> int: return count diff --git a/cycode/cli/utils/get_api_client.py b/cycode/cli/utils/get_api_client.py index 7bbfa2d9..110d528b 100644 --- a/cycode/cli/utils/get_api_client.py +++ b/cycode/cli/utils/get_api_client.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Tuple, Union +from typing import TYPE_CHECKING, Optional, Union import click @@ -6,6 +6,8 @@ from cycode.cyclient.client_creator import create_report_client, create_scan_client if TYPE_CHECKING: + import typer + from cycode.cyclient.report_client import ReportClient from cycode.cyclient.scan_client import ScanClient @@ -23,18 +25,19 @@ def _get_cycode_client( return create_client_func(client_id, client_secret, hide_response_log) -def get_scan_cycode_client( - client_id: Optional[str] = None, client_secret: Optional[str] = None, hide_response_log: bool = True -) -> 'ScanClient': +def get_scan_cycode_client(ctx: 'typer.Context') -> 'ScanClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') + hide_response_log = not ctx.obj.get('show_secret', False) return _get_cycode_client(create_scan_client, client_id, client_secret, hide_response_log) -def get_report_cycode_client( - client_id: Optional[str] = None, client_secret: Optional[str] = None, hide_response_log: bool = True -) -> 'ReportClient': +def get_report_cycode_client(ctx: 'typer.Context', hide_response_log: bool = True) -> 'ReportClient': + client_id = ctx.obj.get('client_id') + client_secret = ctx.obj.get('client_secret') return _get_cycode_client(create_report_client, client_id, client_secret, hide_response_log) -def _get_configured_credentials() -> Tuple[str, str]: +def _get_configured_credentials() -> tuple[str, str]: credentials_manager = CredentialsManager() return credentials_manager.get_credentials() diff --git a/cycode/cli/utils/git_proxy.py b/cycode/cli/utils/git_proxy.py index c46d016b..beaafdd0 100644 --- a/cycode/cli/utils/git_proxy.py +++ b/cycode/cli/utils/git_proxy.py @@ -1,9 +1,9 @@ import types from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Optional, Type +from typing import TYPE_CHECKING, Optional _GIT_ERROR_MESSAGE = """ -Cycode CLI needs the git executable to be installed on the system. +Cycode CLI needs the Git executable to be installed on the system. Git executable must be available in the PATH. Git 1.7.x or newer is required. You can help Cycode CLI to locate the Git executable @@ -31,10 +31,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: ... @abstractmethod - def get_invalid_git_repository_error(self) -> Type[BaseException]: ... + def get_invalid_git_repository_error(self) -> type[BaseException]: ... @abstractmethod - def get_git_command_error(self) -> Type[BaseException]: ... + def get_git_command_error(self) -> type[BaseException]: ... class _DummyGitProxy(_AbstractGitProxy): @@ -44,10 +44,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: raise RuntimeError(_GIT_ERROR_MESSAGE) - def get_invalid_git_repository_error(self) -> Type[BaseException]: + def get_invalid_git_repository_error(self) -> type[BaseException]: return GitProxyError - def get_git_command_error(self) -> Type[BaseException]: + def get_git_command_error(self) -> type[BaseException]: return GitProxyError @@ -58,10 +58,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: return git.NULL_TREE - def get_invalid_git_repository_error(self) -> Type[BaseException]: + def get_invalid_git_repository_error(self) -> type[BaseException]: return git.InvalidGitRepositoryError - def get_git_command_error(self) -> Type[BaseException]: + def get_git_command_error(self) -> type[BaseException]: return git.GitCommandError @@ -87,10 +87,10 @@ def get_repo(self, path: Optional['PathLike'] = None, *args, **kwargs) -> 'Repo' def get_null_tree(self) -> object: return self._git_proxy.get_null_tree() - def get_invalid_git_repository_error(self) -> Type[BaseException]: + def get_invalid_git_repository_error(self) -> type[BaseException]: return self._git_proxy.get_invalid_git_repository_error() - def get_git_command_error(self) -> Type[BaseException]: + def get_git_command_error(self) -> type[BaseException]: return self._git_proxy.get_git_command_error() diff --git a/cycode/cli/utils/ignore_utils.py b/cycode/cli/utils/ignore_utils.py index f44b6024..e8994e46 100644 --- a/cycode/cli/utils/ignore_utils.py +++ b/cycode/cli/utils/ignore_utils.py @@ -38,16 +38,12 @@ import contextlib import os.path import re +from collections.abc import Generator, Iterable from os import PathLike from typing import ( Any, BinaryIO, - Dict, - Generator, - Iterable, - List, Optional, - Tuple, Union, ) @@ -98,7 +94,6 @@ def translate(pat: bytes) -> bytes: Originally copied from fnmatch in Python 2.7, but modified for Dulwich to cope with features in Git ignore patterns. """ - res = b'(?ms)' if b'/' not in pat[:-1]: @@ -131,6 +126,7 @@ def read_ignore_patterns(f: BinaryIO) -> Iterable[bytes]: Args: f: File-like object to read from Returns: List of patterns + """ for line in f: line = line.rstrip(b'\r\n') @@ -160,6 +156,7 @@ def match_pattern(path: bytes, pattern: bytes, ignore_case: bool = False) -> boo ignore_case: Whether to do case-sensitive matching Returns: bool indicating whether the pattern matched + """ return Pattern(pattern, ignore_case).match(path) @@ -200,6 +197,7 @@ def match(self, path: bytes) -> bool: Args: path: Path to match (relative to ignore location) Returns: boolean + """ return bool(self._re.match(path)) @@ -219,7 +217,7 @@ def __init__( for pattern in patterns: self.append_pattern(pattern) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: d = { 'patterns': [str(p) for p in self._patterns], 'ignore_case': self._ignore_case, @@ -242,6 +240,7 @@ def find_matching(self, path: Union[bytes, str]) -> Iterable[Pattern]: path: Path to match Returns: Iterator over iterators + """ if not isinstance(path, bytes): path = os.fsencode(path) @@ -284,7 +283,7 @@ class IgnoreFilterManager: def __init__( self, path: str, - global_filters: List[IgnoreFilter], + global_filters: list[IgnoreFilter], ignore_file_name: Optional[str] = None, ignore_case: bool = False, ) -> None: @@ -303,7 +302,7 @@ def __init__( def __repr__(self) -> str: return f'{type(self).__name__}({self._top_path}, {self._global_filters!r}, {self._ignore_case!r})' - def to_dict(self, include_path_filters: bool = True) -> Dict[str, Any]: + def to_dict(self, include_path_filters: bool = True) -> dict[str, Any]: d = { 'path': self._top_path, 'global_filters': [f.to_dict() for f in self._global_filters], @@ -337,7 +336,7 @@ def _load_path(self, path: str) -> Optional[IgnoreFilter]: p = os.path.join(self._top_path, path, self._ignore_file_name) try: self._path_filters[path] = IgnoreFilter.from_path(p, self._ignore_case) - except IOError: + except OSError: self._path_filters[path] = None return self._path_filters[path] @@ -348,6 +347,7 @@ def _find_matching(self, path: str) -> Iterable[Pattern]: path: Path to check Returns: Iterator over Pattern instances + """ if os.path.isabs(path): raise ValueError(f'{path} is an absolute path') @@ -379,6 +379,7 @@ def is_ignored(self, path: str) -> Optional[bool]: True if the path matches an ignore pattern, False if the path is explicitly not ignored, or None if the file does not match any patterns. + """ if hasattr(path, '__fspath__'): path = path.__fspath__() @@ -387,10 +388,8 @@ def is_ignored(self, path: str) -> Optional[bool]: return matches[-1].is_exclude return None - def walk(self, **kwargs) -> Generator[Tuple[str, List[str], List[str]], None, None]: - """A wrapper for os.walk() without ignored files and subdirectories. - kwargs are passed to walk().""" - + def walk(self, **kwargs) -> Generator[tuple[str, list[str], list[str]], None, None]: + """Wrap os.walk() without ignored files and subdirectories and kwargs are passed to walk.""" for dirpath, dirnames, filenames in os.walk(self.path, topdown=True, **kwargs): rel_dirpath = '' if dirpath == self.path else os.path.relpath(dirpath, self.path) @@ -413,6 +412,7 @@ def build( ignore_case: bool = False, ) -> 'IgnoreFilterManager': """Create a IgnoreFilterManager from patterns and paths. + Args: path: The root path for ignore checks. global_ignore_file_paths: A list of file paths to load patterns from. @@ -421,8 +421,10 @@ def build( global_patterns: Global patterns to ignore. ignore_file_name: The per-directory ignore file name. ignore_case: Whether to ignore case in matching. + Returns: A `IgnoreFilterManager` object + """ if not global_ignore_file_paths: global_ignore_file_paths = [] diff --git a/cycode/cli/utils/jwt_utils.py b/cycode/cli/utils/jwt_utils.py index 7bb7df62..c87b7c48 100644 --- a/cycode/cli/utils/jwt_utils.py +++ b/cycode/cli/utils/jwt_utils.py @@ -1,11 +1,11 @@ -from typing import Optional, Tuple +from typing import Optional import jwt _JWT_PAYLOAD_POSSIBLE_USER_ID_FIELD_NAMES = ('userId', 'internalId', 'token-user-id') -def get_user_and_tenant_ids_from_access_token(access_token: str) -> Tuple[Optional[str], Optional[str]]: +def get_user_and_tenant_ids_from_access_token(access_token: str) -> tuple[Optional[str], Optional[str]]: payload = jwt.decode(access_token, options={'verify_signature': False}) user_id = None diff --git a/cycode/cli/utils/path_utils.py b/cycode/cli/utils/path_utils.py index a2d8816b..7d525e56 100644 --- a/cycode/cli/utils/path_utils.py +++ b/cycode/cli/utils/path_utils.py @@ -1,18 +1,18 @@ import json import os -from functools import lru_cache -from typing import TYPE_CHECKING, AnyStr, List, Optional, Union +from functools import cache +from typing import TYPE_CHECKING, AnyStr, Optional, Union -import click +import typer from binaryornot.helpers import is_binary_string -from cycode.cyclient import logger +from cycode.cli.logger import logger if TYPE_CHECKING: from os import PathLike -@lru_cache(maxsize=None) +@cache def is_sub_path(path: str, sub_path: str) -> bool: try: common_path = os.path.commonpath([get_absolute_path(path), get_absolute_path(sub_path)]) @@ -35,7 +35,7 @@ def _get_starting_chunk(filename: str, length: int = 1024) -> Optional[bytes]: try: with open(filename, 'rb') as f: return f.read(length) - except IOError as e: + except OSError as e: logger.debug('Failed to read the starting chunk from file: %s', filename, exc_info=e) return None @@ -68,7 +68,7 @@ def get_file_dir(path: str) -> str: return os.path.dirname(path) -def get_immediate_subdirectories(path: str) -> List[str]: +def get_immediate_subdirectories(path: str) -> list[str]: return [f.name for f in os.scandir(path) if f.is_dir()] @@ -78,7 +78,7 @@ def join_paths(path: str, filename: str) -> str: def get_file_content(file_path: Union[str, 'PathLike']) -> Optional[AnyStr]: try: - with open(file_path, 'r', encoding='UTF-8') as f: + with open(file_path, encoding='UTF-8') as f: return f.read() except (FileNotFoundError, UnicodeDecodeError): return None @@ -106,8 +106,8 @@ def concat_unique_id(filename: str, unique_id: str) -> str: return os.path.join(unique_id, filename) -def get_path_from_context(context: click.Context) -> Optional[str]: - path = context.params.get('path') - if path is None and 'paths' in context.params: - path = context.params['paths'][0] +def get_path_from_context(ctx: typer.Context) -> Optional[str]: + path = ctx.params.get('path') + if path is None and 'paths' in ctx.params: + path = ctx.params['paths'][0] return path diff --git a/cycode/cli/utils/progress_bar.py b/cycode/cli/utils/progress_bar.py index 623222d7..7c2de487 100644 --- a/cycode/cli/utils/progress_bar.py +++ b/cycode/cli/utils/progress_bar.py @@ -1,18 +1,15 @@ from abc import ABC, abstractmethod from enum import auto -from typing import TYPE_CHECKING, Dict, NamedTuple, Optional +from typing import NamedTuple, Optional -import click +from rich.progress import BarColumn, Progress, SpinnerColumn, TaskProgressColumn, TextColumn, TimeElapsedColumn +from cycode.cli.console import console from cycode.cli.utils.enum_utils import AutoCountEnum -from cycode.cyclient.config import get_logger - -if TYPE_CHECKING: - from click._termui_impl import ProgressBar - from click.termui import V as ProgressBarValue +from cycode.logger import get_logger # use LOGGING_LEVEL=DEBUG env var to see debug logs of this module -logger = get_logger('progress bar', control_level_in_runtime=False) +logger = get_logger('Progress Bar', control_level_in_runtime=False) class ProgressBarSection(AutoCountEnum): @@ -32,8 +29,16 @@ class ProgressBarSectionInfo(NamedTuple): _PROGRESS_BAR_LENGTH = 100 +_PROGRESS_BAR_COLUMNS = ( + SpinnerColumn(), + TextColumn('[progress.description]{task.description}'), + TextColumn('{task.fields[right_side_label]}'), + BarColumn(bar_width=None), + TaskProgressColumn(), + TimeElapsedColumn(), +) -ProgressBarSections = Dict[ProgressBarSection, ProgressBarSectionInfo] +ProgressBarSections = dict[ProgressBarSection, ProgressBarSectionInfo] class ScanProgressBarSection(ProgressBarSection): @@ -91,12 +96,6 @@ class BaseProgressBar(ABC): def __init__(self, *args, **kwargs) -> None: pass - @abstractmethod - def __enter__(self) -> 'BaseProgressBar': ... - - @abstractmethod - def __exit__(self, *args, **kwargs) -> None: ... - @abstractmethod def start(self) -> None: ... @@ -110,19 +109,13 @@ def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> def update(self, section: 'ProgressBarSection') -> None: ... @abstractmethod - def update_label(self, label: Optional[str] = None) -> None: ... + def update_right_side_label(self, label: Optional[str] = None) -> None: ... class DummyProgressBar(BaseProgressBar): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - def __enter__(self) -> 'DummyProgressBar': - return self - - def __exit__(self, *args, **kwargs) -> None: - pass - def start(self) -> None: pass @@ -135,7 +128,7 @@ def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> def update(self, section: 'ProgressBarSection') -> None: pass - def update_label(self, label: Optional[str] = None) -> None: + def update_right_side_label(self, label: Optional[str] = None) -> None: pass @@ -145,36 +138,34 @@ def __init__(self, progress_bar_sections: ProgressBarSections) -> None: self._progress_bar_sections = progress_bar_sections - self._progress_bar_context_manager = click.progressbar( - length=_PROGRESS_BAR_LENGTH, - item_show_func=self._progress_bar_item_show_func, - update_min_steps=0, - ) - self._progress_bar: Optional['ProgressBar'] = None - self._run = False - - self._section_lengths: Dict[ProgressBarSection, int] = {} - self._section_values: Dict[ProgressBarSection, int] = {} + self._section_lengths: dict[ProgressBarSection, int] = {} + self._section_values: dict[ProgressBarSection, int] = {} self._current_section_value = 0 self._current_section: ProgressBarSectionInfo = _get_initial_section(self._progress_bar_sections) + self._current_right_side_label = '' - def __enter__(self) -> 'CompositeProgressBar': - self._progress_bar = self._progress_bar_context_manager.__enter__() - self._run = True - return self + self._progress_bar = Progress(*_PROGRESS_BAR_COLUMNS, console=console, refresh_per_second=5, transient=True) + self._progress_bar_task_id = self._progress_bar.add_task( + description=self._current_section.label, + total=_PROGRESS_BAR_LENGTH, + right_side_label=self._current_right_side_label, + ) - def __exit__(self, *args, **kwargs) -> None: - self._progress_bar_context_manager.__exit__(*args, **kwargs) - self._run = False + def _progress_bar_update(self, advance: int = 0) -> None: + self._progress_bar.update( + self._progress_bar_task_id, + advance=advance, + description=self._current_section.label, + right_side_label=self._current_right_side_label, + refresh=True, + ) def start(self) -> None: - if not self._run: - self.__enter__() + self._progress_bar.start() def stop(self) -> None: - if self._run: - self.__exit__(None, None, None) + self._progress_bar.stop() def set_section_length(self, section: 'ProgressBarSection', length: int = 0) -> None: logger.debug('Calling set_section_length, %s', {'section': str(section), 'length': length}) @@ -190,7 +181,7 @@ def _get_section_length(self, section: 'ProgressBarSection') -> int: return section_info.stop_percent - section_info.start_percent def _skip_section(self, section: 'ProgressBarSection') -> None: - self._progress_bar.update(self._get_section_length(section)) + self._progress_bar_update(self._get_section_length(section)) self._maybe_update_current_section() def _increment_section_value(self, section: 'ProgressBarSection', value: int) -> None: @@ -204,14 +195,14 @@ def _increment_section_value(self, section: 'ProgressBarSection', value: int) -> ) def _rerender_progress_bar(self) -> None: - """Used to update label right after changing the progress bar section.""" - self._progress_bar.update(0) + """Use to update label right after changing the progress bar section.""" + self._progress_bar_update() def _increment_progress(self, section: 'ProgressBarSection') -> None: increment_value = self._get_increment_progress_value(section) self._current_section_value += increment_value - self._progress_bar.update(increment_value) + self._progress_bar_update(increment_value) def _maybe_update_current_section(self) -> None: if not self._current_section.section.has_next(): @@ -237,13 +228,7 @@ def _get_increment_progress_value(self, section: 'ProgressBarSection') -> int: return expected_value - self._current_section_value - def _progress_bar_item_show_func(self, _: Optional['ProgressBarValue'] = None) -> str: - return self._current_section.label - def update(self, section: 'ProgressBarSection', value: int = 1) -> None: - if not self._progress_bar: - raise ValueError('Progress bar is not initialized. Call start() first or use "with" statement.') - if section not in self._section_lengths: raise ValueError(f'{section} section is not initialized. Call set_section_length() first.') if section is not self._current_section.section: @@ -255,12 +240,9 @@ def update(self, section: 'ProgressBarSection', value: int = 1) -> None: self._increment_progress(section) self._maybe_update_current_section() - def update_label(self, label: Optional[str] = None) -> None: - if not self._progress_bar: - raise ValueError('Progress bar is not initialized. Call start() first or use "with" statement.') - - self._progress_bar.label = label or '' - self._progress_bar.render_progress() + def update_right_side_label(self, label: Optional[str] = None) -> None: + self._current_right_side_label = f'({label})' if label else '' + self._progress_bar_update() def get_progress_bar(*, hidden: bool, sections: ProgressBarSections) -> BaseProgressBar: @@ -284,9 +266,9 @@ def get_progress_bar(*, hidden: bool, sections: ProgressBarSections) -> BaseProg for _i in range(section_capacity): time.sleep(0.01) - bar.update_label(f'{bar_section} {_i}/{section_capacity}') + bar.update_right_side_label(f'{bar_section} {_i}/{section_capacity}') bar.update(bar_section) - bar.update_label() + bar.update_right_side_label() bar.stop() diff --git a/cycode/cli/utils/scan_batch.py b/cycode/cli/utils/scan_batch.py index 4019b7b0..8bfd7ed0 100644 --- a/cycode/cli/utils/scan_batch.py +++ b/cycode/cli/utils/scan_batch.py @@ -1,17 +1,20 @@ import os from multiprocessing.pool import ThreadPool -from typing import TYPE_CHECKING, Callable, Dict, List, Tuple +from typing import TYPE_CHECKING, Callable from cycode.cli import consts from cycode.cli.models import Document from cycode.cli.utils.progress_bar import ScanProgressBarSection -from cycode.cyclient import logger +from cycode.logger import get_logger if TYPE_CHECKING: from cycode.cli.models import CliError, LocalScanResult from cycode.cli.utils.progress_bar import BaseProgressBar +logger = get_logger('Batching') + + def _get_max_batch_size(scan_type: str) -> int: logger.debug( 'You can customize the batch size by setting the environment variable "%s"', @@ -42,8 +45,8 @@ def _get_max_batch_files_count(_: str) -> int: def split_documents_into_batches( scan_type: str, - documents: List[Document], -) -> List[List[Document]]: + documents: list[Document], +) -> list[list[Document]]: max_size = _get_max_batch_size(scan_type) max_files_count = _get_max_batch_files_count(scan_type) @@ -104,11 +107,11 @@ def _get_threads_count() -> int: def run_parallel_batched_scan( - scan_function: Callable[[List[Document]], Tuple[str, 'CliError', 'LocalScanResult']], + scan_function: Callable[[list[Document]], tuple[str, 'CliError', 'LocalScanResult']], scan_type: str, - documents: List[Document], + documents: list[Document], progress_bar: 'BaseProgressBar', -) -> Tuple[Dict[str, 'CliError'], List['LocalScanResult']]: +) -> tuple[dict[str, 'CliError'], list['LocalScanResult']]: # batching is disabled for SCA; requested by Mor batches = [documents] if scan_type == consts.SCA_SCAN_TYPE else split_documents_into_batches(scan_type, documents) @@ -121,8 +124,8 @@ def run_parallel_batched_scan( # the progress bar could be significant improved (be more dynamic) in the future threads_count = _get_threads_count() - local_scan_results: List['LocalScanResult'] = [] - cli_errors: Dict[str, 'CliError'] = {} + local_scan_results: list[LocalScanResult] = [] + cli_errors: dict[str, CliError] = {} logger.debug('Running parallel batched scan, %s', {'threads_count': threads_count, 'batches_count': len(batches)}) diff --git a/cycode/cli/utils/scan_utils.py b/cycode/cli/utils/scan_utils.py index 77866c4b..8c9dcca7 100644 --- a/cycode/cli/utils/scan_utils.py +++ b/cycode/cli/utils/scan_utils.py @@ -1,11 +1,11 @@ -import click +import typer -def set_issue_detected(context: click.Context, issue_detected: bool) -> None: - context.obj['issue_detected'] = issue_detected +def set_issue_detected(ctx: typer.Context, issue_detected: bool) -> None: + ctx.obj['issue_detected'] = issue_detected -def is_scan_failed(context: click.Context) -> bool: - did_fail = context.obj.get('did_fail') - issue_detected = context.obj.get('issue_detected') +def is_scan_failed(ctx: typer.Context) -> bool: + did_fail = ctx.obj.get('did_fail') + issue_detected = ctx.obj.get('issue_detected') return did_fail or issue_detected diff --git a/cycode/cli/sentry.py b/cycode/cli/utils/sentry.py similarity index 98% rename from cycode/cli/sentry.py rename to cycode/cli/utils/sentry.py index e132bcf8..16b2a982 100644 --- a/cycode/cli/sentry.py +++ b/cycode/cli/utils/sentry.py @@ -11,8 +11,8 @@ from cycode import __version__ from cycode.cli import consts +from cycode.cli.logger import logger from cycode.cli.utils.jwt_utils import get_user_and_tenant_ids_from_access_token -from cycode.cyclient import logger from cycode.cyclient.config import on_premise_installation # when Sentry is blocked on the machine, we want to keep clean output without retries warnings diff --git a/cycode/cli/utils/shell_executor.py b/cycode/cli/utils/shell_executor.py index 5ac79518..2529890b 100644 --- a/cycode/cli/utils/shell_executor.py +++ b/cycode/cli/utils/shell_executor.py @@ -1,17 +1,22 @@ import subprocess -from typing import List, Optional, Union +from typing import Optional, Union import click +import typer -from cycode.cyclient import logger +from cycode.logger import get_logger _SUBPROCESS_DEFAULT_TIMEOUT_SEC = 60 +logger = get_logger('SHELL') + + def shell( - command: Union[str, List[str]], + command: Union[str, list[str]], timeout: int = _SUBPROCESS_DEFAULT_TIMEOUT_SEC, working_directory: Optional[str] = None, + silent_exc_info: bool = False, ) -> Optional[str]: logger.debug('Executing shell command: %s', command) @@ -19,13 +24,19 @@ def shell( result = subprocess.run( # noqa: S603 command, cwd=working_directory, timeout=timeout, check=True, capture_output=True ) + logger.debug('Shell command executed successfully') return result.stdout.decode('UTF-8').strip() except subprocess.CalledProcessError as e: - logger.debug('Error occurred while running shell command', exc_info=e) + if not silent_exc_info: + logger.debug('Error occurred while running shell command', exc_info=e) except subprocess.TimeoutExpired as e: - raise click.Abort(f'Command "{command}" timed out') from e + logger.debug('Command timed out', exc_info=e) + raise typer.Abort(f'Command "{command}" timed out') from e except Exception as e: + if not silent_exc_info: + logger.debug('Unhandled exception occurred while running shell command', exc_info=e) + raise click.ClickException(f'Unhandled exception: {e}') from e return None diff --git a/cycode/cli/utils/string_utils.py b/cycode/cli/utils/string_utils.py index 9dce0026..c3c0c6c6 100644 --- a/cycode/cli/utils/string_utils.py +++ b/cycode/cli/utils/string_utils.py @@ -62,6 +62,6 @@ def shortcut_dependency_paths(dependency_paths_list: str) -> str: result += dependency_paths else: result += f'{dependencies[0]} -> ... -> {dependencies[-1]}' - result += '\n\n' + result += '\n' return result.rstrip().rstrip(',') diff --git a/cycode/cli/utils/task_timer.py b/cycode/cli/utils/task_timer.py index 29e65dc8..4b5e903e 100644 --- a/cycode/cli/utils/task_timer.py +++ b/cycode/cli/utils/task_timer.py @@ -1,19 +1,18 @@ from _thread import interrupt_main from threading import Event, Thread from types import TracebackType -from typing import Callable, Dict, List, Optional, Type +from typing import Callable, Optional class FunctionContext: - def __init__(self, function: Callable, args: Optional[List] = None, kwargs: Optional[Dict] = None) -> None: + def __init__(self, function: Callable, args: Optional[list] = None, kwargs: Optional[dict] = None) -> None: self.function = function self.args = args or [] self.kwargs = kwargs or {} class TimerThread(Thread): - """ - Custom thread class for executing timer in the background + """Custom thread class for executing timer in the background. Members: timeout - the amount of time to count until timeout in seconds @@ -43,8 +42,7 @@ def _call_quit_function(self) -> None: class TimeoutAfter: - """ - A task wrapper for controlling how much time a task should be run before timing out + """A task wrapper for controlling how much time a task should be run before timing out. Use Example: with TimeoutAfter(5, repeat_function=FunctionContext(x), repeat_interval=2): @@ -66,7 +64,7 @@ def __enter__(self) -> None: self.timer.start() def __exit__( - self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + self, exc_type: Optional[type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] ) -> None: if self.timeout: self.timer.stop() diff --git a/cycode/cli/commands/version/version_checker.py b/cycode/cli/utils/version_checker.py similarity index 89% rename from cycode/cli/commands/version/version_checker.py rename to cycode/cli/utils/version_checker.py index c5ec9d4f..8fd1d005 100644 --- a/cycode/cli/commands/version/version_checker.py +++ b/cycode/cli/utils/version_checker.py @@ -2,18 +2,17 @@ import re import time from pathlib import Path -from typing import List, Optional, Tuple - -import click +from typing import Optional +from cycode.cli.console import console from cycode.cli.user_settings.configuration_manager import ConfigurationManager from cycode.cli.utils.path_utils import get_file_content from cycode.cyclient.cycode_client_base import CycodeClientBase def _compare_versions( - current_parts: List[int], - latest_parts: List[int], + current_parts: list[int], + latest_parts: list[int], current_is_pre: bool, latest_is_pre: bool, latest_version: str, @@ -34,6 +33,7 @@ def _compare_versions( Returns: str | None: The latest version string if an update is recommended, None if no update is needed + """ # If current is stable and latest is pre-release, don't suggest update if not current_is_pre and latest_is_pre: @@ -56,6 +56,7 @@ def _compare_versions( class VersionChecker(CycodeClientBase): PYPI_API_URL = 'https://pypi.org/pypi' PYPI_PACKAGE_NAME = 'cycode' + PYPI_REQUEST_TIMEOUT = 1 GIT_CHANGELOG_URL_PREFIX = 'https://github.com/cycodehq/cycode-cli/releases/tag/v' @@ -82,16 +83,19 @@ def get_latest_version(self) -> Optional[str]: Returns: str | None: The latest version string if successful, None if the request fails or the version information is not available. + """ try: - response = self.get(f'{self.PYPI_PACKAGE_NAME}/json') + response = self.get( + f'{self.PYPI_PACKAGE_NAME}/json', timeout=self.PYPI_REQUEST_TIMEOUT, hide_response_content_log=True + ) data = response.json() return data.get('info', {}).get('version') except Exception: return None @staticmethod - def _parse_version(version: str) -> Tuple[List[int], bool]: + def _parse_version(version: str) -> tuple[list[int], bool]: """Parse version string into components and identify if it's a pre-release. Extracts numeric version components and determines if the version is a pre-release @@ -104,6 +108,7 @@ def _parse_version(version: str) -> Tuple[List[int], bool]: tuple: A tuple containing: - List[int]: List of numeric version components - bool: True if this is a pre-release version, False otherwise + """ version_parts = [int(x) for x in re.findall(r'\d+', version)] is_prerelease = 'dev' in version @@ -122,6 +127,7 @@ def _should_check_update(self, is_prerelease: bool) -> bool: Returns: bool: True if an update check should be performed, False otherwise + """ if not os.path.exists(self.cache_file): return True @@ -148,7 +154,7 @@ def _update_last_check(self) -> None: os.makedirs(os.path.dirname(self.cache_file), exist_ok=True) with open(self.cache_file, 'w', encoding='UTF-8') as f: f.write(str(time.time())) - except IOError: + except OSError: pass def check_for_update(self, current_version: str, use_cache: bool = True) -> Optional[str]: @@ -163,6 +169,7 @@ def check_for_update(self, current_version: str, use_cache: bool = True) -> Opti Returns: str | None: The latest version string if an update is recommended, None if no update is needed or if check should be skipped + """ current_parts, current_is_pre = self._parse_version(current_version) @@ -180,7 +187,7 @@ def check_for_update(self, current_version: str, use_cache: bool = True) -> Opti latest_parts, latest_is_pre = self._parse_version(latest_version) return _compare_versions(current_parts, latest_parts, current_is_pre, latest_is_pre, latest_version) - def check_and_notify_update(self, current_version: str, use_color: bool = True, use_cache: bool = True) -> None: + def check_and_notify_update(self, current_version: str, use_cache: bool = True) -> None: """Check for updates and display a notification if a new version is available. Performs the version check and displays a formatted message with update instructions @@ -191,19 +198,19 @@ def check_and_notify_update(self, current_version: str, use_color: bool = True, Args: current_version: Current version of the CLI - use_color: If True, use colored output in the terminal use_cache: If True, use the cached timestamp to determine if an update check is needed + """ latest_version = self.check_for_update(current_version, use_cache) should_update = bool(latest_version) if should_update: update_message = ( - '\nNew version of cycode available! ' - f"{click.style(current_version, fg='yellow')} → {click.style(latest_version, fg='bright_blue')}\n" - f"Changelog: {click.style(f'{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}', fg='bright_blue')}\n" - f"Run {click.style('pip install --upgrade cycode', fg='green')} to update\n" + '\nNew release of Cycode CLI is available: ' + f'[red]{current_version}[/] -> [green]{latest_version}[/]\n' + f'Changelog: [bright_blue]{self.GIT_CHANGELOG_URL_PREFIX}{latest_version}[/]\n' + f'To update, run: [green]pip install --upgrade cycode[/]\n' ) - click.echo(update_message, color=use_color) + console.print(update_message) version_checker = VersionChecker() diff --git a/cycode/cli/utils/yaml_utils.py b/cycode/cli/utils/yaml_utils.py index 251b6c24..c89e1a5c 100644 --- a/cycode/cli/utils/yaml_utils.py +++ b/cycode/cli/utils/yaml_utils.py @@ -1,10 +1,21 @@ import os -from typing import Any, Dict, Hashable, TextIO +from collections.abc import Hashable +from typing import Any, TextIO import yaml -def _yaml_safe_load(file: TextIO) -> Dict[Hashable, Any]: +def _deep_update(source: dict[Hashable, Any], overrides: dict[Hashable, Any]) -> dict[Hashable, Any]: + for key, value in overrides.items(): + if isinstance(value, dict) and value: + source[key] = _deep_update(source.get(key, {}), value) + else: + source[key] = overrides[key] + + return source + + +def _yaml_safe_load(file: TextIO) -> dict[Hashable, Any]: # loader.get_single_data could return None loaded_file = yaml.safe_load(file) if loaded_file is None: @@ -13,28 +24,18 @@ def _yaml_safe_load(file: TextIO) -> Dict[Hashable, Any]: return loaded_file -def read_file(filename: str) -> Dict[Hashable, Any]: +def read_yaml_file(filename: str) -> dict[Hashable, Any]: if not os.path.exists(filename): return {} - with open(filename, 'r', encoding='UTF-8') as file: + with open(filename, encoding='UTF-8') as file: return _yaml_safe_load(file) -def write_file(filename: str, content: Dict[Hashable, Any]) -> None: +def write_yaml_file(filename: str, content: dict[Hashable, Any]) -> None: with open(filename, 'w', encoding='UTF-8') as file: yaml.safe_dump(content, file) -def update_file(filename: str, content: Dict[Hashable, Any]) -> None: - write_file(filename, _deep_update(read_file(filename), content)) - - -def _deep_update(source: Dict[Hashable, Any], overrides: Dict[Hashable, Any]) -> Dict[Hashable, Any]: - for key, value in overrides.items(): - if isinstance(value, dict) and value: - source[key] = _deep_update(source.get(key, {}), value) - else: - source[key] = overrides[key] - - return source +def update_yaml_file(filename: str, content: dict[Hashable, Any]) -> None: + write_yaml_file(filename, _deep_update(read_yaml_file(filename), content)) diff --git a/cycode/config.py b/cycode/config.py new file mode 100644 index 00000000..f4306b31 --- /dev/null +++ b/cycode/config.py @@ -0,0 +1,45 @@ +import logging +import os +from typing import Optional +from urllib.parse import urlparse + +from cycode.cli import consts +from cycode.cyclient import config_dev + +DEFAULT_CONFIGURATION = { + consts.TIMEOUT_ENV_VAR_NAME: 300, + consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, + config_dev.DEV_MODE_ENV_VAR_NAME: 'false', +} + +configuration = dict(DEFAULT_CONFIGURATION, **os.environ) + + +def get_val_as_string(key: str) -> str: + return configuration.get(key) + + +def get_val_as_bool(key: str, default: bool = False) -> bool: + if key not in configuration: + return default + + return configuration[key].lower() in {'true', '1', 'yes', 'y', 'on', 'enabled'} + + +def get_val_as_int(key: str) -> Optional[int]: + val = configuration.get(key) + if not val: + return None + + try: + return int(val) + except ValueError: + return None + + +def is_valid_url(url: str) -> bool: + try: + parsed_url = urlparse(url) + return all([parsed_url.scheme, parsed_url.netloc]) + except ValueError: + return False diff --git a/cycode/cyclient/__init__.py b/cycode/cyclient/__init__.py index 9bea26e9..e69de29b 100644 --- a/cycode/cyclient/__init__.py +++ b/cycode/cyclient/__init__.py @@ -1,5 +0,0 @@ -from cycode.cyclient.config import logger - -__all__ = [ - 'logger', -] diff --git a/cycode/cyclient/auth_client.py b/cycode/cyclient/auth_client.py index 20c80d13..1df7ad9b 100644 --- a/cycode/cyclient/auth_client.py +++ b/cycode/cyclient/auth_client.py @@ -1,9 +1,9 @@ from typing import Optional -from requests import Response +from requests import Request, Response from cycode.cli.exceptions.custom_exceptions import HttpUnauthorizedError, RequestHttpError -from cycode.cyclient import models +from cycode.cyclient import config, models from cycode.cyclient.cycode_client import CycodeClient @@ -13,6 +13,11 @@ class AuthClient: def __init__(self) -> None: self.cycode_client = CycodeClient() + @staticmethod + def build_login_url(code_challenge: str, session_id: str) -> str: + query_params = {'source': 'cycode_cli', 'code_challenge': code_challenge, 'session_id': session_id} + return Request(url=f'{config.cycode_app_url}/account/sign-in', params=query_params).prepare().url + def start_session(self, code_challenge: str) -> models.AuthenticationSession: path = f'{self.AUTH_CONTROLLER_PATH}/start' body = {'code_challenge': code_challenge} diff --git a/cycode/cyclient/config.py b/cycode/cyclient/config.py index 37183195..ec21efb4 100644 --- a/cycode/cyclient/config.py +++ b/cycode/cyclient/config.py @@ -1,102 +1,9 @@ -import logging -import os -import sys -from typing import NamedTuple, Optional, Set, Union -from urllib.parse import urlparse - from cycode.cli import consts from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.config import get_val_as_bool, get_val_as_int, get_val_as_string, is_valid_url from cycode.cyclient import config_dev +from cycode.cyclient.logger import logger - -def _set_io_encodings() -> None: - # set io encoding (for Windows) - sys.stdout.reconfigure(encoding='UTF-8') - sys.stderr.reconfigure(encoding='UTF-8') - - -_set_io_encodings() - -# logs -logging.basicConfig( - stream=sys.stderr, - level=logging.INFO, - format='%(asctime)s [%(name)s] %(levelname)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', -) -logging.getLogger('urllib3').setLevel(logging.WARNING) -logging.getLogger('werkzeug').setLevel(logging.WARNING) -logging.getLogger('schedule').setLevel(logging.WARNING) -logging.getLogger('kubernetes').setLevel(logging.WARNING) -logging.getLogger('binaryornot').setLevel(logging.WARNING) -logging.getLogger('chardet').setLevel(logging.WARNING) -logging.getLogger('git.cmd').setLevel(logging.WARNING) -logging.getLogger('git.util').setLevel(logging.WARNING) - -# configs -DEFAULT_CONFIGURATION = { - consts.TIMEOUT_ENV_VAR_NAME: 300, - consts.LOGGING_LEVEL_ENV_VAR_NAME: logging.INFO, - config_dev.DEV_MODE_ENV_VAR_NAME: 'false', -} - -configuration = dict(DEFAULT_CONFIGURATION, **os.environ) - - -class CreatedLogger(NamedTuple): - logger: logging.Logger - control_level_in_runtime: bool - - -_CREATED_LOGGERS: Set[CreatedLogger] = set() - - -def get_logger_level() -> Optional[Union[int, str]]: - config_level = get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) - return logging.getLevelName(config_level) - - -def get_logger(logger_name: Optional[str] = None, control_level_in_runtime: bool = True) -> logging.Logger: - new_logger = logging.getLogger(logger_name) - new_logger.setLevel(get_logger_level()) - - _CREATED_LOGGERS.add(CreatedLogger(logger=new_logger, control_level_in_runtime=control_level_in_runtime)) - - return new_logger - - -def set_logging_level(level: int) -> None: - for created_logger in _CREATED_LOGGERS: - if created_logger.control_level_in_runtime: - created_logger.logger.setLevel(level) - - -def get_val_as_string(key: str) -> str: - return configuration.get(key) - - -def get_val_as_bool(key: str, default: str = '') -> bool: - val = configuration.get(key, default) - return val.lower() in {'true', '1'} - - -def get_val_as_int(key: str) -> Optional[int]: - val = configuration.get(key) - if val: - return int(val) - - return None - - -def is_valid_url(url: str) -> bool: - try: - parsed_url = urlparse(url) - return all([parsed_url.scheme, parsed_url.netloc]) - except ValueError: - return False - - -logger = get_logger('cycode cli') configuration_manager = ConfigurationManager() cycode_api_url = configuration_manager.get_cycode_api_url() @@ -107,6 +14,14 @@ def is_valid_url(url: str) -> bool: cycode_api_url = consts.DEFAULT_CYCODE_API_URL +cycode_app_url = configuration_manager.get_cycode_app_url() +if not is_valid_url(cycode_app_url): + logger.warning( + 'Invalid Cycode APP URL: %s, using default value (%s)', cycode_app_url, consts.DEFAULT_CYCODE_APP_URL + ) + cycode_app_url = consts.DEFAULT_CYCODE_APP_URL + + def _is_on_premise_installation(cycode_domain: str) -> bool: return not cycode_api_url.endswith(cycode_domain) diff --git a/cycode/cyclient/config.yaml b/cycode/cyclient/config.yaml deleted file mode 100644 index 7b8c1bdc..00000000 --- a/cycode/cyclient/config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -cycode: - api: - base_url: http://api.cycode.com - time_out: 30 -# base_url: http://localhost:5048 #local configuration diff --git a/cycode/cyclient/cycode_client_base.py b/cycode/cyclient/cycode_client_base.py index 3024de89..4b2e2698 100644 --- a/cycode/cyclient/cycode_client_base.py +++ b/cycode/cyclient/cycode_client_base.py @@ -1,11 +1,12 @@ import os import platform import ssl -from typing import Callable, ClassVar, Dict, Optional +from typing import TYPE_CHECKING, Callable, ClassVar, Optional import requests from requests import Response, exceptions from requests.adapters import HTTPAdapter +from tenacity import retry, retry_if_exception, stop_after_attempt, wait_random_exponential from cycode.cli.exceptions.custom_exceptions import ( HttpUnauthorizedError, @@ -13,10 +14,14 @@ RequestError, RequestHttpError, RequestSslError, - RequestTimeout, + RequestTimeoutError, ) -from cycode.cyclient import config, logger +from cycode.cyclient import config from cycode.cyclient.headers import get_cli_user_agent, get_correlation_id +from cycode.cyclient.logger import logger + +if TYPE_CHECKING: + from tenacity import RetryCallState class SystemStorageSslContext(HTTPAdapter): @@ -44,8 +49,49 @@ def _get_request_function() -> Callable: return session.request +_REQUEST_ERRORS_TO_RETRY = ( + RequestTimeoutError, + RequestConnectionError, + exceptions.ChunkedEncodingError, + exceptions.ContentDecodingError, +) +_RETRY_MAX_ATTEMPTS = 3 +_RETRY_STOP_STRATEGY = stop_after_attempt(_RETRY_MAX_ATTEMPTS) +_RETRY_WAIT_STRATEGY = wait_random_exponential(multiplier=1, min=2, max=10) + + +def _retry_before_sleep(retry_state: 'RetryCallState') -> None: + exception_name = 'None' + if retry_state.outcome.failed: + exception = retry_state.outcome.exception() + exception_name = f'{exception.__class__.__name__}' + + logger.debug( + 'Retrying request after error: %s. Attempt %s of %s. Upcoming sleep: %s', + exception_name, + retry_state.attempt_number, + _RETRY_MAX_ATTEMPTS, + retry_state.upcoming_sleep, + ) + + +def _should_retry_exception(exception: BaseException) -> bool: + if 'PYTEST_CURRENT_TEST' in os.environ: + # We are running under pytest, don't retry + return False + + # Don't retry client errors (400, 401, etc.) + if isinstance(exception, RequestHttpError): + return not exception.status_code < 500 + + is_request_error = isinstance(exception, _REQUEST_ERRORS_TO_RETRY) + is_server_error = isinstance(exception, RequestHttpError) and exception.status_code >= 500 + + return is_request_error or is_server_error + + class CycodeClientBase: - MANDATORY_HEADERS: ClassVar[Dict[str, str]] = { + MANDATORY_HEADERS: ClassVar[dict[str, str]] = { 'User-Agent': get_cli_user_agent(), 'X-Correlation-Id': get_correlation_id(), } @@ -71,6 +117,13 @@ def put(self, url_path: str, body: Optional[dict] = None, headers: Optional[dict def get(self, url_path: str, headers: Optional[dict] = None, **kwargs) -> Response: return self._execute(method='get', endpoint=url_path, headers=headers, **kwargs) + @retry( + retry=retry_if_exception(_should_retry_exception), + stop=_RETRY_STOP_STRATEGY, + wait=_RETRY_WAIT_STRATEGY, + reraise=True, + before_sleep=_retry_before_sleep, + ) def _execute( self, method: str, @@ -107,7 +160,7 @@ def _execute( except Exception as e: self._handle_exception(e) - def get_request_headers(self, additional_headers: Optional[dict] = None, **kwargs) -> Dict[str, str]: + def get_request_headers(self, additional_headers: Optional[dict] = None, **kwargs) -> dict[str, str]: if additional_headers is None: return self.MANDATORY_HEADERS.copy() return {**self.MANDATORY_HEADERS, **additional_headers} @@ -117,7 +170,7 @@ def build_full_url(self, url: str, endpoint: str) -> str: def _handle_exception(self, e: Exception) -> None: if isinstance(e, exceptions.Timeout): - raise RequestTimeout from e + raise RequestTimeoutError from e if isinstance(e, exceptions.HTTPError): raise self._get_http_exception(e) from e if isinstance(e, exceptions.SSLError): diff --git a/cycode/cyclient/cycode_dev_based_client.py b/cycode/cyclient/cycode_dev_based_client.py index 347797c3..d8fe1cab 100644 --- a/cycode/cyclient/cycode_dev_based_client.py +++ b/cycode/cyclient/cycode_dev_based_client.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Optional from cycode.cyclient.config import dev_tenant_id from cycode.cyclient.cycode_client_base import CycodeClientBase @@ -12,7 +12,7 @@ class CycodeDevBasedClient(CycodeClientBase): def __init__(self, api_url: str) -> None: super().__init__(api_url) - def get_request_headers(self, additional_headers: Optional[dict] = None, **_) -> Dict[str, str]: + def get_request_headers(self, additional_headers: Optional[dict] = None, **_) -> dict[str, str]: headers = super().get_request_headers(additional_headers=additional_headers) headers['X-Tenant-Id'] = dev_tenant_id diff --git a/cycode/cyclient/headers.py b/cycode/cyclient/headers.py index c6983d32..5d10f69b 100644 --- a/cycode/cyclient/headers.py +++ b/cycode/cyclient/headers.py @@ -4,9 +4,9 @@ from cycode import __version__ from cycode.cli import consts -from cycode.cli.sentry import add_correlation_id_to_scope from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cyclient import logger +from cycode.cli.utils.sentry import add_correlation_id_to_scope +from cycode.cyclient.logger import logger def get_cli_user_agent() -> str: @@ -35,6 +35,7 @@ def get_correlation_id(self) -> str: Used across all requests to correlate logs and metrics. It doesn't depend on client instances. Lifetime is the same as the process. + """ if self._id is None: # example: 16fd2706-8baf-433b-82eb-8c7fada847da diff --git a/cycode/cyclient/logger.py b/cycode/cyclient/logger.py new file mode 100644 index 00000000..b36f036f --- /dev/null +++ b/cycode/cyclient/logger.py @@ -0,0 +1,3 @@ +from cycode.logger import get_logger + +logger = get_logger('CyClient') diff --git a/cycode/cyclient/models.py b/cycode/cyclient/models.py index 2433ef6c..ed649644 100644 --- a/cycode/cyclient/models.py +++ b/cycode/cyclient/models.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Optional from marshmallow import EXCLUDE, Schema, fields, post_load @@ -33,6 +33,15 @@ def __repr__(self) -> str: f'detection_rule_id:{self.detection_rule_id}' ) + @property + def has_alert(self) -> bool: + """Check if the detection has an alert. + + For example, for SCA, it means that the detection is a package vulnerability. + Otherwise, it is a license. + """ + return 'alert' in self.detection_details + class DetectionSchema(Schema): class Meta: @@ -47,36 +56,23 @@ class Meta: detection_rule_id = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> Detection: + def build_dto(self, data: dict[str, Any], **_) -> Detection: return Detection(**data) class DetectionsPerFile(Schema): - def __init__(self, file_name: str, detections: List[Detection], commit_id: Optional[str] = None) -> None: + def __init__(self, file_name: str, detections: list[Detection], commit_id: Optional[str] = None) -> None: super().__init__() self.file_name = file_name self.detections = detections self.commit_id = commit_id -class DetectionsPerFileSchema(Schema): - class Meta: - unknown = EXCLUDE - - file_name = fields.String() - detections = fields.List(fields.Nested(DetectionSchema)) - commit_id = fields.String(allow_none=True) - - @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'DetectionsPerFile': - return DetectionsPerFile(**data) - - class ZippedFileScanResult(Schema): def __init__( self, did_detect: bool, - detections_per_file: List[DetectionsPerFile], + detections_per_file: list[DetectionsPerFile], report_url: Optional[str] = None, scan_id: Optional[str] = None, err: Optional[str] = None, @@ -89,27 +85,12 @@ def __init__( self.err = err -class ZippedFileScanResultSchema(Schema): - class Meta: - unknown = EXCLUDE - - did_detect = fields.Boolean() - scan_id = fields.String() - report_url = fields.String(allow_none=True) - detections_per_file = fields.List(fields.Nested(DetectionsPerFileSchema)) - err = fields.String() - - @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ZippedFileScanResult': - return ZippedFileScanResult(**data) - - class ScanResult(Schema): def __init__( self, did_detect: bool, scan_id: Optional[str] = None, - detections: Optional[List[Detection]] = None, + detections: Optional[list[Detection]] = None, err: Optional[str] = None, ) -> None: super().__init__() @@ -129,7 +110,7 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanResult': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanResult': return ScanResult(**data) @@ -148,7 +129,7 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanInitializationResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanInitializationResponse': return ScanInitializationResponse(**data) @@ -182,7 +163,7 @@ class ScanReportUrlResponseSchema(Schema): report_url = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanReportUrlResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanReportUrlResponse': return ScanReportUrlResponse(**data) @@ -199,12 +180,12 @@ class Meta: err = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ScanDetailsResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ScanDetailsResponse': return ScanDetailsResponse(**data) class K8SResource: - def __init__(self, name: str, resource_type: str, namespace: str, content: Dict) -> None: + def __init__(self, name: str, resource_type: str, namespace: str, content: dict) -> None: super().__init__() self.name = name self.type = resource_type @@ -229,7 +210,7 @@ def to_json(self) -> dict: # FIXME(MarshalX): rename to to_dict? class ResourcesCollection: - def __init__(self, resource_type: str, namespace: str, resources: List[K8SResource], total_count: int) -> None: + def __init__(self, resource_type: str, namespace: str, resources: list[K8SResource], total_count: int) -> None: super().__init__() self.type = resource_type self.namespace = namespace @@ -268,7 +249,7 @@ def __init__(self, name: str, kind: str) -> None: self.kind = kind def __str__(self) -> str: - return 'Name: {0}, Kind: {1}'.format(self.name, self.kind) + return f'Name: {self.name}, Kind: {self.kind}' class AuthenticationSession(Schema): @@ -284,7 +265,7 @@ class Meta: session_id = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'AuthenticationSession': + def build_dto(self, data: dict[str, Any], **_) -> 'AuthenticationSession': return AuthenticationSession(**data) @@ -305,7 +286,7 @@ class Meta: description = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ApiToken': + def build_dto(self, data: dict[str, Any], **_) -> 'ApiToken': return ApiToken(**data) @@ -324,7 +305,7 @@ class Meta: api_token = fields.Nested(ApiTokenSchema, allow_none=True) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'ApiTokenGenerationPollingResponse': + def build_dto(self, data: dict[str, Any], **_) -> 'ApiTokenGenerationPollingResponse': return ApiTokenGenerationPollingResponse(**data) @@ -335,7 +316,7 @@ class UserAgentOptionScheme(Schema): env_version = fields.String(required=True) # ex. 1.78.2 @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'UserAgentOption': + def build_dto(self, data: dict[str, Any], **_) -> 'UserAgentOption': return UserAgentOption(**data) @@ -377,7 +358,7 @@ class Meta: size = fields.Integer() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> SbomReportStorageDetails: + def build_dto(self, data: dict[str, Any], **_) -> SbomReportStorageDetails: return SbomReportStorageDetails(**data) @@ -401,13 +382,13 @@ class Meta: storage_details = fields.Nested(SbomReportStorageDetailsSchema, allow_none=True) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> ReportExecution: + def build_dto(self, data: dict[str, Any], **_) -> ReportExecution: return ReportExecution(**data) @dataclass class SbomReport: - report_executions: List[ReportExecution] + report_executions: list[ReportExecution] class RequestedSbomReportResultSchema(Schema): @@ -417,7 +398,7 @@ class Meta: report_executions = fields.List(fields.Nested(ReportExecutionSchema)) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> SbomReport: + def build_dto(self, data: dict[str, Any], **_) -> SbomReport: return SbomReport(**data) @@ -433,13 +414,13 @@ class Meta: severity = fields.String() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> ClassificationData: + def build_dto(self, data: dict[str, Any], **_) -> ClassificationData: return ClassificationData(**data) @dataclass class DetectionRule: - classification_data: List[ClassificationData] + classification_data: list[ClassificationData] detection_rule_id: str custom_remediation_guidelines: Optional[str] = None remediation_guidelines: Optional[str] = None @@ -461,14 +442,14 @@ class Meta: display_name = fields.String(allow_none=True) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> DetectionRule: + def build_dto(self, data: dict[str, Any], **_) -> DetectionRule: return DetectionRule(**data) @dataclass class ScanResultsSyncFlow: id: str - detection_messages: List[Dict] + detection_messages: list[dict] class ScanResultsSyncFlowSchema(Schema): @@ -479,7 +460,7 @@ class Meta: detection_messages = fields.List(fields.Dict()) @post_load - def build_dto(self, data: Dict[str, Any], **_) -> ScanResultsSyncFlow: + def build_dto(self, data: dict[str, Any], **_) -> ScanResultsSyncFlow: return ScanResultsSyncFlow(**data) @@ -517,5 +498,5 @@ class Meta: ai_large_language_model = fields.Boolean() @post_load - def build_dto(self, data: Dict[str, Any], **_) -> 'SupportedModulesPreferences': + def build_dto(self, data: dict[str, Any], **_) -> 'SupportedModulesPreferences': return SupportedModulesPreferences(**data) diff --git a/cycode/cyclient/report_client.py b/cycode/cyclient/report_client.py index fa8e0c3f..e8107827 100644 --- a/cycode/cyclient/report_client.py +++ b/cycode/cyclient/report_client.py @@ -1,6 +1,6 @@ import dataclasses import json -from typing import List, Optional +from typing import Optional from requests import Response @@ -97,5 +97,5 @@ def parse_requested_sbom_report_response(response: Response) -> models.SbomRepor return models.RequestedSbomReportResultSchema().load(response.json()) @staticmethod - def parse_execution_status_response(response: Response) -> List[models.ReportExecutionSchema]: + def parse_execution_status_response(response: Response) -> list[models.ReportExecutionSchema]: return models.ReportExecutionSchema().load(response.json(), many=True) diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index 31abba17..e0bf8131 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -1,5 +1,7 @@ import json -from typing import TYPE_CHECKING, List, Optional, Set, Union +from copy import deepcopy +from typing import TYPE_CHECKING, Union +from uuid import UUID from requests import Response @@ -21,34 +23,12 @@ def __init__( self.scan_cycode_client = scan_cycode_client self.scan_config = scan_config - self._SCAN_SERVICE_CONTROLLER_PATH = 'api/v1/scan' self._SCAN_SERVICE_CLI_CONTROLLER_PATH = 'api/v1/cli-scan' - - self._DETECTIONS_SERVICE_CONTROLLER_PATH = 'api/v1/detections' self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH = 'api/v1/detections/cli' - self._POLICIES_SERVICE_CONTROLLER_PATH_V3 = 'api/v3/policies' self._hide_response_log = hide_response_log - def get_scan_controller_path(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if not should_use_scan_service and scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - # we don't use async flow for IaC scan yet - return self._SCAN_SERVICE_CONTROLLER_PATH - if not should_use_scan_service and scan_type == consts.SECRET_SCAN_TYPE: - # if a secret scan goes to detector directly, we should not use CLI controller. - # CLI controller belongs to the scan service only - return self._SCAN_SERVICE_CONTROLLER_PATH - - return self._SCAN_SERVICE_CLI_CONTROLLER_PATH - - def get_detections_service_controller_path(self, scan_type: str) -> str: - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - # we don't use async flow for IaC scan yet - return self._DETECTIONS_SERVICE_CONTROLLER_PATH - - return self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH - @staticmethod def get_scan_flow_type(should_use_sync_flow: bool = False) -> str: if should_use_sync_flow: @@ -56,13 +36,10 @@ def get_scan_flow_type(should_use_sync_flow: bool = False) -> str: return '' - def get_scan_service_url_path( - self, scan_type: str, should_use_scan_service: bool = False, should_use_sync_flow: bool = False - ) -> str: - service_path = self.scan_config.get_service_name(scan_type, should_use_scan_service) - controller_path = self.get_scan_controller_path(scan_type, should_use_scan_service) + def get_scan_service_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: + service_path = self.scan_config.get_service_name(scan_type) flow_type = self.get_scan_flow_type(should_use_sync_flow) - return f'{service_path}/{controller_path}{flow_type}' + return f'{service_path}/{self._SCAN_SERVICE_CLI_CONTROLLER_PATH}{flow_type}' def content_scan(self, scan_type: str, file_name: str, content: str, is_git_diff: bool = True) -> models.ScanResult: path = f'{self.get_scan_service_url_path(scan_type)}/content' @@ -72,27 +49,6 @@ def content_scan(self, scan_type: str, file_name: str, content: str, is_git_diff ) return self.parse_scan_response(response) - def get_zipped_file_scan_url_path(self, scan_type: str) -> str: - return f'{self.get_scan_service_url_path(scan_type)}/zipped-file' - - def zipped_file_scan( - self, scan_type: str, zip_file: InMemoryZip, scan_id: str, scan_parameters: dict, is_git_diff: bool = False - ) -> models.ZippedFileScanResult: - files = {'file': ('multiple_files_scan.zip', zip_file.read())} - - response = self.scan_cycode_client.post( - url_path=self.get_zipped_file_scan_url_path(scan_type), - data={'scan_id': scan_id, 'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)}, - files=files, - hide_response_content_log=self._hide_response_log, - ) - - return self.parse_zipped_file_scan_response(response) - - def get_scan_report_url(self, scan_id: str, scan_type: str) -> models.ScanReportUrlResponse: - response = self.scan_cycode_client.get(url_path=self.get_scan_report_url_path(scan_id, scan_type)) - return models.ScanReportUrlResponseSchema().build_dto(response.json()) - def get_scan_aggregation_report_url(self, aggregation_id: str, scan_type: str) -> models.ScanReportUrlResponse: response = self.scan_cycode_client.get( url_path=self.get_scan_aggregation_report_url_path(aggregation_id, scan_type) @@ -102,16 +58,12 @@ def get_scan_aggregation_report_url(self, aggregation_id: str, scan_type: str) - def get_zipped_file_scan_async_url_path(self, scan_type: str, should_use_sync_flow: bool = False) -> str: async_scan_type = self.scan_config.get_async_scan_type(scan_type) async_entity_type = self.scan_config.get_async_entity_type(scan_type) - scan_service_url_path = self.get_scan_service_url_path( - scan_type, should_use_scan_service=True, should_use_sync_flow=should_use_sync_flow - ) + scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_sync_flow=should_use_sync_flow) return f'{scan_service_url_path}/{async_scan_type}/{async_entity_type}' def get_zipped_file_scan_sync_url_path(self, scan_type: str) -> str: server_scan_type = self.scan_config.get_async_scan_type(scan_type) - scan_service_url_path = self.get_scan_service_url_path( - scan_type, should_use_scan_service=True, should_use_sync_flow=True - ) + scan_service_url_path = self.get_scan_service_url_path(scan_type, should_use_sync_flow=True) return f'{scan_service_url_path}/{server_scan_type}/repository' def zipped_file_scan_sync( @@ -123,6 +75,7 @@ def zipped_file_scan_sync( ) -> models.ScanResultsSyncFlow: files = {'file': ('multiple_files_scan.zip', zip_file.read())} + scan_parameters = deepcopy(scan_parameters) # avoid mutating the original dict if 'report' in scan_parameters: del scan_parameters['report'] # BE raises validation error instead of ignoring it @@ -179,16 +132,10 @@ def multiple_zipped_file_scan_async( return models.ScanInitializationResponseSchema().load(response.json()) def get_scan_details_path(self, scan_type: str, scan_id: str) -> str: - return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service=True)}/{scan_id}' - - def get_scan_report_url_path(self, scan_id: str, scan_type: str) -> str: - return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service=True)}/reportUrl/{scan_id}' + return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}' def get_scan_aggregation_report_url_path(self, aggregation_id: str, scan_type: str) -> str: - return ( - f'{self.get_scan_service_url_path(scan_type, should_use_scan_service=True)}' - f'/reportUrlByAggregationId/{aggregation_id}' - ) + return f'{self.get_scan_service_url_path(scan_type)}/reportUrlByAggregationId/{aggregation_id}' def get_scan_details(self, scan_type: str, scan_id: str) -> models.ScanDetailsResponse: path = self.get_scan_details_path(scan_type, scan_id) @@ -210,8 +157,8 @@ def get_supported_modules_preferences(self) -> models.SupportedModulesPreference def get_ai_remediation_path(detection_id: str) -> str: return f'scm-remediator/api/v1/ContentRemediation/preview/{detection_id}' - def get_ai_remediation(self, detection_id: str, *, fix: bool = False) -> str: - path = self.get_ai_remediation_path(detection_id) + def get_ai_remediation(self, detection_id: UUID, *, fix: bool = False) -> str: + path = self.get_ai_remediation_path(detection_id.hex) data = { 'resolving_parameters': { @@ -231,7 +178,7 @@ def get_ai_remediation(self, detection_id: str, *, fix: bool = False) -> str: @staticmethod def _get_policy_type_by_scan_type(scan_type: str) -> str: scan_type_to_policy_type = { - consts.INFRA_CONFIGURATION_SCAN_TYPE: 'IaC', + consts.IAC_SCAN_TYPE: 'IaC', consts.SCA_SCAN_TYPE: 'SCA', consts.SECRET_SCAN_TYPE: 'SecretDetection', consts.SAST_SCAN_TYPE: 'SAST', @@ -243,10 +190,10 @@ def _get_policy_type_by_scan_type(scan_type: str) -> str: return scan_type_to_policy_type[scan_type] @staticmethod - def parse_detection_rules_response(response: Response) -> List[models.DetectionRule]: + def parse_detection_rules_response(response: Response) -> list[models.DetectionRule]: return models.DetectionRuleSchema().load(response.json(), many=True) - def get_detection_rules(self, detection_rules_ids: Union[Set[str], List[str]]) -> List[models.DetectionRule]: + def get_detection_rules(self, detection_rules_ids: Union[set[str], list[str]]) -> list[models.DetectionRule]: response = self.scan_cycode_client.get( url_path=self.get_detection_rules_path(), params={'ids': detection_rules_ids}, @@ -255,21 +202,13 @@ def get_detection_rules(self, detection_rules_ids: Union[Set[str], List[str]]) - return self.parse_detection_rules_response(response) - def get_scan_detections_path(self, scan_type: str) -> str: - return f'{self.scan_config.get_detections_prefix()}/{self.get_detections_service_controller_path(scan_type)}' + def get_scan_detections_path(self) -> str: + return f'{self.scan_config.get_detections_prefix()}/{self._DETECTIONS_SERVICE_CLI_CONTROLLER_PATH}' - @staticmethod - def get_scan_detections_list_path_suffix(scan_type: str) -> str: - # we don't use async flow for IaC scan yet - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - return '' - - return '/detections' + def get_scan_detections_list_path(self) -> str: + return f'{self.get_scan_detections_path()}/detections' - def get_scan_detections_list_path(self, scan_type: str) -> str: - return f'{self.get_scan_detections_path(scan_type)}{self.get_scan_detections_list_path_suffix(scan_type)}' - - def get_scan_raw_detections(self, scan_type: str, scan_id: str) -> List[dict]: + def get_scan_raw_detections(self, scan_id: str) -> list[dict]: params = {'scan_id': scan_id} page_size = 200 @@ -283,7 +222,7 @@ def get_scan_raw_detections(self, scan_type: str, scan_id: str) -> List[dict]: params['page_number'] = page_number response = self.scan_cycode_client.get( - url_path=self.get_scan_detections_list_path(scan_type), + url_path=self.get_scan_detections_list_path(), params=params, hide_response_content_log=self._hide_response_log, ).json() @@ -294,45 +233,15 @@ def get_scan_raw_detections(self, scan_type: str, scan_id: str) -> List[dict]: return raw_detections - def commit_range_zipped_file_scan( - self, scan_type: str, zip_file: InMemoryZip, scan_id: str - ) -> models.ZippedFileScanResult: - url_path = f'{self.get_scan_service_url_path(scan_type)}/commit-range-zipped-file' - files = {'file': ('multiple_files_scan.zip', zip_file.read())} - response = self.scan_cycode_client.post( - url_path=url_path, data={'scan_id': scan_id}, files=files, hide_response_content_log=self._hide_response_log - ) - return self.parse_zipped_file_scan_response(response) + def get_report_scan_status_path(self, scan_type: str, scan_id: str) -> str: + return f'{self.get_scan_service_url_path(scan_type)}/{scan_id}/status' - def get_report_scan_status_path(self, scan_type: str, scan_id: str, should_use_scan_service: bool = False) -> str: - return f'{self.get_scan_service_url_path(scan_type, should_use_scan_service)}/{scan_id}/status' - - def report_scan_status( - self, scan_type: str, scan_id: str, scan_status: dict, should_use_scan_service: bool = False - ) -> None: + def report_scan_status(self, scan_type: str, scan_id: str, scan_status: dict) -> None: self.scan_cycode_client.post( - url_path=self.get_report_scan_status_path( - scan_type, scan_id, should_use_scan_service=should_use_scan_service - ), + url_path=self.get_report_scan_status_path(scan_type, scan_id), body=scan_status, ) @staticmethod def parse_scan_response(response: Response) -> models.ScanResult: return models.ScanResultSchema().load(response.json()) - - @staticmethod - def parse_zipped_file_scan_response(response: Response) -> models.ZippedFileScanResult: - return models.ZippedFileScanResultSchema().load(response.json()) - - @staticmethod - def get_service_name(scan_type: str) -> Optional[str]: - # TODO(MarshalX): get_service_name should be removed from ScanClient? Because it exists in ScanConfig - if scan_type == consts.SECRET_SCAN_TYPE: - return 'secret' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - return 'iac' - if scan_type == consts.SCA_SCAN_TYPE or scan_type == consts.SAST_SCAN_TYPE: - return 'scans' - - return None diff --git a/cycode/cyclient/scan_config_base.py b/cycode/cyclient/scan_config_base.py index 1ff1da6c..d60068ce 100644 --- a/cycode/cyclient/scan_config_base.py +++ b/cycode/cyclient/scan_config_base.py @@ -5,13 +5,13 @@ class ScanConfigBase(ABC): @abstractmethod - def get_service_name(self, scan_type: str, should_use_scan_service: bool = False) -> str: ... + def get_service_name(self, scan_type: str) -> str: ... @staticmethod def get_async_scan_type(scan_type: str) -> str: if scan_type == consts.SECRET_SCAN_TYPE: return 'Secrets' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: + if scan_type == consts.IAC_SCAN_TYPE: return 'InfraConfiguration' return scan_type.upper() @@ -28,32 +28,16 @@ def get_detections_prefix(self) -> str: ... class DevScanConfig(ScanConfigBase): - def get_service_name(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if should_use_scan_service: - return '5004' - if scan_type == consts.SECRET_SCAN_TYPE: - return '5025' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - return '5026' - - # sca and sast - return '5004' + def get_service_name(self, scan_type: str) -> str: + return '5004' # scan service def get_detections_prefix(self) -> str: - return '5016' + return '5016' # detections service class DefaultScanConfig(ScanConfigBase): - def get_service_name(self, scan_type: str, should_use_scan_service: bool = False) -> str: - if should_use_scan_service: - return 'scans' - if scan_type == consts.SECRET_SCAN_TYPE: - return 'secret' - if scan_type == consts.INFRA_CONFIGURATION_SCAN_TYPE: - return 'iac' - - # sca and sast - return 'scans' + def get_service_name(self, scan_type: str) -> str: + return 'scans' # scan service def get_detections_prefix(self) -> str: return 'detections' diff --git a/cycode/logger.py b/cycode/logger.py new file mode 100644 index 00000000..0ec6023f --- /dev/null +++ b/cycode/logger.py @@ -0,0 +1,65 @@ +import logging +import sys +from typing import NamedTuple, Optional, Union + +import click +import typer +from rich.logging import RichHandler + +from cycode.cli import consts +from cycode.cli.console import console_err +from cycode.config import get_val_as_string + + +def _set_io_encodings() -> None: + # set io encoding (for Windows) + sys.stdout.reconfigure(encoding='UTF-8') + sys.stderr.reconfigure(encoding='UTF-8') + + +_set_io_encodings() + +_RICH_LOGGING_HANDLER = RichHandler(console=console_err, rich_tracebacks=True, tracebacks_suppress=[click, typer]) + +logging.basicConfig( + level=logging.INFO, + format='[%(name)s] %(message)s', + handlers=[_RICH_LOGGING_HANDLER], +) + +logging.getLogger('urllib3').setLevel(logging.WARNING) +logging.getLogger('werkzeug').setLevel(logging.WARNING) +logging.getLogger('schedule').setLevel(logging.WARNING) +logging.getLogger('kubernetes').setLevel(logging.WARNING) +logging.getLogger('binaryornot').setLevel(logging.WARNING) +logging.getLogger('chardet').setLevel(logging.WARNING) +logging.getLogger('git.cmd').setLevel(logging.WARNING) +logging.getLogger('git.util').setLevel(logging.WARNING) + + +class CreatedLogger(NamedTuple): + logger: logging.Logger + control_level_in_runtime: bool + + +_CREATED_LOGGERS: set[CreatedLogger] = set() + + +def get_logger_level() -> Optional[Union[int, str]]: + config_level = get_val_as_string(consts.LOGGING_LEVEL_ENV_VAR_NAME) + return logging.getLevelName(config_level) + + +def get_logger(logger_name: Optional[str] = None, control_level_in_runtime: bool = True) -> logging.Logger: + new_logger = logging.getLogger(logger_name) + new_logger.setLevel(get_logger_level()) + + _CREATED_LOGGERS.add(CreatedLogger(logger=new_logger, control_level_in_runtime=control_level_in_runtime)) + + return new_logger + + +def set_logging_level(level: int) -> None: + for created_logger in _CREATED_LOGGERS: + if created_logger.control_level_in_runtime: + created_logger.logger.setLevel(level) diff --git a/poetry.lock b/poetry.lock index c97b44a9..65e6a971 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "altgraph" @@ -6,6 +6,8 @@ version = "0.17.4" description = "Python graph (network) package" optional = false python-versions = "*" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, @@ -17,6 +19,7 @@ version = "1.3.0" description = "Better dates & times for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "arrow-1.3.0-py3-none-any.whl", hash = "sha256:c728b120ebc00eb84e01882a6f5e7927a53960aa990ce7dd2b10f39005a67f80"}, {file = "arrow-1.3.0.tar.gz", hash = "sha256:d4540617648cb5f895730f1ad8c82a65f2dad0166f57b75f3ca54759c4d67a85"}, @@ -36,6 +39,7 @@ version = "0.4.4" description = "Ultra-lightweight pure Python package to check if a file is binary or text." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "binaryornot-0.4.4-py2.py3-none-any.whl", hash = "sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4"}, {file = "binaryornot-0.4.4.tar.gz", hash = "sha256:359501dfc9d40632edc9fac890e19542db1a287bbcfa58175b66658392018061"}, @@ -46,13 +50,14 @@ chardet = ">=3.0.2" [[package]] name = "certifi" -version = "2024.8.30" +version = "2025.4.26" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" +groups = ["main", "test"] files = [ - {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, - {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, + {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, + {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, ] [[package]] @@ -61,6 +66,7 @@ version = "5.2.0" description = "Universal encoding detector for Python 3" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, @@ -68,127 +74,116 @@ files = [ [[package]] name = "charset-normalizer" -version = "3.4.0" +version = "3.4.1" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.7" +groups = ["main", "test"] files = [ - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win32.whl", hash = "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc"}, - {file = "charset_normalizer-3.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win32.whl", hash = "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99"}, - {file = "charset_normalizer-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win32.whl", hash = "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7"}, - {file = "charset_normalizer-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win32.whl", hash = "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67"}, - {file = "charset_normalizer-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win32.whl", hash = "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149"}, - {file = "charset_normalizer-3.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win32.whl", hash = "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613"}, - {file = "charset_normalizer-3.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win32.whl", hash = "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2"}, - {file = "charset_normalizer-3.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca"}, - {file = "charset_normalizer-3.4.0-py3-none-any.whl", hash = "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079"}, - {file = "charset_normalizer-3.4.0.tar.gz", hash = "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f"}, + {file = "charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b"}, + {file = "charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35"}, + {file = "charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407"}, + {file = "charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win32.whl", hash = "sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487"}, + {file = "charset_normalizer-3.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e"}, + {file = "charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5"}, + {file = "charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765"}, + {file = "charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85"}, + {file = "charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3"}, ] [[package]] name = "click" -version = "8.1.7" +version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, - {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, + {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, + {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, ] [package.dependencies] @@ -200,6 +195,7 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "test"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -211,6 +207,7 @@ version = "7.2.7" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, @@ -275,7 +272,7 @@ files = [ ] [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] [[package]] name = "dunamai" @@ -283,6 +280,7 @@ version = "1.21.2" description = "Dynamic version generation" optional = false python-versions = ">=3.5" +groups = ["executable"] files = [ {file = "dunamai-1.21.2-py3-none-any.whl", hash = "sha256:87db76405bf9366f9b4925ff5bb1db191a9a1bd9f9693f81c4d3abb8298be6f0"}, {file = "dunamai-1.21.2.tar.gz", hash = "sha256:05827fb5f032f5596bfc944b23f613c147e676de118681f3bb1559533d8a65c4"}, @@ -297,6 +295,8 @@ version = "1.2.2" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["test"] +markers = "python_version < \"3.11\"" files = [ {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, @@ -307,13 +307,14 @@ test = ["pytest (>=6)"] [[package]] name = "gitdb" -version = "4.0.11" +version = "4.0.12" description = "Git Object Database" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, - {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, + {file = "gitdb-4.0.12-py3-none-any.whl", hash = "sha256:67073e15955400952c6565cc3e707c554a4eea2e428946f7a4c162fab9bd9bcf"}, + {file = "gitdb-4.0.12.tar.gz", hash = "sha256:5ef71f855d191a3326fcfbc0d5da835f26b13fbcba60c32c21091c349ffdb571"}, ] [package.dependencies] @@ -321,21 +322,22 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.43" +version = "3.1.44" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "GitPython-3.1.43-py3-none-any.whl", hash = "sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff"}, - {file = "GitPython-3.1.43.tar.gz", hash = "sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c"}, + {file = "GitPython-3.1.44-py3-none-any.whl", hash = "sha256:9e0e10cda9bed1ee64bc9a6de50e7e38a9c9943241cd7f585f6df3ed28011110"}, + {file = "gitpython-3.1.44.tar.gz", hash = "sha256:c87e30b26253bf5418b01b0660f818967f3c503193838337fe5e573331249269"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" [package.extras] -doc = ["sphinx (==4.3.2)", "sphinx-autodoc-typehints", "sphinx-rtd-theme", "sphinxcontrib-applehelp (>=1.0.2,<=1.0.4)", "sphinxcontrib-devhelp (==1.0.2)", "sphinxcontrib-htmlhelp (>=2.0.0,<=2.0.1)", "sphinxcontrib-qthelp (==1.0.3)", "sphinxcontrib-serializinghtml (==1.1.5)"] -test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions"] +doc = ["sphinx (>=7.1.2,<7.2)", "sphinx-autodoc-typehints", "sphinx_rtd_theme"] +test = ["coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock ; python_version < \"3.8\"", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "typing-extensions ; python_version < \"3.11\""] [[package]] name = "idna" @@ -343,6 +345,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main", "test"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -353,36 +356,39 @@ all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2 [[package]] name = "importlib-metadata" -version = "8.5.0" +version = "8.7.0" description = "Read metadata from Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["executable"] +markers = "python_version < \"3.10\"" files = [ - {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, - {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, + {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, + {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, ] [package.dependencies] zipp = ">=3.20" [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] perf = ["ipython"] -test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] [[package]] name = "iniconfig" -version = "2.0.0" +version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +groups = ["test"] files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, ] [[package]] @@ -391,6 +397,8 @@ version = "1.16.3" description = "Mach-O header analysis and editing" optional = false python-versions = "*" +groups = ["executable"] +markers = "python_version < \"3.13\" and sys_platform == \"darwin\"" files = [ {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, @@ -405,6 +413,7 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -429,6 +438,7 @@ version = "3.22.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, @@ -448,6 +458,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -459,6 +470,7 @@ version = "4.0.3" description = "Rolling backport of unittest.mock for all Pythons" optional = false python-versions = ">=3.6" +groups = ["test"] files = [ {file = "mock-4.0.3-py3-none-any.whl", hash = "sha256:122fcb64ee37cfad5b3f48d7a7d51875d7031aaf3d8be7c42e2bee25044eee62"}, {file = "mock-4.0.3.tar.gz", hash = "sha256:7d3fbbde18228f4ff2f1f119a45cdffa458b4c0dee32eb4d2bb2f82554bac7bc"}, @@ -471,13 +483,14 @@ test = ["pytest (<5.4)", "pytest-cov"] [[package]] name = "packaging" -version = "24.2" +version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "executable", "test"] files = [ - {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, - {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, ] [[package]] @@ -486,6 +499,7 @@ version = "1.18.1" description = "Library to parse and apply unified diffs." optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "patch-ng-1.18.1.tar.gz", hash = "sha256:52fd46ee46f6c8667692682c1fd7134edc65a2d2d084ebec1d295a6087fc0291"}, ] @@ -496,6 +510,8 @@ version = "2024.8.26" description = "Python PE parsing module" optional = false python-versions = ">=3.6.0" +groups = ["executable"] +markers = "python_version < \"3.13\" and sys_platform == \"win32\"" files = [ {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, @@ -507,6 +523,7 @@ version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" +groups = ["test"] files = [ {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, @@ -518,24 +535,26 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyfakefs" -version = "5.7.2" +version = "5.7.4" description = "pyfakefs implements a fake file system that mocks the Python file system modules." optional = false python-versions = ">=3.7" +groups = ["test"] files = [ - {file = "pyfakefs-5.7.2-py3-none-any.whl", hash = "sha256:e1527b0e8e4b33be52f0b024ca1deb269c73eecd68457c6b0bf608d6dab12ebd"}, - {file = "pyfakefs-5.7.2.tar.gz", hash = "sha256:40da84175c5af8d9c4f3b31800b8edc4af1e74a212671dd658b21cc881c60000"}, + {file = "pyfakefs-5.7.4-py3-none-any.whl", hash = "sha256:3e763d700b91c54ade6388be2cfa4e521abc00e34f7defb84ee511c73031f45f"}, + {file = "pyfakefs-5.7.4.tar.gz", hash = "sha256:4971e65cc80a93a1e6f1e3a4654909c0c493186539084dc9301da3d68c8878fe"}, ] [[package]] name = "pygments" -version = "2.18.0" +version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, ] [package.extras] @@ -547,6 +566,8 @@ version = "5.13.2" description = "PyInstaller bundles a Python application and all its dependencies into a single package." optional = false python-versions = "<3.13,>=3.7" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ {file = "pyinstaller-5.13.2-py3-none-macosx_10_13_universal2.whl", hash = "sha256:16cbd66b59a37f4ee59373a003608d15df180a0d9eb1a29ff3bfbfae64b23d0f"}, {file = "pyinstaller-5.13.2-py3-none-manylinux2014_aarch64.whl", hash = "sha256:8f6dd0e797ae7efdd79226f78f35eb6a4981db16c13325e962a83395c0ec7420"}, @@ -576,29 +597,32 @@ hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] [[package]] name = "pyinstaller-hooks-contrib" -version = "2024.10" +version = "2025.3" description = "Community maintained hooks for PyInstaller" optional = false python-versions = ">=3.8" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ - {file = "pyinstaller_hooks_contrib-2024.10-py3-none-any.whl", hash = "sha256:ad47db0e153683b4151e10d231cb91f2d93c85079e78d76d9e0f57ac6c8a5e10"}, - {file = "pyinstaller_hooks_contrib-2024.10.tar.gz", hash = "sha256:8a46655e5c5b0186b5e527399118a9b342f10513eb1425c483fa4f6d02e8800c"}, + {file = "pyinstaller_hooks_contrib-2025.3-py3-none-any.whl", hash = "sha256:70cba46b1a6b82ae9104f074c25926e31f3dde50ff217434d1d660355b949683"}, + {file = "pyinstaller_hooks_contrib-2025.3.tar.gz", hash = "sha256:af129da5cd6219669fbda360e295cc822abac55b7647d03fec63a8fcf0a608cf"}, ] [package.dependencies] -importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} +importlib_metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} packaging = ">=22.0" setuptools = ">=42.0.0" [[package]] name = "pyjwt" -version = "2.9.0" +version = "2.10.1" description = "JSON Web Token implementation in Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["main"] files = [ - {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, - {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] [package.extras] @@ -613,6 +637,7 @@ version = "7.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "pytest-7.3.2-py3-none-any.whl", hash = "sha256:cdcbd012c9312258922f8cd3f1b62a6580fdced17db6014896053d47cddf9295"}, {file = "pytest-7.3.2.tar.gz", hash = "sha256:ee990a3cc55ba808b80795a79944756f315c67c12b56abd3ac993a7b8c17030b"}, @@ -635,6 +660,7 @@ version = "3.10.0" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, @@ -652,6 +678,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -666,6 +693,8 @@ version = "0.2.3" description = "A (partial) reimplementation of pywin32 using ctypes/cffi" optional = false python-versions = ">=3.6" +groups = ["executable"] +markers = "python_version < \"3.13\" and sys_platform == \"win32\"" files = [ {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, @@ -677,6 +706,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["main", "test"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -739,6 +769,7 @@ version = "2.32.3" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main", "test"] files = [ {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, @@ -760,6 +791,7 @@ version = "0.23.3" description = "A utility library for mocking out the `requests` Python library." optional = false python-versions = ">=3.7" +groups = ["test"] files = [ {file = "responses-0.23.3-py3-none-any.whl", hash = "sha256:e6fbcf5d82172fecc0aa1860fd91e58cbfd96cee5e96da5b63fa6eb3caa10dd3"}, {file = "responses-0.23.3.tar.gz", hash = "sha256:205029e1cb334c21cb4ec64fc7599be48b859a0fd381a42443cdd600bfe8b16a"}, @@ -772,7 +804,7 @@ types-PyYAML = "*" urllib3 = ">=1.25.10,<3.0" [package.extras] -tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli ; python_version < \"3.11\"", "tomli-w", "types-requests"] [[package]] name = "rich" @@ -780,6 +812,7 @@ version = "13.9.4" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, @@ -795,40 +828,42 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "ruff" -version = "0.6.9" +version = "0.11.7" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ - {file = "ruff-0.6.9-py3-none-linux_armv6l.whl", hash = "sha256:064df58d84ccc0ac0fcd63bc3090b251d90e2a372558c0f057c3f75ed73e1ccd"}, - {file = "ruff-0.6.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:140d4b5c9f5fc7a7b074908a78ab8d384dd7f6510402267bc76c37195c02a7ec"}, - {file = "ruff-0.6.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53fd8ca5e82bdee8da7f506d7b03a261f24cd43d090ea9db9a1dc59d9313914c"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645d7d8761f915e48a00d4ecc3686969761df69fb561dd914a773c1a8266e14e"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eae02b700763e3847595b9d2891488989cac00214da7f845f4bcf2989007d577"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d5ccc9e58112441de8ad4b29dcb7a86dc25c5f770e3c06a9d57e0e5eba48829"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:417b81aa1c9b60b2f8edc463c58363075412866ae4e2b9ab0f690dc1e87ac1b5"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c866b631f5fbce896a74a6e4383407ba7507b815ccc52bcedabb6810fdb3ef7"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b118afbb3202f5911486ad52da86d1d52305b59e7ef2031cea3425142b97d6f"}, - {file = "ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67267654edc23c97335586774790cde402fb6bbdb3c2314f1fc087dee320bfa"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3ef0cc774b00fec123f635ce5c547dac263f6ee9fb9cc83437c5904183b55ceb"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:12edd2af0c60fa61ff31cefb90aef4288ac4d372b4962c2864aeea3a1a2460c0"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:55bb01caeaf3a60b2b2bba07308a02fca6ab56233302406ed5245180a05c5625"}, - {file = "ruff-0.6.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:925d26471fa24b0ce5a6cdfab1bb526fb4159952385f386bdcc643813d472039"}, - {file = "ruff-0.6.9-py3-none-win32.whl", hash = "sha256:eb61ec9bdb2506cffd492e05ac40e5bc6284873aceb605503d8494180d6fc84d"}, - {file = "ruff-0.6.9-py3-none-win_amd64.whl", hash = "sha256:785d31851c1ae91f45b3d8fe23b8ae4b5170089021fbb42402d811135f0b7117"}, - {file = "ruff-0.6.9-py3-none-win_arm64.whl", hash = "sha256:a9641e31476d601f83cd602608739a0840e348bda93fec9f1ee816f8b6798b93"}, - {file = "ruff-0.6.9.tar.gz", hash = "sha256:b076ef717a8e5bc819514ee1d602bbdca5b4420ae13a9cf61a0c0a4f53a2baa2"}, + {file = "ruff-0.11.7-py3-none-linux_armv6l.whl", hash = "sha256:d29e909d9a8d02f928d72ab7837b5cbc450a5bdf578ab9ebee3263d0a525091c"}, + {file = "ruff-0.11.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dd1fb86b168ae349fb01dd497d83537b2c5541fe0626e70c786427dd8363aaee"}, + {file = "ruff-0.11.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d3d7d2e140a6fbbc09033bce65bd7ea29d6a0adeb90b8430262fbacd58c38ada"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4809df77de390a1c2077d9b7945d82f44b95d19ceccf0c287c56e4dc9b91ca64"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3a0c2e169e6b545f8e2dba185eabbd9db4f08880032e75aa0e285a6d3f48201"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49b888200a320dd96a68e86736cf531d6afba03e4f6cf098401406a257fcf3d6"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2b19cdb9cf7dae00d5ee2e7c013540cdc3b31c4f281f1dacb5a799d610e90db4"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64e0ee994c9e326b43539d133a36a455dbaab477bc84fe7bfbd528abe2f05c1e"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bad82052311479a5865f52c76ecee5d468a58ba44fb23ee15079f17dd4c8fd63"}, + {file = "ruff-0.11.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7940665e74e7b65d427b82bffc1e46710ec7f30d58b4b2d5016e3f0321436502"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:169027e31c52c0e36c44ae9a9c7db35e505fee0b39f8d9fca7274a6305295a92"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:305b93f9798aee582e91e34437810439acb28b5fc1fee6b8205c78c806845a94"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a681db041ef55550c371f9cd52a3cf17a0da4c75d6bd691092dfc38170ebc4b6"}, + {file = "ruff-0.11.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:07f1496ad00a4a139f4de220b0c97da6d4c85e0e4aa9b2624167b7d4d44fd6b6"}, + {file = "ruff-0.11.7-py3-none-win32.whl", hash = "sha256:f25dfb853ad217e6e5f1924ae8a5b3f6709051a13e9dad18690de6c8ff299e26"}, + {file = "ruff-0.11.7-py3-none-win_amd64.whl", hash = "sha256:0a931d85959ceb77e92aea4bbedfded0a31534ce191252721128f77e5ae1f98a"}, + {file = "ruff-0.11.7-py3-none-win_arm64.whl", hash = "sha256:778c1e5d6f9e91034142dfd06110534ca13220bfaad5c3735f6cb844654f6177"}, + {file = "ruff-0.11.7.tar.gz", hash = "sha256:655089ad3224070736dc32844fde783454f8558e71f501cb207485fe4eee23d4"}, ] [[package]] name = "sentry-sdk" -version = "2.19.2" +version = "2.27.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ - {file = "sentry_sdk-2.19.2-py2.py3-none-any.whl", hash = "sha256:ebdc08228b4d131128e568d696c210d846e5b9d70aa0327dec6b1272d9d40b84"}, - {file = "sentry_sdk-2.19.2.tar.gz", hash = "sha256:467df6e126ba242d39952375dd816fbee0f217d119bf454a8ce74cf1e7909e8d"}, + {file = "sentry_sdk-2.27.0-py2.py3-none-any.whl", hash = "sha256:c58935bfff8af6a0856d37e8adebdbc7b3281c2b632ec823ef03cd108d216ff0"}, + {file = "sentry_sdk-2.27.0.tar.gz", hash = "sha256:90f4f883f9eff294aff59af3d58c2d1b64e3927b28d5ada2b9b41f5aeda47daf"}, ] [package.dependencies] @@ -872,27 +907,43 @@ sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] +statsig = ["statsig (>=0.55.3)"] tornado = ["tornado (>=6)"] +unleash = ["UnleashClient (>=6.0.1)"] [[package]] name = "setuptools" -version = "75.3.0" +version = "80.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["executable"] +markers = "python_version < \"3.13\"" files = [ - {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, - {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, + {file = "setuptools-80.0.0-py3-none-any.whl", hash = "sha256:a38f898dcd6e5380f4da4381a87ec90bd0a7eec23d204a5552e80ee3cab6bd27"}, + {file = "setuptools-80.0.0.tar.gz", hash = "sha256:c40a5b3729d58dd749c0f08f1a07d134fb8a0a3d7f87dc33e7c5e1f762138650"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.12.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] + +[[package]] +name = "shellingham" +version = "1.5.4" +description = "Tool to Detect Surrounding Shell" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, +] [[package]] name = "six" @@ -900,6 +951,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -907,32 +959,40 @@ files = [ [[package]] name = "smmap" -version = "5.0.1" +version = "5.0.2" description = "A pure Python implementation of a sliding window memory map manager" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ - {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, - {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, + {file = "smmap-5.0.2-py3-none-any.whl", hash = "sha256:b30115f0def7d7531d22a0fb6502488d879e75b260a9db4d0819cfb25403af5e"}, + {file = "smmap-5.0.2.tar.gz", hash = "sha256:26ea65a03958fa0c8a1c7e8c7a58fdc77221b8910f6be2131affade476898ad5"}, ] [[package]] -name = "texttable" -version = "1.7.0" -description = "module to create simple ASCII tables" +name = "tenacity" +version = "9.0.0" +description = "Retry code until it succeeds" optional = false -python-versions = "*" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "texttable-1.7.0-py2.py3-none-any.whl", hash = "sha256:72227d592c82b3d7f672731ae73e4d1f88cd8e2ef5b075a7a7f01a23a3743917"}, - {file = "texttable-1.7.0.tar.gz", hash = "sha256:2d2068fb55115807d3ac77a4ca68fa48803e84ebb0ee2340f858107a36522638"}, + {file = "tenacity-9.0.0-py3-none-any.whl", hash = "sha256:93de0c98785b27fcf659856aa9f54bfbd399e29969b0621bc7f762bd441b4539"}, + {file = "tenacity-9.0.0.tar.gz", hash = "sha256:807f37ca97d62aa361264d497b0e31e92b8027044942bfa756160d908320d73b"}, ] +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tomli" version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["test"] +markers = "python_version < \"3.11\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -968,12 +1028,31 @@ files = [ {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, ] +[[package]] +name = "typer" +version = "0.15.3" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "typer-0.15.3-py3-none-any.whl", hash = "sha256:c86a65ad77ca531f03de08d1b9cb67cd09ad02ddddf4b34745b5008f43b239bd"}, + {file = "typer-0.15.3.tar.gz", hash = "sha256:818873625d0569653438316567861899f7e9972f2e6e0c16dab608345ced713c"}, +] + +[package.dependencies] +click = ">=8.0.0" +rich = ">=10.11.0" +shellingham = ">=1.3.0" +typing-extensions = ">=3.7.4.3" + [[package]] name = "types-python-dateutil" version = "2.9.0.20241206" description = "Typing stubs for python-dateutil" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "types_python_dateutil-2.9.0.20241206-py3-none-any.whl", hash = "sha256:e248a4bc70a486d3e3ec84d0dc30eec3a5f979d6e7ee4123ae043eedbb987f53"}, {file = "types_python_dateutil-2.9.0.20241206.tar.gz", hash = "sha256:18f493414c26ffba692a72369fea7a154c502646301ebfe3d56a04b3767284cb"}, @@ -981,24 +1060,26 @@ files = [ [[package]] name = "types-pyyaml" -version = "6.0.12.20240917" +version = "6.0.12.20250402" description = "Typing stubs for PyYAML" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["test"] files = [ - {file = "types-PyYAML-6.0.12.20240917.tar.gz", hash = "sha256:d1405a86f9576682234ef83bcb4e6fff7c9305c8b1fbad5e0bcd4f7dbdc9c587"}, - {file = "types_PyYAML-6.0.12.20240917-py3-none-any.whl", hash = "sha256:392b267f1c0fe6022952462bf5d6523f31e37f6cea49b14cee7ad634b6301570"}, + {file = "types_pyyaml-6.0.12.20250402-py3-none-any.whl", hash = "sha256:652348fa9e7a203d4b0d21066dfb00760d3cbd5a15ebb7cf8d33c88a49546681"}, + {file = "types_pyyaml-6.0.12.20250402.tar.gz", hash = "sha256:d7c13c3e6d335b6af4b0122a01ff1d270aba84ab96d1a1a1063ecba3e13ec075"}, ] [[package]] name = "typing-extensions" -version = "4.12.2" +version = "4.13.2" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, - {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, + {file = "typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c"}, + {file = "typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"}, ] [[package]] @@ -1007,36 +1088,39 @@ version = "1.26.19" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +groups = ["main", "test"] files = [ {file = "urllib3-1.26.19-py2.py3-none-any.whl", hash = "sha256:37a0344459b199fce0e80b0d3569837ec6b6937435c5244e7fd73fa6006830f3"}, {file = "urllib3-1.26.19.tar.gz", hash = "sha256:3e3d753a8618b86d7de333b4223005f68720bcd6a7d2bcb9fbd2229ec7c1e429"}, ] [package.extras] -brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and platform_python_implementation == \"CPython\"", "brotli (>=1.0.9) ; python_version >= \"3\" and platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\"", "brotlipy (>=0.6.0) ; os_name == \"nt\" and python_version < \"3\""] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "zipp" -version = "3.20.2" +version = "3.21.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +groups = ["executable"] +markers = "python_version < \"3.10\"" files = [ - {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, - {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, + {file = "zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931"}, + {file = "zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] [metadata] -lock-version = "2.0" -python-versions = ">=3.8,<3.14" -content-hash = "e91a6f9b7e080cea351f9073ef333afe026df6172b95fba5477af67f15c96000" +lock-version = "2.1" +python-versions = ">=3.9,<3.14" +content-hash = "14f258101aa534aadfc871aa5082ad773aa99873587c21c0598567435bfa5d9a" diff --git a/process_executable_file.py b/process_executable_file.py index ad4d702a..367bb18d 100755 --- a/process_executable_file.py +++ b/process_executable_file.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -""" -Used in the GitHub Actions workflow (build_executable.yml) to process the executable file. +"""Used in the GitHub Actions workflow (build_executable.yml) to process the executable file. + This script calculates hash and renames executable file depending on the OS, arch, and build mode. It also creates a file with the hash of the executable file. It uses SHA256 algorithm to calculate the hash. @@ -15,7 +15,7 @@ import shutil from pathlib import Path from string import Template -from typing import List, Tuple, Union +from typing import Union _ARCHIVE_FORMAT = 'zip' _HASH_FILE_EXT = '.sha256' @@ -27,7 +27,7 @@ _WINDOWS = 'windows' _WINDOWS_EXECUTABLE_SUFFIX = '.exe' -DirHashes = List[Tuple[str, str]] +DirHashes = list[tuple[str, str]] def get_hash_of_file(file_path: Union[str, Path]) -> str: @@ -35,7 +35,7 @@ def get_hash_of_file(file_path: Union[str, Path]) -> str: return hashlib.sha256(f.read()).hexdigest() -def get_hashes_of_many_files(root: str, file_paths: List[str]) -> DirHashes: +def get_hashes_of_many_files(root: str, file_paths: list[str]) -> DirHashes: hashes = [] for file_path in file_paths: diff --git a/pyinstaller.spec b/pyinstaller.spec index cb3382d4..39b8588f 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -23,7 +23,6 @@ with open(_INIT_FILE_PATH, 'w', encoding='UTF-8') as file: a = Analysis( scripts=['cycode/cli/main.py'], - datas=[('cycode/cli/config.yaml', 'cycode/cli'), ('cycode/cyclient/config.yaml', 'cycode/cyclient')], excludes=['tests'], ) diff --git a/pyproject.toml b/pyproject.toml index 42511ec8..755d8207 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -24,10 +23,10 @@ classifiers = [ ] [tool.poetry.scripts] -cycode = "cycode.cli.main:main_cli" +cycode = "cycode.cli.app:app" [tool.poetry.dependencies] -python = ">=3.8,<3.14" +python = ">=3.9,<3.14" click = ">=8.1.0,<8.2.0" colorama = ">=0.4.3,<0.5.0" pyyaml = ">=6.0,<7.0" @@ -35,13 +34,14 @@ marshmallow = ">=3.15.0,<3.23.0" # 3.23 dropped support for Python 3.8 gitpython = ">=3.1.30,<3.2.0" arrow = ">=1.0.0,<1.4.0" binaryornot = ">=0.4.4,<0.5.0" -texttable = ">=1.6.7,<1.8.0" requests = ">=2.32.2,<3.0" urllib3 = "1.26.19" # lock v1 to avoid issues with openssl and old Python versions (<3.9.11) on macOS sentry-sdk = ">=2.8.0,<3.0" pyjwt = ">=2.8.0,<3.0" rich = ">=13.9.4, <14" patch-ng = "1.18.1" +typer = "^0.15.3" +tenacity = ">=9.0.0,<9.1.0" [tool.poetry.group.test.dependencies] mock = ">=4.0.3,<4.1.0" @@ -56,7 +56,7 @@ pyinstaller = {version=">=5.13.2,<5.14.0", python=">=3.8,<3.13"} dunamai = ">=1.18.0,<1.22.0" [tool.poetry.group.dev.dependencies] -ruff = "0.6.9" +ruff = "0.11.7" [tool.pytest.ini_options] log_cli = true @@ -73,7 +73,7 @@ style = "pep440" [tool.ruff] line-length = 120 -target-version = "py38" +target-version = "py39" [tool.ruff.lint] extend-select = [ @@ -81,6 +81,7 @@ extend-select = [ "W", # pycodestyle warnings "F", # Pyflakes "I", # isort + "N", # pep8 naming "C90", # flake8-comprehensions "B", # flake8-bugbear "Q", # flake8-quotes @@ -100,19 +101,26 @@ extend-select = [ "RSE", "RUF", "SIM", + "T10", "T20", - "TCH", "TID", "YTT", + "LOG", "G", + "UP", + "DTZ", + "PYI", + "PT", + "SLOT", + "TC", ] ignore = [ "ANN002", # Missing type annotation for `*args` "ANN003", # Missing type annotation for `**kwargs` - "ANN101", # Missing type annotation for `self` in method - "ANN102", # Missing type annotation for `cls` in classmethod "ANN401", # Dynamically typed expressions (typing.Any) "ISC001", # Conflicts with ruff format + "S105", # False positives + "PT012", # `pytest.raises()` block should contain a single simple statement ] [tool.ruff.lint.flake8-quotes] diff --git a/tests/cli/commands/configure/test_configure_command.py b/tests/cli/commands/configure/test_configure_command.py index c5ae2b9c..5ed94c1d 100644 --- a/tests/cli/commands/configure/test_configure_command.py +++ b/tests/cli/commands/configure/test_configure_command.py @@ -1,8 +1,8 @@ from typing import TYPE_CHECKING -from click.testing import CliRunner +from typer.testing import CliRunner -from cycode.cli.commands.configure.configure_command import configure_command +from cycode.cli.app import app if TYPE_CHECKING: from pytest_mock import MockerFixture @@ -30,7 +30,7 @@ def test_configure_command_no_exist_values_in_file(mocker: 'MockerFixture') -> N # side effect - multiple return values, each item in the list represents return of a call mocker.patch( - 'click.prompt', + 'typer.prompt', side_effect=[api_url_user_input, app_url_user_input, client_id_user_input, client_secret_user_input], ) @@ -45,7 +45,7 @@ def test_configure_command_no_exist_values_in_file(mocker: 'MockerFixture') -> N ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) @@ -75,7 +75,7 @@ def test_configure_command_update_current_configs_in_files(mocker: 'MockerFixtur # side effect - multiple return values, each item in the list represents return of a call mocker.patch( - 'click.prompt', + 'typer.prompt', side_effect=[api_url_user_input, app_url_user_input, client_id_user_input, client_secret_user_input], ) @@ -90,7 +90,7 @@ def test_configure_command_update_current_configs_in_files(mocker: 'MockerFixtur ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(client_id_user_input, client_secret_user_input) @@ -108,13 +108,13 @@ def test_set_credentials_update_only_client_id(mocker: 'MockerFixture') -> None: ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=['', '', client_id_user_input, '']) + mocker.patch('typer.prompt', side_effect=['', '', client_id_user_input, '']) mocked_update_credentials = mocker.patch( 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(client_id_user_input, current_client_id) @@ -131,13 +131,13 @@ def test_configure_command_update_only_client_secret(mocker: 'MockerFixture') -> ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=['', '', '', client_secret_user_input]) + mocker.patch('typer.prompt', side_effect=['', '', '', client_secret_user_input]) mocked_update_credentials = mocker.patch( 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_credentials.assert_called_once_with(current_client_id, client_secret_user_input) @@ -154,13 +154,13 @@ def test_configure_command_update_only_api_url(mocker: 'MockerFixture') -> None: ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=[api_url_user_input, '', '', '']) + mocker.patch('typer.prompt', side_effect=[api_url_user_input, '', '', '']) mocked_update_api_base_url = mocker.patch( 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert mocked_update_api_base_url.assert_called_once_with(api_url_user_input) @@ -177,13 +177,13 @@ def test_configure_command_should_not_update_credentials(mocker: 'MockerFixture' ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=['', '', client_id_user_input, client_secret_user_input]) + mocker.patch('typer.prompt', side_effect=['', '', client_id_user_input, client_secret_user_input]) mocked_update_credentials = mocker.patch( 'cycode.cli.user_settings.credentials_manager.CredentialsManager.update_credentials' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert assert not mocked_update_credentials.called @@ -204,7 +204,7 @@ def test_configure_command_should_not_update_config_file(mocker: 'MockerFixture' ) # side effect - multiple return values, each item in the list represents return of a call - mocker.patch('click.prompt', side_effect=[api_url_user_input, app_url_user_input, '', '']) + mocker.patch('typer.prompt', side_effect=[api_url_user_input, app_url_user_input, '', '']) mocked_update_api_base_url = mocker.patch( 'cycode.cli.user_settings.config_file_manager.ConfigFileManager.update_api_base_url' ) @@ -213,7 +213,7 @@ def test_configure_command_should_not_update_config_file(mocker: 'MockerFixture' ) # Act - CliRunner().invoke(configure_command) + CliRunner().invoke(app, ['configure']) # Assert assert not mocked_update_api_base_url.called diff --git a/tests/cli/commands/scan/test_code_scanner.py b/tests/cli/commands/scan/test_code_scanner.py index 2c15dd3d..3151684e 100644 --- a/tests/cli/commands/scan/test_code_scanner.py +++ b/tests/cli/commands/scan/test_code_scanner.py @@ -1,7 +1,7 @@ import os from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import _does_severity_match_severity_threshold +from cycode.cli.apps.scan.code_scanner import _does_severity_match_severity_threshold from cycode.cli.files_collector.excluder import _is_file_relevant_for_sca_scan from cycode.cli.files_collector.path_documents import _generate_document from cycode.cli.models import Document @@ -42,7 +42,7 @@ def test_generate_document() -> None: }""" iac_document = Document(path, content, is_git_diff) - generated_document = _generate_document(path, consts.INFRA_CONFIGURATION_SCAN_TYPE, content, is_git_diff) + generated_document = _generate_document(path, consts.IAC_SCAN_TYPE, content, is_git_diff) assert iac_document.path == generated_document.path assert iac_document.content == generated_document.content assert iac_document.is_git_diff_format == generated_document.is_git_diff_format @@ -68,7 +68,7 @@ def test_generate_document() -> None: } """ - generated_tfplan_document = _generate_document(path, consts.INFRA_CONFIGURATION_SCAN_TYPE, content, is_git_diff) + generated_tfplan_document = _generate_document(path, consts.IAC_SCAN_TYPE, content, is_git_diff) assert isinstance(generated_tfplan_document, Document) assert generated_tfplan_document.path.endswith('.tf') diff --git a/tests/cli/commands/test_check_latest_version_on_close.py b/tests/cli/commands/test_check_latest_version_on_close.py index 189973b4..eccadf93 100644 --- a/tests/cli/commands/test_check_latest_version_on_close.py +++ b/tests/cli/commands/test_check_latest_version_on_close.py @@ -1,15 +1,16 @@ from unittest.mock import patch import pytest -from click.testing import CliRunner +from typer.testing import CliRunner from cycode import __version__ -from cycode.cli.commands.main_cli import main_cli -from cycode.cli.commands.version.version_checker import VersionChecker +from cycode.cli.app import app +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.utils.version_checker import VersionChecker from tests.conftest import CLI_ENV_VARS _NEW_LATEST_VERSION = '999.0.0' # Simulate a newer version available -_UPDATE_MESSAGE_PART = 'new version of cycode available' +_UPDATE_MESSAGE_PART = 'new release of cycode cli is available' @patch.object(VersionChecker, 'check_for_update') @@ -17,8 +18,8 @@ def test_version_check_with_json_output(mock_check_update: patch) -> None: # When output is JSON, version check should be skipped mock_check_update.return_value = _NEW_LATEST_VERSION - args = ['--output', 'json', 'version'] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + args = ['--output', OutputTypeOption.JSON, 'version'] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # Version check message should not be present in JSON output assert _UPDATE_MESSAGE_PART not in result.output.lower() @@ -28,7 +29,7 @@ def test_version_check_with_json_output(mock_check_update: patch) -> None: @pytest.fixture def mock_auth_info() -> 'patch': # Mock the authorization info to avoid API calls - with patch('cycode.cli.commands.status.status_command.get_authorization_info', return_value=None) as mock: + with patch('cycode.cli.apps.auth.auth_common.get_authorization_info', return_value=None) as mock: yield mock @@ -38,7 +39,7 @@ def test_version_check_for_special_commands(mock_check_update: patch, mock_auth_ # Version and status commands should always check the version without cache mock_check_update.return_value = _NEW_LATEST_VERSION - result = CliRunner().invoke(main_cli, [command], env=CLI_ENV_VARS) + result = CliRunner().invoke(app, [command], env=CLI_ENV_VARS) # Version information should be present in output assert _UPDATE_MESSAGE_PART in result.output.lower() @@ -52,7 +53,7 @@ def test_version_check_with_text_output(mock_check_update: patch) -> None: mock_check_update.return_value = _NEW_LATEST_VERSION args = ['version'] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # Version check message should be present in JSON output assert _UPDATE_MESSAGE_PART in result.output.lower() @@ -64,7 +65,7 @@ def test_version_check_disabled(mock_check_update: patch) -> None: mock_check_update.return_value = _NEW_LATEST_VERSION args = ['--no-update-notifier', 'version'] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # Version check message should not be present assert _UPDATE_MESSAGE_PART not in result.output.lower() diff --git a/tests/cli/commands/test_main_command.py b/tests/cli/commands/test_main_command.py index 7e588cf2..db8fe86b 100644 --- a/tests/cli/commands/test_main_command.py +++ b/tests/cli/commands/test_main_command.py @@ -4,14 +4,14 @@ import pytest import responses -from click.testing import CliRunner +from typer.testing import CliRunner from cycode.cli import consts -from cycode.cli.commands.main_cli import main_cli +from cycode.cli.app import app +from cycode.cli.cli_types import OutputTypeOption from cycode.cli.utils.git_proxy import git_proxy from tests.conftest import CLI_ENV_VARS, TEST_FILES_PATH, ZIP_CONTENT_PATH -from tests.cyclient.mocked_responses.scan_client import mock_scan_responses -from tests.cyclient.test_scan_client import get_zipped_file_scan_response, get_zipped_file_scan_url +from tests.cyclient.mocked_responses.scan_client import mock_scan_async_responses _PATH_TO_SCAN = TEST_FILES_PATH.joinpath('zip_content').absolute() @@ -28,17 +28,17 @@ def _is_json(plain: str) -> bool: @responses.activate -@pytest.mark.parametrize('output', ['text', 'json']) +@pytest.mark.parametrize('output', [OutputTypeOption.TEXT, OutputTypeOption.JSON]) def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token_response: responses.Response) -> None: scan_type = consts.SECRET_SCAN_TYPE scan_id = uuid4() - mock_scan_responses(responses, scan_type, scan_client, scan_id, ZIP_CONTENT_PATH) - responses.add(get_zipped_file_scan_response(get_zipped_file_scan_url(scan_type, scan_client), ZIP_CONTENT_PATH)) + mock_scan_async_responses(responses, scan_type, scan_client, scan_id, ZIP_CONTENT_PATH) responses.add(api_token_response) args = ['--output', output, 'scan', '--soft-fail', 'path', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + env = {'PYTEST_TEST_UNIQUE_ID': str(scan_id), **CLI_ENV_VARS} + result = CliRunner().invoke(app, args, env=env) except_json = output == 'json' @@ -46,24 +46,21 @@ def test_passing_output_option(output: str, scan_client: 'ScanClient', api_token if except_json: output = json.loads(result.output) - assert 'scan_id' in output + assert 'scan_ids' in output else: - assert 'Scan ID' in result.output + assert 'violation:' in result.output @responses.activate def test_optional_git_with_path_scan(scan_client: 'ScanClient', api_token_response: responses.Response) -> None: - mock_scan_responses(responses, consts.SECRET_SCAN_TYPE, scan_client, uuid4(), ZIP_CONTENT_PATH) - responses.add( - get_zipped_file_scan_response(get_zipped_file_scan_url(consts.SECRET_SCAN_TYPE, scan_client), ZIP_CONTENT_PATH) - ) + mock_scan_async_responses(responses, consts.SECRET_SCAN_TYPE, scan_client, uuid4(), ZIP_CONTENT_PATH) responses.add(api_token_response) # fake env without Git executable git_proxy._set_dummy_git_proxy() args = ['--output', 'json', 'scan', 'path', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # do NOT expect error about not found Git executable assert 'GIT_PYTHON_GIT_EXECUTABLE' not in result.output @@ -79,8 +76,8 @@ def test_required_git_with_path_repository(scan_client: 'ScanClient', api_token_ # fake env without Git executable git_proxy._set_dummy_git_proxy() - args = ['--output', 'json', 'scan', 'repository', str(_PATH_TO_SCAN)] - result = CliRunner().invoke(main_cli, args, env=CLI_ENV_VARS) + args = ['--output', OutputTypeOption.JSON, 'scan', 'repository', str(_PATH_TO_SCAN)] + result = CliRunner().invoke(app, args, env=CLI_ENV_VARS) # expect error about not found Git executable assert 'GIT_PYTHON_GIT_EXECUTABLE' in result.output diff --git a/tests/cli/commands/version/test_version_checker.py b/tests/cli/commands/version/test_version_checker.py index eb0b9bd9..14d6150e 100644 --- a/tests/cli/commands/version/test_version_checker.py +++ b/tests/cli/commands/version/test_version_checker.py @@ -7,7 +7,7 @@ if TYPE_CHECKING: from pathlib import Path -from cycode.cli.commands.version.version_checker import VersionChecker +from cycode.cli.utils.version_checker import VersionChecker @pytest.fixture @@ -71,7 +71,7 @@ def test_should_check_update_prerelease_daily(self, version_checker_cached: 'Ver assert version_checker_cached._should_check_update(is_prerelease=True) is True @pytest.mark.parametrize( - 'current_version, latest_version, expected_result', + ('current_version', 'latest_version', 'expected_result'), [ # Stable version comparisons ('1.2.3', '1.2.4', '1.2.4'), # Higher patch version diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py index 82a44bb0..ce72e9de 100644 --- a/tests/cli/exceptions/test_handle_scan_errors.py +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -1,25 +1,31 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import click import pytest -from click import ClickException +import typer from requests import Response +from rich.traceback import Traceback +from cycode.cli.cli_types import OutputTypeOption +from cycode.cli.console import console_err from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.printers import ConsolePrinter from cycode.cli.utils.git_proxy import git_proxy if TYPE_CHECKING: from _pytest.monkeypatch import MonkeyPatch -@pytest.fixture() -def ctx() -> click.Context: - return click.Context(click.Command('path'), obj={'verbose': False, 'output': 'text'}) +@pytest.fixture +def ctx() -> typer.Context: + ctx = typer.Context(click.Command('path'), obj={'verbose': False, 'output': OutputTypeOption.TEXT}) + ctx.obj['console_printer'] = ConsolePrinter(ctx) + return ctx @pytest.mark.parametrize( - 'exception, expected_soft_fail', + ('exception', 'expected_soft_fail'), [ (custom_exceptions.RequestHttpError(400, 'msg', Response()), True), (custom_exceptions.ScanAsyncError('msg'), True), @@ -30,7 +36,7 @@ def ctx() -> click.Context: ], ) def test_handle_exception_soft_fail( - ctx: click.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool + ctx: typer.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool ) -> None: with ctx: handle_scan_exception(ctx, exception) @@ -39,16 +45,16 @@ def test_handle_exception_soft_fail( assert ctx.obj.get('soft_fail') is expected_soft_fail -def test_handle_exception_unhandled_error(ctx: click.Context) -> None: - with ctx, pytest.raises(SystemExit): +def test_handle_exception_unhandled_error(ctx: typer.Context) -> None: + with ctx, pytest.raises(typer.Exit): handle_scan_exception(ctx, ValueError('test')) assert ctx.obj.get('did_fail') is True assert ctx.obj.get('soft_fail') is None -def test_handle_exception_click_error(ctx: click.Context) -> None: - with ctx, pytest.raises(ClickException): +def test_handle_exception_click_error(ctx: typer.Context) -> None: + with ctx, pytest.raises(click.ClickException): handle_scan_exception(ctx, click.ClickException('test')) assert ctx.obj.get('did_fail') is True @@ -56,14 +62,19 @@ def test_handle_exception_click_error(ctx: click.Context) -> None: def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: - ctx = click.Context(click.Command('path'), obj={'verbose': True, 'output': 'text'}) + ctx = typer.Context(click.Command('path'), obj={'verbose': True, 'output': OutputTypeOption.TEXT}) + ctx.obj['console_printer'] = ConsolePrinter(ctx) error_text = 'test' - def mock_secho(msg: str, *_, **__) -> None: - assert error_text in msg or 'Correlation ID:' in msg + def mock_console_print(obj: Any, *_, **__) -> None: + if isinstance(obj, str): + assert 'Correlation ID:' in obj + else: + assert isinstance(obj, Traceback) + assert error_text in str(obj.trace) - monkeypatch.setattr(click, 'secho', mock_secho) + monkeypatch.setattr(console_err, 'print', mock_console_print) - with pytest.raises(SystemExit): + with pytest.raises(typer.Exit): handle_scan_exception(ctx, ValueError(error_text)) diff --git a/tests/cli/files_collector/test_walk_ignore.py b/tests/cli/files_collector/test_walk_ignore.py index b771cdf9..12b9d428 100644 --- a/tests/cli/files_collector/test_walk_ignore.py +++ b/tests/cli/files_collector/test_walk_ignore.py @@ -1,6 +1,6 @@ import os from os.path import normpath -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from cycode.cli.files_collector.walk_ignore import ( _collect_top_level_ignore_files, @@ -95,7 +95,7 @@ def test_collect_top_level_ignore_files(fs: 'FakeFilesystem') -> None: fs.create_file('/home/user/project/.gitignore', contents='*.pyc\n*.log') -def _collect_walk_ignore_files(path: str) -> List[str]: +def _collect_walk_ignore_files(path: str) -> list[str]: files = [] for root, _, filenames in walk_ignore(path): for filename in filenames: diff --git a/tests/cli/models/test_severity.py b/tests/cli/models/test_severity.py index 332f987c..a59d5751 100644 --- a/tests/cli/models/test_severity.py +++ b/tests/cli/models/test_severity.py @@ -1,24 +1,11 @@ -from cycode.cli.models import Severity - - -def test_try_get_value() -> None: - assert Severity.try_get_value('info') == -1 - assert Severity.try_get_value('iNfO') == -1 - - assert Severity.try_get_value('INFO') == -1 - assert Severity.try_get_value('LOW') == 0 - assert Severity.try_get_value('MEDIUM') == 1 - assert Severity.try_get_value('HIGH') == 2 - assert Severity.try_get_value('CRITICAL') == 3 - - assert Severity.try_get_value('NON_EXISTENT') is None +from cycode.cli.cli_types import SeverityOption def test_get_member_weight() -> None: - assert Severity.get_member_weight('INFO') == -1 - assert Severity.get_member_weight('LOW') == 0 - assert Severity.get_member_weight('MEDIUM') == 1 - assert Severity.get_member_weight('HIGH') == 2 - assert Severity.get_member_weight('CRITICAL') == 3 + assert SeverityOption.get_member_weight('INFO') == 0 + assert SeverityOption.get_member_weight('LOW') == 1 + assert SeverityOption.get_member_weight('MEDIUM') == 2 + assert SeverityOption.get_member_weight('HIGH') == 3 + assert SeverityOption.get_member_weight('CRITICAL') == 4 - assert Severity.get_member_weight('NON_EXISTENT') == -2 + assert SeverityOption.get_member_weight('NON_EXISTENT') == -1 diff --git a/tests/cyclient/mocked_responses/scan_client.py b/tests/cyclient/mocked_responses/scan_client.py index 87643001..1726e74c 100644 --- a/tests/cyclient/mocked_responses/scan_client.py +++ b/tests/cyclient/mocked_responses/scan_client.py @@ -9,53 +9,6 @@ from tests.conftest import MOCKED_RESPONSES_PATH -def get_zipped_file_scan_url(scan_type: str, scan_client: ScanClient) -> str: - api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_zipped_file_scan_url_path(scan_type) - return f'{api_url}/{service_url}' - - -def get_zipped_file_scan_response( - url: str, zip_content_path: Path, scan_id: Optional[UUID] = None -) -> responses.Response: - if not scan_id: - scan_id = uuid4() - - json_response = { - 'did_detect': True, - 'scan_id': str(scan_id), # not always as expected due to _get_scan_id and passing scan_id to cxt of CLI - 'detections_per_file': [ - { - 'file_name': str(zip_content_path.joinpath('secrets.py')), - 'commit_id': None, - 'detections': [ - { - 'detection_type_id': '12345678-418f-47ee-abb0-012345678901', - 'detection_rule_id': '12345678-aea1-4304-a6e9-012345678901', - 'message': "Secret of type 'Slack Token' was found in filename 'secrets.py'", - 'type': 'slack-token', - 'is_research': False, - 'detection_details': { - 'sha512': 'sha hash', - 'length': 55, - 'start_position': 19, - 'line': 0, - 'committed_at': '0001-01-01T00:00:00+00:00', - 'file_path': str(zip_content_path), - 'file_name': 'secrets.py', - 'file_extension': '.py', - 'should_resolve_upon_branch_deletion': False, - }, - } - ], - } - ], - 'report_url': None, - } - - return responses.Response(method=responses.POST, url=url, json=json_response, status=200) - - def get_zipped_file_scan_async_url(scan_type: str, scan_client: ScanClient) -> str: api_url = scan_client.scan_cycode_client.api_url service_url = scan_client.get_zipped_file_scan_async_url_path(scan_type) @@ -73,15 +26,9 @@ def get_zipped_file_scan_async_response(url: str, scan_id: Optional[UUID] = None return responses.Response(method=responses.POST, url=url, json=json_response, status=200) -def get_scan_details_url(scan_id: Optional[UUID], scan_client: ScanClient) -> str: - api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_scan_details_path(str(scan_id)) - return f'{api_url}/{service_url}' - - -def get_scan_report_url(scan_id: Optional[UUID], scan_client: ScanClient, scan_type: str) -> str: +def get_scan_details_url(scan_type: str, scan_id: Optional[UUID], scan_client: ScanClient) -> str: api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_scan_report_url_path(str(scan_id), scan_type) + service_url = scan_client.get_scan_details_path(scan_type, str(scan_id)) return f'{api_url}/{service_url}' @@ -91,14 +38,6 @@ def get_scan_aggregation_report_url(aggregation_id: Optional[UUID], scan_client: return f'{api_url}/{service_url}' -def get_scan_report_url_response(url: str, scan_id: Optional[UUID] = None) -> responses.Response: - if not scan_id: - scan_id = uuid4() - json_response = {'report_url': f'https://app.domain/on-demand-scans/{scan_id}'} - - return responses.Response(method=responses.GET, url=url, json=json_response, status=200) - - def get_scan_aggregation_report_url_response(url: str, aggregation_id: Optional[UUID] = None) -> responses.Response: if not aggregation_id: aggregation_id = uuid4() @@ -135,10 +74,10 @@ def get_scan_details_response(url: str, scan_id: Optional[UUID] = None) -> respo return responses.Response(method=responses.GET, url=url, json=json_response, status=200) -def get_scan_detections_url(scan_client: ScanClient, scan_type: str) -> str: +def get_scan_detections_url(scan_client: ScanClient) -> str: api_url = scan_client.scan_cycode_client.api_url - service_url = scan_client.get_scan_detections_path(scan_type) - return f'{api_url}/{service_url}' + path = scan_client.get_scan_detections_list_path() + return f'{api_url}/{path}' def get_scan_detections_response(url: str, scan_id: UUID, zip_content_path: Path) -> responses.Response: @@ -181,20 +120,7 @@ def mock_scan_async_responses( responses_module.add( get_zipped_file_scan_async_response(get_zipped_file_scan_async_url(scan_type, scan_client), scan_id) ) - responses_module.add(get_scan_details_response(get_scan_details_url(scan_id, scan_client), scan_id)) - responses_module.add(get_detection_rules_response(get_detection_rules_url(scan_client))) - responses_module.add( - get_scan_detections_response(get_scan_detections_url(scan_client, scan_type), scan_id, zip_content_path) - ) - responses_module.add(get_report_scan_status_response(get_report_scan_status_url(scan_type, scan_id, scan_client))) - - -def mock_scan_responses( - responses_module: responses, scan_type: str, scan_client: ScanClient, scan_id: UUID, zip_content_path: Path -) -> None: - responses_module.add( - get_zipped_file_scan_response(get_zipped_file_scan_url(scan_type, scan_client), zip_content_path) - ) + responses_module.add(get_scan_details_response(get_scan_details_url(scan_type, scan_id, scan_client), scan_id)) responses_module.add(get_detection_rules_response(get_detection_rules_url(scan_client))) + responses_module.add(get_scan_detections_response(get_scan_detections_url(scan_client), scan_id, zip_content_path)) responses_module.add(get_report_scan_status_response(get_report_scan_status_url(scan_type, scan_id, scan_client))) - responses_module.add(get_scan_report_url_response(get_scan_report_url(scan_id, scan_client, scan_type))) diff --git a/tests/cyclient/scan_config/test_default_scan_config.py b/tests/cyclient/scan_config/test_default_scan_config.py index 75b305b5..987c6c78 100644 --- a/tests/cyclient/scan_config/test_default_scan_config.py +++ b/tests/cyclient/scan_config/test_default_scan_config.py @@ -5,11 +5,10 @@ def test_get_service_name() -> None: default_scan_config = DefaultScanConfig() - assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == 'secret' - assert default_scan_config.get_service_name(consts.INFRA_CONFIGURATION_SCAN_TYPE) == 'iac' + assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == 'scans' + assert default_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == 'scans' assert default_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' assert default_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' - assert default_scan_config.get_service_name(consts.SECRET_SCAN_TYPE, True) == 'scans' def test_get_detections_prefix() -> None: diff --git a/tests/cyclient/scan_config/test_dev_scan_config.py b/tests/cyclient/scan_config/test_dev_scan_config.py index 63c99169..f1cd484c 100644 --- a/tests/cyclient/scan_config/test_dev_scan_config.py +++ b/tests/cyclient/scan_config/test_dev_scan_config.py @@ -5,11 +5,10 @@ def test_get_service_name() -> None: dev_scan_config = DevScanConfig() - assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == '5025' - assert dev_scan_config.get_service_name(consts.INFRA_CONFIGURATION_SCAN_TYPE) == '5026' + assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE) == '5004' + assert dev_scan_config.get_service_name(consts.IAC_SCAN_TYPE) == '5004' assert dev_scan_config.get_service_name(consts.SCA_SCAN_TYPE) == '5004' assert dev_scan_config.get_service_name(consts.SAST_SCAN_TYPE) == '5004' - assert dev_scan_config.get_service_name(consts.SECRET_SCAN_TYPE, should_use_scan_service=True) == '5004' def test_get_detections_prefix() -> None: diff --git a/tests/cyclient/test_auth_client.py b/tests/cyclient/test_auth_client.py index 1d7e6683..24d9b096 100644 --- a/tests/cyclient/test_auth_client.py +++ b/tests/cyclient/test_auth_client.py @@ -3,8 +3,8 @@ import responses from requests import Timeout -from cycode.cli.commands.auth.auth_manager import AuthManager -from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestTimeout +from cycode.cli.apps.auth.auth_manager import AuthManager +from cycode.cli.exceptions.custom_exceptions import CycodeError, RequestTimeoutError from cycode.cyclient.auth_client import AuthClient from cycode.cyclient.models import ( ApiTokenGenerationPollingResponse, @@ -73,7 +73,7 @@ def test_start_session_timeout(client: AuthClient, start_url: str, code_challeng responses.add(responses.POST, start_url, body=timeout_error) - with pytest.raises(RequestTimeout): + with pytest.raises(RequestTimeoutError): client.start_session(code_challenge) diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index 2b8fc3f3..d6928118 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -1,113 +1,109 @@ import os -from typing import List, Tuple from uuid import uuid4 import pytest import requests import responses -from requests import Timeout -from requests.exceptions import ProxyError +from requests.exceptions import ConnectionError as RequestsConnectionError -from cycode.cli import consts -from cycode.cli.config import config +from cycode.cli.cli_types import ScanTypeOption from cycode.cli.exceptions.custom_exceptions import ( CycodeError, HttpUnauthorizedError, RequestConnectionError, - RequestTimeout, + RequestTimeoutError, ) from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip -from cycode.cli.files_collector.zip_documents import zip_documents from cycode.cli.models import Document from cycode.cyclient.scan_client import ScanClient from tests.conftest import ZIP_CONTENT_PATH from tests.cyclient.mocked_responses.scan_client import ( - get_scan_report_url, - get_scan_report_url_response, - get_zipped_file_scan_response, - get_zipped_file_scan_url, + get_scan_aggregation_report_url, + get_scan_aggregation_report_url_response, + get_scan_details_response, + get_scan_details_url, + get_zipped_file_scan_async_response, + get_zipped_file_scan_async_url, ) -def zip_scan_resources(scan_type: str, scan_client: ScanClient) -> Tuple[str, InMemoryZip]: - url = get_zipped_file_scan_url(scan_type, scan_client) +def zip_scan_resources(scan_type: ScanTypeOption, scan_client: ScanClient) -> tuple[str, InMemoryZip]: + url = get_zipped_file_scan_async_url(scan_type, scan_client) zip_file = get_test_zip_file(scan_type) return url, zip_file -def get_test_zip_file(scan_type: str) -> InMemoryZip: +def get_test_zip_file(scan_type: ScanTypeOption) -> InMemoryZip: # TODO(MarshalX): refactor scan_disk_files in code_scanner.py to reuse method here instead of this - test_documents: List[Document] = [] + test_documents: list[Document] = [] for root, _, files in os.walk(ZIP_CONTENT_PATH): for name in files: path = os.path.join(root, name) - with open(path, 'r', encoding='UTF-8') as f: + with open(path, encoding='UTF-8') as f: test_documents.append(Document(path, f.read(), is_git_diff_format=False)) - return zip_documents(scan_type, test_documents) - + from cycode.cli.files_collector.zip_documents import zip_documents -def test_get_service_name(scan_client: ScanClient) -> None: - # TODO(MarshalX): get_service_name should be removed from ScanClient? Because it exists in ScanConfig - assert scan_client.get_service_name(consts.SECRET_SCAN_TYPE) == 'secret' - assert scan_client.get_service_name(consts.INFRA_CONFIGURATION_SCAN_TYPE) == 'iac' - assert scan_client.get_service_name(consts.SCA_SCAN_TYPE) == 'scans' - assert scan_client.get_service_name(consts.SAST_SCAN_TYPE) == 'scans' + return zip_documents(scan_type, test_documents) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan(scan_type: str, scan_client: ScanClient, api_token_response: responses.Response) -> None: +def test_zipped_file_scan_async( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test the zipped_file_scan_async method for the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) expected_scan_id = uuid4() responses.add(api_token_response) # mock token based client - responses.add(get_zipped_file_scan_response(url, ZIP_CONTENT_PATH, expected_scan_id)) + responses.add(get_zipped_file_scan_async_response(url, expected_scan_id)) - zipped_file_scan_response = scan_client.zipped_file_scan( - scan_type, zip_file, scan_id=str(expected_scan_id), scan_parameters={} - ) - assert zipped_file_scan_response.scan_id == str(expected_scan_id) + scan_initialization_response = scan_client.zipped_file_scan_async(zip_file, scan_type, scan_parameters={}) + assert scan_initialization_response.scan_id == str(expected_scan_id) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_get_scan_report_url(scan_type: str, scan_client: ScanClient, api_token_response: responses.Response) -> None: +def test_get_scan_report_url( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test getting the scan report URL for the async flow.""" scan_id = uuid4() - url = get_scan_report_url(scan_id, scan_client, scan_type) + url = get_scan_aggregation_report_url(scan_id, scan_client, scan_type) responses.add(api_token_response) # mock token based client - responses.add(get_scan_report_url_response(url, scan_id)) + responses.add(get_scan_aggregation_report_url_response(url, scan_id)) - scan_report_url_response = scan_client.get_scan_report_url(str(scan_id), scan_type) - assert scan_report_url_response.report_url == 'https://app.domain/on-demand-scans/{scan_id}'.format(scan_id=scan_id) + scan_report_url_response = scan_client.get_scan_aggregation_report_url(str(scan_id), scan_type) + assert scan_report_url_response.report_url == f'https://app.domain/cli-logs-aggregation/{scan_id}' -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_unauthorized_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response +def test_zipped_file_scan_async_unauthorized_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test handling of unauthorized errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=url, status=401) + responses.add(method=responses.POST, url=url, status=401, body='Unauthorized') with pytest.raises(HttpUnauthorizedError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) assert e_info.value.status_code == 401 -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_bad_request_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response +def test_zipped_file_scan_async_bad_request_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: + """Test handling of bad request errors in the async flow.""" url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex expected_status_code = 400 expected_response_text = 'Bad Request' @@ -116,48 +112,59 @@ def test_zipped_file_scan_bad_request_error( responses.add(method=responses.POST, url=url, status=expected_status_code, body=expected_response_text) with pytest.raises(CycodeError) as e_info: - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) assert e_info.value.status_code == expected_status_code assert e_info.value.error_message == expected_response_text -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_timeout_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response +def test_zipped_file_scan_async_timeout_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: - scan_url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex + """Test handling of timeout errors in the async flow.""" + url, zip_file = zip_scan_resources(scan_type, scan_client) + + timeout_error = requests.exceptions.Timeout('Connection timed out') + + responses.add(api_token_response) # mock token based client + responses.add(method=responses.POST, url=url, body=timeout_error) - responses.add(responses.POST, scan_url, status=504) + with pytest.raises(RequestTimeoutError): + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) - timeout_response = requests.post(scan_url, timeout=5) - if timeout_response.status_code == 504: - """bypass SAST""" - responses.reset() +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) +@responses.activate +def test_zipped_file_scan_async_connection_error( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response +) -> None: + """Test handling of connection errors in the async flow.""" + url, zip_file = zip_scan_resources(scan_type, scan_client) - timeout_error = Timeout() - timeout_error.response = timeout_response + # Create a connection error response + connection_error = RequestsConnectionError('Connection refused') responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=scan_url, body=timeout_error, status=504) + responses.add(method=responses.POST, url=url, body=connection_error) - with pytest.raises(RequestTimeout): - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + with pytest.raises(RequestConnectionError): + scan_client.zipped_file_scan_async(zip_file=zip_file, scan_type=scan_type, scan_parameters={}) -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate -def test_zipped_file_scan_connection_error( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response +def test_get_scan_details( + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: - url, zip_file = zip_scan_resources(scan_type, scan_client) - expected_scan_id = uuid4().hex + """Test getting scan details in the async flow.""" + scan_id = uuid4() + url = get_scan_details_url(scan_type, scan_id, scan_client) responses.add(api_token_response) # mock token based client - responses.add(method=responses.POST, url=url, body=ProxyError()) + responses.add(get_scan_details_response(url, scan_id)) - with pytest.raises(RequestConnectionError): - scan_client.zipped_file_scan(scan_type, zip_file, scan_id=expected_scan_id, scan_parameters={}) + scan_details_response = scan_client.get_scan_details(scan_type, str(scan_id)) + assert scan_details_response.id == str(scan_id) + assert scan_details_response.scan_status == 'Completed' diff --git a/tests/test_code_scanner.py b/tests/test_code_scanner.py index d0ae939a..9ef09123 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_code_scanner.py @@ -5,19 +5,16 @@ import responses from cycode.cli import consts -from cycode.cli.commands.scan.code_scanner import ( +from cycode.cli.apps.scan.code_scanner import ( _try_get_aggregation_report_url_if_needed, - _try_get_report_url_if_needed, ) -from cycode.cli.config import config +from cycode.cli.cli_types import ScanTypeOption from cycode.cli.files_collector.excluder import _is_relevant_file_to_scan from cycode.cyclient.scan_client import ScanClient from tests.conftest import TEST_FILES_PATH from tests.cyclient.mocked_responses.scan_client import ( get_scan_aggregation_report_url, get_scan_aggregation_report_url_response, - get_scan_report_url, - get_scan_report_url_response, ) @@ -26,31 +23,9 @@ def test_is_relevant_file_to_scan_sca() -> None: assert _is_relevant_file_to_scan(consts.SCA_SCAN_TYPE, path) is True -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) -def test_try_get_report_url_if_needed_return_none(scan_type: str, scan_client: ScanClient) -> None: - scan_id = uuid4().hex - result = _try_get_report_url_if_needed(scan_client, scan_id, consts.SECRET_SCAN_TYPE, scan_parameters={}) - assert result is None - - -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) -@responses.activate -def test_try_get_report_url_if_needed_return_result( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response -) -> None: - scan_id = uuid4() - url = get_scan_report_url(scan_id, scan_client, scan_type) - responses.add(api_token_response) # mock token based client - responses.add(get_scan_report_url_response(url, scan_id)) - - scan_report_url_response = scan_client.get_scan_report_url(str(scan_id), scan_type) - result = _try_get_report_url_if_needed(scan_client, str(scan_id), scan_type, scan_parameters={'report': True}) - assert result == scan_report_url_response.report_url - - -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( - scan_type: str, scan_client: ScanClient + scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: aggregation_id = uuid4().hex scan_parameter = {'aggregation_id': aggregation_id} @@ -58,19 +33,19 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( assert result is None -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) def test_try_get_aggregation_report_url_if_no_aggregation_id_needed_return_none( - scan_type: str, scan_client: ScanClient + scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: scan_parameter = {'report': True} result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result is None -@pytest.mark.parametrize('scan_type', config['scans']['supported_scans']) +@pytest.mark.parametrize('scan_type', list(ScanTypeOption)) @responses.activate def test_try_get_aggregation_report_url_if_needed_return_result( - scan_type: str, scan_client: ScanClient, api_token_response: responses.Response + scan_type: ScanTypeOption, scan_client: ScanClient, api_token_response: responses.Response ) -> None: aggregation_id = uuid4() scan_parameter = {'report': True, 'aggregation_id': aggregation_id} diff --git a/tests/test_performance_get_all_files.py b/tests/test_performance_get_all_files.py index 60155261..b0e8653d 100644 --- a/tests/test_performance_get_all_files.py +++ b/tests/test_performance_get_all_files.py @@ -3,17 +3,17 @@ import os import timeit from pathlib import Path -from typing import Dict, List, Tuple, Union +from typing import Union logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -def filter_files(paths: List[Union[Path, str]]) -> List[str]: +def filter_files(paths: list[Union[Path, str]]) -> list[str]: return [str(path) for path in paths if os.path.isfile(path)] -def get_all_files_glob(path: Union[Path, str]) -> List[str]: +def get_all_files_glob(path: Union[Path, str]) -> list[str]: # DOESN'T RETURN HIDDEN FILES. CAN'T BE USED # and doesn't show the best performance if not str(path).endswith(os.sep): @@ -22,7 +22,7 @@ def get_all_files_glob(path: Union[Path, str]) -> List[str]: return filter_files(glob.glob(f'{path}**', recursive=True)) -def get_all_files_walk(path: str) -> List[str]: +def get_all_files_walk(path: str) -> list[str]: files = [] for root, _, filenames in os.walk(path): @@ -32,7 +32,7 @@ def get_all_files_walk(path: str) -> List[str]: return files -def get_all_files_listdir(path: str) -> List[str]: +def get_all_files_listdir(path: str) -> list[str]: files = [] def _(sub_path: str) -> None: @@ -50,12 +50,12 @@ def _(sub_path: str) -> None: return files -def get_all_files_rglob(path: str) -> List[str]: +def get_all_files_rglob(path: str) -> list[str]: return filter_files(list(Path(path).rglob(r'*'))) def test_get_all_files_performance(test_files_path: str) -> None: - results: Dict[str, Tuple[int, float]] = {} + results: dict[str, tuple[int, float]] = {} for func in { get_all_files_rglob, get_all_files_listdir, diff --git a/tests/user_settings/test_configuration_manager.py b/tests/user_settings/test_configuration_manager.py index 50251340..5aa7f6a8 100644 --- a/tests/user_settings/test_configuration_manager.py +++ b/tests/user_settings/test_configuration_manager.py @@ -1,6 +1,5 @@ from typing import TYPE_CHECKING, Optional - -from mock import Mock +from unittest.mock import Mock from cycode.cli.consts import DEFAULT_CYCODE_API_URL from cycode.cli.user_settings.configuration_manager import ConfigurationManager diff --git a/tests/utils/test_ignore_utils.py b/tests/utils/test_ignore_utils.py index 563c11a9..6988e1aa 100644 --- a/tests/utils/test_ignore_utils.py +++ b/tests/utils/test_ignore_utils.py @@ -87,9 +87,9 @@ def test_translate(self) -> None: for pattern, regex in TRANSLATE_TESTS: if re.escape(b'/') == b'/': regex = regex.replace(b'\\/', b'/') - assert ( - translate(pattern) == regex - ), f'orig pattern: {pattern!r}, regex: {translate(pattern)!r}, expected: {regex!r}' + assert translate(pattern) == regex, ( + f'orig pattern: {pattern!r}, regex: {translate(pattern)!r}, expected: {regex!r}' + ) def test_read_file(self) -> None: f = BytesIO( diff --git a/tests/utils/test_string_utils.py b/tests/utils/test_string_utils.py index 60d10efa..8c94fceb 100644 --- a/tests/utils/test_string_utils.py +++ b/tests/utils/test_string_utils.py @@ -3,5 +3,5 @@ def test_shortcut_dependency_paths_list_single_dependencies() -> None: dependency_paths = 'A, A -> B, A -> B -> C' - expected_result = 'A\n\nA -> B\n\nA -> ... -> C' + expected_result = 'A\nA -> B\nA -> ... -> C' assert shortcut_dependency_paths(dependency_paths) == expected_result