diff --git a/cycode/cli/auth/__init__.py b/cycode/cli/commands/__init__.py similarity index 100% rename from cycode/cli/auth/__init__.py rename to cycode/cli/commands/__init__.py diff --git a/tests/cli/helpers/__init__.py b/cycode/cli/commands/auth/__init__.py similarity index 100% rename from tests/cli/helpers/__init__.py rename to cycode/cli/commands/auth/__init__.py diff --git a/cycode/cli/auth/auth_command.py b/cycode/cli/commands/auth/auth_command.py similarity index 95% rename from cycode/cli/auth/auth_command.py rename to cycode/cli/commands/auth/auth_command.py index c7853068..51eb212b 100644 --- a/cycode/cli/auth/auth_command.py +++ b/cycode/cli/commands/auth/auth_command.py @@ -2,7 +2,7 @@ import click -from cycode.cli.auth.auth_manager import AuthManager +from cycode.cli.commands.auth.auth_manager import AuthManager from cycode.cli.exceptions.custom_exceptions import AuthProcessError, HttpUnauthorizedError, NetworkError from cycode.cli.models import CliError, CliErrors, CliResult from cycode.cli.printers import ConsolePrinter @@ -15,7 +15,7 @@ invoke_without_command=True, short_help='Authenticate your machine to associate the CLI with your Cycode account.' ) @click.pass_context -def authenticate(context: click.Context) -> None: +def auth_command(context: click.Context) -> None: """Authenticates your machine.""" if context.invoked_subcommand is not None: # if it is a subcommand, do nothing @@ -33,7 +33,7 @@ def authenticate(context: click.Context) -> None: _handle_exception(context, e) -@authenticate.command( +@auth_command.command( name='check', short_help='Checks that your machine is associating the CLI with your Cycode account.' ) @click.pass_context diff --git a/cycode/cli/auth/auth_manager.py b/cycode/cli/commands/auth/auth_manager.py similarity index 100% rename from cycode/cli/auth/auth_manager.py rename to cycode/cli/commands/auth/auth_manager.py diff --git a/cycode/cli/commands/main_cli.py b/cycode/cli/commands/main_cli.py new file mode 100644 index 00000000..e418cf7f --- /dev/null +++ b/cycode/cli/commands/main_cli.py @@ -0,0 +1,79 @@ +import logging +from typing import Optional + +import click + +from cycode.cli.commands.auth.auth_command import auth_command +from cycode.cli.commands.configure.configure_command import configure_command +from cycode.cli.commands.ignore.ignore_command import ignore_command +from cycode.cli.commands.report.report_command import report_command +from cycode.cli.commands.scan.scan_command import scan_command +from cycode.cli.commands.version.version_command import version_command +from cycode.cli.consts import ( + CLI_CONTEXT_SETTINGS, +) +from cycode.cli.user_settings.configuration_manager import ConfigurationManager +from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar +from cycode.cyclient.config import set_logging_level +from cycode.cyclient.cycode_client_base import CycodeClientBase +from cycode.cyclient.models import UserAgentOptionScheme + + +@click.group( + commands={ + 'scan': scan_command, + 'report': report_command, + 'configure': configure_command, + 'ignore': ignore_command, + 'auth': auth_command, + 'version': version_command, + }, + context_settings=CLI_CONTEXT_SETTINGS, +) +@click.option( + '--verbose', + '-v', + is_flag=True, + default=False, + help='Show detailed logs.', +) +@click.option( + '--no-progress-meter', + is_flag=True, + default=False, + help='Do not show the progress meter.', +) +@click.option( + '--output', + '-o', + default='text', + help='Specify the output type (the default is text).', + type=click.Choice(['text', 'json', 'table']), +) +@click.option( + '--user-agent', + default=None, + help='Characteristic JSON object that lets servers identify the application.', + type=str, +) +@click.pass_context +def main_cli( + context: click.Context, verbose: bool, no_progress_meter: bool, output: str, user_agent: Optional[str] +) -> None: + context.ensure_object(dict) + configuration_manager = ConfigurationManager() + + verbose = verbose or configuration_manager.get_verbose_flag() + context.obj['verbose'] = verbose + if verbose: + set_logging_level(logging.DEBUG) + + context.obj['output'] = output + if output == 'json': + no_progress_meter = True + + context.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) + + if user_agent: + user_agent_option = UserAgentOptionScheme().loads(user_agent) + CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) diff --git a/cycode/cli/commands/report/sbom/path/__init__.py b/cycode/cli/commands/report/sbom/path/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/report/sbom/sbom_path_command.py b/cycode/cli/commands/report/sbom/path/path_command.py similarity index 94% rename from cycode/cli/commands/report/sbom/sbom_path_command.py rename to cycode/cli/commands/report/sbom/path/path_command.py index 23062cab..323eb62b 100644 --- a/cycode/cli/commands/report/sbom/sbom_path_command.py +++ b/cycode/cli/commands/report/sbom/path/path_command.py @@ -4,7 +4,7 @@ from cycode.cli import consts from cycode.cli.commands.report.sbom.common import create_sbom_report, send_report_feedback -from cycode.cli.commands.report.sbom.handle_errors import handle_report_exception +from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception from cycode.cli.files_collector.path_documents import get_relevant_document from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents @@ -15,7 +15,7 @@ @click.command(short_help='Generate SBOM report for provided path in the command.') @click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) @click.pass_context -def sbom_path_command(context: click.Context, path: str) -> None: +def path_command(context: click.Context, path: str) -> None: client = get_report_cycode_client() report_parameters = context.obj['report_parameters'] output_format = report_parameters.output_format diff --git a/cycode/cli/commands/report/sbom/repository_url/__init__.py b/cycode/cli/commands/report/sbom/repository_url/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/report/sbom/sbom_repository_url_command.py b/cycode/cli/commands/report/sbom/repository_url/repository_url_command.py similarity index 92% rename from cycode/cli/commands/report/sbom/sbom_repository_url_command.py rename to cycode/cli/commands/report/sbom/repository_url/repository_url_command.py index 4d5ee4a3..4f54cac1 100644 --- a/cycode/cli/commands/report/sbom/sbom_repository_url_command.py +++ b/cycode/cli/commands/report/sbom/repository_url/repository_url_command.py @@ -3,7 +3,7 @@ import click from cycode.cli.commands.report.sbom.common import create_sbom_report, send_report_feedback -from cycode.cli.commands.report.sbom.handle_errors import handle_report_exception +from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception from cycode.cli.utils.get_api_client import get_report_cycode_client from cycode.cli.utils.progress_bar import SbomReportProgressBarSection @@ -11,7 +11,7 @@ @click.command(short_help='Generate SBOM report for provided repository URI in the command.') @click.argument('uri', nargs=1, type=str, required=True) @click.pass_context -def sbom_repository_url_command(context: click.Context, uri: str) -> None: +def repository_url_command(context: click.Context, uri: str) -> None: progress_bar = context.obj['progress_bar'] progress_bar.start() progress_bar.set_section_length(SbomReportProgressBarSection.PREPARE_LOCAL_FILES) diff --git a/cycode/cli/commands/report/sbom/sbom_command.py b/cycode/cli/commands/report/sbom/sbom_command.py index ecfd2782..870f4e0c 100644 --- a/cycode/cli/commands/report/sbom/sbom_command.py +++ b/cycode/cli/commands/report/sbom/sbom_command.py @@ -3,16 +3,16 @@ import click -from cycode.cli.commands.report.sbom.sbom_path_command import sbom_path_command -from cycode.cli.commands.report.sbom.sbom_repository_url_command import sbom_repository_url_command +from cycode.cli.commands.report.sbom.path.path_command import path_command +from cycode.cli.commands.report.sbom.repository_url.repository_url_command import repository_url_command from cycode.cli.config import config from cycode.cyclient.report_client import ReportParameters @click.group( commands={ - 'path': sbom_path_command, - 'repository_url': sbom_repository_url_command, + 'path': path_command, + 'repository_url': repository_url_command, }, short_help='Generate SBOM report for remote repository by url or local directory by path.', ) diff --git a/cycode/cli/commands/scan/__init__.py b/cycode/cli/commands/scan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/code_scanner.py b/cycode/cli/commands/scan/code_scanner.py similarity index 77% rename from cycode/cli/code_scanner.py rename to cycode/cli/commands/scan/code_scanner.py index 1801d63c..be1014de 100644 --- a/cycode/cli/code_scanner.py +++ b/cycode/cli/commands/scan/code_scanner.py @@ -3,34 +3,30 @@ import os import sys import time -import traceback from platform import platform from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Tuple from uuid import UUID, uuid4 import click -from git import NULL_TREE, InvalidGitRepositoryError, Repo +from git import NULL_TREE, Repo from cycode.cli import consts -from cycode.cli.ci_integrations import get_commit_range from cycode.cli.config import configuration_manager from cycode.cli.exceptions import custom_exceptions +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.files_collector.path_documents import get_relevant_document from cycode.cli.files_collector.repository_documents import ( - calculate_pre_receive_commit_range, get_commit_range_modified_documents, - get_diff_file_content, get_diff_file_path, - get_git_repository_tree_file_entries, get_pre_commit_modified_documents, parse_commit_range, ) from cycode.cli.files_collector.sca import sca_code_scanner from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions from cycode.cli.files_collector.zip_documents import zip_documents -from cycode.cli.models import CliError, CliErrors, Document, DocumentDetections, LocalScanResult, Severity +from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult, Severity from cycode.cli.printers import ConsolePrinter from cycode.cli.utils import scan_utils from cycode.cli.utils.path_utils import ( @@ -39,7 +35,6 @@ from cycode.cli.utils.progress_bar import ScanProgressBarSection from cycode.cli.utils.scan_batch import run_parallel_batched_scan from cycode.cli.utils.scan_utils import set_issue_detected -from cycode.cli.utils.task_timer import TimeoutAfter from cycode.cyclient import logger from cycode.cyclient.config import set_logging_level from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult @@ -51,221 +46,6 @@ start_scan_time = time.time() -@click.command(short_help='Scan the git repository including its history.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.option( - '--branch', - '-b', - default=None, - help='Branch to scan, if not set scanning the default branch', - type=str, - required=False, -) -@click.pass_context -def scan_repository(context: click.Context, path: str, branch: str) -> None: - try: - logger.debug('Starting repository scan process, %s', {'path': path, 'branch': branch}) - - scan_type = context.obj['scan_type'] - monitor = context.obj.get('monitor') - if monitor and scan_type != consts.SCA_SCAN_TYPE: - raise click.ClickException('Monitor flag is currently supported for SCA scan type only') - - progress_bar = context.obj['progress_bar'] - progress_bar.start() - - file_entries = list(get_git_repository_tree_file_entries(path, branch)) - progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries)) - - documents_to_scan = [] - for file in file_entries: - # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - - file_path = file.path if monitor else get_path_by_os(os.path.join(path, file.path)) - documents_to_scan.append(Document(file_path, file.data_stream.read().decode('UTF-8', errors='replace'))) - - documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - - perform_pre_scan_documents_actions(context, scan_type, documents_to_scan, is_git_diff=False) - - logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) - scan_documents( - context, documents_to_scan, is_git_diff=False, scan_parameters=get_scan_parameters(context, path) - ) - except Exception as e: - _handle_exception(context, e) - - -@click.command(short_help='Scan all the commits history in this git repository.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.option( - '--commit_range', - '-r', - help='Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1)', - type=click.STRING, - default='--all', - required=False, -) -@click.pass_context -def scan_repository_commit_history(context: click.Context, path: str, commit_range: str) -> None: - try: - logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) - scan_commit_range(context, path=path, commit_range=commit_range) - except Exception as e: - _handle_exception(context, e) - - -def scan_commit_range( - context: click.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None -) -> None: - scan_type = context.obj['scan_type'] - - progress_bar = context.obj['progress_bar'] - progress_bar.start() - - if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: - raise click.ClickException(f'Commit range scanning for {str.upper(scan_type)} is not supported') - - if scan_type == consts.SCA_SCAN_TYPE: - return scan_sca_commit_range(context, path, commit_range) - - documents_to_scan = [] - commit_ids_to_scan = [] - - repo = Repo(path) - total_commits_count = int(repo.git.rev_list('--count', commit_range)) - logger.debug(f'Calculating diffs for {total_commits_count} commits in the commit range {commit_range}') - - progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count) - - scanned_commits_count = 0 - for commit in repo.iter_commits(rev=commit_range): - if _does_reach_to_max_commits_to_scan_limit(commit_ids_to_scan, max_commits_count): - logger.debug(f'Reached to max commits to scan count. Going to scan only {max_commits_count} last commits') - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count - scanned_commits_count) - break - - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - - commit_id = commit.hexsha - commit_ids_to_scan.append(commit_id) - parent = commit.parents[0] if commit.parents else NULL_TREE - diff = commit.diff(parent, create_patch=True, R=True) - commit_documents_to_scan = [] - for blob in diff: - blob_path = get_path_by_os(os.path.join(path, get_diff_file_path(blob))) - commit_documents_to_scan.append( - Document( - path=blob_path, - content=blob.diff.decode('UTF-8', errors='replace'), - is_git_diff_format=True, - unique_id=commit_id, - ) - ) - - logger.debug( - 'Found all relevant files in commit %s', - {'path': path, 'commit_range': commit_range, 'commit_id': commit_id}, - ) - - documents_to_scan.extend(exclude_irrelevant_documents_to_scan(scan_type, commit_documents_to_scan)) - scanned_commits_count += 1 - - logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) - logger.debug('Starting to scan commit range (It may take a few minutes)') - - scan_documents(context, documents_to_scan, is_git_diff=True, is_commit_range=True) - return None - - -@click.command( - short_help='Execute scan in a CI environment which relies on the ' - 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' -) -@click.pass_context -def scan_ci(context: click.Context) -> None: - scan_commit_range(context, path=os.getcwd(), commit_range=get_commit_range()) - - -@click.command(short_help='Scan the files in the path provided in the command.') -@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) -@click.pass_context -def scan_path(context: click.Context, path: str) -> None: - progress_bar = context.obj['progress_bar'] - progress_bar.start() - - logger.debug('Starting path scan process, %s', {'path': path}) - scan_disk_files(context, path) - - -@click.command(short_help='Use this command to scan any content that was not committed yet.') -@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_commit_scan(context: click.Context, ignored_args: List[str]) -> None: - scan_type = context.obj['scan_type'] - - progress_bar = context.obj['progress_bar'] - progress_bar.start() - - if scan_type == consts.SCA_SCAN_TYPE: - scan_sca_pre_commit(context) - return - - diff_files = Repo(os.getcwd()).index.diff('HEAD', create_patch=True, R=True) - - progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files)) - - documents_to_scan = [] - for file in diff_files: - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))) - - documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - scan_documents(context, documents_to_scan, is_git_diff=True) - - -@click.command(short_help='Use this command to scan commits on the server side before pushing them to the repository.') -@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) -@click.pass_context -def pre_receive_scan(context: click.Context, ignored_args: List[str]) -> None: - try: - scan_type = context.obj['scan_type'] - if scan_type != consts.SECRET_SCAN_TYPE: - raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') - - if should_skip_pre_receive_scan(): - logger.info( - 'A scan has been skipped as per your request.' - ' Please note that this may leave your system vulnerable to secrets that have not been detected' - ) - return - - if is_verbose_mode_requested_in_pre_receive_scan(): - enable_verbose_mode(context) - logger.debug('Verbose mode enabled, all log levels will be displayed') - - command_scan_type = context.info_name - timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) - with TimeoutAfter(timeout): - if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: - raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') - - branch_update_details = parse_pre_receive_input() - commit_range = calculate_pre_receive_commit_range(branch_update_details) - if not commit_range: - logger.info( - 'No new commits found for pushed branch, %s', {'branch_update_details': branch_update_details} - ) - return - - max_commits_to_scan = configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type) - scan_commit_range(context, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) - perform_post_pre_receive_scan_actions(context) - except Exception as e: - _handle_exception(context, e) - - def scan_sca_pre_commit(context: click.Context) -> None: scan_type = context.obj['scan_type'] scan_parameters = get_default_scan_parameters(context) @@ -312,7 +92,7 @@ def scan_disk_files(context: click.Context, path: str) -> None: perform_pre_scan_documents_actions(context, scan_type, documents) scan_documents(context, documents, scan_parameters=scan_parameters) except Exception as e: - _handle_exception(context, e) + handle_scan_exception(context, e) def set_issue_detected_by_scan_results(context: click.Context, scan_results: List[LocalScanResult]) -> None: @@ -349,7 +129,7 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local scan_completed = True except Exception as e: - error = _handle_exception(context, e, return_exception=True) + error = handle_scan_exception(context, e, return_exception=True) error_message = str(e) if local_scan_result: @@ -384,6 +164,69 @@ def _scan_batch_thread_func(batch: List[Document]) -> Tuple[str, CliError, Local return _scan_batch_thread_func +def scan_commit_range( + context: click.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None +) -> None: + scan_type = context.obj['scan_type'] + + progress_bar = context.obj['progress_bar'] + progress_bar.start() + + if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: + raise click.ClickException(f'Commit range scanning for {str.upper(scan_type)} is not supported') + + if scan_type == consts.SCA_SCAN_TYPE: + return scan_sca_commit_range(context, path, commit_range) + + documents_to_scan = [] + commit_ids_to_scan = [] + + repo = Repo(path) + total_commits_count = int(repo.git.rev_list('--count', commit_range)) + logger.debug(f'Calculating diffs for {total_commits_count} commits in the commit range {commit_range}') + + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count) + + scanned_commits_count = 0 + for commit in repo.iter_commits(rev=commit_range): + if _does_reach_to_max_commits_to_scan_limit(commit_ids_to_scan, max_commits_count): + logger.debug(f'Reached to max commits to scan count. Going to scan only {max_commits_count} last commits') + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count - scanned_commits_count) + break + + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + + commit_id = commit.hexsha + commit_ids_to_scan.append(commit_id) + parent = commit.parents[0] if commit.parents else NULL_TREE + diff = commit.diff(parent, create_patch=True, R=True) + commit_documents_to_scan = [] + for blob in diff: + blob_path = get_path_by_os(os.path.join(path, get_diff_file_path(blob))) + commit_documents_to_scan.append( + Document( + path=blob_path, + content=blob.diff.decode('UTF-8', errors='replace'), + is_git_diff_format=True, + unique_id=commit_id, + ) + ) + + logger.debug( + 'Found all relevant files in commit %s', + {'path': path, 'commit_range': commit_range, 'commit_id': commit_id}, + ) + + documents_to_scan.extend(exclude_irrelevant_documents_to_scan(scan_type, commit_documents_to_scan)) + scanned_commits_count += 1 + + logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) + logger.debug('Starting to scan commit range (It may take a few minutes)') + + scan_documents(context, documents_to_scan, is_git_diff=True, is_commit_range=True) + return None + + def scan_documents( context: click.Context, documents_to_scan: List[Document], @@ -472,7 +315,7 @@ def scan_commit_range_documents( scan_completed = True except Exception as e: - _handle_exception(context, e) + handle_scan_exception(context, e) error_message = str(e) zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size @@ -827,75 +670,6 @@ def _get_document_by_file_name( return None -def _handle_exception(context: click.Context, e: Exception, *, return_exception: bool = False) -> Optional[CliError]: - context.obj['did_fail'] = True - - if context.obj['verbose']: - click.secho(f'Error: {traceback.format_exc()}', fg='red') - - errors: CliErrors = { - custom_exceptions.NetworkError: CliError( - soft_fail=True, - code='cycode_error', - message='Cycode was unable to complete this scan. ' - 'Please try again by executing the `cycode scan` command', - ), - custom_exceptions.ScanAsyncError: CliError( - soft_fail=True, - code='scan_error', - message='Cycode was unable to complete this scan. ' - 'Please try again by executing the `cycode scan` command', - ), - custom_exceptions.HttpUnauthorizedError: CliError( - soft_fail=True, - code='auth_error', - message='Unable to authenticate to Cycode, your token is either invalid or has expired. ' - 'Please re-generate your token and reconfigure it by running the `cycode configure` command', - ), - custom_exceptions.ZipTooLargeError: CliError( - soft_fail=True, - code='zip_too_large_error', - message='The path you attempted to scan exceeds the current maximum scanning size cap (10MB). ' - 'Please try ignoring irrelevant paths using the `cycode ignore --by-path` command ' - 'and execute the scan again', - ), - custom_exceptions.TfplanKeyError: CliError( - soft_fail=True, - code='key_error', - message=f'\n{e!s}\n' - 'A crucial field is missing in your terraform plan file. ' - 'Please make sure that your file is well formed ' - 'and execute the scan again', - ), - InvalidGitRepositoryError: CliError( - soft_fail=False, - code='invalid_git_error', - message='The path you supplied does not correlate to a git repository. ' - 'If you still wish to scan this path, use: `cycode scan path `', - ), - } - - if type(e) in errors: - error = errors[type(e)] - - if error.soft_fail is True: - context.obj['soft_fail'] = True - - if return_exception: - return error - - ConsolePrinter(context).print_error(error) - return None - - if return_exception: - return CliError(code='unknown_error', message=str(e)) - - if isinstance(e, click.ClickException): - raise e - - raise click.ClickException(str(e)) - - def _report_scan_status( cycode_client: 'ScanClient', scan_type: str, diff --git a/cycode/cli/commands/scan/commit_history/__init__.py b/cycode/cli/commands/scan/commit_history/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/scan/commit_history/commit_history_command.py b/cycode/cli/commands/scan/commit_history/commit_history_command.py new file mode 100644 index 00000000..f7db9404 --- /dev/null +++ b/cycode/cli/commands/scan/commit_history/commit_history_command.py @@ -0,0 +1,24 @@ +import click + +from cycode.cli.commands.scan.code_scanner import scan_commit_range +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cyclient import logger + + +@click.command(short_help='Scan all the commits history in this git repository.') +@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) +@click.option( + '--commit_range', + '-r', + help='Scan a commit range in this git repository, by default cycode scans all commit history (example: HEAD~1)', + type=click.STRING, + default='--all', + required=False, +) +@click.pass_context +def commit_history_command(context: click.Context, path: str, commit_range: str) -> None: + try: + logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) + scan_commit_range(context, path=path, commit_range=commit_range) + except Exception as e: + handle_scan_exception(context, e) diff --git a/cycode/cli/commands/scan/path/__init__.py b/cycode/cli/commands/scan/path/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/scan/path/path_command.py b/cycode/cli/commands/scan/path/path_command.py new file mode 100644 index 00000000..7098016e --- /dev/null +++ b/cycode/cli/commands/scan/path/path_command.py @@ -0,0 +1,15 @@ +import click + +from cycode.cli.commands.scan.code_scanner import scan_disk_files +from cycode.cyclient import logger + + +@click.command(short_help='Scan the files in the path provided in the command.') +@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) +@click.pass_context +def path_command(context: click.Context, path: str) -> None: + progress_bar = context.obj['progress_bar'] + progress_bar.start() + + logger.debug('Starting path scan process, %s', {'path': path}) + scan_disk_files(context, path) diff --git a/cycode/cli/commands/scan/pre_commit/__init__.py b/cycode/cli/commands/scan/pre_commit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/scan/pre_commit/pre_commit_command.py b/cycode/cli/commands/scan/pre_commit/pre_commit_command.py new file mode 100644 index 00000000..a758f0f5 --- /dev/null +++ b/cycode/cli/commands/scan/pre_commit/pre_commit_command.py @@ -0,0 +1,44 @@ +import os +from typing import List + +import click +from git import Repo + +from cycode.cli import consts +from cycode.cli.commands.scan.code_scanner import scan_documents, scan_sca_pre_commit +from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan +from cycode.cli.files_collector.repository_documents import ( + get_diff_file_content, + get_diff_file_path, +) +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import ( + get_path_by_os, +) +from cycode.cli.utils.progress_bar import ScanProgressBarSection + + +@click.command(short_help='Use this command to scan any content that was not committed yet.') +@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def pre_commit_command(context: click.Context, ignored_args: List[str]) -> None: + scan_type = context.obj['scan_type'] + + progress_bar = context.obj['progress_bar'] + progress_bar.start() + + if scan_type == consts.SCA_SCAN_TYPE: + scan_sca_pre_commit(context) + return + + diff_files = Repo(os.getcwd()).index.diff('HEAD', create_patch=True, R=True) + + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files)) + + documents_to_scan = [] + for file in diff_files: + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))) + + documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) + scan_documents(context, documents_to_scan, is_git_diff=True) diff --git a/cycode/cli/commands/scan/pre_receive/__init__.py b/cycode/cli/commands/scan/pre_receive/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/scan/pre_receive/pre_receive_command.py b/cycode/cli/commands/scan/pre_receive/pre_receive_command.py new file mode 100644 index 00000000..71cd82c1 --- /dev/null +++ b/cycode/cli/commands/scan/pre_receive/pre_receive_command.py @@ -0,0 +1,62 @@ +import os +from typing import List + +import click + +from cycode.cli import consts +from cycode.cli.commands.scan.code_scanner import ( + enable_verbose_mode, + is_verbose_mode_requested_in_pre_receive_scan, + parse_pre_receive_input, + perform_post_pre_receive_scan_actions, + scan_commit_range, + should_skip_pre_receive_scan, +) +from cycode.cli.config import configuration_manager +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.repository_documents import ( + calculate_pre_receive_commit_range, +) +from cycode.cli.utils.task_timer import TimeoutAfter +from cycode.cyclient import logger + + +@click.command(short_help='Use this command to scan commits on the server side before pushing them to the repository.') +@click.argument('ignored_args', nargs=-1, type=click.UNPROCESSED) +@click.pass_context +def pre_receive_command(context: click.Context, ignored_args: List[str]) -> None: + try: + scan_type = context.obj['scan_type'] + if scan_type != consts.SECRET_SCAN_TYPE: + raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') + + if should_skip_pre_receive_scan(): + logger.info( + 'A scan has been skipped as per your request.' + ' Please note that this may leave your system vulnerable to secrets that have not been detected' + ) + return + + if is_verbose_mode_requested_in_pre_receive_scan(): + enable_verbose_mode(context) + logger.debug('Verbose mode enabled, all log levels will be displayed') + + command_scan_type = context.info_name + timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) + with TimeoutAfter(timeout): + if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: + raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') + + branch_update_details = parse_pre_receive_input() + commit_range = calculate_pre_receive_commit_range(branch_update_details) + if not commit_range: + logger.info( + 'No new commits found for pushed branch, %s', {'branch_update_details': branch_update_details} + ) + return + + max_commits_to_scan = configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type) + scan_commit_range(context, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) + perform_post_pre_receive_scan_actions(context) + except Exception as e: + handle_scan_exception(context, e) diff --git a/cycode/cli/commands/scan/repository/__init__.py b/cycode/cli/commands/scan/repository/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/scan/repository/repisotiry_command.py b/cycode/cli/commands/scan/repository/repisotiry_command.py new file mode 100644 index 00000000..38aab411 --- /dev/null +++ b/cycode/cli/commands/scan/repository/repisotiry_command.py @@ -0,0 +1,60 @@ +import os + +import click + +from cycode.cli import consts +from cycode.cli.commands.scan.code_scanner import get_scan_parameters, scan_documents +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.excluder import exclude_irrelevant_documents_to_scan +from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries +from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions +from cycode.cli.models import Document +from cycode.cli.utils.path_utils import get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cyclient import logger + + +@click.command(short_help='Scan the git repository including its history.') +@click.argument('path', nargs=1, type=click.Path(exists=True, resolve_path=True), required=True) +@click.option( + '--branch', + '-b', + default=None, + help='Branch to scan, if not set scanning the default branch', + type=str, + required=False, +) +@click.pass_context +def repisotiry_command(context: click.Context, path: str, branch: str) -> None: + try: + logger.debug('Starting repository scan process, %s', {'path': path, 'branch': branch}) + + scan_type = context.obj['scan_type'] + monitor = context.obj.get('monitor') + if monitor and scan_type != consts.SCA_SCAN_TYPE: + raise click.ClickException('Monitor flag is currently supported for SCA scan type only') + + progress_bar = context.obj['progress_bar'] + progress_bar.start() + + file_entries = list(get_git_repository_tree_file_entries(path, branch)) + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(file_entries)) + + documents_to_scan = [] + for file in file_entries: + # FIXME(MarshalX): probably file could be tree or submodule too. we expect blob only + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + + file_path = file.path if monitor else get_path_by_os(os.path.join(path, file.path)) + documents_to_scan.append(Document(file_path, file.data_stream.read().decode('UTF-8', errors='replace'))) + + documents_to_scan = exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) + + perform_pre_scan_documents_actions(context, scan_type, documents_to_scan, is_git_diff=False) + + logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) + scan_documents( + context, documents_to_scan, is_git_diff=False, scan_parameters=get_scan_parameters(context, path) + ) + except Exception as e: + handle_scan_exception(context, e) diff --git a/cycode/cli/commands/scan/scan_ci/__init__.py b/cycode/cli/commands/scan/scan_ci/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/ci_integrations.py b/cycode/cli/commands/scan/scan_ci/ci_integrations.py similarity index 100% rename from cycode/cli/ci_integrations.py rename to cycode/cli/commands/scan/scan_ci/ci_integrations.py diff --git a/cycode/cli/commands/scan/scan_ci/scan_ci_command.py b/cycode/cli/commands/scan/scan_ci/scan_ci_command.py new file mode 100644 index 00000000..594aad63 --- /dev/null +++ b/cycode/cli/commands/scan/scan_ci/scan_ci_command.py @@ -0,0 +1,15 @@ +import os + +import click + +from cycode.cli.commands.scan.code_scanner import scan_commit_range +from cycode.cli.commands.scan.scan_ci.ci_integrations import get_commit_range + + +@click.command( + short_help='Execute scan in a CI environment which relies on the ' + 'CYCODE_TOKEN and CYCODE_REPO_LOCATION environment variables' +) +@click.pass_context +def scan_ci_command(context: click.Context) -> None: + scan_commit_range(context, path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/commands/scan/scan_command.py b/cycode/cli/commands/scan/scan_command.py new file mode 100644 index 00000000..920b8be2 --- /dev/null +++ b/cycode/cli/commands/scan/scan_command.py @@ -0,0 +1,159 @@ +import sys +from typing import List + +import click + +from cycode.cli.commands.scan.commit_history.commit_history_command import commit_history_command +from cycode.cli.commands.scan.path.path_command import path_command +from cycode.cli.commands.scan.pre_commit.pre_commit_command import pre_commit_command +from cycode.cli.commands.scan.pre_receive.pre_receive_command import pre_receive_command +from cycode.cli.commands.scan.repository.repisotiry_command import repisotiry_command +from cycode.cli.config import config +from cycode.cli.consts import ( + ISSUE_DETECTED_STATUS_CODE, + NO_ISSUES_STATUS_CODE, + SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, +) +from cycode.cli.models import Severity +from cycode.cli.utils import scan_utils +from cycode.cli.utils.get_api_client import get_scan_cycode_client + + +@click.group( + commands={ + 'repository': repisotiry_command, + 'commit_history': commit_history_command, + 'path': path_command, + 'pre_commit': pre_commit_command, + 'pre_receive': pre_receive_command, + }, + short_help='Scan the content for Secrets/IaC/SCA/SAST violations. ' + 'You`ll need to specify which scan type to perform: ci/commit_history/path/repository/etc.', +) +@click.option( + '--scan-type', + '-t', + default='secret', + help='Specify the type of scan you wish to execute (the default is Secrets)', + type=click.Choice(config['scans']['supported_scans']), +) +@click.option( + '--secret', + default=None, + help='Specify a Cycode client secret for this specific scan execution.', + type=str, + required=False, +) +@click.option( + '--client-id', + default=None, + help='Specify a Cycode client ID for this specific scan execution.', + type=str, + required=False, +) +@click.option( + '--show-secret', is_flag=True, default=False, help='Show Secrets in plain text.', type=bool, required=False +) +@click.option( + '--soft-fail', + is_flag=True, + default=False, + help='Run the scan without failing; always return a non-error status code.', + type=bool, + required=False, +) +@click.option( + '--severity-threshold', + default=None, + help='Show violations only for the specified level or higher (supported for SCA scan types only).', + type=click.Choice([e.name for e in Severity]), + required=False, +) +@click.option( + '--sca-scan', + default=None, + help='Specify the type of SCA scan you wish to execute (the default is both).', + multiple=True, + type=click.Choice(config['scans']['supported_sca_scans']), +) +@click.option( + '--monitor', + is_flag=True, + default=False, + help='Used for SCA scan types only; when specified, the scan results are recorded in the Discovery module.', + type=bool, + required=False, +) +@click.option( + '--report', + is_flag=True, + default=False, + help='When specified, generates a violations report. A link to the report will be displayed in the console output.', + type=bool, + required=False, +) +@click.option( + f'--{SCA_SKIP_RESTORE_DEPENDENCIES_FLAG}', + is_flag=True, + default=False, + help='When specified, Cycode will not run restore command. Will scan direct dependencies ONLY!', + type=bool, + required=False, +) +@click.pass_context +def scan_command( + context: click.Context, + scan_type: str, + secret: str, + client_id: str, + show_secret: bool, + soft_fail: bool, + severity_threshold: str, + sca_scan: List[str], + monitor: bool, + report: bool, + no_restore: bool, +) -> int: + """Scans for Secrets, IaC, SCA or SAST violations.""" + if show_secret: + context.obj['show_secret'] = show_secret + else: + context.obj['show_secret'] = config['result_printer']['default']['show_secret'] + + if soft_fail: + context.obj['soft_fail'] = soft_fail + else: + context.obj['soft_fail'] = config['soft_fail'] + + context.obj['client'] = get_scan_cycode_client(client_id, secret, not context.obj['show_secret']) + context.obj['scan_type'] = scan_type + context.obj['severity_threshold'] = severity_threshold + context.obj['monitor'] = monitor + context.obj['report'] = report + context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore + + _sca_scan_to_context(context, sca_scan) + + return 1 + + +def _sca_scan_to_context(context: click.Context, sca_scan_user_selected: List[str]) -> None: + for sca_scan_option_selected in sca_scan_user_selected: + context.obj[sca_scan_option_selected] = True + + +@scan_command.result_callback() +@click.pass_context +def finalize(context: click.Context, *_, **__) -> None: + progress_bar = context.obj.get('progress_bar') + if progress_bar: + progress_bar.stop() + + if context.obj['soft_fail']: + sys.exit(0) + + exit_code = NO_ISSUES_STATUS_CODE + if scan_utils.is_scan_failed(context): + exit_code = ISSUE_DETECTED_STATUS_CODE + + sys.exit(exit_code) diff --git a/cycode/cli/commands/version/__init__.py b/cycode/cli/commands/version/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cycode/cli/commands/version/version_command.py b/cycode/cli/commands/version/version_command.py new file mode 100644 index 00000000..55755e24 --- /dev/null +++ b/cycode/cli/commands/version/version_command.py @@ -0,0 +1,22 @@ +import json + +import click + +from cycode import __version__ +from cycode.cli.consts import PROGRAM_NAME + + +@click.command(short_help='Show the CLI version and exit.') +@click.pass_context +def version_command(context: click.Context) -> None: + output = context.obj['output'] + + prog = PROGRAM_NAME + ver = __version__ + + message = f'{prog}, version {ver}' + if output == 'json': + message = json.dumps({'name': prog, 'version': ver}) + + click.echo(message, color=context.color) + context.exit() diff --git a/cycode/cli/commands/report/sbom/handle_errors.py b/cycode/cli/exceptions/handle_report_sbom_errors.py similarity index 100% rename from cycode/cli/commands/report/sbom/handle_errors.py rename to cycode/cli/exceptions/handle_report_sbom_errors.py diff --git a/cycode/cli/exceptions/handle_scan_errors.py b/cycode/cli/exceptions/handle_scan_errors.py new file mode 100644 index 00000000..1ee026f8 --- /dev/null +++ b/cycode/cli/exceptions/handle_scan_errors.py @@ -0,0 +1,80 @@ +import traceback +from typing import Optional + +import click +from git import InvalidGitRepositoryError + +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.models import CliError, CliErrors +from cycode.cli.printers import ConsolePrinter + + +def handle_scan_exception( + context: click.Context, e: Exception, *, return_exception: bool = False +) -> Optional[CliError]: + context.obj['did_fail'] = True + + if context.obj['verbose']: + click.secho(f'Error: {traceback.format_exc()}', fg='red') + + errors: CliErrors = { + custom_exceptions.NetworkError: CliError( + soft_fail=True, + code='cycode_error', + message='Cycode was unable to complete this scan. ' + 'Please try again by executing the `cycode scan` command', + ), + custom_exceptions.ScanAsyncError: CliError( + soft_fail=True, + code='scan_error', + message='Cycode was unable to complete this scan. ' + 'Please try again by executing the `cycode scan` command', + ), + custom_exceptions.HttpUnauthorizedError: CliError( + soft_fail=True, + code='auth_error', + message='Unable to authenticate to Cycode, your token is either invalid or has expired. ' + 'Please re-generate your token and reconfigure it by running the `cycode configure` command', + ), + custom_exceptions.ZipTooLargeError: CliError( + soft_fail=True, + code='zip_too_large_error', + message='The path you attempted to scan exceeds the current maximum scanning size cap (10MB). ' + 'Please try ignoring irrelevant paths using the `cycode ignore --by-path` command ' + 'and execute the scan again', + ), + custom_exceptions.TfplanKeyError: CliError( + soft_fail=True, + code='key_error', + message=f'\n{e!s}\n' + 'A crucial field is missing in your terraform plan file. ' + 'Please make sure that your file is well formed ' + 'and execute the scan again', + ), + InvalidGitRepositoryError: CliError( + soft_fail=False, + code='invalid_git_error', + message='The path you supplied does not correlate to a git repository. ' + 'If you still wish to scan this path, use: `cycode scan path `', + ), + } + + if type(e) in errors: + error = errors[type(e)] + + if error.soft_fail is True: + context.obj['soft_fail'] = True + + if return_exception: + return error + + ConsolePrinter(context).print_error(error) + return None + + if return_exception: + return CliError(code='unknown_error', message=str(e)) + + if isinstance(e, click.ClickException): + raise e + + raise click.ClickException(str(e)) diff --git a/cycode/cli/main.py b/cycode/cli/main.py index 082cca3a..b27c98e2 100644 --- a/cycode/cli/main.py +++ b/cycode/cli/main.py @@ -1,253 +1,4 @@ -import json -import logging -import sys -from typing import List, Optional - -import click - -from cycode import __version__ -from cycode.cli import code_scanner -from cycode.cli.auth.auth_command import authenticate -from cycode.cli.commands.configure.configure_command import configure_command -from cycode.cli.commands.ignore.ignore_command import ignore_command -from cycode.cli.commands.report.report_command import report_command -from cycode.cli.config import config -from cycode.cli.consts import ( - CLI_CONTEXT_SETTINGS, - ISSUE_DETECTED_STATUS_CODE, - NO_ISSUES_STATUS_CODE, - PROGRAM_NAME, - SCA_SKIP_RESTORE_DEPENDENCIES_FLAG, -) -from cycode.cli.models import Severity -from cycode.cli.user_settings.configuration_manager import ConfigurationManager -from cycode.cli.utils import scan_utils -from cycode.cli.utils.get_api_client import get_scan_cycode_client -from cycode.cli.utils.progress_bar import SCAN_PROGRESS_BAR_SECTIONS, get_progress_bar -from cycode.cyclient.config import set_logging_level -from cycode.cyclient.cycode_client_base import CycodeClientBase -from cycode.cyclient.models import UserAgentOptionScheme - - -@click.group( - commands={ - 'repository': code_scanner.scan_repository, - 'commit_history': code_scanner.scan_repository_commit_history, - 'path': code_scanner.scan_path, - 'pre_commit': code_scanner.pre_commit_scan, - 'pre_receive': code_scanner.pre_receive_scan, - }, - short_help='Scan the content for Secrets/IaC/SCA/SAST violations. ' - 'You`ll need to specify which scan type to perform: ci/commit_history/path/repository/etc.', -) -@click.option( - '--scan-type', - '-t', - default='secret', - help='Specify the type of scan you wish to execute (the default is Secrets)', - type=click.Choice(config['scans']['supported_scans']), -) -@click.option( - '--secret', - default=None, - help='Specify a Cycode client secret for this specific scan execution.', - type=str, - required=False, -) -@click.option( - '--client-id', - default=None, - help='Specify a Cycode client ID for this specific scan execution.', - type=str, - required=False, -) -@click.option( - '--show-secret', is_flag=True, default=False, help='Show Secrets in plain text.', type=bool, required=False -) -@click.option( - '--soft-fail', - is_flag=True, - default=False, - help='Run the scan without failing; always return a non-error status code.', - type=bool, - required=False, -) -@click.option( - '--severity-threshold', - default=None, - help='Show violations only for the specified level or higher (supported for SCA scan types only).', - type=click.Choice([e.name for e in Severity]), - required=False, -) -@click.option( - '--sca-scan', - default=None, - help='Specify the type of SCA scan you wish to execute (the default is both).', - multiple=True, - type=click.Choice(config['scans']['supported_sca_scans']), -) -@click.option( - '--monitor', - is_flag=True, - default=False, - help='Used for SCA scan types only; when specified, the scan results are recorded in the Discovery module.', - type=bool, - required=False, -) -@click.option( - '--report', - is_flag=True, - default=False, - help='When specified, generates a violations report. A link to the report will be displayed in the console output.', - type=bool, - required=False, -) -@click.option( - f'--{SCA_SKIP_RESTORE_DEPENDENCIES_FLAG}', - is_flag=True, - default=False, - help='When specified, Cycode will not run restore command. Will scan direct dependencies ONLY!', - type=bool, - required=False, -) -@click.pass_context -def code_scan( - context: click.Context, - scan_type: str, - secret: str, - client_id: str, - show_secret: bool, - soft_fail: bool, - severity_threshold: str, - sca_scan: List[str], - monitor: bool, - report: bool, - no_restore: bool, -) -> int: - """Scans for Secrets, IaC, SCA or SAST violations.""" - if show_secret: - context.obj['show_secret'] = show_secret - else: - context.obj['show_secret'] = config['result_printer']['default']['show_secret'] - - if soft_fail: - context.obj['soft_fail'] = soft_fail - else: - context.obj['soft_fail'] = config['soft_fail'] - - context.obj['client'] = get_scan_cycode_client(client_id, secret, not context.obj['show_secret']) - context.obj['scan_type'] = scan_type - context.obj['severity_threshold'] = severity_threshold - context.obj['monitor'] = monitor - context.obj['report'] = report - context.obj[SCA_SKIP_RESTORE_DEPENDENCIES_FLAG] = no_restore - - _sca_scan_to_context(context, sca_scan) - - return 1 - - -@code_scan.result_callback() -@click.pass_context -def finalize(context: click.Context, *_, **__) -> None: - progress_bar = context.obj.get('progress_bar') - if progress_bar: - progress_bar.stop() - - if context.obj['soft_fail']: - sys.exit(0) - - exit_code = NO_ISSUES_STATUS_CODE - if _should_fail_scan(context): - exit_code = ISSUE_DETECTED_STATUS_CODE - - sys.exit(exit_code) - - -@click.command(short_help='Show the CLI version and exit.') -@click.pass_context -def version(context: click.Context) -> None: - output = context.obj['output'] - - prog = PROGRAM_NAME - ver = __version__ - - message = f'{prog}, version {ver}' - if output == 'json': - message = json.dumps({'name': prog, 'version': ver}) - - click.echo(message, color=context.color) - context.exit() - - -@click.group( - commands={ - 'scan': code_scan, - 'report': report_command, - 'configure': configure_command, - 'ignore': ignore_command, - 'auth': authenticate, - 'version': version, - }, - context_settings=CLI_CONTEXT_SETTINGS, -) -@click.option( - '--verbose', - '-v', - is_flag=True, - default=False, - help='Show detailed logs.', -) -@click.option( - '--no-progress-meter', - is_flag=True, - default=False, - help='Do not show the progress meter.', -) -@click.option( - '--output', - '-o', - default='text', - help='Specify the output type (the default is text).', - type=click.Choice(['text', 'json', 'table']), -) -@click.option( - '--user-agent', - default=None, - help='Characteristic JSON object that lets servers identify the application.', - type=str, -) -@click.pass_context -def main_cli( - context: click.Context, verbose: bool, no_progress_meter: bool, output: str, user_agent: Optional[str] -) -> None: - context.ensure_object(dict) - configuration_manager = ConfigurationManager() - - verbose = verbose or configuration_manager.get_verbose_flag() - context.obj['verbose'] = verbose - if verbose: - set_logging_level(logging.DEBUG) - - context.obj['output'] = output - if output == 'json': - no_progress_meter = True - - context.obj['progress_bar'] = get_progress_bar(hidden=no_progress_meter, sections=SCAN_PROGRESS_BAR_SECTIONS) - - if user_agent: - user_agent_option = UserAgentOptionScheme().loads(user_agent) - CycodeClientBase.enrich_user_agent(user_agent_option.user_agent_suffix) - - -def _should_fail_scan(context: click.Context) -> bool: - return scan_utils.is_scan_failed(context) - - -def _sca_scan_to_context(context: click.Context, sca_scan_user_selected: List[str]) -> None: - for sca_scan_option_selected in sca_scan_user_selected: - context.obj[sca_scan_option_selected] = True - +from cycode.cli.commands.main_cli import main_cli if __name__ == '__main__': main_cli() diff --git a/tests/cli/commands/__init__.py b/tests/cli/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/commands/configure/__init__.py b/tests/cli/commands/configure/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/test_configure_command.py b/tests/cli/commands/configure/test_configure_command.py similarity index 100% rename from tests/cli/test_configure_command.py rename to tests/cli/commands/configure/test_configure_command.py diff --git a/tests/cli/commands/scan/__init__.py b/tests/cli/commands/scan/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/test_code_scanner.py b/tests/cli/commands/scan/test_code_scanner.py similarity index 55% rename from tests/cli/test_code_scanner.py rename to tests/cli/commands/scan/test_code_scanner.py index f4fe4f69..c993958c 100644 --- a/tests/cli/test_code_scanner.py +++ b/tests/cli/commands/scan/test_code_scanner.py @@ -1,76 +1,10 @@ import os -from typing import TYPE_CHECKING - -import click -import pytest -from click import ClickException -from git import InvalidGitRepositoryError -from requests import Response from cycode.cli import consts -from cycode.cli.code_scanner import _handle_exception -from cycode.cli.exceptions import custom_exceptions from cycode.cli.files_collector.excluder import _is_file_relevant_for_sca_scan from cycode.cli.files_collector.path_documents import _generate_document from cycode.cli.models import Document -if TYPE_CHECKING: - from _pytest.monkeypatch import MonkeyPatch - - -@pytest.fixture() -def ctx() -> click.Context: - return click.Context(click.Command('path'), obj={'verbose': False, 'output': 'text'}) - - -@pytest.mark.parametrize( - 'exception, expected_soft_fail', - [ - (custom_exceptions.NetworkError(400, 'msg', Response()), True), - (custom_exceptions.ScanAsyncError('msg'), True), - (custom_exceptions.HttpUnauthorizedError('msg', Response()), True), - (custom_exceptions.ZipTooLargeError(1000), True), - (custom_exceptions.TfplanKeyError('msg'), True), - (InvalidGitRepositoryError(), None), - ], -) -def test_handle_exception_soft_fail( - ctx: click.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool -) -> None: - with ctx: - _handle_exception(ctx, exception) - - assert ctx.obj.get('did_fail') is True - assert ctx.obj.get('soft_fail') is expected_soft_fail - - -def test_handle_exception_unhandled_error(ctx: click.Context) -> None: - with ctx, pytest.raises(ClickException): - _handle_exception(ctx, ValueError('test')) - - assert ctx.obj.get('did_fail') is True - assert ctx.obj.get('soft_fail') is None - - -def test_handle_exception_click_error(ctx: click.Context) -> None: - with ctx, pytest.raises(ClickException): - _handle_exception(ctx, click.ClickException('test')) - - assert ctx.obj.get('did_fail') is True - assert ctx.obj.get('soft_fail') is None - - -def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: - ctx = click.Context(click.Command('path'), obj={'verbose': True, 'output': 'text'}) - - def mock_secho(msg: str, *_, **__) -> None: - assert 'Error:' in msg - - monkeypatch.setattr(click, 'secho', mock_secho) - - with ctx, pytest.raises(ClickException): - _handle_exception(ctx, ValueError('test')) - def test_is_file_relevant_for_sca_scan() -> None: path = os.path.join('some_package', 'node_modules', 'package.json') diff --git a/tests/cli/test_main.py b/tests/cli/commands/test_main_command.py similarity index 96% rename from tests/cli/test_main.py rename to tests/cli/commands/test_main_command.py index 3f41ed6a..d74a2c40 100644 --- a/tests/cli/test_main.py +++ b/tests/cli/commands/test_main_command.py @@ -6,7 +6,7 @@ import responses from click.testing import CliRunner -from cycode.cli.main import main_cli +from cycode.cli.commands.main_cli import main_cli from tests.conftest import CLI_ENV_VARS, TEST_FILES_PATH, ZIP_CONTENT_PATH from tests.cyclient.mocked_responses.scan_client import mock_scan_responses from tests.cyclient.test_scan_client import get_zipped_file_scan_response, get_zipped_file_scan_url diff --git a/tests/cli/exceptions/__init__.py b/tests/cli/exceptions/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/exceptions/test_handle_scan_errors.py b/tests/cli/exceptions/test_handle_scan_errors.py new file mode 100644 index 00000000..7d63802b --- /dev/null +++ b/tests/cli/exceptions/test_handle_scan_errors.py @@ -0,0 +1,67 @@ +from typing import TYPE_CHECKING + +import click +import pytest +from click import ClickException +from git import InvalidGitRepositoryError +from requests import Response + +from cycode.cli.exceptions import custom_exceptions +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception + +if TYPE_CHECKING: + from _pytest.monkeypatch import MonkeyPatch + + +@pytest.fixture() +def ctx() -> click.Context: + return click.Context(click.Command('path'), obj={'verbose': False, 'output': 'text'}) + + +@pytest.mark.parametrize( + 'exception, expected_soft_fail', + [ + (custom_exceptions.NetworkError(400, 'msg', Response()), True), + (custom_exceptions.ScanAsyncError('msg'), True), + (custom_exceptions.HttpUnauthorizedError('msg', Response()), True), + (custom_exceptions.ZipTooLargeError(1000), True), + (custom_exceptions.TfplanKeyError('msg'), True), + (InvalidGitRepositoryError(), None), + ], +) +def test_handle_exception_soft_fail( + ctx: click.Context, exception: custom_exceptions.CycodeError, expected_soft_fail: bool +) -> None: + with ctx: + handle_scan_exception(ctx, exception) + + assert ctx.obj.get('did_fail') is True + assert ctx.obj.get('soft_fail') is expected_soft_fail + + +def test_handle_exception_unhandled_error(ctx: click.Context) -> None: + with ctx, pytest.raises(ClickException): + handle_scan_exception(ctx, ValueError('test')) + + assert ctx.obj.get('did_fail') is True + assert ctx.obj.get('soft_fail') is None + + +def test_handle_exception_click_error(ctx: click.Context) -> None: + with ctx, pytest.raises(ClickException): + handle_scan_exception(ctx, click.ClickException('test')) + + assert ctx.obj.get('did_fail') is True + assert ctx.obj.get('soft_fail') is None + + +def test_handle_exception_verbose(monkeypatch: 'MonkeyPatch') -> None: + ctx = click.Context(click.Command('path'), obj={'verbose': True, 'output': 'text'}) + + def mock_secho(msg: str, *_, **__) -> None: + assert 'Error:' in msg + + monkeypatch.setattr(click, 'secho', mock_secho) + + with ctx, pytest.raises(ClickException): + handle_scan_exception(ctx, ValueError('test')) diff --git a/tests/cli/files_collector/iac/__init__.py b/tests/cli/files_collector/iac/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/cli/helpers/test_tf_content_generator.py b/tests/cli/files_collector/iac/test_tf_content_generator.py similarity index 100% rename from tests/cli/helpers/test_tf_content_generator.py rename to tests/cli/files_collector/iac/test_tf_content_generator.py diff --git a/tests/cyclient/test_auth_client.py b/tests/cyclient/test_auth_client.py index 0a34d3b2..51618e3c 100644 --- a/tests/cyclient/test_auth_client.py +++ b/tests/cyclient/test_auth_client.py @@ -3,6 +3,7 @@ import responses from requests import Timeout +from cycode.cli.commands.auth.auth_manager import AuthManager from cycode.cli.exceptions.custom_exceptions import CycodeError from cycode.cyclient.auth_client import AuthClient from cycode.cyclient.models import ( @@ -14,16 +15,12 @@ @pytest.fixture(scope='module') def code_challenge() -> str: - from cycode.cli.auth.auth_manager import AuthManager - code_challenge, _ = AuthManager()._generate_pkce_code_pair() return code_challenge @pytest.fixture(scope='module') def code_verifier() -> str: - from cycode.cli.auth.auth_manager import AuthManager - _, code_verifier = AuthManager()._generate_pkce_code_pair() return code_verifier diff --git a/tests/cyclient/test_scan_client.py b/tests/cyclient/test_scan_client.py index 6df7b544..69d657b2 100644 --- a/tests/cyclient/test_scan_client.py +++ b/tests/cyclient/test_scan_client.py @@ -8,10 +8,10 @@ from requests import Timeout from requests.exceptions import ProxyError -from cycode.cli.code_scanner import zip_documents from cycode.cli.config import config from cycode.cli.exceptions.custom_exceptions import CycodeError, HttpUnauthorizedError from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip +from cycode.cli.files_collector.zip_documents import zip_documents from cycode.cli.models import Document from cycode.cyclient.scan_client import ScanClient from tests.conftest import ZIP_CONTENT_PATH