From ed5d6583433d7bb094b5fcde3abe39ab80e76327 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Mon, 24 Jul 2023 16:33:19 +0200 Subject: [PATCH 01/15] chore: improved logging --- main.py | 53 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/main.py b/main.py index bb1d09e..bab8ab5 100644 --- a/main.py +++ b/main.py @@ -5,12 +5,15 @@ import random import json import yaml +import logging from pathlib import Path from typing import Dict, Any, Tuple, Callable, Sequence, Optional, List NUM_RETRIES = 3 +logging.basicConfig(level=logging.INFO) + class CouldNotAuthenticateException(Exception): pass @@ -46,10 +49,12 @@ def _retry_request( for retry_num in range(num_retries): if retry_num > 0: sleep_time = rand() + retry_num - print( + logging.info( f"Sleeping {sleep_time} seconds before retry " - f"{retry_num} of {num_retries - 1} after {reason}" + f"{retry_num} of {num_retries - 1}" ) + if reason: + logging.info(reason) sleep(sleep_time) response = requests.request( method, @@ -70,7 +75,7 @@ def _retry_request( reason = f"response status code: {response.status_code}, for message: {error_message}" continue if not validation_function(response): - print("Response did not pass the validation, retrying...") + logging.info("Response did not pass the validation, retrying...") continue return response raise CouldNotRetryRequestException() @@ -93,6 +98,7 @@ def _download_report( def login(username: str, password: str) -> requests.Response: + logging.debug("Sending request to retrieve login token") try: return _retry_request( "GET", @@ -100,10 +106,10 @@ def login(username: str, password: str) -> requests.Response: auth=(username, password), ) except CouldNotAuthenticateException: - print("Credentials are not correct, please check the configuration.") + logging.error("Credentials are not correct, please check the configuration.") sys.exit(1) except CouldNotRetryRequestException: - print("Could not get response after all retries, exiting...") + logging.error("Could not get response after all retries, exiting...") sys.exit(1) @@ -114,6 +120,7 @@ def validate(token: str, build: Path, payload: Dict[str, str]) -> requests.Respo (build.name, open(build.as_posix(), "rb"), "application/octet-stream"), ) ] + logging.debug(f"Sending package {build.name} for validation") try: response = _retry_request( "POST", @@ -126,22 +133,26 @@ def validate(token: str, build: Path, payload: Dict[str, str]) -> requests.Respo ) return response except CouldNotAuthenticateException: - print("Credentials are not correct, please check the configuration.") + logging.error("Credentials are not correct, please check the configuration.") sys.exit(1) except CouldNotRetryRequestException: - print("Could not get response after all retries, exiting...") + logging.error("Could not get response after all retries, exiting...") sys.exit(1) def submit(token: str, request_id: str) -> requests.Response: def _validate_validation_status(response: requests.Response) -> bool: - return response.json()["status"] == "SUCCESS" + is_successful = response.json()["status"] == "SUCCESS" + if is_successful: + logging.info('Response status is not "SUCCESS"') + return is_successful # appinspect api needs some time to process the request # if the response status will be "PROCESSING" wait 60s and make another call # there is a problem with pycov marking this line as not covered - excluded from coverage try: + logging.debug("Submitting package") return _retry_request( # pragma: no cover "GET", f"https://appinspect.splunk.com/v1/app/validate/status/{request_id}", @@ -153,22 +164,24 @@ def _validate_validation_status(response: requests.Response) -> bool: validation_function=_validate_validation_status, ) except CouldNotAuthenticateException: - print("Credentials are not correct, please check the configuration.") + logging.error("Credentials are not correct, please check the configuration.") sys.exit(1) except CouldNotRetryRequestException: - print("Could not get response after all retries, exiting...") + logging.error("Could not get response after all retries, exiting...") sys.exit(1) def download_json_report( token: str, request_id: str, payload: Dict[str, Any] ) -> requests.Response: + logging.debug("Downloading response in json format") return _download_report( token=token, request_id=request_id, payload=payload, response_type="json" ) def download_and_save_html_report(token: str, request_id: str, payload: Dict[str, Any]): + logging.debug("Downloading report in html format") response = _download_report( token=token, request_id=request_id, payload=payload, response_type="html" ) @@ -178,6 +191,9 @@ def download_and_save_html_report(token: str, request_id: str, payload: Dict[str def get_appinspect_failures_list(response_dict: Dict[str, Any]) -> List[str]: + logging.debug( + f"Parsing json respnose to find failed checks\n response: {response_dict}" + ) reports = response_dict["reports"] groups = reports[0]["groups"] @@ -187,7 +203,7 @@ def get_appinspect_failures_list(response_dict: Dict[str, Any]) -> List[str]: for check in group["checks"]: if check["result"] == "failure": failed_tests_list.append(check["name"]) - print(f"Failed appinspect check for name: {check['name']}\n") + logging.debug(f"Failed appinspect check for name: {check['name']}\n") return failed_tests_list @@ -196,26 +212,25 @@ def read_yaml_as_dict(filename_path: Path) -> Dict[str, str]: try: out_dict = yaml.safe_load(file) except yaml.YAMLError as e: - print(f"Can not read yaml file named {filename_path}") + logging.error(f"Can not read yaml file named {filename_path}") raise e return out_dict if out_dict else {} def compare_failures(failures: List[str], expected: List[str]): if sorted(failures) != sorted(expected): - print( + logging.error( "Appinspect failures doesn't match appinspect.expect file, check for exceptions file" ) raise AppinspectFailures def parse_results(results: Dict[str, Any]): - print(results) print("\n======== AppInspect Api Results ========") for metric, count in results["info"].items(): print(f"{metric:>15} : {count: <4}") if results["info"]["error"] > 0 or results["info"]["failure"] > 0: - print("\nError or failures found in App Inspect\n") + logging.error("\nError or failures found in App Inspect\n") raise AppinspectChecksFailuresException @@ -236,8 +251,8 @@ def compare_against_known_failures(response_json: Dict[str, Any], exceptions_fil expected_failures = list(read_yaml_as_dict(exceptions_file_path).keys()) compare_failures(failures, expected_failures) else: - print( - "ERROR: File `.appinspect_api.expect.yaml` not found, please create `.appinspect_api.except.yaml` file with exceptions\n" # noqa: E501 + logging.error( + f"File `{exceptions_file_path.name}` not found, please create `{exceptions_file_path.name}` file with exceptions\n" # noqa: E501 ) sys.exit(1) @@ -261,12 +276,12 @@ def main(argv: Optional[Sequence[str]] = None): login_response = login(args.username, args.password) token = login_response.json()["data"]["token"] - print("Successfully received token") + logging.info("Successfully received token") payload = build_payload(args.included_tags, args.excluded_tags) validate_response = validate(token, build, payload) - print(f"Successfully sent package for validation using {payload}") + logging.info(f"Successfully sent package for validation using {payload}") request_id = validate_response.json()["request_id"] submit_response = submit(token, request_id) From d5ff95cd2ed9c7906e9b0158835430c01469899f Mon Sep 17 00:00:00 2001 From: mbruzda Date: Tue, 25 Jul 2023 15:36:59 +0200 Subject: [PATCH 02/15] test: fixed tests for improved logging --- test/unit/test_main.py | 99 ++++++++++-------------------------------- 1 file changed, 23 insertions(+), 76 deletions(-) diff --git a/test/unit/test_main.py b/test/unit/test_main.py index 14f94ec..88dd56d 100644 --- a/test/unit/test_main.py +++ b/test/unit/test_main.py @@ -34,28 +34,23 @@ def test_login_success(mock_requests): @mock.patch.object(main, "_retry_request") -def test_login_when_credentials_are_not_ok(mock_retry_request, capsys): +def test_login_when_credentials_are_not_ok(mock_retry_request, caplog): mock_retry_request.side_effect = main.CouldNotAuthenticateException with pytest.raises(SystemExit): main.login("username", "password") - captured = capsys.readouterr() - - assert ( - captured.out == "Credentials are not correct, please check the configuration.\n" - ) + assert "Credentials are not correct, please check the configuration." in caplog.text @mock.patch.object(main, "_retry_request") -def test_login_cant_retry_request(mock_retry_request, capsys): +def test_login_cant_retry_request(mock_retry_request, caplog): mock_retry_request.side_effect = main.CouldNotRetryRequestException with pytest.raises(SystemExit): main.login("username", "password") - captured = capsys.readouterr() - assert captured.out == "Could not get response after all retries, exiting...\n" + assert "Could not get response after all retries, exiting...\n" in caplog.text @mock.patch("main.requests") @@ -89,7 +84,7 @@ def test_validate_success(mock_requests, tmp_path): @mock.patch.object(main, "_retry_request") -def test_validate_invalid_token(mock_retry_request, capsys, tmp_path): +def test_validate_invalid_token(mock_retry_request, caplog, tmp_path): mock_retry_request.side_effect = main.CouldNotAuthenticateException file = tmp_path / "test.spl" @@ -98,14 +93,13 @@ def test_validate_invalid_token(mock_retry_request, capsys, tmp_path): with pytest.raises(SystemExit): main.validate(token="token", build=file, payload={}) - captured = capsys.readouterr() assert ( - captured.out == "Credentials are not correct, please check the configuration.\n" + "Credentials are not correct, please check the configuration.\n" in caplog.text ) @mock.patch.object(main, "_retry_request") -def test_validate_count_retry(mock_retry_request, capsys, tmp_path): +def test_validate_count_retry(mock_retry_request, caplog, tmp_path): mock_retry_request.side_effect = main.CouldNotRetryRequestException file = tmp_path / "test.spl" @@ -114,8 +108,7 @@ def test_validate_count_retry(mock_retry_request, capsys, tmp_path): with pytest.raises(SystemExit): main.validate(token="token", build=file, payload={}) - captured = capsys.readouterr() - assert captured.out == "Could not get response after all retries, exiting...\n" + assert "Could not get response after all retries, exiting...\n" in caplog.text @mock.patch("main.requests") @@ -155,27 +148,25 @@ def test_submit_success(mock_requests): @mock.patch.object(main, "_retry_request") -def test_submit_invalid_token(mock_retry_request, capsys): +def test_submit_invalid_token(mock_retry_request, caplog): mock_retry_request.side_effect = main.CouldNotAuthenticateException with pytest.raises(SystemExit): main.submit(token="invalid_token", request_id="1234-1234") - captured = capsys.readouterr() assert ( - captured.out == "Credentials are not correct, please check the configuration.\n" + "Credentials are not correct, please check the configuration.\n" in caplog.text ) @mock.patch.object(main, "_retry_request") -def test_submit_cant_retry_request(mock_retry_request, capsys): +def test_submit_cant_retry_request(mock_retry_request, caplog): mock_retry_request.side_effect = main.CouldNotRetryRequestException with pytest.raises(SystemExit): main.submit(token="invalid_token", request_id="1234-1234") - captured = capsys.readouterr() - assert captured.out == "Could not get response after all retries, exiting...\n" + assert "Could not get response after all retries, exiting...\n" in caplog.text @pytest.mark.parametrize( @@ -192,33 +183,7 @@ def test_build_payload(included, excluded, payload): assert test_payload == payload -# @mock.patch("main.requests") -# def test_download_html(mock_requests): -# mock_response = mock.MagicMock() -# -# sample_html = """ -# -# -# -# Sample HTML -# -# -#

This is sample HTML

-# -# -# """ -# -# mock_response.text = sample_html -# mock_response.status_code = 200 -# mock_requests.request.return_value = mock_response -# -# main.download_html_report("token", "123-123-123", {}) -# -# with open("./AppInspect_response.html") as test_output: -# assert test_output.read() == sample_html - - -def test_parse_results_errors(capsys): +def test_parse_results_errors(): results = {"info": {"error": 1, "failure": 1}} with pytest.raises(main.AppinspectChecksFailuresException): main.parse_results(results) @@ -230,11 +195,14 @@ def test_parse_results_no_errors(capsys): main.parse_results(results) captured = capsys.readouterr() - assert "{'info': {'error': 0, 'failure': 0}}\n" in captured.out + assert ( + "\n======== AppInspect Api Results ========\n error : 0 \n failure : 0 \n" + in captured.out + ) @mock.patch("main.requests") -def test_retry_request_always_400(mock_requests, capsys): +def test_retry_request_always_400(mock_requests): mock_response = mock.MagicMock() response_input_json = {"msg": "Invalid request"} mock_response.json.return_value = response_input_json @@ -246,16 +214,9 @@ def test_retry_request_always_400(mock_requests, capsys): method="GET", url="http://test", sleep=lambda _: 0.0, rand=lambda: 0.0 ) - captured = capsys.readouterr() - - assert ( - "Sleeping 1.0 seconds before retry 1 of 2 after response status code: 400, for message: Invalid request\n" - in captured.out - ) - @mock.patch("main.requests") -def test_retry_request_message_key_in_response(mock_requests, capsys): +def test_retry_request_message_key_in_response(mock_requests): mock_response = mock.MagicMock() response_input_json = {"message": "message key instead of msg"} mock_response.json.return_value = response_input_json @@ -270,9 +231,6 @@ def test_retry_request_message_key_in_response(mock_requests, capsys): rand=lambda: 0.0, ) - captured = capsys.readouterr() - assert "message key instead of msg" in captured.out - @mock.patch("main.requests") def test_retry_request_error_401(mock_requests, capsys): @@ -287,7 +245,7 @@ def test_retry_request_error_401(mock_requests, capsys): @mock.patch("main.requests") -def test_retry_request_did_not_pass_validation(mock_requests, capsys): +def test_retry_request_did_not_pass_validation(mock_requests): mock_response = mock.MagicMock() response_input_json = {"message": "message key instead of msg"} mock_response.json.return_value = response_input_json @@ -303,13 +261,9 @@ def test_retry_request_did_not_pass_validation(mock_requests, capsys): validation_function=lambda _: False, ) - captured = capsys.readouterr() - - assert "Response did not pass the validation, retrying..." in captured.out - @mock.patch("main.requests") -def test_retry_request_501_then_200(mock_request, capsys): +def test_retry_request_501_then_200(mock_request): mock_response_501 = mock.MagicMock() response_input_json_501 = {"status_code": 501, "message": "should be retried"} mock_response_501.json.return_value = response_input_json_501 @@ -337,13 +291,7 @@ def test_retry_request_501_then_200(mock_request, capsys): response = main._retry_request("user", "password") - captured = capsys.readouterr() - assert response.status_code == 200 - assert ( - "retry 1 of 2 after response status code: 501, for message: should be retried" - in captured.out - ) @mock.patch("main.download_and_save_html_report") @@ -721,7 +669,7 @@ def test_compare_failures_fails(): @mock.patch("yaml.safe_load") -def test_read_yaml_as_dict_incorrect_yaml(mock_safe_load, capsys, tmp_path): +def test_read_yaml_as_dict_incorrect_yaml(mock_safe_load, caplog, tmp_path): mock_safe_load.side_effect = yaml.YAMLError file_path = tmp_path / "foo.yaml" file_path.write_text("test") @@ -729,8 +677,7 @@ def test_read_yaml_as_dict_incorrect_yaml(mock_safe_load, capsys, tmp_path): with pytest.raises(yaml.YAMLError): main.read_yaml_as_dict(file_path) - captured = capsys.readouterr() - assert captured.out == f"Can not read yaml file named {file_path}\n" + assert f"Can not read yaml file named {file_path}\n" in caplog.text def test_compare_known_failures_no_exceptions(tmp_path): From cf6838c091a79461be62fceda6b542e50cbaa8cf Mon Sep 17 00:00:00 2001 From: mbruzda Date: Tue, 25 Jul 2023 15:38:15 +0200 Subject: [PATCH 03/15] chore: improved logging --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 74faaf3..1b7c647 100644 --- a/main.py +++ b/main.py @@ -271,7 +271,7 @@ def main(argv: Optional[Sequence[str]] = None): args = parser.parse_args(argv) - print( + logging.info( f"app_path={args.app_path}, included_tags={args.included_tags}, excluded_tags={args.excluded_tags}" ) build = Path(args.app_path) From efaefb8942dde9c8e053f751f9d1a6960546da29 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Wed, 26 Jul 2023 14:06:32 +0200 Subject: [PATCH 04/15] fix: added log_level parameter to action --- action.yml | 4 ++++ entrypoint.sh | 4 ++-- main.py | 5 +++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/action.yml b/action.yml index 14cbedb..248659b 100644 --- a/action.yml +++ b/action.yml @@ -19,6 +19,10 @@ inputs: description: comma seperated list of tags to be excluded from appinspect scans default: "" required: false + log_level: + description: severity for python logging ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") + default: "INFO" + required: false runs: using: "docker" image: docker://ghcr.io/splunk/appinspect-api-action/appinspect-api-action:v3.0.0 diff --git a/entrypoint.sh b/entrypoint.sh index 33b4c4a..823de7f 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -3,6 +3,6 @@ ADDON_NAME=$(ls $INPUT_APP_PATH) ADDON_FULL_PATH="$INPUT_APP_PATH/$ADDON_NAME" -echo "$INPUT_USERNAME" "$INPUT_PASSWORD" "$ADDON_FULL_PATH" "$INPUT_INCLUDED_TAGS" "$INPUT_EXCLUDED_TAGS" +echo "$INPUT_USERNAME" "$INPUT_PASSWORD" "$ADDON_FULL_PATH" "$INPUT_INCLUDED_TAGS" "$INPUT_EXCLUDED_TAGS" "$INPUT_LOG_LEVEL" -python3 /main.py "$INPUT_USERNAME" "$INPUT_PASSWORD" "$ADDON_FULL_PATH" "$INPUT_INCLUDED_TAGS" "$INPUT_EXCLUDED_TAGS" \ No newline at end of file +python3 /main.py "$INPUT_USERNAME" "$INPUT_PASSWORD" "$ADDON_FULL_PATH" "$INPUT_INCLUDED_TAGS" "$INPUT_EXCLUDED_TAGS" "$INPUT_LOG_LEVEL" \ No newline at end of file diff --git a/main.py b/main.py index 1b7c647..1d3fcc6 100644 --- a/main.py +++ b/main.py @@ -12,8 +12,6 @@ NUM_RETRIES = 3 -logging.basicConfig(level=logging.INFO) - class CouldNotAuthenticateException(Exception): pass @@ -266,11 +264,14 @@ def main(argv: Optional[Sequence[str]] = None): parser.add_argument("app_path") parser.add_argument("included_tags") parser.add_argument("excluded_tags") + parser.add_argument("log_level") appinspect_expect_filename = ".appinspect_api.expect.yaml" args = parser.parse_args(argv) + logging.basicConfig(level=args.log_level) + logging.info( f"app_path={args.app_path}, included_tags={args.included_tags}, excluded_tags={args.excluded_tags}" ) From 4defbaa2fe7ac0d0f682b5ae2f8b1fef800171e5 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Wed, 26 Jul 2023 14:17:11 +0200 Subject: [PATCH 05/15] test: fix unit tests --- test/unit/test_main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit/test_main.py b/test/unit/test_main.py index 88dd56d..54ccdb1 100644 --- a/test/unit/test_main.py +++ b/test/unit/test_main.py @@ -387,7 +387,7 @@ def test_main_errors_in_except_file( download_mock_response.status_code = 200 mock_download_and_save_html_report.request.return_value = download_mock_response - main.main(["user", "pass", "build", "i_tag", "e_tag"]) + main.main(["user", "pass", "build", "i_tag", "e_tag", "DEBUG"]) @mock.patch("main.download_json_report") @@ -501,7 +501,7 @@ def test_main_failures_file_does_not_exist( mock_download_json_report.return_value = mock_json_response with pytest.raises(SystemExit): - main.main(["user", "pass", "build", "i_tag", "e_tag"]) + main.main(["user", "pass", "build", "i_tag", "e_tag", "DEBUG"]) @mock.patch("main.validate") @@ -531,7 +531,7 @@ def test_main_invalid_token(mock_login, mock_validate): mock_validate.side_effect = main.CouldNotAuthenticateException with pytest.raises(main.CouldNotAuthenticateException): - main.main(["user", "pass", "build", "i_tag", "e_tag"]) + main.main(["user", "pass", "build", "i_tag", "e_tag", "DEBUG"]) @mock.patch("main.login") @@ -539,7 +539,7 @@ def test_main_api_down_cant_retry_request(mock_login): mock_login.side_effect = main.CouldNotRetryRequestException with pytest.raises(main.CouldNotRetryRequestException): - main.main(["user", "pass", "build", "i_tag", "e_tag"]) + main.main(["user", "pass", "build", "i_tag", "e_tag", "DEBUG"]) @mock.patch("main.requests") From 2499b8bae47c4f71503e9bb8a0a91b4c83a9fb01 Mon Sep 17 00:00:00 2001 From: Marcin Bruzda <94437843+mbruzda-splunk@users.noreply.github.com> Date: Thu, 27 Jul 2023 10:40:02 +0200 Subject: [PATCH 06/15] docs: update readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fc8d296..42c0ca3 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ jobs: | `app_path` | Path to the directory where addon is located, without filename | **required** | | | `included_tags` | Comma separated list of [tags](#reference-docs) to include in appinspect job | | None | | `excluded_tags` | Comma separated list of [tags](#reference-docs) to exclude from appinspect job | | None | +| `log_level' | Python logging level for action | | `INFO | You can explicitly include and exclude tags from a validation by including additional options in your request. Specifically, using the included_tags and excluded_tags options includes and excludes the tags you specify from a validation. If no tags are specified all checks will be done and no tags are excluded from the validation. From 26a3a8284a6c22977c7a15f17107ae719c462e75 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Thu, 27 Jul 2023 10:57:36 +0200 Subject: [PATCH 07/15] test: use Dockefile --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 248659b..2a418cb 100644 --- a/action.yml +++ b/action.yml @@ -25,4 +25,4 @@ inputs: required: false runs: using: "docker" - image: docker://ghcr.io/splunk/appinspect-api-action/appinspect-api-action:v3.0.0 + image: Dockerfile From 9545e7a805ae203ccd0e9b99d4a84d3040db2e4d Mon Sep 17 00:00:00 2001 From: mbruzda Date: Thu, 27 Jul 2023 14:16:22 +0200 Subject: [PATCH 08/15] chore: remove newlines --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 1d3fcc6..33c754a 100644 --- a/main.py +++ b/main.py @@ -228,7 +228,7 @@ def parse_results(results: Dict[str, Any]): for metric, count in results["info"].items(): print(f"{metric:>15} : {count: <4}") if results["info"]["error"] > 0 or results["info"]["failure"] > 0: - logging.error("\nError or failures found in App Inspect\n") + logging.warning("Error or failures found in App Inspect") raise AppinspectChecksFailuresException From 51689d07342efcb235f220aa2038bebc735fdd18 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Mon, 31 Jul 2023 14:14:48 +0200 Subject: [PATCH 09/15] fix: more logging improvements --- main.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 33c754a..9739fdb 100644 --- a/main.py +++ b/main.py @@ -118,7 +118,7 @@ def validate(token: str, build: Path, payload: Dict[str, str]) -> requests.Respo (build.name, open(build.as_posix(), "rb"), "application/octet-stream"), ) ] - logging.debug(f"Sending package {build.name} for validation") + logging.debug(f"Sending package `{build.name}` for validation") try: response = _retry_request( "POST", @@ -142,7 +142,9 @@ def submit(token: str, request_id: str) -> requests.Response: def _validate_validation_status(response: requests.Response) -> bool: is_successful = response.json()["status"] == "SUCCESS" if is_successful: - logging.info('Response status is not "SUCCESS"') + logging.debug( + f'Response status is `{response.json()["status"]}`, "SUCCESS" expected.' + ) return is_successful # appinspect api needs some time to process the request @@ -190,7 +192,7 @@ def download_and_save_html_report(token: str, request_id: str, payload: Dict[str def get_appinspect_failures_list(response_dict: Dict[str, Any]) -> List[str]: logging.debug( - f"Parsing json respnose to find failed checks\n response: {response_dict}" + f"Parsing json response to find failed checks\n response: {response_dict}" ) reports = response_dict["reports"] groups = reports[0]["groups"] @@ -220,6 +222,8 @@ def compare_failures(failures: List[str], expected: List[str]): logging.error( "Appinspect failures doesn't match appinspect.expect file, check for exceptions file" ) + logging.debug(f"Appinspect failures: {failures}") + logging.debug(f"Expected failures: {expected}") raise AppinspectFailures @@ -228,7 +232,7 @@ def parse_results(results: Dict[str, Any]): for metric, count in results["info"].items(): print(f"{metric:>15} : {count: <4}") if results["info"]["error"] > 0 or results["info"]["failure"] > 0: - logging.warning("Error or failures found in App Inspect") + logging.warning("Error or failures found in AppInspect Report") raise AppinspectChecksFailuresException @@ -243,6 +247,9 @@ def build_payload(included_tags: str, excluded_tags: str) -> Dict[str, str]: def compare_against_known_failures(response_json: Dict[str, Any], exceptions_file_path): + logging.debug( + f"Comparing AppInspect Failures with `{exceptions_file_path.name}` file" + ) failures = get_appinspect_failures_list(response_json) if exceptions_file_path.exists(): @@ -279,15 +286,18 @@ def main(argv: Optional[Sequence[str]] = None): login_response = login(args.username, args.password) token = login_response.json()["data"]["token"] - logging.info("Successfully received token") + logging.debug("Successfully received token") payload = build_payload(args.included_tags, args.excluded_tags) + logging.debug(f"Validation payload: {payload}") validate_response = validate(token, build, payload) - logging.info(f"Successfully sent package for validation using {payload}") + logging.debug(f"Successfully sent package for validation using {payload}") request_id = validate_response.json()["request_id"] submit_response = submit(token, request_id) + logging.info("Successfully submitted and validated package") + download_and_save_html_report(token, request_id, payload) # if this is true it compares the exceptions and results From 6f056917b8d9b645c5115826cb72a1af11981a2a Mon Sep 17 00:00:00 2001 From: Marcin Bruzda <94437843+mbruzda-splunk@users.noreply.github.com> Date: Mon, 31 Jul 2023 15:33:35 +0200 Subject: [PATCH 10/15] chore: Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 42c0ca3..7817ec9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ jobs: | `app_path` | Path to the directory where addon is located, without filename | **required** | | | `included_tags` | Comma separated list of [tags](#reference-docs) to include in appinspect job | | None | | `excluded_tags` | Comma separated list of [tags](#reference-docs) to exclude from appinspect job | | None | -| `log_level' | Python logging level for action | | `INFO | +| `log_level` | Python logging level for action | | `INFO | You can explicitly include and exclude tags from a validation by including additional options in your request. Specifically, using the included_tags and excluded_tags options includes and excludes the tags you specify from a validation. If no tags are specified all checks will be done and no tags are excluded from the validation. From e777dba5439d4447f158a7001ccfc8bfc7940e33 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Mon, 31 Jul 2023 15:36:01 +0200 Subject: [PATCH 11/15] chore: remove big json from debug logs --- main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/main.py b/main.py index 9739fdb..e404fb6 100644 --- a/main.py +++ b/main.py @@ -191,9 +191,7 @@ def download_and_save_html_report(token: str, request_id: str, payload: Dict[str def get_appinspect_failures_list(response_dict: Dict[str, Any]) -> List[str]: - logging.debug( - f"Parsing json response to find failed checks\n response: {response_dict}" - ) + logging.debug("Parsing json response to find failed checks\n") reports = response_dict["reports"] groups = reports[0]["groups"] From 899feb4af86ad584c041f7435a634f90139b1ade Mon Sep 17 00:00:00 2001 From: mbruzda Date: Tue, 1 Aug 2023 10:51:51 +0200 Subject: [PATCH 12/15] chore: change debug to info in compare --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index e404fb6..3f12765 100644 --- a/main.py +++ b/main.py @@ -245,7 +245,7 @@ def build_payload(included_tags: str, excluded_tags: str) -> Dict[str, str]: def compare_against_known_failures(response_json: Dict[str, Any], exceptions_file_path): - logging.debug( + logging.info( f"Comparing AppInspect Failures with `{exceptions_file_path.name}` file" ) failures = get_appinspect_failures_list(response_json) From 81eebc09a61d85314412f3a6d8f7da023f0b2a64 Mon Sep 17 00:00:00 2001 From: Marcin Bruzda <94437843+mbruzda-splunk@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:42:54 +0200 Subject: [PATCH 13/15] chore: typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7817ec9..61fb97b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ jobs: | `app_path` | Path to the directory where addon is located, without filename | **required** | | | `included_tags` | Comma separated list of [tags](#reference-docs) to include in appinspect job | | None | | `excluded_tags` | Comma separated list of [tags](#reference-docs) to exclude from appinspect job | | None | -| `log_level` | Python logging level for action | | `INFO | +| `log_level` | Python logging level for action | | `INFO` | You can explicitly include and exclude tags from a validation by including additional options in your request. Specifically, using the included_tags and excluded_tags options includes and excludes the tags you specify from a validation. If no tags are specified all checks will be done and no tags are excluded from the validation. From 542d1495fe14c12250035da784db042d7651df74 Mon Sep 17 00:00:00 2001 From: mbruzda Date: Wed, 2 Aug 2023 14:19:41 +0200 Subject: [PATCH 14/15] chore: change logging when appinspect action fails --- main.py | 11 +++++++---- test/unit/test_main.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 3f12765..111e84b 100644 --- a/main.py +++ b/main.py @@ -217,9 +217,6 @@ def read_yaml_as_dict(filename_path: Path) -> Dict[str, str]: def compare_failures(failures: List[str], expected: List[str]): if sorted(failures) != sorted(expected): - logging.error( - "Appinspect failures doesn't match appinspect.expect file, check for exceptions file" - ) logging.debug(f"Appinspect failures: {failures}") logging.debug(f"Expected failures: {expected}") raise AppinspectFailures @@ -252,7 +249,13 @@ def compare_against_known_failures(response_json: Dict[str, Any], exceptions_fil if exceptions_file_path.exists(): expected_failures = list(read_yaml_as_dict(exceptions_file_path).keys()) - compare_failures(failures, expected_failures) + try: + compare_failures(failures, expected_failures) + except AppinspectFailures: + logging.error( + "Appinspect failures doesn't match appinspect.expect file, check for exceptions file" + ) + sys.exit(1) else: logging.error( f"File `{exceptions_file_path.name}` not found, please create `{exceptions_file_path.name}` file with exceptions\n" # noqa: E501 diff --git a/test/unit/test_main.py b/test/unit/test_main.py index 54ccdb1..f72bdc1 100644 --- a/test/unit/test_main.py +++ b/test/unit/test_main.py @@ -699,7 +699,7 @@ def test_compare_known_failures_no_exceptions(tmp_path): exceptions_file = tmp_path / "foo.yaml" exceptions_file.write_text(exceptions_content) - with pytest.raises(main.AppinspectFailures): + with pytest.raises(SystemExit): main.compare_against_known_failures(response_json, exceptions_file) From dc1ed90a15f6577ec63ff897ca44f676aad40cce Mon Sep 17 00:00:00 2001 From: mbruzda Date: Wed, 2 Aug 2023 15:50:41 +0200 Subject: [PATCH 15/15] chore: typo --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 111e84b..22dda5c 100644 --- a/main.py +++ b/main.py @@ -253,7 +253,7 @@ def compare_against_known_failures(response_json: Dict[str, Any], exceptions_fil compare_failures(failures, expected_failures) except AppinspectFailures: logging.error( - "Appinspect failures doesn't match appinspect.expect file, check for exceptions file" + "Appinspect failures don't match appinspect.expect file, check for exceptions file" ) sys.exit(1) else: