From 94c2ebea1434007c4d7db17d5081158dfb35be03 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Tue, 12 Mar 2024 12:44:21 +0000 Subject: [PATCH] Change secret replacement so that it is not re-processed by urlencode in query parameters --- dispatcher/backend/src/common/constants.py | 2 +- .../src/tests/unit/routes/test_utils.py | 104 +++++++++--------- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/dispatcher/backend/src/common/constants.py b/dispatcher/backend/src/common/constants.py index d367bb84..c03408d8 100644 --- a/dispatcher/backend/src/common/constants.py +++ b/dispatcher/backend/src/common/constants.py @@ -75,7 +75,7 @@ SLACK_ICON = os.getenv("SLACK_ICON") # string to replace hidden secrets with -SECRET_REPLACEMENT = "********" # nosec +SECRET_REPLACEMENT = "--------" # nosec # ### # workers whitelist management diff --git a/dispatcher/backend/src/tests/unit/routes/test_utils.py b/dispatcher/backend/src/tests/unit/routes/test_utils.py index 8b0a896d..62f67d08 100644 --- a/dispatcher/backend/src/tests/unit/routes/test_utils.py +++ b/dispatcher/backend/src/tests/unit/routes/test_utils.py @@ -29,12 +29,12 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, "upload": None, @@ -62,12 +62,12 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -94,12 +94,12 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -127,13 +127,13 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", "flag_missing_in_commang": "some_value", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -156,7 +156,7 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", "flag_missing_in_commang": "some_value", }, }, @@ -195,23 +195,23 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], "str_command": ( 'kolibri2zim --name="khanacademy_en_all" ' - '--optimization-cache="********"' + '--optimization-cache="--------"' ), }, "container": { "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -240,14 +240,14 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, }, "container": { "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -276,14 +276,14 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, }, "container": { "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -312,14 +312,14 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, }, "container": { "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -358,16 +358,16 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], "str_command": ( 'kolibri2zim --name="khanacademy_en_all" ' - '--optimization-cache="********"' + '--optimization-cache="--------"' ), }, "container": { @@ -375,7 +375,7 @@ "something", "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], }, }, @@ -426,16 +426,16 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, "command": [ "kolibri2zim", '--name="khanacademy_en_all"', - '--optimization-cache="********"', + '--optimization-cache="--------"', ], "str_command": ( 'kolibri2zim --name="khanacademy_en_all" ' - '--optimization-cache="********"' + '--optimization-cache="--------"' ), }, "upload": { @@ -443,8 +443,8 @@ "expiration": 60, "upload_uri": ( "s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs" ), }, @@ -452,8 +452,8 @@ "expiration": 20, "upload_uri": ( "s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-artifacts" ), }, @@ -486,15 +486,15 @@ "task_name": "kolibri", "flags": { "name": "khanacademy_en_all", - "optimization-cache": "********", + "optimization-cache": "--------", }, }, "i_am_not_a_real": { "response_but": { "please_clean_me": ( "something\nwhat s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs what\n" "something\n" ), @@ -573,68 +573,68 @@ def test_remove_secrets(response, expected_response): "something" ), "please_clean_me8": ( - " ftp://username:password@hostname:123/path not encoded?param=value" + " ftp://username:password@hostname:123/path not encoded?param=val%26ue" "#anchor " ), }, { "please_clean_me1": ( "s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs" ), "please_clean_me2": ( "s3://s3.us-west-1.wasabisys.com/" "?bucketName=org-kiwix-zimfarm-logs" - "&keyId=********" - "&secretAccessKey=********" + "&keyId=--------" + "&secretAccessKey=--------" ), "please_clean_me3": ( "s3://s3.us-west-1.wasabisys.com/" "?bucketName=org-kiwix-zimfarm-logs" - "&keyId=********" - "&secretAccessKey=********" + "&keyId=--------" + "&secretAccessKey=--------" "&something=somevalue" ), "please_clean_me4": ( "s3://s3.us-west-1.wasabisys.com/" "?bucketName=org-kiwix-zimfarm-logs" - "&secretAccessKey=********" + "&secretAccessKey=--------" "&something=somevalue" - "&keyId=********" + "&keyId=--------" "&something2=somevalue2" ), "please_clean_me5": ( " s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs" ), "please_clean_me6": ( "s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs " ), "please_clean_me7": ( "something s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" - "&secretAccessKey=********" + "?keyId=--------" + "&secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs \n" "something s3://s3.us-west-1.wasabisys.com/" - "?secretAccessKey=********" + "?secretAccessKey=--------" "&bucketName=org-kiwix-zimfarm-logs \n" "something s3://s3.us-west-1.wasabisys.com/" - "?keyId=********" + "?keyId=--------" "&bucketName=org-kiwix-zimfarm-logs \n" "something s3://s3.us-west-1.wasabisys.com/" "?bucketName=org-kiwix-zimfarm-logs" - "&keyId=******** \n" + "&keyId=-------- \n" "something" ), "please_clean_me8": ( - " ftp://username:********@hostname:123/path not encoded?param=value" + " ftp://username:--------@hostname:123/path not encoded?param=val%26ue" "#anchor " ), },