Merge pull request #8 from gabrielweiz/feature/mask-sensitive-data-log

adding settings options with regex to be able to mask sensitive data

Author: VMRuiz

README.md

@@ -27,6 +27,7 @@ When you run your spider you will see a log like below when spider is closing:
 
 * `SETTINGS_LOGGING_REGEX` - Add a regular expression to only show some settings - for example `SETTINGS_LOGGING_REGEX = "SPIDERMON"` will show settings with SPIDERMON in their name.
 * `SETTINGS_LOGGING_INDENT` - Add indentation to make log more human-readable.
+* `MASKED_SENSITIVE_SETTINGS_ENABLED` - Default is `True` - if settings logging is enabled it will mask the value of settings that may be sensitive (password, apikey). For example AWS_SECRET_ACCESS_KEY will have their value shown as **********
 
 ## Advanced
 

src/scrapy_settings_log/__init__.py

@@ -9,6 +9,12 @@
 
 logger = logging.getLogger(__name__)
 
+DEFAULT_REGEXES = [
+    ".*(?i)(api[\W_]*key).*",  # apikey and variations e.g: shub_apikey or SC_APIKEY
+    ".*(?i)(AWS[\W_]*(SECRET[\W_]*)?(ACCESS)?[\W_]*(KEY|ACCESS[\W_]*KEY))",  # AWS_SECRET_ACCESS_KEY and variations
+    ".*(?i)([\W_]*password[\W_]*).*"  # password word
+]
+
 
 def prepare_for_json_serialization(obj):
     """Prepare the obj recursively for JSON serialization.
@@ -51,6 +57,11 @@ def spider_closed(self, spider):
         regex = settings.get("SETTINGS_LOGGING_REGEX")
         if regex is not None:
             settings = {k: v for k, v in settings.items() if re.search(regex, k)}
+        if spider.settings.getbool("MASKED_SENSITIVE_SETTINGS_ENABLED", True):
+            regex_list = spider.settings.getlist("MASKED_SENSITIVE_SETTINGS_REGEX_LIST", DEFAULT_REGEXES)
+            for reg in regex_list:
+                updated_settings = {k: '**********' if v else v for k, v in settings.items() if re.match(reg, k)}
+                settings = {**settings, **updated_settings}
 
         self.output_settings(settings, spider)
 

tests/test_code.py

@@ -103,3 +103,95 @@ class CustomClass:
         logger.spider_closed(spider)
 
     assert '{"DUMMY_CUSTOM_CLASS": {"CustomClass": "/foo/bar"}}' in caplog.text
+
+
+def test_log_all_should_not_return_apikey_value_by_default(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "SHUB_APIKEY": 'apikey_value1',
+        "shub_apikey": 'apikey_value2',
+        "api_key": 'apikey_value3',
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    assert '"SHUB_APIKEY": "**********"' in caplog.text
+    assert '"shub_apikey": "**********"' in caplog.text
+    assert '"api_key": "**********"' in caplog.text
+    assert 'apikey_value' not in caplog.text
+
+
+def test_log_all_should_return_apikey_value_if_MASKED_SENSITIVE_SETTINGS_ENABLED_is_false(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "APIKEY": 'apikey_value',
+        "MASKED_SENSITIVE_SETTINGS_ENABLED": False,
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    assert '"APIKEY": "apikey_value"' in caplog.text
+
+
+def test_log_all_should_not_return_aws_secret_key_value_by_default(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "AWS_SECRET_ACCESS_KEY": 'secret_value1',
+        "aws_secret_access_key": 'secret_value2',
+        "aws_access_key": 'secret_value2',
+        "AWS_SECRET_KEY": 'secret_value2',
+        "aws_secret_key": 'secret_value2',
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    assert '"AWS_SECRET_ACCESS_KEY": "**********"' in caplog.text
+    assert '"aws_secret_access_key": "**********"' in caplog.text
+    assert '"aws_access_key": "**********"' in caplog.text
+    assert '"AWS_SECRET_KEY": "**********"' in caplog.text
+    assert '"aws_secret_key": "**********"' in caplog.text
+    assert 'secret_value' not in caplog.text
+
+
+def test_log_all_should_not_return_password_value_by_default(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "test_password": 'secret_value1',
+        "PASSWORD_TEST": 'secret_value2',
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    assert '"test_password": "**********"' in caplog.text
+    assert '"PASSWORD_TEST": "**********"' in caplog.text
+    assert 'secret_value' not in caplog.text
+
+
+def test_log_all_should_return_only_the_custom_regex_data_masked_if_MASKED_SENSITIVE_SETTINGS_REGEX_LIST_configured(caplog):
+    settings = {
+        "SETTINGS_LOGGING_ENABLED": True,
+        "MASKED_SENSITIVE_SETTINGS_REGEX_LIST": ["apppppppikey"],
+        "APIKEY": 'apikey_value1',
+        "apppppppikey": 'some_random_value',
+    }
+
+    spider = MockSpider(settings)
+    logger = SpiderSettingsLogging()
+    with caplog.at_level(logging.INFO):
+        logger.spider_closed(spider)
+
+    assert 'apikey_value1' in caplog.text
+    assert '"apppppppikey": "**********"' in caplog.text
+    assert 'some_random_value' not in caplog.text