Merge pull request #45 from Skyscanner/html

HTML plugin
Skyscanner · Mar 6, 2021 · 031d6be · 031d6be
2 parents 3a9e2c8 + b5b6731
commit 031d6be
Show file tree

Hide file tree

Showing 10 changed files with 67 additions and 4 deletions.
diff --git a/requirements.txt b/requirements.txt
@@ -5,14 +5,13 @@
 #    make freeze
 #
 astroid==2.5.1            # via whispers (setup.py)
+beautifulsoup4==4.9.3     # via whispers (setup.py)
 jproperties==2.1.0        # via whispers (setup.py)
-lazy-object-proxy==1.5.2  # via astroid
 luhn==0.2.0               # via whispers (setup.py)
 lxml==4.6.2               # via whispers (setup.py)
 python-levenshtein==0.12.2  # via whispers (setup.py)
 pyyaml==5.4.1             # via whispers (setup.py)
 six==1.15.0               # via astroid, jproperties
-wrapt==1.12.1             # via astroid
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/setup.py b/setup.py
@@ -9,7 +9,7 @@ def get_version():
     return import_module("whispers.__version__").__version__
 
 
-install_requires = ["luhn==0.2.0", "lxml==4.6.2", "pyyaml==5.3.1", "astroid==2.4.2", "jproperties==2.1.0", "python-levenshtein==0.12.0"]
+install_requires = ["luhn>=0.2.0", "lxml>=4.6.2", "pyyaml>=5.3.1", "astroid>=2.4.2", "jproperties>=2.1.0", "python-levenshtein>=0.12.0", "beautifulsoup4>=4.9.3"]
 
 dev_requires = [
     "black>=19.10b0",

diff --git a/tests/fixtures/language.html b/tests/fixtures/language.html
@@ -0,0 +1,16 @@
+<html>
+<head>
+    <title>Whispers HTML fixture</title>
+</head>
+<body>
+    <!-- TODO: hardcoded comment 01 -->
+    <!-- 
+        TODO: hardcoded comment 02    
+    -->
+    <!-- 
+        TODO: hardcoded comment 03
+        has multiple lines 
+    -->
+    This is not a hardcoded comment
+</body>
+</html>
diff --git a/tests/unit/test_secrets.py b/tests/unit/test_secrets.py
@@ -84,6 +84,7 @@ def test_detection_by_key(src, keys):
         ("language.java", 3),
         ("language.go", 9),
         ("language.php", 4),
+        ("language.html", 3),
         ("plaintext.txt", 2),
         ("uri.yml", 2),
         ("java.properties", 3),

diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
@@ -19,9 +19,24 @@
     similar_strings,
     simple_string,
     strip_string,
+    truncate_all_space,
 )
 
 
+@pytest.mark.parametrize(
+    ("rawstr", "expected"),
+    [
+        ("", ""),
+        ("whis\npers", "whis pers"),
+        ("whis\tpers", "whis pers"),
+        ("whis\n\n\n\npers", "whis pers"),
+        ("whis\n       pers", "whis pers"),
+    ],
+)
+def test_truncate_all_space(rawstr, expected):
+    assert truncate_all_space(rawstr) == expected
+
+
 @pytest.mark.parametrize(
     "rawstr",
     [

diff --git a/whispers/__version__.py b/whispers/__version__.py
@@ -1,3 +1,3 @@
-VERSION = (1, 4, 3)
+VERSION = (1, 4, 4)
 
 __version__ = ".".join(map(str, VERSION))
diff --git a/whispers/plugins/__init__.py b/whispers/plugins/__init__.py
@@ -5,6 +5,7 @@
 from whispers.plugins.config import Config
 from whispers.plugins.dockerfile import Dockerfile
 from whispers.plugins.go import Go
+from whispers.plugins.html import Html
 from whispers.plugins.htpasswd import Htpasswd
 from whispers.plugins.java import Java
 from whispers.plugins.javascript import Javascript
@@ -67,6 +68,8 @@ def load_plugin(self) -> Optional[object]:
             return Htpasswd()
         elif self.filetype == "txt":
             return Plaintext()
+        elif self.filetype.startswith("htm"):
+            return Html()
         elif self.filetype == "py":
             return Python()
         elif self.filetype == "js":

diff --git a/whispers/plugins/html.py b/whispers/plugins/html.py
@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from bs4 import BeautifulSoup, Comment
+
+from whispers.utils import truncate_all_space
+
+
+class Html:
+    def pairs(self, filepath: Path):
+        soup = BeautifulSoup(filepath.read_text(), "lxml")
+        comments = soup.find_all(text=lambda t: isinstance(t, Comment))
+        for comment in comments:
+            comment = truncate_all_space(comment)
+            if len(comment):
+                yield "comment", comment
diff --git a/whispers/rules/comments.yml b/whispers/rules/comments.yml
@@ -0,0 +1,8 @@
+comments:
+  description: Potential information leak through comments
+  message: Comment
+  severity: MINOR
+  key:
+    regex: "^comment$"
+    ignorecase: False
+    minlen: 1
diff --git a/whispers/utils.py b/whispers/utils.py
@@ -14,6 +14,12 @@
 escaped_chars = str.maketrans({"'": r"\'", '"': r"\""})
 
 
+def truncate_all_space(value: str) -> str:
+    if not value:
+        return ""
+    return re.sub(r"\s+", " ", value)
+
+
 def strip_string(value: str) -> str:
     """
     Strips leading and trailing quotes and spaces