don't index javascript URIs and URL fragments

s0md3v · May 1, 2019 · c960849 · c960849
1 parent 6a29f2c
commit c960849
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/core/utils.py b/core/utils.py
@@ -38,6 +38,8 @@ def is_link(url, processed, files):
         bool If `url` should be crawled
     """
     if url not in processed:
+        if url.startswith('#') or url.startswith('javascript:'):
+            return False
         is_file = url.endswith(BAD_TYPES)
         if is_file:
             files.add(url)