fix and update test

Querent-ai · Sep 2, 2023 · 1810c97 · 1810c97
1 parent 2450b79
commit 1810c97
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 1 deletion.
diff --git a/querent/tools/web_page_extractor.py b/querent/tools/web_page_extractor.py
@@ -271,6 +271,7 @@ def extract_links(self, url):
                     if (
                         link_href.startswith(url)
                         and link_href not in self.crawled_urls
+                        and link_href != url
                     ):
                         links.append(link_href)
                 return links

diff --git a/tests/test_webscrapper.py b/tests/test_webscrapper.py
@@ -23,7 +23,7 @@ def test_fs_collector_factory():
 
 
 def test_scrapping_data():
-    uri = Uri("https://asecuritysite.com/")
+    uri = Uri("https://protocolstreams.xyz/")
     resolver = CollectorResolver()
     webscrapperConfig = WebScraperConfig(website_url=uri.uri)
     collector = resolver.resolve(uri, webscrapperConfig)