Image Crawler Added

Author: digitaldreams

Crawler/Image.py

@@ -0,0 +1,51 @@
+import urllib.parse
+from Crawler.ImgFinder import ImgFinder
+import Models.Queue.Image
+
+
+class Image:
+    def __init__(self, page_url):
+
+        image = Models.Queue.Image.Image
+        image.fetch()
+        self.image = image
+
+        self.page_url = page_url
+        urlres = urllib.parse.urlparse(page_url)
+        self.base_url = urlres.netloc
+        self.images = set()
+
+    def add(self, link):
+        full_url = self.sanitize_url(link)
+        if full_url:
+            self.images.add(full_url)
+        return self
+
+    def fetch_links(self):
+        """
+        Get all the anchor tag url from the website
+        :return:
+        """
+        img_finder = ImgFinder(self.page_url)
+        img_finder.feed(img_finder.html_string())
+        self.images = img_finder.get_values()
+        return self.images
+
+    def links(self):
+        return self.images
+
+    def _merge_links(self):
+        for lk in self.images:
+            self.image.add(lk)
+        return self.image.links
+
+    def save(self):
+        self._merge_links()
+        self.image.save()
+
+    def save_links(self):
+        self.fetch_links()
+        self.save()
+
+
+

Crawler/ImgFinder.py

@@ -0,0 +1,6 @@
+from UrlFinder import UrlFinder
+
+
+class ImgFinder(UrlFinder):
+    def __init__(self, page_url):
+        UrlFinder.__init__(self, page_url, 'img', 'src')

Crawler/__pycache__/Image.cpython-36.pyc

@@ -2,7 +2,7 @@
 import urllib.parse
 import urllib.request
 
-from Image import Finder
+from Crawler import ImgFinder
 
 
 def gather_img_src(page_url) -> object:
@@ -13,7 +13,7 @@ def gather_img_src(page_url) -> object:
     """
     try:
         html = html_string(page_url)
-        finder = Finder.ImgFinder(page_url)
+        finder = ImgFinder.ImgFinder(page_url)
         finder.feed(html)
     except Exception as e:
         print(str(e))

Crawler/__pycache__/ImgFinder.cpython-36.pyc

@@ -2,7 +2,12 @@
 from Models.Complete.Image import Image
 import os.path;
 from Crawler.Page import Page
+from Crawler.Image import Image
+
 if __name__ == '__main__':
+    """
     page=Page('https://gopostie.com')
-    page.save()
-    page.save_links()
+    page.save_links()
+    """
+    img = Image('https://gopostie.com')
+    img.save_links()