[SCRAPER]: include title on scraped

Author: amadolid

jaseci_ai_kit/jac_misc/jac_misc/scraper/async_scraper.py

@@ -94,7 +94,13 @@ async def scrape(
     pages: list, pre_configs: list = [], detailed: bool = False, target: str = None
 ):
     content = ""
-    urls = {"scanned": [], "scanned_urls": set(), "scraped": [], "crawled": set()}
+    urls = {
+        "url": pages[0]["goto"]["url"],
+        "scanned": [],
+        "scanned_urls": set(),
+        "scraped": {},
+        "crawled": set(),
+    }
 
     ws = Client()
     trigger_id = uuid4()
@@ -140,6 +146,7 @@ async def scrape(
 
     if detailed:
         return {
+            "url": urls["url"],
             "content": content,
             "scanned": urls["scanned"],
             "scraped": urls["scraped"],
@@ -202,7 +209,9 @@ async def getters(page: Page, specss: list[dict], urls: dict):
             if expression:
                 print(f"[getters]: getting content from {page.url}")
                 content += await page.evaluate(f"() =>{expression}")
-            add_url(page, urls, expression)
+                add_url(page, urls, await page.title())
+            else:
+                add_url(page, urls)
 
             await run_scripts(page, post, urls)
 

jaseci_ai_kit/jac_misc/jac_misc/scraper/sync_scraper.py

@@ -45,7 +45,13 @@ def scrape(
     pages: list, pre_configs: list = [], detailed: bool = False, target: str = None
 ):
     content = ""
-    urls = {"scanned": [], "scanned_urls": set(), "scraped": [], "crawled": set()}
+    urls = {
+        "url": pages[0]["goto"]["url"],
+        "scanned": [],
+        "scanned_urls": set(),
+        "scraped": {},
+        "crawled": set(),
+    }
 
     trigger_id = uuid4()
 
@@ -89,6 +95,7 @@ def scrape(
 
     if detailed:
         return {
+            "url": urls["url"],
             "content": content,
             "scanned": urls["scanned"],
             "scraped": urls["scraped"],
@@ -150,7 +157,9 @@ def getters(page: Page, specss: list[dict], urls: dict):
             if expression:
                 print(f"[getters]: getting content from {page.url}")
                 content += page.evaluate(f"() =>{expression}")
-            add_url(page, urls, expression)
+                add_url(page, urls, page.title())
+            else:
+                add_url(page, urls)
 
             run_scripts(page, post, urls)
 

jaseci_ai_kit/jac_misc/jac_misc/scraper/utils.py

@@ -2,7 +2,7 @@
 from copy import deepcopy
 
 
-def add_url(page, urls: dict, scraped: bool = False, error: str = None):
+def add_url(page, urls: dict, title: str = None, error: str = None):
     url = page.url
     source = page.source
     if url:
@@ -16,8 +16,8 @@ def add_url(page, urls: dict, scraped: bool = False, error: str = None):
                 scan["source"] = source
             urls["scanned"].append(scan)
 
-        if scraped and url not in urls["scraped"]:
-            urls["scraped"].append(url)
+        if title and url not in urls["scraped"]:
+            urls["scraped"][url] = title
 
 
 def add_crawl(pages: list, pre_configs: list, urls: dict, url: str, def_crawl: dict):