Skip to content

Commit

Permalink
Fix unicode errors in web_search
Browse files Browse the repository at this point in the history
  • Loading branch information
hlohaus committed Jan 24, 2025
1 parent 76b34e6 commit 9d6777e
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions g4f/tools/web_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ def scrape_text(html: str, max_words: int = None, add_source=True, count_images:
count_images -= 1
continue

for line in paragraph.text.splitlines():
for line in paragraph.get_text(" ").splitlines():
words = [word for word in line.split() if word]
count = len(words)
if not count:
Expand Down Expand Up @@ -136,7 +136,7 @@ async def fetch_and_scrape(session: ClientSession, url: str, max_words: int = No
return cache_file.read_text()
async with session.get(url) as response:
if response.status == 200:
html = await response.text()
html = await response.text(errors="replace")
text = "".join(scrape_text(html, max_words, add_source))
with open(cache_file, "w") as f:
f.write(text)
Expand Down

0 comments on commit 9d6777e

Please sign in to comment.