Skip to content

Commit

Permalink
Added requests session retry feature
Browse files Browse the repository at this point in the history
  • Loading branch information
mohamedmujtabaraza committed Apr 1, 2022
1 parent f7e0c26 commit 18894e4
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 4 deletions.
Binary file modified src/__pycache__/app.cpython-38.pyc
Binary file not shown.
14 changes: 10 additions & 4 deletions src/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import re

import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
import validators
import requests_random_user_agent
from bs4 import BeautifulSoup
Expand Down Expand Up @@ -51,8 +53,12 @@
'Referer': 'https://www.google.com'
}

s = requests.Session()
s.headers.update(HEADERS)
session = requests.Session()
session.headers.update(HEADERS)
retry = Retry(total=5, connect=3, backoff_factor=0.5)
adapter = HTTPAdapter(max_retries=retry)
session.mount('http://', adapter)
session.mount('https://', adapter)


def get_webpage(word_url):
Expand All @@ -64,8 +70,8 @@ def get_webpage(word_url):
# print("Found")
break
if not r_text:
print(s.headers['User-Agent'], s.headers['Referer'])
r_text = s.get(word_url).text
print(session.headers['User-Agent'], session.headers['Referer'])
r_text = session.get(word_url, verify=False).text
CONTAINER['requests'].append((word_url, r_text))
return r_text

Expand Down

0 comments on commit 18894e4

Please sign in to comment.