diff --git a/twitter/tweety.py b/twitter/tweety.py index f0650e8..14597d3 100644 --- a/twitter/tweety.py +++ b/twitter/tweety.py @@ -1,5 +1,6 @@ from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By from selenium import webdriver from textblob import TextBlob from bs4 import BeautifulSoup @@ -17,15 +18,18 @@ class tweets: :returns: Returns all the tweets. """ - def __init__(self, keyword): + def __init__(self, keyword, ChromeOptions=None, BS4parser='lxml'): start_time = datetime.now() - options = Options() + if ChromeOptions is not None: + options = ChromeOptions + else: + options = Options() options.headless = True options=options browser = webdriver.Chrome(options=options) browser.get("https://twitter.com/search?q=" + keyword) - while browser.find_element_by_tag_name('div'): + while browser.find_element(By.TAG_NAME, 'div'): browser.execute_script("window.scrollTo(0, document.body.scrollHeight);") time_delta = datetime.now() - start_time sys.stdout.write('\r' + str("calculating time") + " " + str(time_delta.seconds) + " " + "seconds taken to parse all the tweets from twitter" + '\r') @@ -33,7 +37,7 @@ def __init__(self, keyword): if time_delta.seconds >= 150: break - soup = BeautifulSoup(browser.page_source, 'lxml') + soup = BeautifulSoup(browser.page_source, BS4parser) browser.quit() tweets_count = soup.select('.TweetTextSize')