From 89171942743cdbd84ac059ab6c95f595dc056e83 Mon Sep 17 00:00:00 2001 From: Charlie Date: Fri, 9 Apr 2021 06:47:13 -0400 Subject: [PATCH 01/14] Fix bug in crash reporting Only report and raise Exceptions other than KeyboardInterrupt. Add log and `return` to handling of interrupt. Move notification and `raise` from `else:` to `except:` - this would generate an exception when the program exited normally during `--single-shot` --- cli/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 9cc802d0..893e7ced 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -64,8 +64,9 @@ def decorator(*args, **kwargs): try: func(*args, **kwargs) except KeyboardInterrupt: - pass - else: + log.info("Caught Ctrl-C - Exiting...") + return + except: notification_handler.send_notification(f"FairGame has crashed.") raise From 4a054a751284f0dff0fbd680b315143a8ceeacc4 Mon Sep 17 00:00:00 2001 From: Charlie Date: Fri, 9 Apr 2021 06:50:06 -0400 Subject: [PATCH 02/14] Specify window size for headless Failing to specify window size when running headless can result in very small windows and unhelpful screenshots. Chose 1920x1080 because it is currently the most common desktop size. --- utils/selenium_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/selenium_utils.py b/utils/selenium_utils.py index d7121fa6..b368836d 100644 --- a/utils/selenium_utils.py +++ b/utils/selenium_utils.py @@ -153,5 +153,6 @@ def add_cookies_to_session_from_driver(driver, session): def enable_headless(): options.add_argument("--headless") + options.add_argument("--window-size=1920x1080") options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") From e00f34961f0fc3576f95643229dc52cea3237e09 Mon Sep 17 00:00:00 2001 From: Charlie Date: Sun, 11 Apr 2021 06:41:26 -0400 Subject: [PATCH 03/14] Track time to check stock Changed log messages to warnings for error conditions --- cli/cli.py | 4 +++- stores/amazon.py | 57 +++++++++++++++++++++++++++++++----------------- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 893e7ced..3fe14e04 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -63,9 +63,11 @@ def notify_on_crash(func): def decorator(*args, **kwargs): try: func(*args, **kwargs) + except KeyboardInterrupt: - log.info("Caught Ctrl-C - Exiting...") + log.info("Caught ctrl-c; Exiting...") return + except: notification_handler.send_notification(f"FairGame has crashed.") raise diff --git a/stores/amazon.py b/stores/amazon.py index c6383783..99015ad4 100644 --- a/stores/amazon.py +++ b/stores/amazon.py @@ -126,7 +126,9 @@ def __init__( self.single_shot = single_shot self.take_screenshots = not no_screenshots self.start_time = time.time() + self.start_time_check = 0 self.start_time_atc = 0 + self.end_time_atc = 0 self.webdriver_child_pids = [] self.driver = None self.refresh_delay = DEFAULT_REFRESH_DELAY @@ -432,7 +434,7 @@ def run_asins(self, delay): while not found_asin: for i in range(len(self.asin_list)): for asin in self.asin_list[i]: - # start_time = time.time() + self.start_time_check = time.time() if self.log_stock_check: log.info(f"Checking ASIN: {asin}.") if self.check_stock(asin, self.reserve_min[i], self.reserve_max[i]): @@ -526,7 +528,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): ) ) if footer and footer[0].tag_name == "img": - log.info(f"Saw dogs for {asin}. Skipping...") + log.warning(f"Saw dogs for {asin}. Skipping...") return False log.debug(f"After footer page title {self.driver.title}") @@ -698,7 +700,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): if test and (test.text in amazon_config["NO_SELLERS"]): return False if time.time() > timeout: - log.info(f"failed to load page for {asin}, going to next ASIN") + log.warning(f"Failed to load page for {asin}, going to next ASIN") return False timeout = self.get_timeout() @@ -718,7 +720,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): if prices: break if time.time() > timeout: - log.info(f"failed to load prices for {asin}, going to next ASIN") + log.warning(f"Failed to load prices for {asin}, going to next ASIN") return False shipping = [] shipping_prices = [] @@ -757,7 +759,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): break if time.time() > timeout: - log.info(f"failed to load shipping for {asin}, going to next ASIN") + log.warning(f"Failed to load shipping for {asin}, going to next ASIN") return False in_stock = False @@ -829,6 +831,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): ): return True else: + log.error(f"Failed to Add to Cart with offer ID: {offering_id}") self.send_notification( "Failed Add to Cart after {max-atc-retries}", "failed-atc", @@ -868,9 +871,9 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): ): return True else: - log.info("did not add to cart, trying again") + log.warning("Did not add to cart, trying again") if emtpy_cart_elements: - log.info( + log.warning( "Cart appeared empty after clicking Add To Cart button" ) log.debug(f"failed title was {self.driver.title}") @@ -942,14 +945,15 @@ def navigate_pages(self, test): f"Title was blank, checking to find a real title for {timeout_seconds} seconds" ) timeout = self.get_timeout(timeout=timeout_seconds) - while True: + while time.time() <= timeout: if self.driver.title != "": title = self.driver.title log.debug(f"found a real title: {title}.") break - if time.time() > timeout: - log.debug("Time out reached, page title was still blank.") - break + time.sleep(0.05) + else: + log.debug("Time out reached, page title was still blank.") + if title in amazon_config["SIGN_IN_TITLES"]: self.login() elif title in amazon_config["CAPTCHA_PAGE_TITLES"]: @@ -1073,7 +1077,7 @@ def navigate_pages(self, test): self.driver.get(AMAZON_URLS["CART_URL"]) except sel_exceptions.WebDriverException: log.error( - "failed to load cart URL, refreshing and returning to handler" + "Failed to load cart URL, refreshing and returning to handler" ) with self.wait_for_page_content_change(timeout=10): self.driver.refresh() @@ -1254,7 +1258,7 @@ def do_button_click( @debug def handle_home_page(self): - log.info("On home page, trying to get back to checkout") + log.warning("On home page, trying to get back to checkout") button = None tries = 0 maxTries = 10 @@ -1270,9 +1274,9 @@ def handle_home_page(self): if self.do_button_click(button=button): return else: - log.info("Failed to click on cart button") + log.error("Failed to click on cart button") else: - log.info("Could not find cart button after " + str(maxTries) + " tries") + log.error("Could not find cart button after " + str(maxTries) + " tries") # no button found or could not interact with the button self.send_notification( @@ -1284,7 +1288,7 @@ def handle_home_page(self): while self.driver.title == current_page: time.sleep(0.25) if time.time() > timeout: - log.info("user failed to intervene in time, returning to stock check") + log.error("user failed to intervene in time, returning to stock check") self.try_to_checkout = False break @@ -1310,12 +1314,12 @@ def handle_cart(self): except sel_exceptions.NoSuchElementException: pass if self.get_cart_count() == 0: - log.info("You have no items in cart. Going back to stock check.") + log.error("You have no items in cart. Going back to stock check.") self.try_to_checkout = False break if time.time() > timeout: - log.info("couldn't find buttons to proceed to checkout") + log.error("couldn't find buttons to proceed to checkout") self.save_page_source("ptc-error") self.send_notification( "Proceed to Checkout Error Occurred", @@ -1385,9 +1389,15 @@ def handle_checkout(self, test): self.order_retry += 1 return if test: + self.end_time_atc = time.time() log.info(f"Found button {button.text}, but this is a test") log.info("will not try to complete order") - log.info(f"test time took {time.time() - self.start_time_atc} to check out") + log.info( + f" From cart: took {self.end_time_atc - self.start_time_atc} to check out" + ) + log.info( + f" From check: took {self.end_time_atc - self.start_time_check} to check out" + ) self.try_to_checkout = False self.great_success = True if self.single_shot: @@ -1398,7 +1408,14 @@ def handle_checkout(self, test): @debug def handle_order_complete(self): + self.end_time_atc = time.time() log.info("Order Placed.") + log.info( + f" From cart: took {self.end_time_atc - self.start_time_atc} to check out" + ) + log.info( + f" From check: took {self.end_time_atc - self.start_time_check} to check out" + ) self.send_notification("Order placed.", "order-placed", self.take_screenshots) self.notification_handler.play_purchase_sound() self.great_success = True @@ -1611,7 +1628,7 @@ def get_page(self, url): try: self.driver.get(url=url) except sel_exceptions.WebDriverException or sel_exceptions.TimeoutException: - log.error(f"failed to load page at url: {url}") + log.error(f"Failed to load page at url: {url}") return False if check_cart_element: timeout = self.get_timeout() From 1e2e88a592b9cae6d57541373b5aae300beefcd4 Mon Sep 17 00:00:00 2001 From: Charlie Date: Sun, 11 Apr 2021 10:47:20 -0400 Subject: [PATCH 04/14] Ignore all `config/*.json` files Also ignore `tags` for `ctags` users --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 1186b056..5f3368e8 100644 --- a/.gitignore +++ b/.gitignore @@ -10,9 +10,8 @@ .profile .profile-amz* .vscode/ -/config/*_credentials.json +/config/*.json /config/apprise.conf -/config/apprise_config.json Pipfile.lock __pycache__/ build/ @@ -20,4 +19,5 @@ dist/ geckodriver.log html_saves/*.html screenshots/*.png -logs/*.log* \ No newline at end of file +logs/*.log* +tags From a8e6d17f98f30174a981073405b655084b93c23a Mon Sep 17 00:00:00 2001 From: Charlie Date: Sun, 11 Apr 2021 15:41:01 -0400 Subject: [PATCH 05/14] Verify that min price is not greater than max Output error message and exit if reserve minimum price is greater than reserve maximum price. Otherwise, the script will never actually purchase anything because that's how math works. Print price range with message that there were no offers that met it as a further sanity check and to prevent ambiguity as to why the script detected an offer but did not attempt to cart it --- stores/amazon.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/stores/amazon.py b/stores/amazon.py index 99015ad4..fa77dc9b 100644 --- a/stores/amazon.py +++ b/stores/amazon.py @@ -179,10 +179,19 @@ def __init__( "amazon_website", "smile.amazon.com" ) for x in range(self.asin_groups): + if float(config[f"reserve_min_{x + 1}"]) > float( + config[f"reserve_max_{x + 1}"] + ): + log.error("Minimum price must be <= maximum price") + log.error( + f" {float(config[f'reserve_min_{x + 1}']):.2f} > {float(config[f'reserve_max_{x + 1}']):.2f}" + ) + exit(0) + self.asin_list.append(config[f"asin_list_{x + 1}"]) self.reserve_min.append(float(config[f"reserve_min_{x + 1}"])) self.reserve_max.append(float(config[f"reserve_max_{x + 1}"])) - # assert isinstance(self.asin_list, list) + except Exception as e: log.error(f"{e} is missing") log.error( @@ -887,6 +896,23 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): reserve_min=reserve_min, retry=retry + 1, ) + elif reserve_min > (price_float + ship_float): + log.debug( + f" Min ({reserve_min}) > Price ({price_float} + {ship_float} shipping)" + ) + + elif reserve_max < (price_float + ship_float): + log.debug( + f" Max ({reserve_max}) < Price ({price_float} + {ship_float} shipping)" + ) + + else: + log.error("Serious problem with price comparison") + log.error(f" Min: {reserve_min}") + log.error(f" Price: {price_float} + {ship_float} shipping") + log.error(f" Max: {reserve_max}") + + log.info(f"Offers exceed price range ({reserve_min:.2f}-{reserve_max:.2f})") return in_stock def attempt_atc(self, offering_id, max_atc_retries=DEFAULT_MAX_ATC_TRIES): @@ -1692,6 +1718,7 @@ def show_config(self): log.info( f"--Looking for {len(asins)} ASINs between {self.reserve_min[idx]:.2f} and {self.reserve_max[idx]:.2f}" ) + log.info(f"- {asins}") if not presence.enabled: log.info(f"--Discord Presence feature is disabled.") if self.no_image: @@ -1805,7 +1832,7 @@ def get_shipping_costs(tree, free_shipping_string): for free_message in amazon_config["FREE_SHIPPING"] ): # We found some version of "free" inside the span.. but this relies on a match - log.info( + log.debug( f"Assuming free shipping based on this message: '{shipping_span_text}'" ) return FREE_SHIPPING_PRICE From 9d205565eb98ed4fa749f35b1f7fb00e3f08e1d1 Mon Sep 17 00:00:00 2001 From: DakkJaniels <6080734+DakkJaniels@users.noreply.github.com> Date: Mon, 12 Apr 2021 10:36:37 -0400 Subject: [PATCH 06/14] Update cli.py Grab the exception on crash. Removed raise, since we logged the exception. --- cli/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 3fe14e04..6ddade48 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -68,9 +68,9 @@ def decorator(*args, **kwargs): log.info("Caught ctrl-c; Exiting...") return - except: + except Exception as e: + log.debug(e) notification_handler.send_notification(f"FairGame has crashed.") - raise return decorator From 436b1453dd4823cd3d60587314bfd4abb9d5d6be Mon Sep 17 00:00:00 2001 From: Charlie Date: Mon, 12 Apr 2021 10:51:41 -0400 Subject: [PATCH 07/14] Update cli.py Remove unnecessary log statement --- cli/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index 6ddade48..315e25ea 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -65,8 +65,7 @@ def decorator(*args, **kwargs): func(*args, **kwargs) except KeyboardInterrupt: - log.info("Caught ctrl-c; Exiting...") - return + pass except Exception as e: log.debug(e) From e41af4a22517f102b7b718b6bbb15fa034596a30 Mon Sep 17 00:00:00 2001 From: DakkJaniels <6080734+DakkJaniels@users.noreply.github.com> Date: Wed, 14 Apr 2021 13:33:16 -0400 Subject: [PATCH 08/14] Update selenium_utils.py --- utils/selenium_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/selenium_utils.py b/utils/selenium_utils.py index b368836d..e2d4fa0b 100644 --- a/utils/selenium_utils.py +++ b/utils/selenium_utils.py @@ -156,3 +156,4 @@ def enable_headless(): options.add_argument("--window-size=1920x1080") options.add_argument("--no-sandbox") options.add_argument("--disable-dev-shm-usage") + options.add_argument("--disable-gpu") From 45e44a94b69172aad57a4c441e447e44278f05ba Mon Sep 17 00:00:00 2001 From: Charlie Date: Wed, 14 Apr 2021 13:39:47 -0400 Subject: [PATCH 09/14] Add traceback on exception --- cli/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/cli.py b/cli/cli.py index 315e25ea..0aa041a6 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -21,6 +21,7 @@ import platform import shutil import time +import traceback from datetime import datetime from functools import wraps from pathlib import Path @@ -68,7 +69,7 @@ def decorator(*args, **kwargs): pass except Exception as e: - log.debug(e) + log.error(traceback.format_exc()) notification_handler.send_notification(f"FairGame has crashed.") return decorator From 81908f5bbd7f6cbd60066a5ca03aa27c9c555469 Mon Sep 17 00:00:00 2001 From: DakkJaniels <6080734+DakkJaniels@users.noreply.github.com> Date: Wed, 14 Apr 2021 13:45:53 -0400 Subject: [PATCH 10/14] Update fairgame.conf add some more titles --- config/fairgame.conf | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/config/fairgame.conf b/config/fairgame.conf index b21af181..7edf427c 100644 --- a/config/fairgame.conf +++ b/config/fairgame.conf @@ -99,6 +99,11 @@ "CHECKOUT_TITLES": [ "Amazon.com Checkout", "Amazon.co.uk Checkout", + "Amazon.ca Checkout", + "Amazon.es Checkout", + "Amazon.de Checkout", + "Amazon.fr Checkout", + "Amazon.it Checkout", "Place Your Order - Amazon.ca Checkout", "Place Your Order - Amazon.co.uk Checkout", "Amazon.de Checkout", @@ -165,7 +170,14 @@ "Complete your Amazon Prime sign up" ], "OUT_OF_STOCK": [ - "Out of Stock - AmazonSmile Checkout" + "Out of Stock - AmazonSmile Checkout", + "Out of Stock - Amazon.com Checkout", + "Out of Stock - Amazon.ca Checkout", + "Out of Stock - Amazon.de Checkout", + "Out of Stock - Amazon.fr Checkout", + "Out of Stock - Amazon.co.uk Checkout", + "Out of Stock - Amazon.es Checkout", + "Out of Stock - Amazon.it Checkout" ], "NO_SELLERS": [ "Currently, there are no sellers that can deliver this item to your location.", From d46e47d4efd32775e72705be8664d36e6c284cf6 Mon Sep 17 00:00:00 2001 From: DakkJaniels <6080734+DakkJaniels@users.noreply.github.com> Date: Wed, 14 Apr 2021 13:57:30 -0400 Subject: [PATCH 11/14] venv with fairgame, by default, remove traceback from timeout warning to terminal setting venv to fairgame folder to reduce issues with username having a space or special characters --- .gitignore | 1 + __INSTALL (RUN FIRST).bat | 1 + stores/amazon.py | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 5f3368e8..d184ef68 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ html_saves/*.html screenshots/*.png logs/*.log* tags +.venv* \ No newline at end of file diff --git a/__INSTALL (RUN FIRST).bat b/__INSTALL (RUN FIRST).bat index 76a44cea..43fd9c24 100644 --- a/__INSTALL (RUN FIRST).bat +++ b/__INSTALL (RUN FIRST).bat @@ -1,6 +1,7 @@ @echo on pip install pipenv pause +set PIPENV_VENV_IN_PROJECT=1 pipenv install diff --git a/stores/amazon.py b/stores/amazon.py index fa77dc9b..c2b5218a 100644 --- a/stores/amazon.py +++ b/stores/amazon.py @@ -666,12 +666,12 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): ) except sel_exceptions.TimeoutException as te: - log.error("Timed out waiting for offers to render. Skipping...") - log.error(f"URL: {self.driver.current_url}") - log.exception(te) + log.warning("Timed out waiting for offers to render. Skipping...") + log.warning(f"URL: {self.driver.current_url}") + log.debug(te) return False except sel_exceptions.NoSuchElementException: - log.error("Unable to find any offers listing. Skipping...") + log.warning("Unable to find any offers listing. Skipping...") return False except sel_exceptions.ElementClickInterceptedException as e: log.debug( From 799c21271caf0a1814461dad7400ffaf8bed852b Mon Sep 17 00:00:00 2001 From: DakkJaniels <6080734+DakkJaniels@users.noreply.github.com> Date: Wed, 14 Apr 2021 13:59:02 -0400 Subject: [PATCH 12/14] Update version.py Update version number to 0.6.3 --- utils/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/version.py b/utils/version.py index b6748163..cf097504 100644 --- a/utils/version.py +++ b/utils/version.py @@ -27,7 +27,7 @@ # See https://www.python.org/dev/peps/pep-0440/ for specification # See https://www.python.org/dev/peps/pep-0440/#examples-of-compliant-version-schemes for examples -__VERSION = "0.6.2" +__VERSION = "0.6.3" version = Version(__VERSION) From 7e977953d44b23b92db0c01440d7acb2f27d6f26 Mon Sep 17 00:00:00 2001 From: DakkJaniels <6080734+DakkJaniels@users.noreply.github.com> Date: Wed, 14 Apr 2021 14:34:27 -0400 Subject: [PATCH 13/14] (PDP/BuyBox) moved over code based on dev branch. Co-Authored-By: unapproachable <74546317+unapproachable@users.noreply.github.com> --- config/fairgame.conf | 3 + stores/amazon.py | 321 +++++++++++++++++-------------------------- 2 files changed, 129 insertions(+), 195 deletions(-) diff --git a/config/fairgame.conf b/config/fairgame.conf index 7edf427c..0f4c0434 100644 --- a/config/fairgame.conf +++ b/config/fairgame.conf @@ -232,6 +232,9 @@ ], "ATC": [ "//div[@id='aod-pinned-offer' or @id='aod-offer' or @id='olpOfferList']//input[@name='submit.addToCart']" + ], + "ATC_BUY_BOX":[ + "//div[@id='qualifiedBuybox']//input[@id='add-to-cart-button']" ] } } diff --git a/stores/amazon.py b/stores/amazon.py index c2b5218a..c6c21411 100644 --- a/stores/amazon.py +++ b/stores/amazon.py @@ -456,25 +456,8 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): if retry > DEFAULT_MAX_ATC_TRIES: log.info("max add to cart retries hit, returning to asin check") return False - - if self.alt_offers: - if self.checkshipping: - if self.used: - f = furl(self.ACTIVE_OFFER_URL + asin) - else: - f = furl(self.ACTIVE_OFFER_URL + asin + "/ref=olp_f_new&f_new=true") - else: - if self.used: - f = furl(self.ACTIVE_OFFER_URL + asin + "/f_freeShipping=on") - else: - f = furl( - self.ACTIVE_OFFER_URL - + asin - + "/ref=olp_f_new&f_new=true&f_freeShipping=on" - ) - else: - # Force the flyout by default - f = furl(self.ACTIVE_OFFER_URL + asin + "?aod=1") + # load page + f = furl(self.ACTIVE_OFFER_URL + asin) fail_counter = 0 presence.searching_update() @@ -526,50 +509,35 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): timeout = self.get_timeout() atc_buttons = None while True: + buy_box = False # Sanity check to see if we have any offers try: # Wait for the page to load before determining what's in it by looking for the footer - footer: List[WebElement] = WebDriverWait( + offer_container = WebDriverWait( self.driver, timeout=DEFAULT_MAX_TIMEOUT ).until( - lambda d: d.find_elements_by_xpath( - "//div[@class='nav-footer-line'] | //div[@id='navFooter'] | //img[@alt='Dogs of Amazon']" - ) - ) - if footer and footer[0].tag_name == "img": - log.warning(f"Saw dogs for {asin}. Skipping...") - return False - - log.debug(f"After footer page title {self.driver.title}") - log.debug(f" page url: {self.driver.current_url}") - - offers = WebDriverWait(self.driver, timeout=DEFAULT_MAX_TIMEOUT).until( lambda d: d.find_element_by_xpath( "//div[@id='aod-container'] | " - "//div[@id='olpOfferList'] | " "//div[@id='backInStock' or @id='outOfStock'] |" "//span[@data-action='show-all-offers-display'] | " "//input[@name='submit.add-to-cart' and not(//span[@data-action='show-all-offers-display'])]" ) ) offer_count = [] - offer_id = offers.get_attribute("id") + offer_id = offer_container.get_attribute("id") if offer_id == "outOfStock" or offer_id == "backInStock": # No dice... Early out and move on log.info("Item is currently unavailable. Moving on...") return False - - if offer_id == "olpOfferList": - # Offers Page ... count the 'a-row' classes to know how many offers we 'see' - offer_count = self.driver.find_elements_by_xpath( - "//div[@id='olpOfferList']//div[contains(@class, 'olpOffer')]" - ) elif offer_id == "aod-container": # Offer Flyout or Ajax call ... count the 'aod-offer' divs that we 'see' offer_count = self.driver.find_elements_by_xpath( "//div[@id='aod-pinned-offer' or @id='aod-offer']//input[@name='submit.addToCart']" ) - elif offers.get_attribute("data-action") == "show-all-offers-display": + elif ( + offer_container.get_attribute("data-action") + == "show-all-offers-display" + ): # PDP Page # Find the offers link first, just to burn some cycles in case the flyout is loading open_offers_link = None @@ -620,7 +588,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): self.driver, timeout=DEFAULT_MAX_TIMEOUT ).until( lambda d: d.find_element_by_xpath( - "//div[@id='aod-container'] | //div[@id='olpOfferList']" + "//div[@id='aod-container']" ) ) log.debug("Flyout should be open and populated.") @@ -633,25 +601,25 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): else: log.error("Could not open offers link") elif ( - offers.get_attribute("aria-labelledby") + offer_container.get_attribute("aria-labelledby") == "submit.add-to-cart-announce" ): - # This assumes we're on a PDP with only an add to cart button... no offers - log.warning( - "NOT YET IMPLEMENTED: PDP represents only item worth considering. No other sellers available." - " TODO: Parse pricing and Add To Cart from PDP if item qualifies." + # Use the Buy Box as an Offer as a last resort since it is not guaranteed to be a good offer + buy_box = True + offer_count = self.driver.find_elements_by_xpath( + "//div[@id='qualifiedBuybox']//input[@id='add-to-cart-button']" ) else: log.warning( "We found elements, but didn't recognize any of the combinations." ) - log.warning(f"Element found: {offers.tag_name}") + log.warning(f"Element found: {offer_container.tag_name}") attrs = self.driver.execute_script( "var items = {}; " "for (index = 0; index < arguments[0].attributes.length; ++index) " "{ items[arguments[0].attributes[index].name] = arguments[0].attributes[index].value }; " "return items;", - offers, + offer_container, ) log.warning("Dumping element attributes:") for attr in attrs: @@ -678,22 +646,10 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): "Covering element detected... Assuming it's a slow flyout... scanning document again..." ) continue - - atc_buttons = self.get_amazon_elements(key="ATC") - # if not atc_buttons: - # # Sanity check to see if we have a valid page, but no offers: - # offer_count = WebDriverWait(self.driver, timeout=25).until( - # lambda d: d.find_element_by_xpath( - # "//div[@id='aod-offer-list']//input[@id='aod-total-offer-count']" - # ) - # ) - # - # # offer_count = self.driver.find_element_by_xpath( - # # "//div[@id='aod-offer-list']//input[@id='aod-total-offer-count']" - # # ) - # if offer_count.get_attribute("value") == "0": - # log.info("Found zero offers explicitly. Moving to next ASIN.") - # return False + if buy_box: + atc_buttons = self.get_amazon_elements(key="ATC_BUY_BOX") + else: + atc_buttons = self.get_amazon_elements(key="ATC") if atc_buttons: # Early out if we found buttons break @@ -713,67 +669,47 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): return False timeout = self.get_timeout() - flyout_mode = False while True: - prices = self.driver.find_elements_by_xpath( - '//*[@class="a-size-large a-color-price olpOfferPrice a-text-bold"]' - ) - if not prices: - # Try the flyout x-paths + if buy_box: + prices = self.driver.find_elements_by_xpath( + "//span[@id='price_inside_buybox']" + ) + else: prices = self.driver.find_elements_by_xpath( "//div[@id='aod-pinned-offer' or @id='aod-offer']//div[contains(@id, 'aod-price')]//span[@class='a-price']//span[@class='a-offscreen']" ) - if prices: - flyout_mode = True - break if prices: break if time.time() > timeout: - log.warning(f"Failed to load prices for {asin}, going to next ASIN") + log.warning(f"failed to load prices for {asin}, going to next ASIN") return False shipping = [] shipping_prices = [] timeout = self.get_timeout() while True: - if not flyout_mode: - shipping = self.driver.find_elements_by_xpath( - '//*[@class="a-color-secondary"]' - ) - if shipping: - # Convert to prices just in case - for idx, shipping_node in enumerate(shipping): - log.debug(f"Processing shipping node {idx}") - if self.checkshipping: - if amazon_config["SHIPPING_ONLY_IF"] in shipping_node.text: - shipping_prices.append(parse_price("0")) - else: - shipping_prices.append(parse_price(shipping_node.text)) - else: - shipping_prices.append(parse_price("0")) + # Check for offers" + if buy_box: + offer_xpath = "//form[@id='addToCart']" else: - # Check for offers - # offer_xpath = "//div[@id='aod-pinned-offer' or @id='aod-offer']" offer_xpath = ( "//div[@id='aod-offer' and .//input[@name='submit.addToCart']] | " "//div[@id='aod-pinned-offer' and .//input[@name='submit.addToCart']]" ) - offers = self.driver.find_elements_by_xpath(offer_xpath) - for idx, offer in enumerate(offers): - tree = html.fromstring(offer.get_attribute("innerHTML")) - shipping_prices.append( - get_shipping_costs(tree, amazon_config["FREE_SHIPPING"]) - ) + offer_container = self.driver.find_elements_by_xpath(offer_xpath) + for idx, offer in enumerate(offer_container): + tree = html.fromstring(offer.get_attribute("innerHTML")) + shipping_prices.append( + get_shipping_costs(tree, amazon_config["FREE_SHIPPING"]) + ) if shipping_prices: break if time.time() > timeout: - log.warning(f"Failed to load shipping for {asin}, going to next ASIN") + log.warning(f"failed to load shipping for {asin}, going to next ASIN") return False in_stock = False - for shipping_price in shipping_prices: - log.debug(f"\tShipping Price: {shipping_price}") for idx, atc_button in enumerate(atc_buttons): # If the user has specified that they only want free items, we can skip any items @@ -782,7 +718,9 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): continue # Condition check first, using the button to find the form that will divulge the item's condition - if flyout_mode: + # with the assumption that anything in the Buy Box on the PDP *must* be New and therefor will clear + # any condition hurdle. + if not buy_box: condition: List[WebElement] = atc_button.find_elements_by_xpath( "./ancestor::form[@method='post']" ) @@ -799,10 +737,7 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): continue try: - if flyout_mode: - price = parse_price(prices[idx].get_attribute("innerHTML")) - else: - price = parse_price(prices[idx].text) + price = parse_price(prices[idx].get_attribute("innerHTML")) except IndexError: log.debug("Price index error") return False @@ -815,102 +750,98 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): if ship_float is None: ship_float = 0 - if ( - (ship_float + price_float) <= reserve_max - or math.isclose((price_float + ship_float), reserve_max, abs_tol=0.01) - ) and ( - (ship_float + price_float) >= reserve_min - or math.isclose((price_float + ship_float), reserve_min, abs_tol=0.01) - ): - log.info("Item in stock and in reserve range!") - log.info(f"{price_float} + {ship_float} shipping <= {reserve_max}") - log.debug( - f"{reserve_min} <= {price_float} + {ship_float} shipping <= {reserve_max}" - ) - log.info("Adding to cart") - # Get the offering ID - offering_id_elements = atc_button.find_elements_by_xpath( - "./preceding::input[@name='offeringID.1'][1]" - ) - if offering_id_elements: - log.info("Attempting Add To Cart with offer ID...") - offering_id = offering_id_elements[0].get_attribute("value") - if self.attempt_atc( - offering_id, max_atc_retries=DEFAULT_MAX_ATC_TRIES - ): - return True - else: - log.error(f"Failed to Add to Cart with offer ID: {offering_id}") - self.send_notification( - "Failed Add to Cart after {max-atc-retries}", - "failed-atc", - self.take_screenshots, - ) - self.save_page_source("failed-atc") - return False + if ( + (ship_float + price_float) <= reserve_max + or math.isclose((price_float + ship_float), reserve_max, abs_tol=0.01) + ) and ( + (ship_float + price_float) >= reserve_min + or math.isclose((price_float + ship_float), reserve_min, abs_tol=0.01) + ): + log.info(f"Item {asin} in stock and in reserve range!") + log.debug( + f"{reserve_min} <= {price_float} + {ship_float} shipping <= {reserve_max}" + ) + log.info("Adding to cart") + # Get the offering ID + offering_id_elements = atc_button.find_elements_by_xpath( + "./preceding::input[@name='offeringID.1'][1] | ./preceding::input[@id='offerListingID']" + ) + if offering_id_elements: + log.info("Attempting Add To Cart with offer ID...") + offering_id = offering_id_elements[0].get_attribute("value") + if self.attempt_atc(offering_id, max_atc_retries=DEFAULT_MAX_ATC_TRIES): + return True else: - log.error( - "Unable to find offering ID to add to cart. Using legacy mode." + self.send_notification( + "Failed Add to Cart after {max-atc-retries}", + "failed-atc", + self.take_screenshots, ) - self.notification_handler.play_notify_sound() - if self.detailed: - self.send_notification( - message=f"Found Stock ASIN:{asin}", - page_name="Stock Alert", - take_screenshot=self.take_screenshots, - ) - - presence.buy_update() - current_title = self.driver.title - # log.info(f"current page title is {current_title}") - try: - atc_button.click() - except IndexError: - log.debug("Index Error") - return False - self.wait_for_page_change(current_title) - # log.info(f"page title is {self.driver.title}") - emtpy_cart_elements = self.driver.find_elements_by_xpath( - "//div[contains(@class, 'sc-your-amazon-cart-is-empty') or contains(@class, 'sc-empty-cart')]" + self.save_page_source("failed-atc") + return False + else: + log.error( + "Unable to find offering ID to add to cart. Using legacy mode." + ) + self.notification_handler.play_notify_sound() + if self.detailed: + self.send_notification( + message=f"Found Stock ASIN:{asin}", + page_name="Stock Alert", + take_screenshot=self.take_screenshots, ) - if ( - not emtpy_cart_elements - and self.driver.title in amazon_config["SHOPPING_CART_TITLES"] - ): - return True - else: - log.warning("Did not add to cart, trying again") - if emtpy_cart_elements: - log.warning( - "Cart appeared empty after clicking Add To Cart button" - ) - log.debug(f"failed title was {self.driver.title}") - self.send_notification( - "Failed Add to Cart", "failed-atc", self.take_screenshots - ) - self.save_page_source("failed-atc") - in_stock = self.check_stock( - asin=asin, - reserve_max=reserve_max, - reserve_min=reserve_min, - retry=retry + 1, - ) - elif reserve_min > (price_float + ship_float): - log.debug( - f" Min ({reserve_min}) > Price ({price_float} + {ship_float} shipping)" + presence.buy_update() + current_title = self.driver.title + # log.info(f"current page title is {current_title}") + try: + atc_button.click() + except IndexError: + log.debug("Index Error") + return False + self.wait_for_page_change(current_title) + # log.info(f"page title is {self.driver.title}") + emtpy_cart_elements = self.driver.find_elements_by_xpath( + "//div[contains(@class, 'sc-your-amazon-cart-is-empty') or contains(@class, 'sc-empty-cart')]" ) - elif reserve_max < (price_float + ship_float): - log.debug( - f" Max ({reserve_max}) < Price ({price_float} + {ship_float} shipping)" - ) + if ( + not emtpy_cart_elements + and self.driver.title in amazon_config["SHOPPING_CART_TITLES"] + ): + return True + else: + log.warning("Did not add to cart, trying again") + if emtpy_cart_elements: + log.info( + "Cart appeared empty after clicking Add To Cart button" + ) + log.debug(f"failed title was {self.driver.title}") + self.send_notification( + "Failed Add to Cart", "failed-atc", self.take_screenshots + ) + self.save_page_source("failed-atc") + in_stock = self.check_stock( + asin=asin, + reserve_max=reserve_max, + reserve_min=reserve_min, + retry=retry + 1, + ) + elif reserve_min > (price_float + ship_float): + log.debug( + f" Min ({reserve_min}) > Price ({price_float} + {ship_float} shipping)" + ) - else: - log.error("Serious problem with price comparison") - log.error(f" Min: {reserve_min}") - log.error(f" Price: {price_float} + {ship_float} shipping") - log.error(f" Max: {reserve_max}") + elif reserve_max < (price_float + ship_float): + log.debug( + f" Max ({reserve_max}) < Price ({price_float} + {ship_float} shipping)" + ) + + else: + log.error("Serious problem with price comparison") + log.error(f" Min: {reserve_min}") + log.error(f" Price: {price_float} + {ship_float} shipping") + log.error(f" Max: {reserve_max}") log.info(f"Offers exceed price range ({reserve_min:.2f}-{reserve_max:.2f})") return in_stock From ee3b2da143f13b2ec3b278cd994e45d72047c376 Mon Sep 17 00:00:00 2001 From: Charlie Date: Wed, 14 Apr 2021 15:00:51 -0400 Subject: [PATCH 14/14] Log reserve on price match --- stores/amazon.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stores/amazon.py b/stores/amazon.py index c6c21411..5ef5bc52 100644 --- a/stores/amazon.py +++ b/stores/amazon.py @@ -757,9 +757,8 @@ def check_stock(self, asin, reserve_min, reserve_max, retry=0): (ship_float + price_float) >= reserve_min or math.isclose((price_float + ship_float), reserve_min, abs_tol=0.01) ): - log.info(f"Item {asin} in stock and in reserve range!") - log.debug( - f"{reserve_min} <= {price_float} + {ship_float} shipping <= {reserve_max}" + log.info( + f"Item {asin} in stock and in reserve range: {price_float} + {ship_float} shipping <= {reserve_max}" ) log.info("Adding to cart") # Get the offering ID