From a4bc9deeb013e9e748e7c53e3b542466c184dde7 Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Mon, 12 May 2025 13:03:25 -0700 Subject: [PATCH 01/11] Retrieve a item from Ebay api --- .../__pycache__/base_scraper.cpython-39.pyc | Bin 1964 -> 1979 bytes webscraper/api/EbayAPI.py | 23 ++++++++- .../api/__pycache__/interface.cpython-39.pyc | Bin 638 -> 870 bytes webscraper/api/interface.py | 5 ++ webscraper/api/tests/test_ebay_api.py | 45 ++++++++++++++++-- .../src/__pycache__/__init__.cpython-39.pyc | Bin 158 -> 171 bytes .../__pycache__/fetch_utils.cpython-39.pyc | Bin 782 -> 797 bytes .../tests/__pycache__/__init__.cpython-39.pyc | Bin 164 -> 177 bytes .../test_fetch_and_cache.cpython-39.pyc | Bin 2601 -> 2614 bytes 9 files changed, 68 insertions(+), 5 deletions(-) diff --git a/webscraper/ABC/__pycache__/base_scraper.cpython-39.pyc b/webscraper/ABC/__pycache__/base_scraper.cpython-39.pyc index dd1704d439473037f51337091f46e993989391f5..a9f2891eee3baadb7379179f164f7423c9a74b74 100644 GIT binary patch delta 55 zcmZ3(znh;sk(ZZ?0SG4Bi)`dxz{C`mJb5jXf>>5!ZfbFIMrvX~YEcY~*4uoA=>`h` Dr& dict: + response_json = EbayAPI.retrieve_ebay_response( + "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", + query + ) + + try: + # Grab the first item from the results + item = response_json["itemSummaries"][0] + title = item.get("title") + price = item.get("price", {}).get("value") + currency = item.get("price", {}).get("currency") + return {"name": title, "price": price, "currency": currency} + except (KeyError, IndexError): + raise Exception("Could not parse item from eBay response.") + + @staticmethod def retrieve_access_token(): try: response = requests.post("https://api.sandbox.ebay.com/identity/v1/oauth2/token", @@ -32,6 +50,7 @@ def retrieve_access_token(): except Exception as e: raise e + @staticmethod def retrieve_ebay_response(httprequest:str,query:str): auth = EbayAPI.retrieve_access_token() try: diff --git a/webscraper/api/__pycache__/interface.cpython-39.pyc b/webscraper/api/__pycache__/interface.cpython-39.pyc index 4597f1eeb5a3455a46e54e675609dd0afa97dbcf..a0ffbebfbc401e40c6b0808d4078ab69c46bf88a 100644 GIT binary patch delta 509 zcmZutJxc>Y5Z&3^yIejlXcGtsk#yID^nPGekRSvh-66_mcjHBJN!+^>u}P6&e;jrq z_#gZ!R{H}4|AK|Mn-H~dVBRvbZ{8c`i@ZB!>UjmE9cQE30*UqRO-vCz7%x5=l7bJ}7cJUn*+UN;$n>$OgfFRFv+Rja1AGRGmB;5^sc3E}~ zyOmoSiI|0gPK3PNyp2pk1Dxyynq|GDeK!?yCUbf=Rka?{iqKg}`B>(SVre70HZUhr z#aK|u!s|gIqM~lZSOdcMwJwv!t##3Ej}jsKl!?Y4ZpSZVZ?HEvGm;qa4E${VYaz`+ z3?nw5F2+zH7*^em?B6b+@%f)$eHKPK1az<)m zL2A+Djf{GXtdk!w8Zi55a!j7jWStLk075;4#R(E)0TRqWTnu7xFflQLMf@~bZZRh& hB^NP))PUr{!q`l>#bJ}1pHiBWYR3pN2BblN2>?3jEgk>> diff --git a/webscraper/api/interface.py b/webscraper/api/interface.py index af2ef9d..ad91861 100644 --- a/webscraper/api/interface.py +++ b/webscraper/api/interface.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from typing import Dict class ScraperAPIInterface(ABC): @@ -6,3 +7,7 @@ class ScraperAPIInterface(ABC): def get_scraped_data(self, paths: list[str]) -> dict: """Given a list of paths, return scraped results.""" pass + + @abstractmethod + def search_item(self, query: str) -> Dict[str, str]: + pass diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index 7651c13..b8fdbe9 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -9,9 +9,28 @@ def setUp(self): self.EbayAPI = EbayAPI - def test_retrieve_access_token(self): - self.EbayAPI.retrieve_access_token() - self.assertEqual(type(self.EbayAPI.retrieve_access_token()),str) + # def test_retrieve_access_token_real(self): + # token = self.EbayAPI.retrieve_access_token() + # self.assertIsInstance(token, str) + # self.assertGreater(len(token), 0) + + # def test_search_item_real(self): + # result = self.EbayAPI.search_item("macbook") + # self.assertIn("name", result) + # self.assertIn("price", result) + # self.assertIn("currency", result) + # self.assertIsInstance(result["name"], str) + # self.assertTrue(result["price"]) # not None or '' + + @patch("webscraper.api.EbayAPI.requests.post") + def test_retrieve_access_token(self, mock_post): + mock_response = Mock() + mock_response.status_code = 200 + mock_response.json.return_value = {"access_token": "mock_token"} + mock_post.return_value = mock_response + + token = self.EbayAPI.retrieve_access_token() + self.assertEqual(token, "mock_token") @patch("webscraper.api.EbayAPI.requests.post") def test_retrieve_access_token_invalid(self,mock_post): @@ -30,6 +49,26 @@ def test_retrieve_ebay_response_invalid(self,mock_get): self.EbayAPI.retrieve_ebay_response("https://test","item") self.assertRaises(Exception) + @patch("webscraper.api.EbayAPI.EbayAPI.retrieve_ebay_response") + def test_search_item(self, mock_response): + mock_response.return_value = { + "itemSummaries": [ + { + "title": "Test Product", + "price": { + "value": "19.99", + "currency": "USD" + } + } + ] + } + + result = self.EbayAPI.search_item("test") + self.assertEqual(result["name"], "Test Product") + self.assertEqual(result["price"], "19.99") + self.assertEqual(result["currency"], "USD") + + if __name__ == '__main__': unittest.main() \ No newline at end of file diff --git a/webscraper/src/__pycache__/__init__.cpython-39.pyc b/webscraper/src/__pycache__/__init__.cpython-39.pyc index d43fde4adb6f131a6895b08baaa604a0dfb64288..22b3fcaed5ccca0f74c5d3fa776ead4e1317deba 100644 GIT binary patch delta 41 vcmbQoxSEkWk(ZZ?0SG4Bi%jIUVG2l|=&8Vym6)4aoSc!GSddyYF~bZ1$CL|f delta 28 icmZ3@IFFG#k(ZZ?0SN4ua82a4VRCk!=&3NV%oG4pv<6=Q diff --git a/webscraper/src/__pycache__/fetch_utils.cpython-39.pyc b/webscraper/src/__pycache__/fetch_utils.cpython-39.pyc index 8e69c484bfea4d5ab61881109c7886ea75770c10..6358c3a3ad3780fdd9f4b8b103c235a150b73c1c 100644 GIT binary patch delta 155 zcmeBUo6E+X$ji&c00fimMK*FzV`RKFc@d+jk(yYLS~NMI$&FV8D6Yv- a!~>*?_$MD?QWFJb?#1Zpf|1rdBe;udRBeraAxku*qv8AM2d2pJF|3nVnzi#R}R zIUo_mno*LQV^|~)XB!nM0NJ-DmvHP66$P@2#6g52h)@9$nv-2Qodj-il;&lYl%y7y a6oD+iHF-Lx3gfNGdpHdkwI;viv;hDGL^9t1 delta 186 zcmdlcvQmUQk(ZZ?0SF}a^Kayy!N?TqJb5Xj!s1JejEqs6FEa%*F-lH8$Sl0siuC{! zqxj_C?8b}|ll3`7ZKZ&kikLx!B#4j(5i&qRlf8%o#FhmTQLGsyxjBYKa&Wd$kvxb! zxshX!s0c`z7>G~+5y~J!eR2?|lRy+lX Date: Mon, 12 May 2025 13:50:56 -0700 Subject: [PATCH 02/11] fixed main.py so it runs, made minor changes, updated readme on how to run --- README.md | 2 +- webscraper/main.py | 6 ++---- webscraper/src/Cheaper_Scraper.py | 4 ++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d70d692..37c1d57 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Initial Landing page![Initial Landing page](https://github.com/user-attachments/ -To run the scraper, execute the main.py script by running the command -python src/main.py +python main.py -Make sure you are in the webscraper directory when you run the command diff --git a/webscraper/main.py b/webscraper/main.py index 0ae9fe5..c05f95c 100644 --- a/webscraper/main.py +++ b/webscraper/main.py @@ -13,14 +13,12 @@ def main(): # Set up the scraper for a simple legal-to-scrape website - scraper = CheaperScraper("https://books.toscrape.com", - user_agent="CheaperBot/0.1", - delay=2.0) + scraper = CheaperScraper("https://books.toscrape.com", user_agent="CheaperBot/0.1", delay=2.0) # Define which pages you want to scrape (you can use "/" for homepage) pages = ["/"] # Use the scraper to fetch and parse the pages - results = CheaperScraper.scraper.scrape(pages) + results = scraper.scrape(pages) # Show the output in the terminal for path, items in results.items(): diff --git a/webscraper/src/Cheaper_Scraper.py b/webscraper/src/Cheaper_Scraper.py index 5aa7bc9..03ff85e 100644 --- a/webscraper/src/Cheaper_Scraper.py +++ b/webscraper/src/Cheaper_Scraper.py @@ -107,3 +107,7 @@ def scrape(self, paths: List[str]) -> Dict[str, List[str]]: def get_scraped_data(self, paths: List[str]) -> Dict[str, List[str]]: return self.scrape(paths) + + def search_item(self, query: str) -> Dict[str, str]: + raise NotImplementedError("search_item() is not supported by CheaperScraper.") + From 9ab9ad3fe7a07f2ed915850712183732a1f965f8 Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Mon, 19 May 2025 17:27:59 -0700 Subject: [PATCH 03/11] fixed to use real api now from ebay --- webscraper/api/EbayAPI.py | 4 ++ webscraper/api/tests/test_ebay_api.py | 91 ++++++++++++++------------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py index 3f8e2c4..b96bbce 100644 --- a/webscraper/api/EbayAPI.py +++ b/webscraper/api/EbayAPI.py @@ -15,11 +15,14 @@ class EbayAPI(ScraperAPIInterface): @staticmethod def search_item(query: str) -> dict: + # print("🔗 LIVE API HIT: search_item") response_json = EbayAPI.retrieve_ebay_response( "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", query ) + # print("Raw response:", response_json) + try: # Grab the first item from the results item = response_json["itemSummaries"][0] @@ -44,6 +47,7 @@ def retrieve_access_token(): ) access_token = response.json().get("access_token") status_code = response.status_code + # print("🎟️ Token fetched:", access_token) if(status_code == 404): raise Exception("404 error here") return access_token diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index b8fdbe9..6577236 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -2,6 +2,8 @@ from unittest.mock import patch,Mock import requests from webscraper.api.EbayAPI import EbayAPI +from dotenv import load_dotenv +load_dotenv() class EbayTestApi(unittest.TestCase): @@ -9,28 +11,29 @@ def setUp(self): self.EbayAPI = EbayAPI - # def test_retrieve_access_token_real(self): - # token = self.EbayAPI.retrieve_access_token() - # self.assertIsInstance(token, str) - # self.assertGreater(len(token), 0) - - # def test_search_item_real(self): - # result = self.EbayAPI.search_item("macbook") - # self.assertIn("name", result) - # self.assertIn("price", result) - # self.assertIn("currency", result) - # self.assertIsInstance(result["name"], str) - # self.assertTrue(result["price"]) # not None or '' - - @patch("webscraper.api.EbayAPI.requests.post") - def test_retrieve_access_token(self, mock_post): - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = {"access_token": "mock_token"} - mock_post.return_value = mock_response - + def test_retrieve_access_token_real(self): token = self.EbayAPI.retrieve_access_token() - self.assertEqual(token, "mock_token") + self.assertIsInstance(token, str) + self.assertGreater(len(token), 0) + + def test_search_item_real(self): + result = self.EbayAPI.search_item("macbook") + self.assertIn("name", result) + self.assertIn("price", result) + self.assertIn("currency", result) + self.assertIsInstance(result["name"], str) + self.assertTrue(result["price"]) # not None or '' + + + # @patch("webscraper.api.EbayAPI.requests.post") + # def test_retrieve_access_token(self, mock_post): + # mock_response = Mock() + # mock_response.status_code = 200 + # mock_response.json.return_value = {"access_token": "mock_token"} + # mock_post.return_value = mock_response + + # token = self.EbayAPI.retrieve_access_token() + # self.assertEqual(token, "mock_token") @patch("webscraper.api.EbayAPI.requests.post") def test_retrieve_access_token_invalid(self,mock_post): @@ -44,29 +47,29 @@ def test_retrieve_access_token_invalid(self,mock_post): - @patch("webscraper.api.EbayAPI.requests.get") - def test_retrieve_ebay_response_invalid(self,mock_get): - self.EbayAPI.retrieve_ebay_response("https://test","item") - self.assertRaises(Exception) - - @patch("webscraper.api.EbayAPI.EbayAPI.retrieve_ebay_response") - def test_search_item(self, mock_response): - mock_response.return_value = { - "itemSummaries": [ - { - "title": "Test Product", - "price": { - "value": "19.99", - "currency": "USD" - } - } - ] - } - - result = self.EbayAPI.search_item("test") - self.assertEqual(result["name"], "Test Product") - self.assertEqual(result["price"], "19.99") - self.assertEqual(result["currency"], "USD") + # @patch("webscraper.api.EbayAPI.requests.get") + # def test_retrieve_ebay_response_invalid(self,mock_get): + # self.EbayAPI.retrieve_ebay_response("https://test","item") + # self.assertRaises(Exception) + + # @patch("webscraper.api.EbayAPI.EbayAPI.retrieve_ebay_response") + # def test_search_item(self, mock_response): + # mock_response.return_value = { + # "itemSummaries": [ + # { + # "title": "Test Product", + # "price": { + # "value": "19.99", + # "currency": "USD" + # } + # } + # ] + # } + + # result = self.EbayAPI.search_item("test") + # self.assertEqual(result["name"], "Test Product") + # self.assertEqual(result["price"], "19.99") + # self.assertEqual(result["currency"], "USD") From 226a14757b0859854fbc4264419f562b3757e5b7 Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Wed, 21 May 2025 17:06:04 -0700 Subject: [PATCH 04/11] fixed issues, used ORM instead of dict, made it save to local database as well in PostgreSQL --- webscraper/api/EbayAPI.py | 26 ++++++++++++++------ webscraper/api/tests/test_ebay_api.py | 35 ++++++++++++++++++++++----- webscraper/database/db.py | 15 ++++++++++++ webscraper/database/init_db.sql | 7 ++++++ webscraper/database/models.py | 23 ++++++++++++++++++ 5 files changed, 93 insertions(+), 13 deletions(-) create mode 100644 webscraper/database/db.py create mode 100644 webscraper/database/models.py diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py index b96bbce..7d334ed 100644 --- a/webscraper/api/EbayAPI.py +++ b/webscraper/api/EbayAPI.py @@ -3,6 +3,7 @@ from dotenv import load_dotenv import os from webscraper.api.interface import ScraperAPIInterface +from webscraper.database.models import EbayItem load_dotenv() #initialize @@ -14,7 +15,7 @@ class EbayAPI(ScraperAPIInterface): get_user_key = HTTPBasicAuth(client_id_key, client_secret_key) @staticmethod - def search_item(query: str) -> dict: + def search_item(query: str) -> EbayItem: # print("🔗 LIVE API HIT: search_item") response_json = EbayAPI.retrieve_ebay_response( "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", @@ -24,16 +25,27 @@ def search_item(query: str) -> dict: # print("Raw response:", response_json) try: - # Grab the first item from the results item = response_json["itemSummaries"][0] - title = item.get("title") - price = item.get("price", {}).get("value") - currency = item.get("price", {}).get("currency") - return {"name": title, "price": price, "currency": currency} + return EbayItem( + name=item.get("title"), + price=float(item["price"]["value"]), + currency=item["price"]["currency"], + url=item.get("itemWebUrl"), + user_id=None # Set this if you have user tracking + ) + + # Save to database + session = SessionLocal() + session.add(new_item) + session.commit() + session.refresh(new_item) + session.close() + + return new_item + except (KeyError, IndexError): raise Exception("Could not parse item from eBay response.") - @staticmethod def retrieve_access_token(): try: diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index 6577236..c6583ce 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -4,6 +4,8 @@ from webscraper.api.EbayAPI import EbayAPI from dotenv import load_dotenv load_dotenv() +from webscraper.database.db import SessionLocal +from webscraper.database.models import EbayItem class EbayTestApi(unittest.TestCase): @@ -17,13 +19,34 @@ def test_retrieve_access_token_real(self): self.assertGreater(len(token), 0) def test_search_item_real(self): - result = self.EbayAPI.search_item("macbook") - self.assertIn("name", result) - self.assertIn("price", result) - self.assertIn("currency", result) - self.assertIsInstance(result["name"], str) - self.assertTrue(result["price"]) # not None or '' + item = self.EbayAPI.search_item("macbook") + self.assertIsInstance(item.name, str) + self.assertIsInstance(item.price, float) + self.assertIsInstance(item.currency, str) + self.assertTrue(item.url.startswith("http")) + def test_search_item_stores_to_db(self): + session = SessionLocal() + try: + # Run actual API search + result = self.EbayAPI.search_item("macbook") + + # Save result to DB + new_item = EbayItem( + name=result.name, + price=result.price, + currency=result.currency, + url=result.url, + user_id=None # or a test user ID if needed + ) + session.add(new_item) + session.commit() + + # Query the DB to check if the item is saved + item = session.query(EbayItem).filter_by(name=result.name).first() + self.assertIsNotNone(item) + finally: + session.close() # @patch("webscraper.api.EbayAPI.requests.post") # def test_retrieve_access_token(self, mock_post): diff --git a/webscraper/database/db.py b/webscraper/database/db.py new file mode 100644 index 0000000..6ad48de --- /dev/null +++ b/webscraper/database/db.py @@ -0,0 +1,15 @@ +# webscraper/database/db.py + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from webscraper.database.models import Base +from webscraper.database.models import User, EbayItem + + +DATABASE_URL = "postgresql://postgres:cheaper@localhost/cheaper_local" + +engine = create_engine(DATABASE_URL) +SessionLocal = sessionmaker(bind=engine) + +# Create tables if they don't exist +Base.metadata.create_all(bind=engine) diff --git a/webscraper/database/init_db.sql b/webscraper/database/init_db.sql index 5effbec..069122a 100644 --- a/webscraper/database/init_db.sql +++ b/webscraper/database/init_db.sql @@ -1,3 +1,6 @@ +from sqlalchemy import create_engine +from webscraper.database.models import Base + -- Create users table CREATE TABLE users ( id SERIAL PRIMARY KEY, @@ -14,3 +17,7 @@ CREATE TABLE products ( user_id INTEGER NOT NULL, FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE ); + + +engine = create_engine('postgresql://postgres:your_password@localhost/cheaper_local') +Base.metadata.create_all(engine) \ No newline at end of file diff --git a/webscraper/database/models.py b/webscraper/database/models.py new file mode 100644 index 0000000..040093b --- /dev/null +++ b/webscraper/database/models.py @@ -0,0 +1,23 @@ +from sqlalchemy import Column, Integer, String, Float, ForeignKey +from sqlalchemy.ext.declarative import declarative_base + + +Base = declarative_base() + +class EbayItem(Base): + __tablename__ = 'ebay_items' + + id = Column(Integer, primary_key=True) + name = Column(String) + price = Column(Float) + currency = Column(String) + url = Column(String) + user_id = Column(Integer, ForeignKey('users.id'), nullable=True) # if linked to a user + + +class User(Base): + __tablename__ = "users" + + id = Column(Integer, primary_key=True, index=True) + username = Column(String, unique=True, nullable=False) + password = Column(String, nullable=False) # Assume it's already hashed \ No newline at end of file From 71822de6f90abb91fbee7426ce1dd4e95aa32b8e Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Wed, 21 May 2025 17:10:01 -0700 Subject: [PATCH 05/11] removed pyc files --- .../api/__pycache__/interface.cpython-39.pyc | Bin 870 -> 0 bytes .../src/__pycache__/__init__.cpython-39.pyc | Bin 171 -> 0 bytes .../src/__pycache__/fetch_utils.cpython-311.pyc | Bin 1254 -> 0 bytes .../src/__pycache__/fetch_utils.cpython-39.pyc | Bin 797 -> 0 bytes webscraper/src/__pycache__/main.cpython-39.pyc | Bin 862 -> 0 bytes 5 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 webscraper/api/__pycache__/interface.cpython-39.pyc delete mode 100644 webscraper/src/__pycache__/__init__.cpython-39.pyc delete mode 100644 webscraper/src/__pycache__/fetch_utils.cpython-311.pyc delete mode 100644 webscraper/src/__pycache__/fetch_utils.cpython-39.pyc delete mode 100644 webscraper/src/__pycache__/main.cpython-39.pyc diff --git a/webscraper/api/__pycache__/interface.cpython-39.pyc b/webscraper/api/__pycache__/interface.cpython-39.pyc deleted file mode 100644 index a0ffbebfbc401e40c6b0808d4078ab69c46bf88a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 870 zcmah{QH#?+5Z>Kv(p;~V13|bagZL7V;1+lt@qVdm_1_cV6O4e@^1y2V{|$Iq&?=N^Cco7PqR7 zL{t{WM(btNmUtF*B_h-Ik>Hx9v}Q8>>}F3CLCYHd9YuOOIRw6oisk39;5lg z{9j-i_Y7|s42F!+N1W02bQTBB6N_9UcU^g`kg30zG8P-7hF^Gc<7=auIATDMB5-AM944RC7D;bJF!U*D5K(bX#XmM&$ zag1AHZe~tpj7xrUX>Mv>NpXyOW{F2>QcPB2ZfbFIMrvX~YEcY~E>BG=PA&oq6&EGP f#K&jmWtPOp>lIYq;;_lhPbtkwwFBAo8HgDG#563r diff --git a/webscraper/src/__pycache__/fetch_utils.cpython-311.pyc b/webscraper/src/__pycache__/fetch_utils.cpython-311.pyc deleted file mode 100644 index 4a1d1acb538b5e84cd4b055fcc0dbedf1a2dc1af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1254 zcmZ8g&1)M+6o0d`D~;qjGRRbMTxz9Gp*pmpb$mz&0nw)IDJ^wM80i_MyjK{U5A~AY#F%l7nw5zJ#3mW^KthZ)V=-%-i3adHcuw zd=ije<`!xz2Ef1Cm=ZfvP6R5?!2yT3P&}2SNYFN1qe+XDXdiXWrd6~E7{IAx9$0$> zU?0~hIWZn{;f>I*ka`jU&1UF9tm#spWsXqBw#;y2&Me;?ncXFf3<*f`TR$tG;k`>;uJ z@w=T5e){^}y>GMM@kc(F>OuBPuFAD~tD3C{ubJKC4{v5W$?x%6KQ8ENJ=QIgGs1X{ z+YT2pjH`N+d%o(V?g-(DY-M`BW8BQ7T5W6me7R@-JjweWID5Kdo9UB!lk~jtfn`Gyap}v z4X^2v9EKin2q)JiFN7zfgs=M-&_I8+zy3Q#fV4<5< ze^?HX_2HG72$I(a5*f~+xjr)219N?xT7FRsE`8MhYGTB!#iR7{ueWyT;D^5-2I;~u zT^OYcPuZ9)ykuAZVONK2b;MQ!wmME_c6tGO@Ac7KdNg-!0*1AC&W>1WXFhoMvjIh> z<)c$MWNRa~7O=H(D*aSV$7`gCg$Yo~41#^}X(0)R$yDJsJzTm{xIqtZSg7ZiNMTeg zA`250zg1SA=SrcQHjJt5Hr~0=of%T8L{$HuaDq07%gElaTTRdLU4C1vps9Z(xr1y% z2_a)h^`kL{OZ{k!VI{aY$FLk+oRb(BbN!`fYXQj&NoGVc6C*(upM5?76uTRd82tEG W`QSmMp0ocb7?-E}&)wpE1oS`8dpyGc diff --git a/webscraper/src/__pycache__/fetch_utils.cpython-39.pyc b/webscraper/src/__pycache__/fetch_utils.cpython-39.pyc deleted file mode 100644 index 6358c3a3ad3780fdd9f4b8b103c235a150b73c1c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 797 zcmYjPO>fjN5Vhlcb=5+JP=Q;7xa2_91E;FuL(tL#Du}kKSb|n*JiD7J*(_s6bfYXn zD%YMk0#bYAFZs%8{{kn*yA*XIzwwMcSIB_z1ejQAW)F zn`LY@J*S}>0!^%2_MQE7&N%UOTAuT%(?pt|I4x@(No>!>RJ7tkTr$pDapE4aedaRS z;Y~DWQDf-wZ(Zu=UQ)j!_Kvhq-OsM1n#F8`ZP1J~27Ar@_qzuNZ=$#G zu?D3-L_46fVLlo}eXPpp6C7_wP3Mp#J$z;*%`ElUF~Ja038+k<^Af6BH#fHtD~$SA z^^LO`#i8-WnA1&!xHv!=F#v5=G0he9`W5z+PIaw}tKpRL|9EVlW-z~J6BN~8Kzxh` zwtwuQPYf`yV7b`NYFv#J7!JP}u2JsKVYZoA&MMhS%e3h5d(1Ll H79sx&=$g_Z diff --git a/webscraper/src/__pycache__/main.cpython-39.pyc b/webscraper/src/__pycache__/main.cpython-39.pyc deleted file mode 100644 index bd339199e596383c4f2c650179cbba61fecbfd2d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 862 zcmYjPOKaOe5Z;#_ksr7%l+sh^Az*0b?uuX|pPXwb z^jZq>v42U|o^t6A@S${8PD*CkZ)awA9^XiDr{f_I_QxB)U=YS3=b5$+OmvLs8q#Mng9$wul09_Wl)S14$SHn-1-{3==t z1YUfM`pzv>LPs;=`+Zz@55^)cgghQebBxU0y*Qw_un%CX?*JMyaKb5P*L04!b&BV> zLN~aA`g2l|8?r=)=*!EH%;}tI%I%8&K-~FB7tmkD;I2Ej!0%Spf?#y9f3c+*-{9VY zeA}Gc+B&lFwuwu9Hhxw>X zp}Y#;C;CG<3`grv8D5IvUH(wXQJBU_<`>hN8KniO)H47?7$%tfwJn=^w1<1x#a-IQ z@-f`iZWNi=?_A3n0a7Cmt%(G0@ACtaf-Jvp5wd{3Mx(C<>lH w2makZ7Oh7sn`!;8V*;Y6;rIX^HC?<;b6!rw8#7+j1#qbgeQx14b)UNUA1pTAegFUf From 934b311cd35f9e4f7a806aedce86de4ef4d0db72 Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Wed, 21 May 2025 17:16:20 -0700 Subject: [PATCH 06/11] fixed error --- .../ABC/__pycache__/base_scraper.cpython-39.pyc | Bin 1979 -> 0 bytes webscraper/api/EbayAPI.py | 10 +++++----- 2 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 webscraper/ABC/__pycache__/base_scraper.cpython-39.pyc diff --git a/webscraper/ABC/__pycache__/base_scraper.cpython-39.pyc b/webscraper/ABC/__pycache__/base_scraper.cpython-39.pyc deleted file mode 100644 index a9f2891eee3baadb7379179f164f7423c9a74b74..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1979 zcma)7&5qkP5Z3={?e#WEw*`7Az^j2`&xH}R$tFlrVCxk5=|K=6&?2>Bp}&w69Pc%G zgIx0h1^N&@_YrjMsZS6fr_4~6S6gfl(;&?m4(I2a;fT&=6AQ+>pHK4Ex2%7W7!MDK zdob0HurQ0+g;m;-T{@9tBkdG!=|x`YM}8SZfo(mun9IDk7W26M#*Ge`|J<4f|G{l* z?sV?`vj^QVT`4VTs!Oi(n$6wLI~PiK-b0bVzK-`0=Uc%tY#Nh^zP(kd4a>@ZgXE=sKt)L z)4{O?c>(H-hL_LIiLSS*uL)oIz926l4r1aUkSbAK;2psrE|G1$2D%!oPl_5(CR0c8SCGF4kU95Ao-YM zwLeOK_w9(3ZJ|X|V55NhdjVx67i5TXC@}ME`)9HX@5@+W2gl|uH>9LZgN?)JN8AA| zAqNyCSK#WX1zanylN*o=dS~V%= EbayItem: ) # Save to database - session = SessionLocal() - session.add(new_item) - session.commit() - session.refresh(new_item) - session.close() + if store: + session = SessionLocal() + session.add(new_item) + session.commit() + session.refresh(new_item) return new_item From e63bcce4827411efe16f5188cbbe9297fa6895bc Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Wed, 21 May 2025 17:25:54 -0700 Subject: [PATCH 07/11] used orm instead of dict --- webscraper/api/EbayAPI.py | 20 ++++++++++---------- webscraper/api/tests/test_ebay_api.py | 25 +------------------------ webscraper/database/db.py | 15 --------------- webscraper/database/models.py | 23 ----------------------- 4 files changed, 11 insertions(+), 72 deletions(-) delete mode 100644 webscraper/database/db.py delete mode 100644 webscraper/database/models.py diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py index 7538192..aab8dbb 100644 --- a/webscraper/api/EbayAPI.py +++ b/webscraper/api/EbayAPI.py @@ -3,10 +3,19 @@ from dotenv import load_dotenv import os from webscraper.api.interface import ScraperAPIInterface -from webscraper.database.models import EbayItem + load_dotenv() #initialize +class EbayItem: + def __init__(self, name, price, currency, url, user_id=None): + self.name = name + self.price = price + self.currency = currency + self.url = url + self.user_id = user_id + + class EbayAPI(ScraperAPIInterface): client_secret_key = os.getenv("clientsecret") @@ -34,15 +43,6 @@ def search_item(query: str) -> EbayItem: user_id=None # Set this if you have user tracking ) - # Save to database - if store: - session = SessionLocal() - session.add(new_item) - session.commit() - session.refresh(new_item) - - return new_item - except (KeyError, IndexError): raise Exception("Could not parse item from eBay response.") diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index c6583ce..7a2bdeb 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -4,8 +4,7 @@ from webscraper.api.EbayAPI import EbayAPI from dotenv import load_dotenv load_dotenv() -from webscraper.database.db import SessionLocal -from webscraper.database.models import EbayItem + class EbayTestApi(unittest.TestCase): @@ -25,28 +24,6 @@ def test_search_item_real(self): self.assertIsInstance(item.currency, str) self.assertTrue(item.url.startswith("http")) - def test_search_item_stores_to_db(self): - session = SessionLocal() - try: - # Run actual API search - result = self.EbayAPI.search_item("macbook") - - # Save result to DB - new_item = EbayItem( - name=result.name, - price=result.price, - currency=result.currency, - url=result.url, - user_id=None # or a test user ID if needed - ) - session.add(new_item) - session.commit() - - # Query the DB to check if the item is saved - item = session.query(EbayItem).filter_by(name=result.name).first() - self.assertIsNotNone(item) - finally: - session.close() # @patch("webscraper.api.EbayAPI.requests.post") # def test_retrieve_access_token(self, mock_post): diff --git a/webscraper/database/db.py b/webscraper/database/db.py deleted file mode 100644 index 6ad48de..0000000 --- a/webscraper/database/db.py +++ /dev/null @@ -1,15 +0,0 @@ -# webscraper/database/db.py - -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker -from webscraper.database.models import Base -from webscraper.database.models import User, EbayItem - - -DATABASE_URL = "postgresql://postgres:cheaper@localhost/cheaper_local" - -engine = create_engine(DATABASE_URL) -SessionLocal = sessionmaker(bind=engine) - -# Create tables if they don't exist -Base.metadata.create_all(bind=engine) diff --git a/webscraper/database/models.py b/webscraper/database/models.py deleted file mode 100644 index 040093b..0000000 --- a/webscraper/database/models.py +++ /dev/null @@ -1,23 +0,0 @@ -from sqlalchemy import Column, Integer, String, Float, ForeignKey -from sqlalchemy.ext.declarative import declarative_base - - -Base = declarative_base() - -class EbayItem(Base): - __tablename__ = 'ebay_items' - - id = Column(Integer, primary_key=True) - name = Column(String) - price = Column(Float) - currency = Column(String) - url = Column(String) - user_id = Column(Integer, ForeignKey('users.id'), nullable=True) # if linked to a user - - -class User(Base): - __tablename__ = "users" - - id = Column(Integer, primary_key=True, index=True) - username = Column(String, unique=True, nullable=False) - password = Column(String, nullable=False) # Assume it's already hashed \ No newline at end of file From f283331edaaf6198fc38af6ecfc5a22da15f7f71 Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Wed, 21 May 2025 17:31:27 -0700 Subject: [PATCH 08/11] fixed small error everything good now --- webscraper/src/Cheaper_Scraper.py | 7 +++---- .../tests/__pycache__/__init__.cpython-39.pyc | Bin 177 -> 0 bytes .../test_fetch_and_cache.cpython-39.pyc | Bin 2614 -> 0 bytes 3 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 webscraper/src/tests/__pycache__/__init__.cpython-39.pyc delete mode 100644 webscraper/src/tests/__pycache__/test_fetch_and_cache.cpython-39.pyc diff --git a/webscraper/src/Cheaper_Scraper.py b/webscraper/src/Cheaper_Scraper.py index 03ff85e..a7ff65c 100644 --- a/webscraper/src/Cheaper_Scraper.py +++ b/webscraper/src/Cheaper_Scraper.py @@ -9,7 +9,7 @@ from webscraper.api.interface import ScraperAPIInterface from webscraper.src.fetch_utils import cached_get from functools import lru_cache - +from webscraper.api.EbayAPI import EbayItem @@ -108,6 +108,5 @@ def scrape(self, paths: List[str]) -> Dict[str, List[str]]: def get_scraped_data(self, paths: List[str]) -> Dict[str, List[str]]: return self.scrape(paths) - def search_item(self, query: str) -> Dict[str, str]: - raise NotImplementedError("search_item() is not supported by CheaperScraper.") - +def search_item(self, query: str) -> EbayItem: + raise NotImplementedError("search_item() is not supported by CheaperScraper.") diff --git a/webscraper/src/tests/__pycache__/__init__.cpython-39.pyc b/webscraper/src/tests/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 026f1ec99871e1d3aea95496d18791842f0ccee6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 177 zcmYe~<>g`kg30zG8LB|~F^Gc<7=auIATDMB5-AM944RC7D;bJF!U*D5Sh7`2XmM&$ zag1AHZe~tpj7xrUX>Mv>NpXyOW{F2>QcPB2ZfbFIMrvX~YEcY~E>BG=PA&oq6&EGP kl%y5|HN?kf=4F<|$LkeT-r}&y%}*)KNwovn_Zf&80OTAmIRF3v diff --git a/webscraper/src/tests/__pycache__/test_fetch_and_cache.cpython-39.pyc b/webscraper/src/tests/__pycache__/test_fetch_and_cache.cpython-39.pyc deleted file mode 100644 index e5403a8f296015e12e996ce397f1aa3e0bc514e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2614 zcmai0O>Y}T7@pY=d*e7!NP(swl%?tcqb4C0AQW1t6xvdRd>|P4${Gh#Z>R;%I=bd$$I1PT^@C=eK;$?lIvB|2Y%BWc!r^w^fya zs62Ai_mYk0CJdwbHfvO0Vola;@M3YRE91UY+Z{y(dTO1x)0HB6C{2e(9sfCK+{UYK z!bnE@XgPul=Q(S+!WAA$_!kwFzNm@-Wkt-18p^7e7j=|@SilYzi_1Gwn_2I7;j+6$ z?mHX~4u4c9&s%u)`!EJ54)};g+&E*`qGDLnX$~)Tjp^&n_4VC6f27w;t}U*$PTp&H ziWVuVWtUPewdO6_sFTWAz2JqXWxDr*DI5rUwcFWjKhRR?_T9Lbq)*%5=AA)LW=6N~ zCFZ-qZhJrONsa9ib?w9QiQJvKXlvDJ8^F}oMtg)WiZc;euWS9Mg|DS~&|ic*O$W|+ zjSsKBjhQt&VTK=q<>R&dTt{WZ_Su*VKH<;6(%2bs<4zoo{;?;}8v7GQZ9%NL%~yQ= zm4CBqdTFDgXraQlJ1v%C4fy81zMp9mXB}C%+NdI!c4vj3=1-)0;i!OSd9;M3t~8|Y zVu|oakK;5ECH`gf>Iw|wbv(n>x1v2c`~QJ|f)(EcF4%zeuUgO$%40U+uVD`_tebA4 zSNripxhit0SF_x#%HI-TT)<)w43VMeaEe-qc}ESTA_VFj8Is8{tmo}+ldMGBk4?9P zd>+H5$6I%XpPdc(%_U}V)3!)0_Q3a0F>K5xj1GIBk8vRhW!X34E9{wb#&_4aV-NE^ z2*e+`BWEg>%E;TV3fr1WMb+W$C0)oyP-+3sr`XD9QHnL(+bbS{KVDf%lY* zjZ}Aj9mFXFlNjCbOZE7Wm@D;pZ1u9^R8F2mk3fKng2VnhYTq+OGfy-I8w88YppCahe z^9x?fqSI_GLTOPPfr~U6sI=i0^^%?+<3vkc)P9Q7fxM$su58d!Rk%CJ!#kFTEYBhe zdUhR*I@B2-uAL3}_~aR^kNH$-2))n-hq}D7m$W9_1Lv8G(cZ}Y z6KUu#msmMvmiNXiAG0bW|7ey~BlIY$6R>G2<0?E?Fe(^T4$5$b+Ws6R7#d;Ooo^1W z)o$Io6>d=;33rlSl06I&bHi;D8yylUA-@56ez7YN+wO|?QkukCcDlHt+J*o4SI z*v5vPCPUnfMy>GsD#?txfH_Aut?*So$b`B?y)GGhPe}jj1DHnDUMY12^_EXjc|(2p zx_(22s3Y^xc20`wk`)zgTt>Kw*uBkHcI+yqY4U{o< Date: Sat, 24 May 2025 20:30:38 -0700 Subject: [PATCH 09/11] adressed all recent comments --- webscraper/api/EbayAPI.py | 6 +++--- webscraper/api/interface.py | 8 +++++--- webscraper/api/tests/test_ebay_api.py | 21 +++++++++++++++++---- webscraper/src/Cheaper_Scraper.py | 2 -- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py index aab8dbb..c1775bb 100644 --- a/webscraper/api/EbayAPI.py +++ b/webscraper/api/EbayAPI.py @@ -2,7 +2,7 @@ from requests.auth import HTTPBasicAuth from dotenv import load_dotenv import os -from webscraper.api.interface import ScraperAPIInterface +from webscraper.api.interface import EbayABC load_dotenv() #initialize @@ -16,7 +16,7 @@ def __init__(self, name, price, currency, url, user_id=None): self.user_id = user_id -class EbayAPI(ScraperAPIInterface): +class EbayAPI(EbayABC): client_secret_key = os.getenv("clientsecret") client_id_key = os.getenv("clientid") @@ -87,4 +87,4 @@ def retrieve_ebay_response(httprequest:str,query:str): return response.json() except Exception as e: raise e - + diff --git a/webscraper/api/interface.py b/webscraper/api/interface.py index ad91861..72f0cf0 100644 --- a/webscraper/api/interface.py +++ b/webscraper/api/interface.py @@ -1,13 +1,15 @@ from abc import ABC, abstractmethod from typing import Dict +from accounts.models import Product -class ScraperAPIInterface(ABC): + +class EbayABC(ABC): @abstractmethod - def get_scraped_data(self, paths: list[str]) -> dict: + def get_scraped_data(self, paths: list[str]) -> Product: """Given a list of paths, return scraped results.""" pass @abstractmethod - def search_item(self, query: str) -> Dict[str, str]: + def search_item(self, query: str) -> Product: pass diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index 7a2bdeb..784c0e8 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -1,3 +1,10 @@ +import os +import django + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "cheaper.settings") # adjust if your settings module is different +django.setup() + + import unittest from unittest.mock import patch,Mock import requests @@ -24,6 +31,12 @@ def test_search_item_real(self): self.assertIsInstance(item.currency, str) self.assertTrue(item.url.startswith("http")) + def test_search_item_not_found(self): + with self.assertRaises(Exception) as context: + self.EbayAPI.search_item("asdkfjasldfjalskdfj") # nonsense query + + self.assertIn("Could not parse item", str(context.exception)) + # @patch("webscraper.api.EbayAPI.requests.post") # def test_retrieve_access_token(self, mock_post): @@ -47,10 +60,10 @@ def test_retrieve_access_token_invalid(self,mock_post): - # @patch("webscraper.api.EbayAPI.requests.get") - # def test_retrieve_ebay_response_invalid(self,mock_get): - # self.EbayAPI.retrieve_ebay_response("https://test","item") - # self.assertRaises(Exception) + @patch("webscraper.api.EbayAPI.requests.get") + def test_retrieve_ebay_response_invalid(self,mock_get): + self.EbayAPI.retrieve_ebay_response("https://test","item") + self.assertRaises(Exception) # @patch("webscraper.api.EbayAPI.EbayAPI.retrieve_ebay_response") # def test_search_item(self, mock_response): diff --git a/webscraper/src/Cheaper_Scraper.py b/webscraper/src/Cheaper_Scraper.py index a7ff65c..ce83e2f 100644 --- a/webscraper/src/Cheaper_Scraper.py +++ b/webscraper/src/Cheaper_Scraper.py @@ -108,5 +108,3 @@ def scrape(self, paths: List[str]) -> Dict[str, List[str]]: def get_scraped_data(self, paths: List[str]) -> Dict[str, List[str]]: return self.scrape(paths) -def search_item(self, query: str) -> EbayItem: - raise NotImplementedError("search_item() is not supported by CheaperScraper.") From d5e2d6831fdbe916ed21a315d175e4515460216e Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Thu, 29 May 2025 17:40:35 -0700 Subject: [PATCH 10/11] Addresed comments --- webscraper/api/EbayAPI.py | 149 ++++++++++++++------------ webscraper/api/tests/test_ebay_api.py | 11 +- 2 files changed, 89 insertions(+), 71 deletions(-) diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py index c1775bb..23f8cde 100644 --- a/webscraper/api/EbayAPI.py +++ b/webscraper/api/EbayAPI.py @@ -2,10 +2,17 @@ from requests.auth import HTTPBasicAuth from dotenv import load_dotenv import os +import logging from webscraper.api.interface import EbayABC +# Load environment variables and configure logging +load_dotenv() -load_dotenv() #initialize +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s" +) +logger = logging.getLogger(__name__) class EbayItem: def __init__(self, name, price, currency, url, user_id=None): @@ -15,76 +22,82 @@ def __init__(self, name, price, currency, url, user_id=None): self.url = url self.user_id = user_id - class EbayAPI(EbayABC): - - client_secret_key = os.getenv("clientsecret") - client_id_key = os.getenv("clientid") + client_secret_key = os.getenv("clientsecret") + client_id_key = os.getenv("clientid") + get_user_key = HTTPBasicAuth(client_id_key, client_secret_key) - get_user_key = HTTPBasicAuth(client_id_key, client_secret_key) + @staticmethod + def search_item(query: str) -> EbayItem: + """Search for an item on eBay using the query string.""" + if not isinstance(query, str) or not query.strip(): + logger.warning("Invalid query input.") + raise ValueError("Query must be a non-empty string.") + + logger.info(f"Searching eBay for: {query}") + response_json = EbayAPI.retrieve_ebay_response( + "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", query + ) - @staticmethod - def search_item(query: str) -> EbayItem: - # print("🔗 LIVE API HIT: search_item") - response_json = EbayAPI.retrieve_ebay_response( - "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", - query + try: + item = response_json["itemSummaries"][0] + logger.debug(f"Item found: {item}") + return EbayItem( + name=item.get("title"), + price=float(item["price"]["value"]), + currency=item["price"]["currency"], + url=item.get("itemWebUrl"), + user_id=None ) + except (KeyError, IndexError) as e: + logger.error(f"Item not found or response invalid: {response_json}") + raise Exception("Could not parse item from eBay response.") from e - # print("Raw response:", response_json) - - try: - item = response_json["itemSummaries"][0] - return EbayItem( - name=item.get("title"), - price=float(item["price"]["value"]), - currency=item["price"]["currency"], - url=item.get("itemWebUrl"), - user_id=None # Set this if you have user tracking - ) - - except (KeyError, IndexError): - raise Exception("Could not parse item from eBay response.") - - @staticmethod - def retrieve_access_token(): - try: - response = requests.post("https://api.sandbox.ebay.com/identity/v1/oauth2/token", - headers = {"Content-Type":"application/x-www-form-urlencoded"}, - data = { - "grant_type": "client_credentials", - "scope": "https://api.ebay.com/oauth/api_scope" - }, - auth=EbayAPI.get_user_key - ) - access_token = response.json().get("access_token") - status_code = response.status_code - # print("🎟️ Token fetched:", access_token) - if(status_code == 404): - raise Exception("404 error here") - return access_token - except Exception as e: - raise e + @staticmethod + def retrieve_access_token() -> str: + """Fetch access token from eBay API.""" + logger.info("Requesting eBay access token...") + try: + response = requests.post( + "https://api.sandbox.ebay.com/identity/v1/oauth2/token", + headers={"Content-Type": "application/x-www-form-urlencoded"}, + data={ + "grant_type": "client_credentials", + "scope": "https://api.ebay.com/oauth/api_scope" + }, + auth=EbayAPI.get_user_key + ) + response.raise_for_status() + token = response.json().get("access_token") + if not token: + logger.error("Access token missing from response.") + raise Exception("Access token not found in response.") + logger.info("Access token successfully retrieved.") + return token + except requests.exceptions.RequestException as e: + logger.exception("Failed to retrieve token.") + raise - @staticmethod - def retrieve_ebay_response(httprequest:str,query:str): - auth = EbayAPI.retrieve_access_token() - try: - response = requests.get(httprequest, - headers={ - "Authorization": f"Bearer {auth}", - "Content-Type": "application/json" - }, - params= { - "q": query, - "category_tree_id": 0 - } - ) - status_code = response.status_code - if(status_code == 404): - raise Exception("not found 404 error") - - return response.json() - except Exception as e: - raise e - + @staticmethod + def retrieve_ebay_response(httprequest: str, query: str) -> dict: + """Perform GET request to eBay API.""" + auth = EbayAPI.retrieve_access_token() + logger.info(f"Making GET request to eBay API: {httprequest} with query: {query}") + try: + response = requests.get( + httprequest, + headers={ + "Authorization": f"Bearer {auth}", + "Content-Type": "application/json" + }, + params={"q": query, "category_tree_id": 0} + ) + if response.status_code == 429: + logger.warning("Rate limit exceeded.") + raise Exception("Rate limit exceeded.") + response.raise_for_status() + logger.debug(f"Raw eBay API response: {response.text}") + return response.json() + except requests.exceptions.RequestException as e: + logger.exception("Error retrieving eBay response.") + raise diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index 784c0e8..764d0c2 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -61,9 +61,14 @@ def test_retrieve_access_token_invalid(self,mock_post): @patch("webscraper.api.EbayAPI.requests.get") - def test_retrieve_ebay_response_invalid(self,mock_get): - self.EbayAPI.retrieve_ebay_response("https://test","item") - self.assertRaises(Exception) + def test_retrieve_ebay_response_invalid(self, mock_get): + mock_get.side_effect = requests.exceptions.RequestException("Invalid request") + with self.assertRaises(Exception): + self.EbayAPI.retrieve_ebay_response("https://test", "item") + + def test_search_item_empty_query(self): + with self.assertRaises(ValueError): + self.EbayAPI.search_item("") # @patch("webscraper.api.EbayAPI.EbayAPI.retrieve_ebay_response") # def test_search_item(self, mock_response): From 51758b5b177b812d455a764c16f5b0e19f4edeef Mon Sep 17 00:00:00 2001 From: gmanhas12 Date: Sun, 15 Jun 2025 15:13:33 -0700 Subject: [PATCH 11/11] Filtering the response from EbayAPI #34 --- webscraper/api/EbayAPI.py | 41 +++++++++++++++----------- webscraper/api/tests/test_ebay_api.py | 42 +++++++++++++++++++++++---- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py index 23f8cde..715913b 100644 --- a/webscraper/api/EbayAPI.py +++ b/webscraper/api/EbayAPI.py @@ -15,11 +15,12 @@ logger = logging.getLogger(__name__) class EbayItem: - def __init__(self, name, price, currency, url, user_id=None): + def __init__(self, name, price, currency, url, date, user_id=None): self.name = name self.price = price self.currency = currency self.url = url + self.date = date self.user_id = user_id class EbayAPI(EbayABC): @@ -28,30 +29,36 @@ class EbayAPI(EbayABC): get_user_key = HTTPBasicAuth(client_id_key, client_secret_key) @staticmethod - def search_item(query: str) -> EbayItem: - """Search for an item on eBay using the query string.""" + def search_item(query: str) -> list[EbayItem]: + """Search for items on eBay and return a list of EbayItem objects.""" if not isinstance(query, str) or not query.strip(): logger.warning("Invalid query input.") raise ValueError("Query must be a non-empty string.") - + logger.info(f"Searching eBay for: {query}") response_json = EbayAPI.retrieve_ebay_response( "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", query ) + results = [] try: - item = response_json["itemSummaries"][0] - logger.debug(f"Item found: {item}") - return EbayItem( - name=item.get("title"), - price=float(item["price"]["value"]), - currency=item["price"]["currency"], - url=item.get("itemWebUrl"), - user_id=None - ) - except (KeyError, IndexError) as e: - logger.error(f"Item not found or response invalid: {response_json}") - raise Exception("Could not parse item from eBay response.") from e + item_summaries = response_json["itemSummaries"] + for item in item_summaries: + ebay_item = EbayItem( + name=item.get("title"), + price=float(item["price"]["value"]), + currency=item["price"]["currency"], + url=item.get("itemWebUrl"), + date=item.get("itemCreationDate"), + user_id=None + ) + results.append(ebay_item) + return results + except (KeyError, IndexError, TypeError) as e: + logger.error(f"Item list not found or response invalid: {response_json}") + raise Exception("Could not parse items from eBay response.") from e + finally: + logger.debug(f"Search attempt complete for query: {query}") @staticmethod def retrieve_access_token() -> str: @@ -100,4 +107,4 @@ def retrieve_ebay_response(httprequest: str, query: str) -> dict: return response.json() except requests.exceptions.RequestException as e: logger.exception("Error retrieving eBay response.") - raise + raise Exception(f"Error retrieving eBay response: {str(e)}") from e \ No newline at end of file diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py index 764d0c2..18e1917 100644 --- a/webscraper/api/tests/test_ebay_api.py +++ b/webscraper/api/tests/test_ebay_api.py @@ -25,12 +25,42 @@ def test_retrieve_access_token_real(self): self.assertGreater(len(token), 0) def test_search_item_real(self): - item = self.EbayAPI.search_item("macbook") - self.assertIsInstance(item.name, str) - self.assertIsInstance(item.price, float) - self.assertIsInstance(item.currency, str) - self.assertTrue(item.url.startswith("http")) - + items = self.EbayAPI.search_item("macbook") + self.assertIsInstance(items, list) + self.assertGreater(len(items), 0) + self.assertIsInstance(items[0].name, str) + self.assertIsInstance(items[0].price, float) + self.assertIsInstance(items[0].currency, str) + self.assertTrue(items[0].url.startswith("http")) + + @patch("webscraper.api.EbayAPI.requests.get") + def test_search_item_http_500(self, mock_get): + mock_get.return_value.status_code = 500 + mock_get.return_value.raise_for_status.side_effect = requests.exceptions.HTTPError("Internal Server Error") + + with self.assertRaises(Exception) as context: + self.EbayAPI.search_item("macbook") + + self.assertIn("Error retrieving eBay response", str(context.exception)) + + @patch("webscraper.api.EbayAPI.requests.get") + def test_search_item_http_404(self, mock_get): + mock_get.return_value.status_code = 404 + mock_get.return_value.raise_for_status.side_effect = requests.exceptions.HTTPError("Not Found") + + with self.assertRaises(Exception): + self.EbayAPI.search_item("macbook") + + @patch("webscraper.api.EbayAPI.EbayAPI.retrieve_ebay_response") + def test_search_item_no_items_in_response(self, mock_response): + mock_response.return_value = {} # Missing 'itemSummaries' + + with self.assertRaises(Exception) as context: + self.EbayAPI.search_item("macbook") + + self.assertIn("Could not parse items", str(context.exception)) + + def test_search_item_not_found(self): with self.assertRaises(Exception) as context: self.EbayAPI.search_item("asdkfjasldfjalskdfj") # nonsense query