Refactored names, fixed some functions, added background process

.gitignore

@@ -24,7 +24,7 @@ venv/
 .vscode/
 
 # === macOS system files ===
-.DS_Store
+*.DS_Store
 
 # === Pytest and test cache ===
 htmlcov/

webscraper/ABC/Ebay_API.py → cheaper_main/ABC/RetailerApi.py

@@ -1,15 +1,15 @@
 from abc import ABC,abstractmethod
 
-class EbayApi(ABC):
+class RetailerApi(ABC):
 
     @abstractmethod
-    def retrieve_access_token() -> str:
+    def retrieve_access_token(self) -> str:
         """ retrieves the user access token for sandbox environment it's a long line
             of text, numbers, symbols
         """
         pass
 
     @abstractmethod
-    def retrieve_ebay_response(httprequest:str,query:str) -> dict:
+    def retrieve_response(self,httprequest:str,query:str) -> dict:
         """ retrieves a json of large data with category ids, names, parentcategorynodes """
         pass

webscraper/src/Cheaper_Scraper.py → cheaper_main/Scraper/Cheaper_Scraper.py

@@ -4,17 +4,15 @@
 from urllib.parse import urlparse
 import logging
 from typing import Dict, List, Optional
-from webscraper.ABC.base_scraper import BaseScraper
-from webscraper.src.robot_check import RoboCheck
-from webscraper.api.interface import ScraperAPIInterface
-from webscraper.src.fetch_utils import cached_get
+from cheaper_main.ABC.base_scraper import BaseScraper
+from cheaper_main.Scraper.robot_check import RoboCheck
+from cheaper_main.Scraper.fetch_utils import cached_get
 from functools import lru_cache
-from webscraper.api.EbayAPI import EbayItem
 
 
 
 
-class CheaperScraper(BaseScraper, ScraperAPIInterface):
+class CheaperScraper(BaseScraper):
     def __init__(self, base_url: str = "", user_agent: str = "CheaperBot/0.1", delay: float = 2.0) -> None:
         """Initialize the scraper with base parameters.
 

cheaper_main/api/Etsy/EtsyApi.py

@@ -0,0 +1,31 @@
+from cheaper_main.ABC.RetailerApi import RetailerApi
+import requests
+import os
+from generate_code_challenge import generate_code_challenge
+
+
+keystring = os.getenv("etsykeystring")
+sharedsecret = os.getenv("etsysharedsecret")
+
+class Etsy(RetailerApi):
+    def retrieve_access_token(self):
+        # most likely this url will change and I will have a parameter set for it
+        # otherwise this default url will be used for testing purposes and development
+        try:
+            response = requests.post("https://api.etsy.com/v3/public/oauth/token",
+                                     headers={"Content-Type': 'application/x-www-form-urlencoded"},
+                                     data = {"grant_type":"client_credentials",
+                                             "scope":"listings_r",
+                                             "client_id":f"{keystring}",
+                                             "code_challenge":f"{generate_code_challenge.generate_code_challenge()}",
+                                             "code_challenge_method":"S256"
+                                     }
+
+            )
+            if(response.status_code == 200):
+                data = response.json()
+        except Exception as e:
+            raise e
+
+    def retrieve_response(self): 
+            raise NotImplementedError 

cheaper_main/api/Etsy/generate_code_challenge.py

@@ -0,0 +1,15 @@
+import secrets
+import hashlib
+import base64
+
+
+class generate_code_challenge:
+    # Will most likely be used only for APIs that require it
+    # If it gets used more than once I will make an Abstract Base Class
+    def generate_code_challenge() -> str:
+        code_client = secrets.token_urlsafe(64)
+        code_challenge = base64.urlsafe_b64encode(hashlib.sha256(code_client.encode())
+                                              .digest()).rstrip(b'=').decode()
+        return code_challenge
+
+

cheaper_main/api/best_buy_api/best_buy_api.py

@@ -0,0 +1,16 @@
+import os
+from cheaper_main.ABC import RetailerApi
+
+best_buy = os.getenv("bestbuysecret")
+
+
+class best_buy_api(RetailerApi):
+
+    def retrieve_access_token(self):
+
+        return
+
+
+    def retrieve_response(self):
+
+        return

webscraper/api/EbayAPI.py → cheaper_main/api/ebay_api/EbayAPI.py

@@ -3,7 +3,7 @@
 from dotenv import load_dotenv
 import os
 import logging
-from webscraper.api.interface import EbayABC
+from ABC.RetailerApi import RetailerApi
 
 # Load environment variables and configure logging
 load_dotenv()
@@ -22,21 +22,22 @@ def __init__(self, name, price, currency, url, date, user_id=None):
         self.url = url
         self.date = date
         self.user_id = user_id
+        pass
 
-class EbayAPI(EbayABC):
-    client_secret_key = os.getenv("clientsecret")
-    client_id_key = os.getenv("clientid")
-    get_user_key = HTTPBasicAuth(client_id_key, client_secret_key)
+class EbayAPI(RetailerApi):
+    def __init__(self):
+        self.client_secret_key = os.getenv("clientsecret")
+        self.client_id_key = os.getenv("clientid")
+        self.auth = HTTPBasicAuth(self.client_id_key, self.client_secret_key)
 
-    @staticmethod
-    def search_item(query: str) -> list[EbayItem]:
-        """Search for items on eBay and return a list of EbayItem objects."""
+    def search_item(self,query: str) -> EbayItem:
+        """Search for an item on eBay using the query string."""
         if not isinstance(query, str) or not query.strip():
             logger.warning("Invalid query input.")
             raise ValueError("Query must be a non-empty string.")
 
         logger.info(f"Searching eBay for: {query}")
-        response_json = EbayAPI.retrieve_ebay_response(
+        response_json = self.retrieve_response(
             "https://api.sandbox.ebay.com/buy/browse/v1/item_summary/search", query
         )
 
@@ -60,8 +61,7 @@ def search_item(query: str) -> list[EbayItem]:
         finally:
             logger.debug(f"Search attempt complete for query: {query}")
 
-    @staticmethod
-    def retrieve_access_token() -> str:
+    def retrieve_access_token(self) -> str:
         """Fetch access token from eBay API."""
         logger.info("Requesting eBay access token...")
         try:
@@ -72,7 +72,7 @@ def retrieve_access_token() -> str:
                     "grant_type": "client_credentials",
                     "scope": "https://api.ebay.com/oauth/api_scope"
                 },
-                auth=EbayAPI.get_user_key
+                auth=self.auth
             )
             response.raise_for_status()
             token = response.json().get("access_token")
@@ -85,10 +85,9 @@ def retrieve_access_token() -> str:
             logger.exception("Failed to retrieve token.")
             raise
 
-    @staticmethod
-    def retrieve_ebay_response(httprequest: str, query: str) -> dict:
+    def retrieve_response(self,httprequest: str, query: str) -> dict:
         """Perform GET request to eBay API."""
-        auth = EbayAPI.retrieve_access_token()
+        auth = self.retrieve_access_token()
         logger.info(f"Making GET request to eBay API: {httprequest} with query: {query}")
         try:
             response = requests.get(

webscraper/api/routes.py → cheaper_main/api/routes.py

@@ -3,7 +3,7 @@
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
 
 from flask import Flask, jsonify, request
-from webscraper.src.Cheaper_Scraper import CheaperScraper
+from Scraper.Cheaper_Scraper import CheaperScraper
 
 app = Flask(__name__)
 scraper = CheaperScraper(base_url="https://books.toscrape.com")

webscraper/api/tests/test_ebay_api.py → cheaper_main/api/tests/test_ebay_api.py

@@ -8,7 +8,7 @@
 import unittest
 from unittest.mock import patch,Mock
 import requests
-from webscraper.api.EbayAPI import EbayAPI
+from ...api.ebay_api import EbayAPI
 from dotenv import load_dotenv
 load_dotenv()
 

cheaper_main/main.py

@@ -0,0 +1,72 @@
+from flask import Flask, request , jsonify
+import json
+#import time  # for testing
+# i added these imports below because when i ran it it wasnt finding the folders
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from cheaper_main.Scraper.Cheaper_Scraper import CheaperScraper
+
+app = Flask(__name__)
+
+#python main.py will run it in the background git bash
+#to stop put pm2 stop Cheaper in git bash
+@app.route('/')
+def scrape():
+
+    # Set up the scraper for a simple legal-to-scrape website
+    scraper = CheaperScraper("https://books.toscrape.com",
+                             user_agent="CheaperBot/0.1",
+                             delay=2.0)
+
+    # Define which pages you want to scrape (you can use "/" for homepage)
+    pages = ["/"]
+
+    # Use the scraper to fetch and parse the pages
+    results = scraper.scrape(pages)
+
+    # Show the output in the terminal
+    for path, items in results.items():
+        print(f"\nScraped from {path}:")
+        for item in items:
+            print("-", item)
+
+    # Save the output to a JSON file
+    #with open("output.json", "w") as f:
+        #json.dump(results, f, indent=2)
+    return jsonify(results)
+
+@app.route('/api/products/search', methods=['GET'])
+def ebay_search():
+    try:
+        from api.ebay_api.EbayAPI import EbayAPI
+        #instantiate object
+        ebay_api = EbayAPI()
+
+        product = request.args.get('product')
+        #The route will look like this
+        # http://127.0.0.1:5000/api/products/search?product=
+        #after product= type any generic item to receive json like ?product=clothes
+        #put that in the address bar
+
+        print(f"product = {product}")
+        if not product:
+            return jsonify({"error": "missing ?product=parameter"}),400
+        response = ebay_api.search_item(product)
+
+        return jsonify({
+            "name": response.name,
+            "price": response.price,
+            "currency": response.currency,
+            "url": response.url
+        })
+
+    except Exception as e:
+        print("failed to import",e)
+        return jsonify({"error": str(e)}), 500
+
+
+
+
+if __name__ == "__main__":#
+    app.run(debug=True)

cheaper_main/start_service.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+APP_NAME="Cheaper"
+APP_FILE="main.py"
+LOG_DIR="$HOME/CheaperLogs"
+
+# Allow user to override PYTHON_PATH by setting it externally
+if [ -z "$PYTHON_PATH" ]; then
+  PYTHON_PATH=$(command -v python3)
+fi
+
+if [ -z "$PYTHON_PATH" ]; then
+  PYTHON_PATH=$(command -v python)
+fi
+
+# Final fallback for Windows users (optional)
+if [ -z "$PYTHON_PATH" ] && [ -f "/c/Users/$USERNAME/AppData/Local/Programs/Python/Python39/python.exe" ]; then
+  PYTHON_PATH="/c/Users/$USERNAME/AppData/Local/Programs/Python/Python39/python.exe"
+fi
+
+# Validate Python path
+if ! "$PYTHON_PATH" --version > /dev/null 2>&1; then
+  echo "❌ Python not found. Please install it or set PYTHON_PATH manually."
+  exit 1
+fi
+
+echo "✅ Using Python at: $PYTHON_PATH"
+
+# Create log directory
+mkdir -p "$LOG_DIR"
+
+# Start with PM2
+pm2 start "$APP_FILE" \
+  --name "$APP_NAME" \
+  --interpreter="$PYTHON_PATH" \
+  --output "$LOG_DIR/out.log" \
+  --error "$LOG_DIR/err.log" \
+  --watch
+
+pm2 save