diff --git a/.env b/.env deleted file mode 100644 index e69de29..0000000 diff --git a/.gitignore b/.gitignore index e69de29..5aeb0f4 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,3 @@ +*.env +*.pyc +__pycache__/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..b2d293c --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,14 @@ +{ + "python.analysis.extraPaths": [ + "./webscraper/ABC" + ], + "python.testing.unittestArgs": [ + "-v", + "-s", + "./webscraper", + "-p", + "*test*.py" + ], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true +} \ No newline at end of file diff --git a/accounts/__pycache__/__init__.cpython-312.pyc b/accounts/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 228acde..0000000 Binary files a/accounts/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/accounts/__pycache__/admin.cpython-312.pyc b/accounts/__pycache__/admin.cpython-312.pyc deleted file mode 100644 index 5a69a0d..0000000 Binary files a/accounts/__pycache__/admin.cpython-312.pyc and /dev/null differ diff --git a/accounts/__pycache__/apps.cpython-312.pyc b/accounts/__pycache__/apps.cpython-312.pyc deleted file mode 100644 index 33636a4..0000000 Binary files a/accounts/__pycache__/apps.cpython-312.pyc and /dev/null differ diff --git a/accounts/__pycache__/models.cpython-312.pyc b/accounts/__pycache__/models.cpython-312.pyc deleted file mode 100644 index 8e8be97..0000000 Binary files a/accounts/__pycache__/models.cpython-312.pyc and /dev/null differ diff --git a/accounts/migrations/0002_remove_product_name_remove_product_source_url_and_more.py b/accounts/migrations/0002_remove_product_name_remove_product_source_url_and_more.py new file mode 100644 index 0000000..e478a27 --- /dev/null +++ b/accounts/migrations/0002_remove_product_name_remove_product_source_url_and_more.py @@ -0,0 +1,55 @@ +# Generated by Django 5.2 on 2025-05-05 19:14 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("accounts", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="product", + name="name", + ), + migrations.RemoveField( + model_name="product", + name="source_url", + ), + migrations.RemoveField( + model_name="useraccount", + name="password", + ), + migrations.AddField( + model_name="product", + name="product_name", + field=models.CharField(default="Unnamed Product", max_length=255), + ), + migrations.AddField( + model_name="product", + name="url", + field=models.TextField(default="https://example.com"), + ), + migrations.AddField( + model_name="product", + name="user", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="accounts.useraccount", + ), + ), + migrations.AddField( + model_name="useraccount", + name="password_hash", + field=models.CharField(default="defaultpass123", max_length=100), + ), + migrations.AlterField( + model_name="product", + name="price", + field=models.DecimalField(decimal_places=2, default=0.0, max_digits=10), + ), + ] diff --git a/accounts/migrations/__pycache__/0001_initial.cpython-312.pyc b/accounts/migrations/__pycache__/0001_initial.cpython-312.pyc deleted file mode 100644 index 2a79d5d..0000000 Binary files a/accounts/migrations/__pycache__/0001_initial.cpython-312.pyc and /dev/null differ diff --git a/accounts/migrations/__pycache__/__init__.cpython-312.pyc b/accounts/migrations/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index f9a196d..0000000 Binary files a/accounts/migrations/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/accounts/models.py b/accounts/models.py index 3ff6537..c79a3e2 100644 --- a/accounts/models.py +++ b/accounts/models.py @@ -14,7 +14,7 @@ def validate_email(value): class UserAccount(models.Model): email = models.EmailField(max_length=50, unique=True) - password = models.CharField(max_length=100) + password_hash = models.CharField(max_length=100, default='defaultpass123') # added default def clean(self): validate_email(self.email) @@ -22,10 +22,12 @@ def clean(self): def __str__(self): return self.email + class Product(models.Model): - name = models.CharField(max_length=200) - price = models.CharField(max_length=10) - source_url = models.URLField(max_length=150) + product_name = models.CharField(max_length=255, default='Unnamed Product') + price = models.DecimalField(max_digits=10, decimal_places=2, default=0.00) + url = models.TextField(default='https://example.com') + user = models.ForeignKey(UserAccount, on_delete=models.CASCADE, null=True) # optional if user not yet created def __str__(self): - return self.name \ No newline at end of file + return self.product_name diff --git a/accounts/views.py b/accounts/views.py index 91ea44a..2ebc3d3 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -1,3 +1,10 @@ from django.shortcuts import render +from django.http import JsonResponse +from .models import Product + +def product_list(request): + products = Product.objects.all() + data = [{"name": p.product_name, "price": float(p.price), "url": p.url} for p in products] + return JsonResponse(data, safe=False) # Create your views here. diff --git a/cheaper/__pycache__/__init__.cpython-312.pyc b/cheaper/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 65ab433..0000000 Binary files a/cheaper/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/cheaper/__pycache__/settings.cpython-312.pyc b/cheaper/__pycache__/settings.cpython-312.pyc deleted file mode 100644 index 048f033..0000000 Binary files a/cheaper/__pycache__/settings.cpython-312.pyc and /dev/null differ diff --git a/cheaper/__pycache__/urls.cpython-312.pyc b/cheaper/__pycache__/urls.cpython-312.pyc deleted file mode 100644 index 8e83205..0000000 Binary files a/cheaper/__pycache__/urls.cpython-312.pyc and /dev/null differ diff --git a/cheaper/urls.py b/cheaper/urls.py index 49e4fb1..3a5aea7 100644 --- a/cheaper/urls.py +++ b/cheaper/urls.py @@ -16,7 +16,9 @@ """ from django.contrib import admin from django.urls import path +from accounts.views import product_list urlpatterns = [ path('admin/', admin.site.urls), + path('', product_list, name='product_list'), # This sets the homepage ] diff --git a/db.sqlite3 b/db.sqlite3 index 9a49d22..fc5a943 100644 Binary files a/db.sqlite3 and b/db.sqlite3 differ diff --git a/webscraper/ABC/Ebay_API.py b/webscraper/ABC/Ebay_API.py new file mode 100644 index 0000000..2be5a07 --- /dev/null +++ b/webscraper/ABC/Ebay_API.py @@ -0,0 +1,15 @@ +from abc import ABC,abstractmethod + +class EbayApi(ABC): + + @abstractmethod + def retrieve_access_token() -> str: + """ retrieves the user access token for sandbox environment it's a long line + of text, numbers, symbols + """ + pass + + @abstractmethod + def retrieve_ebay_response(httprequest:str,query:str) -> dict: + """ retrieves a json of large data with category ids, names, parentcategorynodes """ + pass \ No newline at end of file diff --git a/webscraper/ABC/__pycache__/base_scraper.cpython-311.pyc b/webscraper/ABC/__pycache__/base_scraper.cpython-311.pyc deleted file mode 100644 index 2dc46e3..0000000 Binary files a/webscraper/ABC/__pycache__/base_scraper.cpython-311.pyc and /dev/null differ diff --git a/webscraper/api/EbayAPI.py b/webscraper/api/EbayAPI.py new file mode 100644 index 0000000..15f78e3 --- /dev/null +++ b/webscraper/api/EbayAPI.py @@ -0,0 +1,55 @@ +import requests +from requests.auth import HTTPBasicAuth +from dotenv import load_dotenv +import os + + +load_dotenv() #initialize + +class EbayAPI: + + client_secret_key = os.getenv("clientsecret") + client_id_key = os.getenv("clientid") + + get_user_key = HTTPBasicAuth(client_id_key, client_secret_key) + + + def retrieve_access_token(): + try: + response = requests.post("https://api.sandbox.ebay.com/identity/v1/oauth2/token", + headers = {"Content-Type":"application/x-www-form-urlencoded"}, + data = { + "grant_type": "client_credentials", + "scope": "https://api.ebay.com/oauth/api_scope" + }, + auth=EbayAPI.get_user_key + ) + access_token = response.json().get("access_token") + status_code = response.status_code + if(status_code == 404): + raise Exception("404 error here") + return access_token + except Exception as e: + raise e + + def retrieve_ebay_response(httprequest:str,query:str): + auth = EbayAPI.retrieve_access_token() + try: + response = requests.get(httprequest, + headers={ + "Authorization": f"Bearer {auth}", + "Content-Type": "application/json" + }, + params= { + "q": query, + "category_tree_id": 0 + } + ) + status_code = response.status_code + if(status_code == 404): + raise Exception("not found 404 error") + + return response.json() + except Exception as e: + raise e + diff --git a/webscraper/api/__pycache__/interface.cpython-311.pyc b/webscraper/api/__pycache__/interface.cpython-311.pyc deleted file mode 100644 index 1b9240d..0000000 Binary files a/webscraper/api/__pycache__/interface.cpython-311.pyc and /dev/null differ diff --git a/webscraper/api/__pycache__/routes.cpython-311.pyc b/webscraper/api/__pycache__/routes.cpython-311.pyc deleted file mode 100644 index 5e18603..0000000 Binary files a/webscraper/api/__pycache__/routes.cpython-311.pyc and /dev/null differ diff --git a/webscraper/api/tests/__pycache__/test_routes.cpython-311.pyc b/webscraper/api/tests/__pycache__/test_routes.cpython-311.pyc deleted file mode 100644 index 1ce37af..0000000 Binary files a/webscraper/api/tests/__pycache__/test_routes.cpython-311.pyc and /dev/null differ diff --git a/webscraper/api/tests/test_ebay_api.py b/webscraper/api/tests/test_ebay_api.py new file mode 100644 index 0000000..7651c13 --- /dev/null +++ b/webscraper/api/tests/test_ebay_api.py @@ -0,0 +1,35 @@ +import unittest +from unittest.mock import patch,Mock +import requests +from webscraper.api.EbayAPI import EbayAPI + +class EbayTestApi(unittest.TestCase): + + def setUp(self): + self.EbayAPI = EbayAPI + + + def test_retrieve_access_token(self): + self.EbayAPI.retrieve_access_token() + self.assertEqual(type(self.EbayAPI.retrieve_access_token()),str) + + @patch("webscraper.api.EbayAPI.requests.post") + def test_retrieve_access_token_invalid(self,mock_post): + mock_response = Mock() + mock_response.status_code = 404 + mock_response.json.return_value ={"error": "not found"} + mock_post.return_value = mock_response + + with self.assertRaises(Exception): + self.EbayAPI.retrieve_access_token() + + + + @patch("webscraper.api.EbayAPI.requests.get") + def test_retrieve_ebay_response_invalid(self,mock_get): + self.EbayAPI.retrieve_ebay_response("https://test","item") + self.assertRaises(Exception) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/webscraper/src/main.py b/webscraper/main.py similarity index 85% rename from webscraper/src/main.py rename to webscraper/main.py index 4a27839..0ae9fe5 100644 --- a/webscraper/src/main.py +++ b/webscraper/main.py @@ -1,4 +1,3 @@ - import json #import time // for testing # i added htese imports below becasue when i ran it it wasnt finding the folders @@ -9,16 +8,19 @@ def main(): + + + + # Set up the scraper for a simple legal-to-scrape website scraper = CheaperScraper("https://books.toscrape.com", - user_agent="CheaperBot/0.1", - delay=2.0) - + user_agent="CheaperBot/0.1", + delay=2.0) # Define which pages you want to scrape (you can use "/" for homepage) pages = ["/"] # Use the scraper to fetch and parse the pages - results = scraper.scrape(pages) + results = CheaperScraper.scraper.scrape(pages) # Show the output in the terminal for path, items in results.items(): diff --git a/webscraper/src/__pycache__/CheaperScraper.cpython-39.pyc b/webscraper/src/__pycache__/CheaperScraper.cpython-39.pyc deleted file mode 100644 index f60b091..0000000 Binary files a/webscraper/src/__pycache__/CheaperScraper.cpython-39.pyc and /dev/null differ diff --git a/webscraper/src/__pycache__/Cheaper_Scraper.cpython-311.pyc b/webscraper/src/__pycache__/Cheaper_Scraper.cpython-311.pyc deleted file mode 100644 index cc3aa5e..0000000 Binary files a/webscraper/src/__pycache__/Cheaper_Scraper.cpython-311.pyc and /dev/null differ diff --git a/webscraper/src/__pycache__/Cheaper_Scraper.cpython-39.pyc b/webscraper/src/__pycache__/Cheaper_Scraper.cpython-39.pyc deleted file mode 100644 index 036324a..0000000 Binary files a/webscraper/src/__pycache__/Cheaper_Scraper.cpython-39.pyc and /dev/null differ diff --git a/webscraper/src/__pycache__/__init__.cpython-311.pyc b/webscraper/src/__pycache__/__init__.cpython-311.pyc deleted file mode 100644 index c420941..0000000 Binary files a/webscraper/src/__pycache__/__init__.cpython-311.pyc and /dev/null differ diff --git a/webscraper/src/__pycache__/robot_check.cpython-311.pyc b/webscraper/src/__pycache__/robot_check.cpython-311.pyc deleted file mode 100644 index e7f181b..0000000 Binary files a/webscraper/src/__pycache__/robot_check.cpython-311.pyc and /dev/null differ diff --git a/webscraper/src/__pycache__/robot_check.cpython-39.pyc b/webscraper/src/__pycache__/robot_check.cpython-39.pyc deleted file mode 100644 index d1557b9..0000000 Binary files a/webscraper/src/__pycache__/robot_check.cpython-39.pyc and /dev/null differ diff --git a/webscraper/src/__pycache__/test_cheaper_scraper.cpython-311.pyc b/webscraper/src/__pycache__/test_cheaper_scraper.cpython-311.pyc deleted file mode 100644 index ef900b2..0000000 Binary files a/webscraper/src/__pycache__/test_cheaper_scraper.cpython-311.pyc and /dev/null differ