API Endpoints with Flask and Unit Testing

environment.yml

@@ -8,6 +8,7 @@ dependencies:
   - numpy
   - pandas
   - requests
+  - flask
   - pip:
     - beautifulsoup4
     - lxml

webscraper/api/interface.py

@@ -0,0 +1,8 @@
+from abc import ABC, abstractmethod
+
+class ScraperAPIInterface(ABC):
+
+    @abstractmethod
+    def get_scraped_data(self, paths: list[str]) -> dict:
+        """Given a list of paths, return scraped results."""
+        pass

webscraper/api/routes.py

@@ -0,0 +1,26 @@
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+
+from flask import Flask, jsonify, request
+from webscraper.src.Cheaper_Scraper import CheaperScraper
+
+app = Flask(__name__)
+scraper = CheaperScraper(base_url="https://books.toscrape.com")
+
+@app.route('/')
+def home():
+    return jsonify({"message": "Welcome to Cheaper API!"})
+
+@app.route('/scrape', methods=['GET'])
+def scrape_books():
+    paths = request.args.getlist('path')
+    if not paths:
+        return jsonify({"error": "No paths provided"}), 400
+
+    results = scraper.get_scraped_data(paths)    
+    return jsonify(results)
+
+if __name__ == '__main__':
+    app.run(debug=True)
+

webscraper/api/tests/test_routes.py

@@ -0,0 +1,26 @@
+import unittest
+from webscraper.api.routes import app
+
+class TestRoutes(unittest.TestCase):
+
+    def setUp(self):
+        self.client = app.test_client()
+        self.client.testing = True
+
+    def test_home_route(self):
+        response = self.client.get('/')
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(b"Welcome to Cheaper API", response.data)
+
+    def test_scrape_no_params(self):
+        response = self.client.get('/scrape')
+        self.assertEqual(response.status_code, 400)
+        self.assertIn(b"No paths provided", response.data)
+
+    def test_scrape_valid_path(self):
+        response = self.client.get('/scrape?path=/catalogue/page-1.html')
+        self.assertEqual(response.status_code, 200)
+        self.assertIsInstance(response.get_json(), dict)
+
+if __name__ == '__main__':
+    unittest.main()

webscraper/src/Cheaper_Scraper.py

@@ -3,12 +3,15 @@
 from bs4 import BeautifulSoup
 import logging
 from typing import Dict, List, Optional
-from ABC.base_scraper import BaseScraper
-from Robot_Check import RoboCheck
+from webscraper.ABC.base_scraper import BaseScraper
+from webscraper.src.robot_check import RoboCheck
+from webscraper.api.interface import ScraperAPIInterface
 
 
-class CheaperScraper(BaseScraper):
-    def __init__(self, base_url:str, user_agent: str= "CheaperBot/0.1", delay: float=2.0) -> None:
+
+
+class CheaperScraper(BaseScraper, ScraperAPIInterface):
+    def __init__(self, base_url: str = "", user_agent: str = "CheaperBot/0.1", delay: float = 2.0) -> None:
         """Initialize the scraper with base parameters.
 
         Args:
@@ -88,4 +91,7 @@ def scrape(self, paths: List[str]) -> Dict[str, List[str]]:
             html = self.fetch(path)
             if html:
                 results[path] = self.parse(html)
-        return results
+        return results
+
+    def get_scraped_data(self, paths: List[str]) -> Dict[str, List[str]]:
+        return self.scrape(paths)