Add getchromedriver.py

Author: Lucas Faudman

pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+authors = [{name = "Lucas Faudman", email = "lucasfaudman@gmail.com"}]
+dynamic = ["version"]
+name = "souperscraper"
+requires-python = ">=3.8"
+readme = "README.md"
+license = { file = "LICENSE.txt" }
+description = "A simple web scraper base combining Beautiful Soup and Selenium"
+keywords = ["web-scraping", "scraping", "beautifulsoup4", "beautifulsoup", "bs4", "selenium", "selenium-webdriver"]
+dependencies = [
+    "selenium",
+    "beautifulsoup4",
+    "requests",
+]
+
+[project.urls]
+Homepage = "https://github.com/LucasFaudman/souper-scraper.git"
+Repository = "https://github.com/LucasFaudman/souper-scraper.git"
+
+[tool.setuptools]
+
+[tool.setuptools_scm]
+version_file = "src/souperscraper/_version.py"

src/souperscraper/__init__.py

@@ -0,0 +1,2 @@
+from .souperscraper import SouperScraper, Keys
+from .getchromedriver import get_chromedriver

src/souperscraper/_version.py

@@ -0,0 +1,16 @@
+# file generated by setuptools_scm
+# don't change, don't track in version control
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple, Union
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.1.dev1+g5b3b1f0'
+__version_tuple__ = version_tuple = (0, 1, 'dev1', 'g5b3b1f0')

src/souperscraper/getchromedriver.py

@@ -0,0 +1,137 @@
+import requests
+import zipfile
+import argparse
+from time import sleep
+from pathlib import Path
+from typing import Optional
+
+DEFAULT_PATH = Path('~/chromedriver/')
+
+
+def options_menu(options, prompt, param_name='option', default=-1):
+    """
+    Display a menu of options and prompt the user to select one.
+    """
+    selected = None
+    while not selected:
+        print(prompt)
+        for i, option in enumerate(options):
+            print(f"({i+1}) {option}")
+        print()
+        default_option = options[default]
+        selected = input(
+            f"Enter 1-{len(options)} to select a {param_name} (default: {options.index(default_option) + 1} {default_option}): ")
+        if not selected:
+            selected = default
+        elif selected in options:
+            selected = options.index(selected)
+        elif not selected.isdigit() or int(selected) < 1 or int(selected) > len(options):
+            print(
+                f"\nInvalid {param_name} number: {selected}. Enter 1-{len(options)}")
+            sleep(2)
+            selected = None
+        else:
+            selected = int(selected) - 1
+
+    return options[selected]
+
+
+def select_chromedriver(version_number: Optional[int]=None, headless: Optional[bool]=None, platform: Optional[str]=None) -> tuple[str, str]:
+    """
+    Select the chromedriver version and platform to download. Return the filename and download URL.
+    """
+    versions_url = 'https://googlechromelabs.github.io/chrome-for-testing/latest-versions-per-milestone-with-downloads.json'
+    versions = requests.get(versions_url).json()
+    milestones = versions['milestones']
+    
+    version_options = [
+        f"Version {ms}" for ms in milestones if milestones[ms]['downloads'].get('chromedriver')]
+    version_q = f"""
+    Which chrome version number do you have installed? 
+    Open chrome and go to:
+        chrome://settings/help
+    you will see a version number like Version 121.0.6167.85 (Official Build) (x86_64)
+    which corresponds to chromedriver version 121
+
+    Available chromedriver versions:"""
+    selected_version = version_number or options_menu(version_options, version_q, 'version number', -1).split(' ')[1]
+    if int(selected_version) >= 120:
+        headless = headless or input(
+            "Do you want to use the headless version of chromedriver? (y/n): ").lower() == 'y'
+    else:
+        headless = False
+    executable = 'chromedriver' if not headless else 'chrome-headless-shell'
+
+    platform_options = [download['platform']
+                        for download in milestones[selected_version]['downloads'][executable]]
+    platform_q = f"""Which platform are you using?
+
+    Available platforms:"""
+    platform = platform or options_menu(platform_options, platform_q, 'platform', 0)
+
+    filename = f"{executable}{selected_version}-{platform}.zip"
+    for download in milestones[selected_version]['downloads'][executable]:
+        if download['platform'] == platform:
+            download_url = download['url']
+            print(f"Found download for {filename} at {download_url}")
+            return filename, download_url
+    
+    raise ValueError(f"Could not find download for {filename}")
+
+
+def download_chromedriver(filename: str, download_url: str, destdir: Optional[Path]=None):
+    """
+    Download the chromedriver zip file from the download URL and save it to the destination directory.
+    """
+    if not destdir:
+        destdir_input = input(f"Where do you want to save {filename}? (default: {DEFAULT_PATH}): ").rstrip('/')
+        if not destdir_input:
+            destdir = DEFAULT_PATH.resolve()
+        else:
+            destdir = Path(destdir_input).resolve()
+    else:
+        destdir = destdir.resolve()
+    
+    if not destdir.exists():
+        print(f"Creating {destdir}...")
+        destdir.mkdir(parents=True)
+
+    print(f"Downloading {filename} from {download_url}...")
+    destpath = destdir / filename
+    with destpath.open('wb') as f:
+        f.write(requests.get(download_url).content)
+    print(f"Downloaded {filename} to {destpath}")
+
+    print(f"Extracting {filename} to {destdir}...")
+    with zipfile.ZipFile(destpath, 'r') as zip_ref:
+        zip_ref.extractall(destdir)
+    
+    executable_path = next(path for path in destdir.rglob('*chrome*') if path.is_file() and path.suffix == '')
+    return executable_path
+
+
+def get_chromedriver() -> Optional[Path]:
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--version', type=int, help='Chromedriver version number')
+    parser.add_argument('-p', '--platform', type=str, help='Platform to download chromedriver for')
+    parser.add_argument('-d', '--destdir', type=Path, help='Directory to save chromedriver to')
+    parser.add_argument('--headless', action='store_true', help='Use headless version of chromedriver')
+    args = parser.parse_args()
+
+    try:
+        filename, download_url = select_chromedriver(args.version, args.headless, args.platform)
+    except ValueError as e:
+        print(e)
+        return None
+    
+    try:
+        executable_path = download_chromedriver(filename, download_url, args.destdir)
+    except Exception as e:
+        print("Failed to download and extract chromedriver. Error: ", e)
+        return None
+    
+    print('Success. Chromedriver executable downloaded and saved to', executable_path)
+    return executable_path
+
+if __name__ == '__main__':
+    get_chromedriver()