Merge pull request #9 from Oaphi/ublock

Make uBlock fully configurable

.gitignore

@@ -1,4 +1,6 @@
-/config.json
+# configuration files
+*config.json
+
 compile_commands.json
 *.xpi
 /stackexchange_*/

README.md

@@ -92,11 +92,12 @@ The downloader does **not** support Docker due to the display requirement.
 
 Exractor CLI supports the following configuration options:
 
-| Short | Long                   | Type     | Default           | Description                                                                                                                                                                |
-| ----- | ---------------------- | -------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `-o`  | `--outputDir <path>`   | Optional | `<cwd>/downloads` | Specifies the directory to download the archives to.                                                                                                                       |
+| Short | Long                   | Type     | Default           | Description                                                                                                                                                                 |
+| ----- | ---------------------- | -------- | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `-o`  | `--outputDir <path>`   | Optional | `<cwd>/downloads` | Specifies the directory to download the archives to.                                                                                                                        |
+| `-k`  | `--keep-consent`       | Optional | `false`           | Whether to keep OneTrust's consent dialog. If set, you are responsible for getting rid of it yourself (uBlock can handle that for you too).                                 |
 | `-s`  | `--skip-loaded <path>` | Optional | -                 | Whether to skip over archives that have already been downloaded. An archive is considered to be downloaded if the output directory has one already & the file is not empty. |
-| -     | `--dry-run`            | Optional | -                 | Whether to actually download the archives. If set, only traverses the network's sites.                                                                                     |
+| -     | `--dry-run`            | Optional | -                 | Whether to actually download the archives. If set, only traverses the network's sites.                                                                                      |
 
 #### Captchas and other misc. barriers
 

config.example.json

@@ -1,7 +1,42 @@
 {
-    "email": "The email you use to log into SE",
-    "password": "The password you use to log into SE",
-    "notifications": {
-        "provider": "string, any of: native,. Can be null, which disables notifications"
+  "email": "The email you use to log into SE",
+  "password": "The password you use to log into SE",
+  "notifications": {
+    "provider": "string, any of: native,. Can be null, which disables notifications"
+  },
+  "ubo": {
+    "download_url": "https://github.com/gorhill/uBlock/releases/download/1.59.0/uBlock0_1.59.0.firefox.signed.xpi",
+    "settings": {
+      "userSettings": {
+        "advancedUserEnabled": true
+      },
+      "hiddenSettings": {},
+      "selectedFilterLists": [
+        "user-filters",
+        "ublock-filters",
+        "ublock-badware",
+        "ublock-privacy",
+        "ublock-unbreak",
+        "ublock-quick-fixes",
+        "easylist",
+        "easyprivacy",
+        "urlhaus-1",
+        "plowe-0",
+        "fanboy-cookiemonster",
+        "ublock-cookies-easylist",
+        "adguard-cookies",
+        "ublock-cookies-adguard"
+      ],
+      "whitelist": ["chrome-extension-scheme", "moz-extension-scheme"],
+      "dynamicFilters": {
+        "toAdd": [],
+        "toRemove": []
+      },
+      "userFilters": {
+        "enabled": true,
+        "trusted": true,
+        "toOverwrite": []
+      }
     }
+  }
 }

sedd/cli.py

@@ -0,0 +1,52 @@
+import argparse
+
+from os import getcwd
+from os.path import join
+
+
+class SEDDCLIArgs(argparse.Namespace):
+    skip_loaded: bool
+    keep_consent: bool
+    output_dir: str
+    dry_run: bool
+
+
+parser = argparse.ArgumentParser(
+    prog="sedd",
+    description="Automatic (unofficial) SE data dump downloader for the anti-community data dump format",
+)
+
+parser.add_argument(
+    "-s", "--skip-loaded",
+    required=False,
+    default=False,
+    action="store_true",
+    dest="skip_loaded"
+)
+
+parser.add_argument(
+    "-k", "--keep-consent",
+    required=False,
+    dest="keep_consent",
+    action="store_true",
+    default=False
+)
+
+parser.add_argument(
+    "-o", "--outputDir",
+    required=False,
+    dest="output_dir",
+    default=join(getcwd(), "downloads")
+)
+
+parser.add_argument(
+    "--dry-run",
+    required=False,
+    default=False,
+    action="store_true",
+    dest="dry_run"
+)
+
+
+def parse_cli_args() -> SEDDCLIArgs:
+    return parser.parse_args()

sedd/config/__init__.py

@@ -0,0 +1,3 @@
+from .config import *
+from .defaults import *
+from .typings import *

sedd/config/config.py

@@ -0,0 +1,52 @@
+from typing import Literal
+
+from json import load
+from os import path, getcwd
+
+from .defaults import default_ubo_url, default_ubo_settings, default_notifications_config, default_ubo_config
+from .typings import SEDDNotificationsConfig, SEDDUboConfig, SEDDUboSettings
+
+
+class SEDDConfig:
+    email: str
+    password: str
+    notifications: SEDDNotificationsConfig
+    ubo: SEDDUboConfig
+
+    def __init__(self, email: str, pwd: str, notifications: SEDDNotificationsConfig, ubo: SEDDUboConfig):
+        self.email = email
+        self.password = pwd
+        self.notifications = notifications
+        self.ubo = ubo
+
+    def get_notifications_provider(self) -> Literal['native'] | None:
+        notifications_config = self.notifications
+        return notifications_config['provider'] if 'provider' in notifications_config else None
+
+    def get_ubo_download_url(self) -> str:
+        ubo_config = self.ubo
+        return ubo_config['download_url'] if 'download_url' in ubo_config else default_ubo_url
+
+    def get_ubo_settings(self) -> SEDDUboSettings:
+        ubo_config = self.ubo
+        return ubo_config['settings'] if 'settings' in ubo_config else default_ubo_settings
+
+
+def load_sedd_config() -> SEDDConfig:
+    config_path = path.join(getcwd(), 'config.json')
+
+    config: SEDDConfig = None
+
+    with open(config_path, "r") as f:
+        config = load(f)
+
+        email = config["email"]
+        password = config["password"]
+
+        notifications = config['notifications'] if 'notifications' in config else default_notifications_config
+
+        ubo = config['ubo'] if 'ubo' in config else default_ubo_config
+
+        config = SEDDConfig(email, password, notifications, ubo)
+
+    return config

sedd/config/defaults.py

@@ -0,0 +1,44 @@
+from .typings import SEDDNotificationsConfig, SEDDUboSettings, SEDDUboConfig
+
+default_ubo_url = "https://github.com/gorhill/uBlock/releases/download/1.59.0/uBlock0_1.59.0.firefox.signed.xpi"
+
+default_notifications_config: SEDDNotificationsConfig = {
+    'provider': None
+}
+
+default_ubo_settings: SEDDUboSettings = {
+    'userSettings': {
+        'advancedUserEnabled': True,
+    },
+    "hiddenSettings": {},
+    "selectedFilterLists": [
+        "user-filters",
+        "ublock-filters",
+        "ublock-badware",
+        "ublock-privacy",
+        "ublock-unbreak",
+        "ublock-quick-fixes",
+        "easylist",
+        "easyprivacy",
+        "urlhaus-1",
+        "plowe-0"
+    ],
+    "whitelist": [
+        "chrome-extension-scheme",
+        "moz-extension-scheme"
+    ],
+    "dynamicFilters": {
+        "toAdd": [],
+        "toRemove": []
+    },
+    "userFilters": {
+        "enabled": False,
+        "trusted": False,
+        "toOverwrite": []
+    }
+}
+
+default_ubo_config: SEDDUboConfig = {
+    'download_url': default_ubo_url,
+    'settings': default_ubo_settings
+}

sedd/config/typings.py

@@ -0,0 +1,25 @@
+from typing import TypedDict, Literal
+
+
+class SEDDNotificationsConfig(TypedDict):
+    provider: Literal['native'] | None
+
+
+class SEDDUboUserFiltersSettings(TypedDict):
+    enabled: bool
+    trusted: bool
+    toOverwrite: list[str]
+
+
+class SEDDUboSettings(TypedDict):
+    userSettings: dict[str, str | bool | int]
+    hiddenSettings: dict[str, str | bool | int]
+    selectedFilterLists: list[str]
+    whitelist: list[str]
+    dynamicFilters: dict[Literal['toAdd'] | Literal['toRemove'], list[str]]
+    userFilters: SEDDUboUserFiltersSettings
+
+
+class SEDDUboConfig(TypedDict):
+    download_url: str
+    settings: SEDDUboSettings | None

sedd/driver.py

@@ -0,0 +1,50 @@
+from os import path, makedirs
+from urllib import request
+from json import dumps
+from uuid import uuid4
+
+from selenium import webdriver
+from selenium.webdriver.firefox.options import Options
+
+from .config import SEDDConfig
+from .ubo import init_ubo_settings
+
+
+def init_output_dir(output_dir: str):
+    if not path.exists(output_dir):
+        makedirs(output_dir)
+
+    print(output_dir)
+
+    return output_dir
+
+
+def init_firefox_driver(config: SEDDConfig, output_dir: str):
+    options = Options()
+    options.enable_downloads = True
+    options.set_preference("browser.download.folderList", 2)
+    options.set_preference("browser.download.manager.showWhenStarting", False)
+    options.set_preference("browser.download.dir", output_dir)
+    options.set_preference(
+        "browser.helperApps.neverAsk.saveToDisk", "application/x-gzip"
+    )
+
+    # our own uuid for uBO so as we don't need to do the dance of inspecing internals
+    ubo_internal_uuid = f"{uuid4()}"
+
+    options.set_preference("extensions.webextensions.uuids", dumps(
+        {"uBlock0@raymondhill.net": ubo_internal_uuid}))
+
+    browser = webdriver.Firefox(options=options)
+
+    ubo_download_url = config.get_ubo_download_url()
+
+    if not path.exists("ubo.xpi"):
+        print(f"Downloading uBO from: {ubo_download_url}")
+        request.urlretrieve(ubo_download_url, "ubo.xpi")
+
+    ubo_id = browser.install_addon("ubo.xpi", temporary=True)
+
+    init_ubo_settings(browser, config, ubo_internal_uuid)
+
+    return browser, ubo_id