Merge pull request #1 from thegreenwebfoundation/ca-add-webserver

Add web server for serving validator over an API

.env.dist

@@ -0,0 +1,5 @@
+# SECURITY WARNING: don't run with the debug turned on in production!
+DEBUG=True
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY=secret

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+name: Run tests
+
+on:
+  push:
+    paths-ignore:
+      - "**.md"
+      - ".gitignore"
+  # make our tests run when we have external PRs
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - ".gitignore"
+
+defaults:
+  run:
+    working-directory: ./
+
+jobs:
+  run_tests:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.11]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install tooling for managing dependencies
+        run: |
+          python -m pip install --upgrade uv wheel
+
+
+      - name: Run tests
+        run: |
+          uv run pytest
+        env:
+          SECRET_KEY: "test"

README.md

@@ -1,31 +1,49 @@
 # Carbon.txt validator
 
-This validator reads carbon.txt files, and validates them against a spec defined on http://carbontxt.org.
-
-
-
+This validator reads carbon.txt files, and validates them against a spec defined
+on http://carbontxt.org.
 
 # Usage
 
 ## With the CLI
 
-Run a validation against a given domain, or file, say if the file is valid TOML, and it confirms to the carbon.txt spec
+Run a validation against a given domain, or file, say if the file is valid TOML,
+and it confirms to the carbon.txt spec.
+
+The following commands assume you are working in a virtual environment:
 
 ```shell
 # parse the carbon.txt file on default paths on some-domain.com
-carbontxt validate domain some-domain.com
+carbon-txt validate domain some-domain.com
 
 # parse a remote file available at https://somedomain.com/path-to-carbon.txt
-carbontxt validate file https://somedomain.com/path-to-carbon.txt
+carbon-txt validate file https://somedomain.com/path-to-carbon.txt
 
 # parse a local file ./path-to-file.com
-carbontxt validate file ./path-to-file.com
+carbon-txt validate file ./path-to-file.com
 
 # pipe the contents of a file into the file validation command as part of a pipeline
 cat ./path-to-file.com | carbontxt validate file
-
 ```
 
+### Using UV
+
+If you are not using a virtual environments, but running `uv`, in a project you
+can run it with `uv run carbon-txt your args`
+
 ## With the HTTP API
 
-(Coming up next)
+You can also validate carbon.txt files sent over an HTTP API.
+
+```shell
+# run the carbon-txt validator as a server using the default django server. Not for production
+carbon-txt serve
+```
+
+For production, [Granian](https://github.com/emmett-framework/granian), a
+performant webserver is bundled. Pass the flag `--server granian` to use it.
+
+```shell
+# run the carbon-txt validator as a server using the production granian server
+carbon-txt serve --server granian
+```

justfile

@@ -0,0 +1,19 @@
+
+
+# load the contents of .env into environment variables
+# override by calling `just --dotenv-filename ENV_FILENAME COMMAND`
+# where ENV_FILENAME is the file containing env vars you want to use instead
+set dotenv-load
+
+default:
+  just --list
+
+test *options:
+  uv run pytest {{ options }}
+
+test-watch *options:
+  uv run pytest-watch -- {{ options }}
+
+
+serve:
+  uv run python ./src/carbon_txt_validator/web/manage.py runserver

pyproject.toml

@@ -12,8 +12,10 @@ readme = "README.md"
 requires-python = ">=3.11"
 classifiers = ["License :: OSI Approved :: Apache Software License"]
 dependencies = [
+    "django-environ>=0.11.2",
     "django-ninja>=1.3.0",
     "dnspython>=2.7.0",
+    "granian>=1.6.2",
     "httpx>=0.27.2",
     "rich>=13.9.2",
     # faster, but not yet supported by python 3.13 yet
@@ -33,6 +35,7 @@ build-backend = "hatchling.build"
 dev-dependencies = [
     "ipdb>=0.13.13",
     "mypy>=1.12.0",
+    "pytest-django>=4.9.0",
     "pytest-watch>=4.2.0",
     "pytest>=8.3.3",
 ]

pytest.ini

@@ -1,3 +1,9 @@
 # pytest.ini
 [pytest]
 pythonpath = src
+addopts = --ds="carbon_txt_validator.web.config.settings.test"
+
+; list the warnings to ignore
+filterwarnings =
+    ignore::django.utils.deprecation.RemovedInDjango60Warning
+    ignore::pydantic.warnings.PydanticDeprecatedSince20

src/carbon_txt_validator/cli.py

@@ -1,12 +1,19 @@
-import typer
+import os
+import sys
+from pathlib import Path
+
 import rich
+import typer
+from django.core.management import execute_from_command_line
 
 from . import finders, parsers_toml
 
 app = typer.Typer()
 validate_app = typer.Typer()
+web_app = typer.Typer()
 app.add_typer(validate_app, name="validate")
 
+
 file_finder = finders.FileFinder()
 parser = parsers_toml.CarbonTxtParser()
 
@@ -44,7 +51,13 @@ def validate_file(
     if file_path == "-":
         content = typer.get_text_stream("stdin").read()
     else:
-        result = file_finder.resolve_uri(file_path)
+        try:
+            result = file_finder.resolve_uri(file_path)
+        except FileNotFoundError as e:
+            full_file_path = Path(file_path).absolute()
+            rich.print(f"No valid carbon.txt file found at {full_file_path}.")
+            return 1
+
         if result:
             rich.print(f"Carbon.txt file found at {result}.")
             content = parser.get_carbon_txt_file(result)
@@ -60,5 +73,61 @@ def validate_file(
     return parsed_result
 
 
+def configure_django(debug=True):
+    """Configure Django settings programmatically"""
+
+    # Get the path to the web directory containing manage.py
+    web_dir = Path(__file__).parent / "web"
+
+    if not web_dir.exists():
+        rich.print(
+            "[red]Error: Could not find web directory containing Django app[/red]"
+        )
+        sys.exit(1)
+
+    # Add the web directory to the Python path
+    sys.path.insert(0, str(web_dir))
+
+    # Set the Django settings module
+    os.environ.setdefault(
+        "DJANGO_SETTINGS_MODULE", "carbon_txt_validator.web.config.settings.production"
+    )
+
+    # Import Django settings after path setup
+    import django
+
+    django.setup()
+
+
+@app.command()
+def serve(
+    host: str = typer.Option("127.0.0.1", help="Host to bind to"),
+    port: int = typer.Option(8000, help="Port to listen on"),
+    debug: bool = typer.Option(True, help="Run in debug mode"),
+    server: str = typer.Option(
+        "django",
+        help="Run in as django server or in production with the granian server",
+    ),
+):
+    """Run the carbon.txt validator web server"""
+
+    # Configure Django first
+    configure_django(debug=debug)
+
+    web_dir = Path(__file__).parent / "web"
+    os.chdir(web_dir)
+
+    if server == "granian":
+        rich.print("Running in production mode with Granian server")
+        rich.print("\n ----------------\n")
+        # Run Granian instead of Django development server
+        os.system(
+            f"granian --interface wsgi carbon_txt_validator.web.config.wsgi:application"
+        )
+    else:
+        # Run Django development server
+        execute_from_command_line(["manage.py", "runserver", f"{host}:{port}"])
+
+
 if __name__ == "__main__":
     app()

src/carbon_txt_validator/finders.py

@@ -31,13 +31,38 @@ def _lookup_dns(self, domain):
         """Try a DNS TXT record lookup for the given domain"""
 
         # look for a TXT record on the domain first
-        # if have it, return that
+        # if there is a valid TXT record it, return that
         try:
-            dns_record = dns.resolver.resolve(domain, "TXT")
-            if dns_record:
-                url = dns_record[0].strings[0].decode()
-                rich.print(url)
-                return url
+            answers = dns.resolver.resolve(domain, "TXT")
+
+            for answer in answers:
+
+                txt_record = answer.to_text().strip('"')
+                if txt_record.startswith("carbon-txt"):
+                    # pull out our url to check
+                    _, txt_record_body = txt_record.split("=")
+
+                    domain_hash_check = txt_record_body.split(" ")
+
+                    # check for delegation with no domain hash
+                    if len(domain_hash_check) == 1:
+                        override_url = domain_hash_check[0]
+                        logger.info(
+                            f"Found an override_url to use from the DNS lookup: {override_url}"
+                        )
+                        return override_url
+
+                    # check for delegation WITH a domain hash
+                    if len(domain_hash_check) == 2:
+                        override_url = domain_hash_check[0]
+
+                        # TODO add verification of domain hash
+                        domain_hash = domain_hash_check[1]
+
+                        logger.info(
+                            f"Found an override_url to use from the DNS lookup: {override_url}"
+                        )
+                        return override_url
 
         except dns.resolver.NoAnswer:
             logger.info("No result from TXT lookup")
@@ -80,6 +105,8 @@ def resolve_uri(self, uri: str) -> str:
         # if there is no http or https scheme, we assume a local file
         if not parsed_uri:
             path_to_file = Path(uri)
+            if not path_to_file.exists():
+                raise FileNotFoundError(f"File not found at {path_to_file.absolute()}")
             return str(path_to_file.resolve())
 
         response = httpx.head(parsed_uri.geturl())

src/carbon_txt_validator/parsers_toml.py

@@ -36,7 +36,12 @@ def validate_as_carbon_txt(self, parsed) -> schemas.CarbonTxtFile:
         Accept a parsed TOML object and return a CarbonTxtFile, validating that
         necessary keys are present and values are of the correct type.
         """
-
-        carb_txt_obj = schemas.CarbonTxtFile(**parsed)
-
-        return carb_txt_obj
+        from pydantic import ValidationError
+        import rich
+
+        try:
+            carb_txt_obj = schemas.CarbonTxtFile(**parsed)
+            return carb_txt_obj
+        except ValidationError as e:
+            rich.print(e)
+            return e

src/carbon_txt_validator/web/config/api.py

@@ -0,0 +1,57 @@
+from ninja import NinjaAPI, Schema
+from django.http import HttpRequest, HttpResponse
+
+from carbon_txt_validator.parsers_toml import CarbonTxtParser
+from carbon_txt_validator.schemas import CarbonTxtFile
+
+# Initialize the NinjaAPI with OpenAPI documentation details
+ninja_api = NinjaAPI(
+    openapi_extra={
+        "info": {
+            "termsOfService": "https://thegreenwebfoundation.org/terms/",
+        }
+    },
+    title="Carbon.txt Validator API",
+    description="API for validating carbon.txt files.",
+)
+
+
+class CarbonTextSubmission(Schema):
+    """
+    Schema for the submission of carbon.txt file contents. We use
+    """
+
+    text_contents: str
+
+
+@ninja_api.post(
+    "/validate/", description="Accept contents of a carbon.txt file and validate it."
+)
+def validate_contents(
+    request: HttpRequest, CarbonTextSubmission: CarbonTextSubmission
+) -> HttpResponse:
+    """
+    Endpoint to validate the contents of a carbon.txt file.
+
+    Args:
+        request: The request object.
+        CarbonTextSubmission: The schema containing the text contents of the carbon.txt file.
+
+    Returns:
+        dict: A dictionary containing the success status and either the validated data or errors.
+    """
+    # Initialize the parser
+    parser = CarbonTxtParser()
+
+    # Parse the TOML contents from the submission
+    parsed = parser.parse_toml(CarbonTextSubmission.text_contents)
+
+    # Validate the parsed contents as a carbon.txt file
+    result = parser.validate_as_carbon_txt(parsed)
+
+    # Check if the result is a valid CarbonTxtFile instance
+    if isinstance(result, CarbonTxtFile):
+        return {"success": True, "data": result}
+
+    # Return errors if validation failed
+    return {"success": False, "errors": result.errors()}