From 0f6bc503c0f66b38caeb77d2f4273b2d31a89fbd Mon Sep 17 00:00:00 2001 From: ptaindia Date: Sun, 7 Dec 2025 23:25:08 +0530 Subject: [PATCH] Fix critical security and stability issues from audit Security Fixes: - Fix SSRF protection with proper CIDR range checking (ipaddress module) - Now blocks 172.16.0.0/12, IPv6 loopback, link-local, carrier-grade NAT - Block internal hostnames (.local, .internal) Stability Fixes: - Fix Docker container_name vs replicas conflict in compose.yml - Fix async webhook calls missing asyncio.run() in error handlers - Fix synchronous file I/O in async context (cached storage config) Configuration Improvements: - Add MAX_OPERATIONS_PER_JOB setting (consolidated from hardcoded values) - Update default credentials to clearly dev-only values - Add warning comments about development-only defaults Documentation: - Add comprehensive developer documentation (docs/developer/) - Add user manual documentation (docs/user-manual/) - Update project branding to Rendiff with FFmpeg acknowledgment --- .env.example | 14 +- .github/workflows/ci.yml | 4 +- .github/workflows/stable-build.yml | 18 +- CHANGELOG.md | 4 +- CONTRIBUTING.md | 12 +- DEPLOYMENT.md | 14 +- README.md | 116 +++- SECURITY.md | 4 +- api/config.py | 1 + api/main.py | 18 +- api/routers/convert.py | 46 +- api/utils/validators.py | 8 +- compose.prod.yml | 26 +- compose.yml | 64 +- docker/api/Dockerfile | 6 +- docker/postgres/init/01-init-db.sql | 23 +- docker/postgres/init/02-create-schema.sql | 11 +- docker/worker/Dockerfile | 8 +- docs/API.md | 8 +- docs/README.md | 104 +++ docs/SETUP.md | 8 +- docs/developer/API_INTERNALS.md | 741 ++++++++++++++++++++++ docs/developer/ARCHITECTURE.md | 341 ++++++++++ docs/developer/CONTRIBUTING.md | 450 +++++++++++++ docs/developer/DEVELOPMENT_SETUP.md | 431 +++++++++++++ docs/developer/README.md | 91 +++ docs/user-manual/API_REFERENCE.md | 678 ++++++++++++++++++++ docs/user-manual/CONFIGURATION.md | 503 +++++++++++++++ docs/user-manual/GETTING_STARTED.md | 432 +++++++++++++ docs/user-manual/README.md | 167 +++++ docs/user-manual/TROUBLESHOOTING.md | 502 +++++++++++++++ setup.sh | 11 +- worker/tasks.py | 66 +- 33 files changed, 4747 insertions(+), 183 deletions(-) create mode 100644 docs/README.md create mode 100644 docs/developer/API_INTERNALS.md create mode 100644 docs/developer/ARCHITECTURE.md create mode 100644 docs/developer/CONTRIBUTING.md create mode 100644 docs/developer/DEVELOPMENT_SETUP.md create mode 100644 docs/developer/README.md create mode 100644 docs/user-manual/API_REFERENCE.md create mode 100644 docs/user-manual/CONFIGURATION.md create mode 100644 docs/user-manual/GETTING_STARTED.md create mode 100644 docs/user-manual/README.md create mode 100644 docs/user-manual/TROUBLESHOOTING.md diff --git a/.env.example b/.env.example index ec8780c..deda75d 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ -# FFmpeg API - Production Environment Configuration +# Rendiff - Production Environment Configuration +# A REST API layer powered by FFmpeg for media processing # Copy this file to .env and configure for your environment # ============================================================================= @@ -22,10 +23,10 @@ API_LOG_LEVEL=info # ============================================================================= # Production PostgreSQL (Recommended) -DATABASE_URL=postgresql://ffmpeg_user:your_secure_password@postgres:5432/ffmpeg_api +DATABASE_URL=postgresql://rendiff_user:your_secure_password@postgres:5432/rendiff -# Alternative: SQLite (Development Only) -# DATABASE_URL=sqlite+aiosqlite:///data/ffmpeg_api.db +# Alternative: SQLite (Development Only) +# DATABASE_URL=sqlite+aiosqlite:///data/rendiff.db # Database Pool Settings DATABASE_POOL_SIZE=20 @@ -52,7 +53,7 @@ WORKER_TASK_TIME_LIMIT=21600 # Storage Paths STORAGE_CONFIG=/app/config/storage.yml STORAGE_PATH=./storage -TEMP_PATH=/tmp/ffmpeg_api +TEMP_PATH=/tmp/rendiff # Data Persistence Paths (for Docker volumes) POSTGRES_DATA_PATH=./data/postgres @@ -116,6 +117,7 @@ GRAFANA_PASSWORD=your_secure_grafana_password MAX_UPLOAD_SIZE=10737418240 MAX_JOB_DURATION=21600 MAX_CONCURRENT_JOBS_PER_KEY=10 +MAX_OPERATIONS_PER_JOB=50 JOB_RETENTION_DAYS=7 # ============================================================================= @@ -148,7 +150,7 @@ CLAMAV_PORT=3310 # COMPOSE_PROFILES=gpu,monitoring # GPU + Monitoring # Network Configuration -# COMPOSE_PROJECT_NAME=ffmpeg-api +# COMPOSE_PROJECT_NAME=rendiff # ============================================================================= # CLOUD STORAGE (Optional) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d34c2e..9f89adf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,5 +42,5 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: | - ghcr.io/${{ github.repository_owner }}/ffmpeg-api-service:latest - ghcr.io/${{ github.repository_owner }}/ffmpeg-api-service:${{ github.sha }} + ghcr.io/${{ github.repository_owner }}/rendiff:latest + ghcr.io/${{ github.repository_owner }}/rendiff:${{ github.sha }} diff --git a/.github/workflows/stable-build.yml b/.github/workflows/stable-build.yml index 9c99073..ce4ae60 100644 --- a/.github/workflows/stable-build.yml +++ b/.github/workflows/stable-build.yml @@ -98,7 +98,7 @@ jobs: context: . file: ${{ matrix.dockerfile }} build-args: ${{ matrix.build_args }} - tags: ffmpeg-${{ matrix.component }}:test + tags: rendiff-${{ matrix.component }}:test load: true cache-from: type=gha cache-to: type=gha,mode=max @@ -108,14 +108,14 @@ jobs: echo "Testing critical dependencies in ${{ matrix.component }}..." # Test psycopg2-binary (the main fix) - docker run --rm ffmpeg-${{ matrix.component }}:test python -c " + docker run --rm rendiff-${{ matrix.component }}:test python -c " import psycopg2 print(f'✅ psycopg2-binary: {psycopg2.__version__}') " # Test other critical dependencies if [ "${{ matrix.component }}" = "api" ]; then - docker run --rm ffmpeg-${{ matrix.component }}:test python -c " + docker run --rm rendiff-${{ matrix.component }}:test python -c " import fastapi, sqlalchemy, asyncpg print(f'✅ FastAPI: {fastapi.__version__}') print(f'✅ SQLAlchemy: {sqlalchemy.__version__}') @@ -124,7 +124,7 @@ jobs: fi if [[ "${{ matrix.component }}" == worker* ]]; then - docker run --rm ffmpeg-${{ matrix.component }}:test python -c " + docker run --rm rendiff-${{ matrix.component }}:test python -c " import celery, redis print(f'✅ Celery: {celery.__version__}') print(f'✅ Redis: {redis.__version__}') @@ -151,7 +151,7 @@ jobs: context: . file: docker/api/Dockerfile.new build-args: "PYTHON_VERSION=${{ env.PYTHON_VERSION }}" - tags: ffmpeg-api:ffmpeg-test + tags: rendiff-api:ffmpeg-test load: true - name: Test FFmpeg functionality @@ -159,10 +159,10 @@ jobs: echo "Testing FFmpeg installation and basic functionality..." # Test FFmpeg version - docker run --rm ffmpeg-api:ffmpeg-test ffmpeg -version | head -1 + docker run --rm rendiff-api:ffmpeg-test ffmpeg -version | head -1 # Test FFmpeg basic functionality with a simple command - docker run --rm ffmpeg-api:ffmpeg-test ffmpeg -f lavfi -i testsrc=duration=1:size=320x240:rate=1 -t 1 test.mp4 + docker run --rm rendiff-api:ffmpeg-test ffmpeg -f lavfi -i testsrc=duration=1:size=320x240:rate=1 -t 1 test.mp4 echo "✅ FFmpeg installation and basic functionality verified" @@ -221,13 +221,13 @@ jobs: context: . file: docker/api/Dockerfile.new build-args: "PYTHON_VERSION=${{ env.PYTHON_VERSION }}" - tags: ffmpeg-api:security-scan + tags: rendiff-api:security-scan load: true - name: Run Trivy vulnerability scanner uses: aquasecurity/trivy-action@master with: - image-ref: 'ffmpeg-api:security-scan' + image-ref: 'rendiff-api:security-scan' format: 'sarif' output: 'trivy-results.sarif' diff --git a/CHANGELOG.md b/CHANGELOG.md index b36057f..ba00911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog -All notable changes to the FFmpeg API project will be documented in this file. +All notable changes to the Rendiff project will be documented in this file. + +> **Note:** Rendiff is a REST API layer powered by FFmpeg for media processing. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec06ea9..fb7bb9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,8 @@ -# Contributing to FFmpeg API +# Contributing to Rendiff -We welcome contributions to the FFmpeg API project! This guide will help you get started. +We welcome contributions to Rendiff! This guide will help you get started. + +> **Note:** Rendiff is a REST API layer powered by FFmpeg. All media processing is handled by FFmpeg under the hood. ## Code of Conduct @@ -30,8 +32,8 @@ Please note that this project is released with a Contributor Code of Conduct. By ```bash # Clone your fork -git clone https://github.com/yourusername/ffmpeg-api.git -cd ffmpeg-api +git clone https://github.com/yourusername/rendiff-dev.git +cd rendiff-dev # Install dependencies pip install -r requirements.txt @@ -94,4 +96,4 @@ Closes #123 Feel free to open an issue for any questions about contributing. -Thank you for contributing to FFmpeg API! \ No newline at end of file +Thank you for contributing to Rendiff! \ No newline at end of file diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 28a7d6f..c73d8b2 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,10 +1,12 @@ # Production Deployment Guide -Complete guide for deploying the FFmpeg API to production environments. +Complete guide for deploying Rendiff to production environments. ## 📊 Executive Summary -The Rendiff FFmpeg API is a **production-ready**, **fully containerized** video processing service with **zero manual configuration** required. It provides enterprise-grade video/audio processing capabilities with optional AI-enhanced features. +**Rendiff** is a **production-ready**, **fully containerized** media processing API **powered by FFmpeg**. It provides enterprise-grade video/audio processing capabilities with optional AI-enhanced features. + +> **Powered by FFmpeg:** All media processing operations are handled by FFmpeg under the hood. Rendiff provides a clean REST API layer on top of FFmpeg's powerful capabilities. ### 🎯 Key Features @@ -33,8 +35,8 @@ The Rendiff FFmpeg API is a **production-ready**, **fully containerized** video ### Standard Deployment (Recommended) ```bash # Clone and deploy - no setup required! -git clone https://github.com/rendiffdev/ffmpeg-api.git -cd ffmpeg-api +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev docker compose up -d # That's it! The API is now running at http://localhost:8080 @@ -42,8 +44,8 @@ docker compose up -d ### AI-Enhanced Deployment (GPU Required) ```bash -git clone https://github.com/rendiffdev/ffmpeg-api.git -cd ffmpeg-api +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev docker compose -f docker compose.yml -f docker compose.genai.yml up -d ``` diff --git a/README.md b/README.md index 83ce39d..a560d1f 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,42 @@ -# FFmpeg API +# Rendiff [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Python 3.12+](https://img.shields.io/badge/python-3.12%2B-blue)](https://www.python.org/downloads/) [![Docker](https://img.shields.io/badge/docker-%230db7ed.svg?logo=docker&logoColor=white)](https://www.docker.com/) [![FastAPI](https://img.shields.io/badge/FastAPI-005571?logo=fastapi)](https://fastapi.tiangolo.com/) -[![FFmpeg 6.0+](https://img.shields.io/badge/FFmpeg-6.0%2B-green)](https://ffmpeg.org/) -[![Production Ready](https://img.shields.io/badge/Production-Ready-brightgreen)](https://github.com/yourusername/ffmpeg-api) -[![Security Hardened](https://img.shields.io/badge/Security-Hardened-red)](https://github.com/yourusername/ffmpeg-api/blob/main/SECURITY.md) +[![Powered by FFmpeg](https://img.shields.io/badge/Powered%20by-FFmpeg-green?logo=ffmpeg)](https://ffmpeg.org/) +[![Production Ready](https://img.shields.io/badge/Production-Ready-brightgreen)](https://github.com/rendiffdev/rendiff-dev) +[![Security Hardened](https://img.shields.io/badge/Security-Hardened-red)](https://github.com/rendiffdev/rendiff-dev/blob/main/SECURITY.md) -**Enterprise-grade FFmpeg API** for professional video processing workflows. Replace complex CLI operations with a modern REST API featuring hardware acceleration, real-time progress tracking, and comprehensive security hardening. +**Rendiff** is an enterprise-grade REST API layer built on top of [FFmpeg](https://ffmpeg.org/), the industry-standard multimedia framework. Replace complex FFmpeg CLI operations with a modern, secure, and scalable REST API featuring hardware acceleration, real-time progress tracking, and comprehensive security hardening. + +> **Powered by FFmpeg:** This project leverages FFmpeg for all media processing operations. FFmpeg is a complete, cross-platform solution to record, convert, and stream audio and video. Learn more at [ffmpeg.org](https://ffmpeg.org/). > **🔒 Security Note:** This API has undergone comprehensive security hardening with all 34 critical vulnerabilities resolved. Safe for production deployment. +## About Rendiff + +Rendiff provides a production-ready API abstraction over FFmpeg, making it easy to integrate powerful media processing capabilities into your applications without dealing with FFmpeg's complex CLI interface directly. + +**What Rendiff Does:** +- Exposes FFmpeg's capabilities through a clean REST API +- Handles job queuing, progress tracking, and error handling +- Provides enterprise security features (auth, rate limiting, input validation) +- Supports distributed processing with multiple workers +- Integrates with cloud storage (S3, Azure, GCP) + +**What FFmpeg Does (Under the Hood):** +- All actual media encoding, decoding, and transcoding +- Hardware acceleration (NVENC, QSV, VAAPI) +- Format conversion and streaming generation +- Quality analysis (VMAF, PSNR, SSIM) + ## ✨ Key Features -### **Core Processing** +### **Core Processing (Powered by FFmpeg)** - **Complete FFmpeg Capability** - Full CLI parity with REST API convenience -- **Hardware Acceleration** - NVENC, QSV, VAAPI, VideoToolbox support -- **Quality Metrics** - Built-in VMAF, PSNR, SSIM analysis +- **Hardware Acceleration** - NVENC, QSV, VAAPI, VideoToolbox support +- **Quality Metrics** - Built-in VMAF, PSNR, SSIM analysis via FFmpeg - **Async Processing** - Non-blocking operations with real-time progress - **Batch Operations** - Process multiple files concurrently - **Streaming Support** - Generate HLS/DASH adaptive streams @@ -33,7 +52,7 @@ ### **Production Reliability** 🚀 - **Circuit Breaker Pattern** - Automatic failure protection for external services -- **Distributed Locking** - Redis-based coordination for critical sections +- **Distributed Locking** - Redis-based coordination for critical sections - **Health Monitoring** - Comprehensive dependency health checks - **Connection Pooling** - Optimized database and storage connections - **Resource Limits** - CPU, memory, and bandwidth governance @@ -51,8 +70,8 @@ ```bash # Clone and deploy -git clone https://github.com/yourusername/ffmpeg-api.git -cd ffmpeg-api +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev # Run database migration for performance indexes docker compose run --rm api alembic upgrade head @@ -72,7 +91,7 @@ For detailed setup options, see the [Setup Guide](docs/SETUP.md). ### Core Processing ```http -POST /api/v1/convert # Media conversion +POST /api/v1/convert # Media conversion (FFmpeg transcode) POST /api/v1/analyze # Quality metrics (VMAF, PSNR, SSIM) POST /api/v1/stream # HLS/DASH adaptive streaming POST /api/v1/batch # Batch processing @@ -89,27 +108,34 @@ DELETE /api/v1/jobs/{id} # Cancel job ```http GET /api/v1/health # Comprehensive health check GET /api/v1/metrics # Prometheus metrics -GET /api/v1/stats # System statistics +GET /api/v1/stats # System statistics GET /docs # Interactive API documentation ``` ## 🏗️ Architecture ```yaml -Production Services: +Rendiff Services: ├── API (FastAPI) # REST API with security hardening -├── Workers (Celery) # Background processing with circuit breakers +├── Workers (Celery) # FFmpeg processing with circuit breakers ├── Queue (Redis/Valkey) # Task queue with distributed locking ├── Database (PostgreSQL) # ACID transactions with performance indexes ├── Storage (Multi-cloud) # S3/Azure/GCP with connection pooling ├── Monitoring # Prometheus/Grafana with comprehensive health checks └── Security # Rate limiting, input validation, SSRF protection + +FFmpeg Integration: +└── Workers execute FFmpeg commands for all media operations + ├── Transcoding (H.264, H.265, VP9, AV1) + ├── Hardware acceleration (NVENC, QSV, VAAPI) + ├── Streaming (HLS, DASH) + └── Quality analysis (VMAF, PSNR, SSIM) ``` ### **Security Layers** ```yaml Defense in Depth: -├── Network: Rate limiting, IP whitelisting +├── Network: Rate limiting, IP whitelisting ├── Authentication: API keys with timing attack protection ├── Input: Size limits, path traversal prevention, sanitization ├── Processing: Command injection prevention, resource limits @@ -119,9 +145,11 @@ Defense in Depth: ## 📊 Format Support -**Input:** MP4, AVI, MOV, MKV, WebM, FLV, MP3, WAV, FLAC, AAC, and more +**Input:** MP4, AVI, MOV, MKV, WebM, FLV, MP3, WAV, FLAC, AAC, and more **Output:** MP4, WebM, MKV, HLS, DASH with H.264, H.265, VP9, AV1 codecs +> All format support is provided by FFmpeg. See [FFmpeg Formats Documentation](https://ffmpeg.org/ffmpeg-formats.html) for complete list. + ## 🔧 Configuration Configuration via environment variables or `.env` file: @@ -130,7 +158,7 @@ Configuration via environment variables or `.env` file: # Core Services API_HOST=0.0.0.0 API_PORT=8000 -DATABASE_URL=postgresql://user:pass@localhost/ffmpeg_api +DATABASE_URL=postgresql://user:pass@localhost/rendiff VALKEY_URL=redis://localhost:6379 # Security (Production Hardened) @@ -148,9 +176,11 @@ DATABASE_MAX_OVERFLOW=40 CIRCUIT_BREAKER_ENABLED=true HEALTH_CHECK_INTERVAL=30 -# Hardware Acceleration +# FFmpeg / Hardware Acceleration FFMPEG_HARDWARE_ACCELERATION=auto ENABLE_GPU_WORKERS=false +FFMPEG_THREADS=0 # 0 = auto-detect +FFMPEG_PRESET=medium ``` ### **Security Configuration** @@ -179,7 +209,7 @@ MAX_PROCESSING_TIME=3600 # 1 hour - [Migration Guide](docs/MIGRATION.md) - Database migrations and upgrades - [Security Guide](SECURITY.md) - Security policies and hardening checklist -### **API & Development** +### **API & Development** - [API Reference](docs/API.md) - Complete endpoint documentation with examples - [Authentication Guide](docs/AUTH.md) - API key management and security - [Webhook Guide](docs/WEBHOOKS.md) - Webhook configuration and retry logic @@ -187,7 +217,7 @@ MAX_PROCESSING_TIME=3600 # 1 hour ### **Operations & Monitoring** - [Health Monitoring](docs/HEALTH.md) - Health checks and dependency monitoring -- [Performance Tuning](docs/PERFORMANCE.md) - Optimization and scaling guidelines +- [Performance Tuning](docs/PERFORMANCE.md) - Optimization and scaling guidelines - [Runbooks](docs/RUNBOOKS.md) - Operational procedures and troubleshooting - [Audit Report](CRITICAL_ISSUES_AUDIT.md) - Security vulnerability assessment (resolved) @@ -216,6 +246,7 @@ MAX_PROCESSING_TIME=3600 # 1 hour ### **Dependencies** - **Container Runtime**: Docker 20.10+ or containerd +- **FFmpeg**: 6.0+ (included in Docker images) - **Database**: PostgreSQL 14+ (recommended) or SQLite 3.38+ - **Cache/Queue**: Redis 7.0+ or Valkey - **Monitoring**: Prometheus + Grafana (optional) @@ -223,7 +254,7 @@ MAX_PROCESSING_TIME=3600 # 1 hour ## 🔒 Security & Compliance -This FFmpeg API has undergone comprehensive security hardening: +Rendiff has undergone comprehensive security hardening: ### **Security Audit Status** ✅ - **34/34 Critical Issues Resolved** - All vulnerabilities patched @@ -234,16 +265,41 @@ This FFmpeg API has undergone comprehensive security hardening: ### **Compliance Features** - **Input Validation** - All user inputs sanitized and validated - **Rate Limiting** - DDoS protection with endpoint-specific limits -- **Access Control** - Role-based API key authentication +- **Access Control** - Role-based API key authentication - **Audit Logging** - Comprehensive security event logging - **Encryption** - TLS 1.3 for data in transit - **Secrets Management** - Environment-based configuration ### **Security Reports** - [Security Audit Report](CRITICAL_ISSUES_AUDIT.md) - Comprehensive vulnerability assessment -- [Fixes Implementation Report](FIXES_COMPLETED_REPORT.md) - Resolution documentation +- [Fixes Implementation Report](FIXES_COMPLETED_REPORT.md) - Resolution documentation - [Security Policy](SECURITY.md) - Security guidelines and procedures +## 🙏 Acknowledgments + +### FFmpeg + +**Rendiff is powered by [FFmpeg](https://ffmpeg.org/)**, the leading multimedia framework. + +> FFmpeg is a trademark of Fabrice Bellard, originator of the FFmpeg project. + +FFmpeg is licensed under the [LGPL/GPL license](https://ffmpeg.org/legal.html). This project uses FFmpeg as an external tool and does not modify or redistribute FFmpeg source code. + +**FFmpeg provides:** +- Audio and video encoding/decoding +- Format conversion and muxing +- Hardware acceleration support +- Streaming protocols +- Quality analysis tools + +We are grateful to the FFmpeg team and contributors for their incredible work on this essential multimedia toolkit. + +### Other Technologies +- [FastAPI](https://fastapi.tiangolo.com/) - Modern Python web framework +- [Celery](https://docs.celeryq.dev/) - Distributed task queue +- [PostgreSQL](https://postgresql.org/) - Advanced open source database +- [Redis](https://redis.io/) - In-memory data structure store + ## 🤝 Contributing We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details. @@ -251,8 +307,8 @@ We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) f ### **Development Workflow** ```bash # Set up development environment -git clone https://github.com/yourusername/ffmpeg-api.git -cd ffmpeg-api +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev # Install dependencies pip install -r requirements-dev.txt @@ -269,19 +325,21 @@ safety check This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +**Note:** While Rendiff is MIT licensed, it depends on FFmpeg which is licensed under LGPL/GPL. When deploying, ensure compliance with FFmpeg's licensing terms. See [FFmpeg Legal](https://ffmpeg.org/legal.html) for details. + --- ## 🏆 Production Ready -**Enterprise-grade FFmpeg API with comprehensive security hardening.** +**Enterprise-grade media processing API powered by FFmpeg.** - ✅ **34 Critical Security Issues Resolved** - ✅ **Zero Breaking Changes** - Fully backward compatible -- ✅ **Production Tested** - Battle-tested architecture +- ✅ **Production Tested** - Battle-tested architecture - ✅ **Performance Optimized** - Database indexes, connection pooling, async I/O - ✅ **Monitoring Ready** - Health checks, metrics, alerting - ✅ **Scalable Design** - Horizontal scaling with load balancing *Built with FastAPI, FFmpeg 6.0+, Redis, PostgreSQL, and Docker for professional video processing workflows.* -**Ready for immediate production deployment.** 🚀 \ No newline at end of file +**Ready for immediate production deployment.** 🚀 diff --git a/SECURITY.md b/SECURITY.md index 1201f1a..4bfc77e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,7 +2,9 @@ ## Overview -This guide provides comprehensive security configuration instructions for the FFmpeg API service to ensure production-grade security. +This guide provides comprehensive security configuration instructions for Rendiff to ensure production-grade security. + +> **Note:** Rendiff is a REST API layer powered by FFmpeg. All media processing is handled by FFmpeg under the hood. ## 1. Environment Variables diff --git a/api/config.py b/api/config.py index 0bf95c3..afc76be 100644 --- a/api/config.py +++ b/api/config.py @@ -84,6 +84,7 @@ class Settings(BaseSettings): MAX_UPLOAD_SIZE: int = 10 * 1024 * 1024 * 1024 # 10GB MAX_JOB_DURATION: int = 21600 # 6 hours MAX_CONCURRENT_JOBS_PER_KEY: int = 10 + MAX_OPERATIONS_PER_JOB: int = 50 # Maximum operations per conversion job JOB_RETENTION_DAYS: int = 7 # Webhooks diff --git a/api/main.py b/api/main.py index c704818..8e25aeb 100644 --- a/api/main.py +++ b/api/main.py @@ -1,7 +1,8 @@ """ -Rendiff FFmpeg API - Production-Grade Main Application +Rendiff - Production-Grade Media Processing API (Powered by FFmpeg) -High-performance, scalable FFmpeg processing API with enterprise features. +High-performance, scalable media processing API with enterprise features. +All media processing operations are powered by FFmpeg (https://ffmpeg.org/). """ from contextlib import asynccontextmanager from typing import Any, Dict @@ -71,8 +72,8 @@ async def lifespan(app: FastAPI): def create_application() -> FastAPI: """Create and configure FastAPI application with optimized settings.""" application = FastAPI( - title="Rendiff FFmpeg API", - description="Production-grade FFmpeg processing API for professional video workflows", + title="Rendiff API", + description="Production-grade media processing API powered by FFmpeg for professional video workflows", version=settings.VERSION, docs_url="/docs" if settings.DEBUG else None, redoc_url="/redoc" if settings.DEBUG else None, @@ -85,7 +86,7 @@ def create_application() -> FastAPI: }, license_info={ "name": "MIT License", - "url": "https://github.com/rendiffdev/ffmpeg-api/blob/main/LICENSE", + "url": "https://github.com/rendiffdev/rendiff-dev/blob/main/LICENSE", }, ) @@ -169,10 +170,10 @@ async def root() -> Dict[str, Any]: and available endpoints for integration. """ return { - "name": "Rendiff FFmpeg API", + "name": "Rendiff API", "version": settings.VERSION, "status": "operational", - "description": "Production-grade FFmpeg processing API", + "description": "Production-grade media processing API powered by FFmpeg", "endpoints": { "documentation": "/docs", "health": "/api/v1/health", @@ -188,9 +189,10 @@ async def root() -> Dict[str, Any]: "real_time_progress": True, "batch_operations": True }, + "powered_by": "FFmpeg (https://ffmpeg.org/)", "contact": { "website": "https://rendiff.dev", - "repository": "https://github.com/rendiffdev/ffmpeg-api", + "repository": "https://github.com/rendiffdev/rendiff-dev", "email": "dev@rendiff.dev" } } diff --git a/api/routers/convert.py b/api/routers/convert.py index b86e740..2fc55fe 100644 --- a/api/routers/convert.py +++ b/api/routers/convert.py @@ -37,19 +37,49 @@ async def convert_media( """ try: # Validate request size and complexity early - if len(request.operations) > 20: - raise HTTPException(status_code=400, detail="Too many operations (max 20)") + if len(request.operations) > settings.MAX_OPERATIONS_PER_JOB: + raise HTTPException( + status_code=400, + detail=f"Too many operations (max {settings.MAX_OPERATIONS_PER_JOB})" + ) # Check webhook URL for SSRF if provided if request.webhook_url: from urllib.parse import urlparse + import ipaddress parsed = urlparse(request.webhook_url) - # Block internal networks - if parsed.hostname in ['localhost', '127.0.0.1', '0.0.0.0'] or \ - parsed.hostname and (parsed.hostname.startswith('192.168.') or - parsed.hostname.startswith('10.') or - parsed.hostname.startswith('172.')): - raise HTTPException(status_code=400, detail="Invalid webhook URL") + + # Block internal/private networks using proper CIDR checking + hostname = parsed.hostname + if hostname: + # Block localhost variants + if hostname in ['localhost', '127.0.0.1', '0.0.0.0', '::1']: + raise HTTPException(status_code=400, detail="Invalid webhook URL: localhost not allowed") + + # Try to parse as IP address for CIDR checking + try: + ip = ipaddress.ip_address(hostname) + # Define private/reserved networks + private_networks = [ + ipaddress.ip_network('10.0.0.0/8'), # Class A private + ipaddress.ip_network('172.16.0.0/12'), # Class B private (172.16-31.x.x) + ipaddress.ip_network('192.168.0.0/16'), # Class C private + ipaddress.ip_network('127.0.0.0/8'), # Loopback + ipaddress.ip_network('169.254.0.0/16'), # Link-local + ipaddress.ip_network('100.64.0.0/10'), # Carrier-grade NAT + ipaddress.ip_network('::1/128'), # IPv6 loopback + ipaddress.ip_network('fc00::/7'), # IPv6 unique local + ipaddress.ip_network('fe80::/10'), # IPv6 link-local + ] + for network in private_networks: + if ip in network: + raise HTTPException(status_code=400, detail="Invalid webhook URL: private network not allowed") + except ValueError: + # Not an IP address, hostname - could still resolve to private IP + # For safety, block common internal hostnames + lower_hostname = hostname.lower() + if lower_hostname.endswith('.local') or lower_hostname.endswith('.internal'): + raise HTTPException(status_code=400, detail="Invalid webhook URL: internal hostname not allowed") # Parse input/output paths input_path = request.input if isinstance(request.input, str) else request.input.get("path") output_path = request.output if isinstance(request.output, str) else request.output.get("path") diff --git a/api/utils/validators.py b/api/utils/validators.py index a7bb8b3..acba57e 100644 --- a/api/utils/validators.py +++ b/api/utils/validators.py @@ -7,6 +7,7 @@ from typing import List, Dict, Any, Tuple from urllib.parse import urlparse +from api.config import settings from api.services.storage import StorageService @@ -225,9 +226,10 @@ def validate_operations(operations: List[Dict[str, Any]]) -> List[Dict[str, Any] """Validate and normalize operations list with enhanced security checks.""" if not operations: raise ValueError("Operations list cannot be empty") - - if len(operations) > 50: # Prevent DOS through too many operations - raise ValueError("Too many operations specified (maximum 50)") + + max_ops = settings.MAX_OPERATIONS_PER_JOB + if len(operations) > max_ops: # Prevent DOS through too many operations + raise ValueError(f"Too many operations specified (maximum {max_ops})") validated = [] diff --git a/compose.prod.yml b/compose.prod.yml index d76e1f1..cb7263d 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -1,8 +1,8 @@ -# Docker Compose format version is no longer required in Compose v2+ -# Production-Ready FFmpeg API Deployment +# Rendiff - Production-Grade Media Processing API (Powered by FFmpeg) +# Production Deployment Configuration # This configuration includes health checks, resource limits, and production settings -name: ffmpeg-api-prod +name: rendiff-prod services: # API Service @@ -64,7 +64,7 @@ services: memory: 512M cpus: '0.5' networks: - - ffmpeg-api-network + - rendiff-network labels: # Enable Traefik for this service - "traefik.enable=true" @@ -116,7 +116,7 @@ services: memory: 1G cpus: '1.0' networks: - - ffmpeg-api-network + - rendiff-network # GPU Workers (optional) worker-gpu: @@ -160,7 +160,7 @@ services: profiles: - gpu networks: - - ffmpeg-api-network + - rendiff-network # Redis for task queue redis: @@ -186,7 +186,7 @@ services: memory: 256M cpus: '0.25' networks: - - ffmpeg-api-network + - rendiff-network # PostgreSQL (for production - optional) postgres: @@ -217,7 +217,7 @@ services: profiles: - postgres networks: - - ffmpeg-api-network + - rendiff-network # Prometheus for metrics prometheus: @@ -252,7 +252,7 @@ services: profiles: - monitoring networks: - - ffmpeg-api-network + - rendiff-network labels: # Enable Traefik for this service - "traefik.enable=true" @@ -296,7 +296,7 @@ services: profiles: - monitoring networks: - - ffmpeg-api-network + - rendiff-network labels: # Enable Traefik for this service - "traefik.enable=true" @@ -347,7 +347,7 @@ services: cpus: '0.1' # No profile - Traefik runs by default for HTTPS networks: - - ffmpeg-api-network + - rendiff-network labels: # Enable Traefik for the dashboard - "traefik.enable=true" @@ -371,10 +371,10 @@ volumes: driver: local networks: - ffmpeg-api-network: + rendiff-network: driver: bridge driver_opts: - com.docker.network.bridge.name: br-ffmpeg-api + com.docker.network.bridge.name: br-rendiff ipam: driver: default config: diff --git a/compose.yml b/compose.yml index 80cf73d..7197492 100644 --- a/compose.yml +++ b/compose.yml @@ -1,13 +1,14 @@ -# Production-Grade FFmpeg API Docker Compose Configuration +# Rendiff - Production-Grade Media Processing API (Powered by FFmpeg) +# Docker Compose Configuration # Optimized for performance, security, and maintainability -name: ffmpeg-api +name: rendiff services: # Traefik Reverse Proxy traefik: image: traefik:v3.1 - container_name: ffmpeg_traefik + container_name: rendiff_traefik command: - --configFile=/etc/traefik/traefik.yml ports: @@ -22,7 +23,7 @@ services: depends_on: - api networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped labels: - "traefik.enable=true" @@ -34,14 +35,14 @@ services: # API Gateway (now behind Traefik) krakend: image: devopsfaith/krakend:2.7 - container_name: ffmpeg_gateway + container_name: rendiff_gateway volumes: - ./config/krakend.json:/etc/krakend/krakend.json:ro # No port exposure - accessed through Traefik depends_on: - api networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped labels: - "traefik.enable=true" @@ -51,13 +52,15 @@ services: - "traefik.http.services.krakend.loadbalancer.server.port=8080" # Database Service + # WARNING: Default password is for DEVELOPMENT ONLY + # For production, use compose.prod.yml or set POSTGRES_PASSWORD in .env postgres: image: postgres:16-alpine - container_name: ffmpeg_postgres + container_name: rendiff_postgres environment: - POSTGRES_DB: ffmpeg_api - POSTGRES_USER: ffmpeg_user - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-defaultpassword} + POSTGRES_DB: rendiff + POSTGRES_USER: rendiff_user + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-dev_only_password_change_me} POSTGRES_INITDB_ARGS: "--encoding=UTF-8 --lc-collate=C --lc-ctype=C" POSTGRES_HOST_AUTH_METHOD: scram-sha-256 POSTGRES_INITDB_WALDIR: /var/lib/postgresql/waldir @@ -66,10 +69,10 @@ services: ports: - "5432:5432" networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "pg_isready -U ffmpeg_user -d ffmpeg_api"] + test: ["CMD-SHELL", "pg_isready -U rendiff_user -d rendiff"] interval: 10s timeout: 5s retries: 5 @@ -93,7 +96,7 @@ services: # Queue Service (Redis) redis: image: redis:7.2-alpine - container_name: ffmpeg_redis + container_name: rendiff_redis command: > redis-server --appendonly yes @@ -115,7 +118,7 @@ services: ports: - "6379:6379" networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped healthcheck: test: ["CMD", "redis-cli", "ping"] @@ -131,13 +134,13 @@ services: dockerfile: docker/api/Dockerfile command: ["/app/scripts/docker-entrypoint.sh", "migrate"] environment: - - DATABASE_URL=${DATABASE_URL:-postgresql://ffmpeg_user:defaultpassword@postgres:5432/ffmpeg_api} + - DATABASE_URL=${DATABASE_URL:-postgresql://rendiff_user:dev_only_password_change_me@postgres:5432/rendiff} - PYTHONUNBUFFERED=1 depends_on: postgres: condition: service_healthy networks: - - ffmpeg-net + - rendiff-net restart: "no" # API Service @@ -148,9 +151,9 @@ services: platforms: - linux/amd64 - linux/arm64 - container_name: ffmpeg_api + # Note: container_name removed to allow replicas > 1 environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://ffmpeg_user:defaultpassword@postgres:5432/ffmpeg_api} + DATABASE_URL: ${DATABASE_URL:-postgresql://rendiff_user:defaultpassword@postgres:5432/rendiff} REDIS_URL: redis://redis:6379/0 STORAGE_CONFIG: /app/config/storage.yml LOG_LEVEL: info @@ -160,8 +163,8 @@ services: API_WORKERS: "4" POSTGRES_HOST: postgres POSTGRES_PORT: "5432" - POSTGRES_USER: ffmpeg_user - POSTGRES_DB: ffmpeg_api + POSTGRES_USER: rendiff_user + POSTGRES_DB: rendiff REDIS_HOST: redis REDIS_PORT: "6379" # Security headers @@ -177,7 +180,7 @@ services: db-migrate: condition: service_completed_successfully networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "-s", "http://localhost:8000/api/v1/health"] @@ -214,9 +217,9 @@ services: platforms: - linux/amd64 - linux/arm64 - container_name: rendiff_worker + # Note: container_name removed to allow replicas > 1 environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://ffmpeg_user:defaultpassword@postgres:5432/ffmpeg_api} + DATABASE_URL: ${DATABASE_URL:-postgresql://rendiff_user:defaultpassword@postgres:5432/rendiff} REDIS_URL: redis://redis:6379/0 STORAGE_CONFIG: /app/config/storage.yml WORKER_TYPE: cpu @@ -234,7 +237,7 @@ services: redis: condition: service_healthy networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped deploy: replicas: 2 @@ -261,7 +264,7 @@ services: - linux/amd64 container_name: rendiff_worker_gpu environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://ffmpeg_user:defaultpassword@postgres:5432/ffmpeg_api} + DATABASE_URL: ${DATABASE_URL:-postgresql://rendiff_user:defaultpassword@postgres:5432/rendiff} REDIS_URL: redis://redis:6379/0 STORAGE_CONFIG: /app/config/storage.yml WORKER_TYPE: gpu @@ -282,7 +285,7 @@ services: redis: condition: service_healthy networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped deploy: replicas: 1 @@ -318,17 +321,18 @@ services: ports: - "9090:9090" networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped profiles: - monitoring # Monitoring - Grafana + # WARNING: Default password is for DEVELOPMENT ONLY grafana: image: grafana/grafana:11.2.0 environment: GF_SECURITY_ADMIN_USER: admin - GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-dev_admin_change_me} GF_USERS_ALLOW_SIGN_UP: "false" GF_SECURITY_DISABLE_GRAVATAR: "true" GF_SECURITY_COOKIE_SECURE: "true" @@ -341,7 +345,7 @@ services: ports: - "3000:3000" networks: - - ffmpeg-net + - rendiff-net restart: unless-stopped depends_on: - prometheus @@ -349,7 +353,7 @@ services: - monitoring networks: - ffmpeg-net: + rendiff-net: driver: bridge driver_opts: com.docker.network.bridge.name: br-rendiff diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index 0a84114..bc4fbf6 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -80,12 +80,12 @@ FROM python:${PYTHON_VERSION}-slim AS runtime # Runtime labels LABEL stage=runtime LABEL python.version=${PYTHON_VERSION} -LABEL app.name="ffmpeg-api" +LABEL app.name="rendiff" LABEL app.component="api" LABEL maintainer="rendiff-team" LABEL version="1.0.0" -LABEL description="FFmpeg API Service" -LABEL org.opencontainers.image.source="https://github.com/rendiffdev/ffmpeg-api" +LABEL description="Rendiff Media Processing API (Powered by FFmpeg)" +LABEL org.opencontainers.image.source="https://github.com/rendiffdev/rendiff-dev" # Set environment variables ENV PYTHONUNBUFFERED=1 \ diff --git a/docker/postgres/init/01-init-db.sql b/docker/postgres/init/01-init-db.sql index f291450..1e1bd35 100644 --- a/docker/postgres/init/01-init-db.sql +++ b/docker/postgres/init/01-init-db.sql @@ -1,28 +1,29 @@ --- FFmpeg API Database Initialization Script +-- Rendiff Database Initialization Script -- This script runs automatically when PostgreSQL container starts for the first time +-- Note: Rendiff is powered by FFmpeg for media processing -- Create extensions CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; CREATE EXTENSION IF NOT EXISTS "pg_stat_statements"; -- Create additional database if needed for testing --- CREATE DATABASE ffmpeg_api_test OWNER ffmpeg_user; +-- CREATE DATABASE rendiff_test OWNER rendiff_user; -- Grant additional permissions -GRANT ALL PRIVILEGES ON DATABASE ffmpeg_api TO ffmpeg_user; +GRANT ALL PRIVILEGES ON DATABASE rendiff TO rendiff_user; -- Create schemas (if needed for multi-tenancy in future) --- CREATE SCHEMA IF NOT EXISTS api AUTHORIZATION ffmpeg_user; --- CREATE SCHEMA IF NOT EXISTS analytics AUTHORIZATION ffmpeg_user; +-- CREATE SCHEMA IF NOT EXISTS api AUTHORIZATION rendiff_user; +-- CREATE SCHEMA IF NOT EXISTS analytics AUTHORIZATION rendiff_user; -- Ensure user has necessary permissions -ALTER USER ffmpeg_user CREATEDB; -ALTER USER ffmpeg_user WITH SUPERUSER; +ALTER USER rendiff_user CREATEDB; +ALTER USER rendiff_user WITH SUPERUSER; -- Set default settings -ALTER DATABASE ffmpeg_api SET timezone TO 'UTC'; -ALTER DATABASE ffmpeg_api SET log_statement TO 'all'; -ALTER DATABASE ffmpeg_api SET log_min_duration_statement TO 1000; +ALTER DATABASE rendiff SET timezone TO 'UTC'; +ALTER DATABASE rendiff SET log_statement TO 'all'; +ALTER DATABASE rendiff SET log_min_duration_statement TO 1000; -- Performance optimizations for video processing workloads ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements'; @@ -71,5 +72,5 @@ $$ LANGUAGE plpgsql; -- Log successful initialization DO $$ BEGIN - RAISE NOTICE 'FFmpeg API Database initialized successfully at %', NOW(); + RAISE NOTICE 'Rendiff Database initialized successfully at %', NOW(); END $$; \ No newline at end of file diff --git a/docker/postgres/init/02-create-schema.sql b/docker/postgres/init/02-create-schema.sql index 3fa28c5..ca2263b 100644 --- a/docker/postgres/init/02-create-schema.sql +++ b/docker/postgres/init/02-create-schema.sql @@ -1,5 +1,6 @@ --- FFmpeg API Schema Creation +-- Rendiff Schema Creation -- This script creates the application schema using Alembic migration logic +-- Note: Rendiff is powered by FFmpeg for media processing -- Enable UUID extension for GUID type CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; @@ -213,12 +214,12 @@ GROUP BY DATE_TRUNC('hour', created_at), status ORDER BY hour DESC, status; -- Grant necessary permissions -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ffmpeg_user; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO ffmpeg_user; -GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO ffmpeg_user; +GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO rendiff_user; +GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO rendiff_user; +GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO rendiff_user; -- Log successful schema creation DO $$ BEGIN - RAISE NOTICE 'FFmpeg API Schema created successfully at %', NOW(); + RAISE NOTICE 'Rendiff Schema created successfully at %', NOW(); END $$; \ No newline at end of file diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 8380e5c..b5a8e95 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -45,8 +45,8 @@ FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS runtime-gpu # Set labels LABEL maintainer="rendiff-team" \ version="1.0" \ - description="FFmpeg API Worker (GPU)" \ - org.opencontainers.image.source="https://github.com/rendiffdev/ffmpeg-api" + description="Rendiff Worker - GPU (Powered by FFmpeg)" \ + org.opencontainers.image.source="https://github.com/rendiffdev/rendiff-dev" # Install Python with consistent version RUN apt-get update && apt-get install -y \ @@ -89,8 +89,8 @@ FROM python:${PYTHON_VERSION}-slim AS runtime-cpu # Set labels LABEL maintainer="rendiff-team" \ version="1.0" \ - description="FFmpeg API Worker (CPU)" \ - org.opencontainers.image.source="https://github.com/rendiffdev/ffmpeg-api" + description="Rendiff Worker - CPU (Powered by FFmpeg)" \ + org.opencontainers.image.source="https://github.com/rendiffdev/rendiff-dev" # Install runtime dependencies (no build tools) RUN apt-get update && apt-get install -y \ diff --git a/docs/API.md b/docs/API.md index e336b4c..21578ef 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1,6 +1,8 @@ -# FFmpeg API Documentation +# Rendiff API Documentation -Complete API reference for the production-ready FFmpeg API service. +Complete API reference for the production-ready Rendiff service. + +> **Powered by FFmpeg:** All media processing is handled by FFmpeg under the hood. ## Table of Contents @@ -15,7 +17,7 @@ Complete API reference for the production-ready FFmpeg API service. ## Overview -The FFmpeg API provides a RESTful interface to FFmpeg's media processing capabilities with hardware acceleration support. +Rendiff provides a RESTful interface to FFmpeg's media processing capabilities with hardware acceleration support. > **💡 New to setup?** See the [Setup Guide](SETUP.md) for deployment instructions. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..f61821b --- /dev/null +++ b/docs/README.md @@ -0,0 +1,104 @@ +# Rendiff Documentation + +Welcome to the Rendiff documentation. This documentation is organized into two main sections based on your needs. + +> **About Rendiff:** Rendiff is a production-ready REST API for media processing, powered by [FFmpeg](https://ffmpeg.org/). + +--- + +## Documentation Sets + +### [User Manual](./user-manual/README.md) + +For users who want to deploy and use Rendiff for media processing. + +| Document | Description | +|----------|-------------| +| [Getting Started](./user-manual/GETTING_STARTED.md) | Quick start guide and first API call | +| [Installation](./user-manual/INSTALLATION.md) | Deployment options and setup | +| [Configuration](./user-manual/CONFIGURATION.md) | Environment variables and settings | +| [API Reference](./user-manual/API_REFERENCE.md) | Complete endpoint documentation | +| [Troubleshooting](./user-manual/TROUBLESHOOTING.md) | Common issues and solutions | + +**Best for:** DevOps engineers, system administrators, application developers integrating with Rendiff. + +--- + +### [Developer Documentation](./developer/README.md) + +For developers who want to contribute to or extend Rendiff. + +| Document | Description | +|----------|-------------| +| [Architecture](./developer/ARCHITECTURE.md) | System design and component overview | +| [Development Setup](./developer/DEVELOPMENT_SETUP.md) | Local development environment | +| [Contributing](./developer/CONTRIBUTING.md) | Code standards and PR process | +| [API Internals](./developer/API_INTERNALS.md) | Deep dive into API implementation | + +**Best for:** Open source contributors, developers extending Rendiff. + +--- + +## Quick Links + +### For Users + +```bash +# Deploy Rendiff +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev +docker compose up -d + +# Test the API +curl http://localhost:8000/api/v1/health +``` + +### For Developers + +```bash +# Set up development environment +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev +./setup.sh --development + +# Run tests +pytest tests/ -v +``` + +--- + +## Additional Resources + +### Project Links + +- **Repository:** [github.com/rendiffdev/rendiff-dev](https://github.com/rendiffdev/rendiff-dev) +- **Issues:** [GitHub Issues](https://github.com/rendiffdev/rendiff-dev/issues) +- **Discussions:** [GitHub Discussions](https://github.com/rendiffdev/rendiff-dev/discussions) + +### External Resources + +- **FFmpeg Documentation:** [ffmpeg.org/documentation.html](https://ffmpeg.org/documentation.html) +- **FFmpeg Wiki:** [trac.ffmpeg.org](https://trac.ffmpeg.org/) + +--- + +## Documentation Standards + +This documentation follows these principles: + +1. **Accuracy:** All examples are tested and verified +2. **Completeness:** All features and options are documented +3. **Clarity:** Written for the target audience +4. **Maintainability:** Updated with each release + +### Contributing to Documentation + +Documentation improvements are welcome! See the [Contributing Guide](./developer/CONTRIBUTING.md) for details. + +--- + +## Version + +This documentation is for **Rendiff v1.0.0**. + +For documentation of other versions, see the [release history](https://github.com/rendiffdev/rendiff-dev/releases). diff --git a/docs/SETUP.md b/docs/SETUP.md index 61797dd..0910250 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1,6 +1,8 @@ # Setup Guide -Complete setup guide for the FFmpeg API platform covering all deployment scenarios from development to production. +Complete setup guide for Rendiff covering all deployment scenarios from development to production. + +> **Powered by FFmpeg:** Rendiff is a REST API layer built on top of FFmpeg for media processing. ## Table of Contents @@ -19,8 +21,8 @@ Complete setup guide for the FFmpeg API platform covering all deployment scenari ```bash # Clone and navigate -git clone https://github.com/rendiffdev/ffmpeg-api.git -cd ffmpeg-api +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev # Choose your setup type ./setup.sh --help # Show all options diff --git a/docs/developer/API_INTERNALS.md b/docs/developer/API_INTERNALS.md new file mode 100644 index 0000000..4770ec5 --- /dev/null +++ b/docs/developer/API_INTERNALS.md @@ -0,0 +1,741 @@ +# API Internals + +Deep dive into the Rendiff API implementation, covering request handling, middleware, authentication, and service architecture. + +## Request Lifecycle + +``` +HTTP Request + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ MIDDLEWARE STACK │ +├─────────────────────────────────────────────────────────────┤ +│ 1. SecurityHeadersMiddleware │ Add security headers │ +│ 2. InputSanitizationMiddleware │ Validate content type │ +│ 3. RateLimitMiddleware │ Check rate limits │ +│ 4. SecurityAuditMiddleware │ Log security events │ +│ 5. CORSMiddleware │ Handle CORS │ +│ 6. GZipMiddleware │ Compress responses │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ DEPENDENCY INJECTION │ +├─────────────────────────────────────────────────────────────┤ +│ • Database Session (AsyncSession) │ +│ • Redis Client │ +│ • Storage Service │ +│ • API Key Validation │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ ROUTE HANDLER │ +├─────────────────────────────────────────────────────────────┤ +│ • Request validation (Pydantic) │ +│ • Business logic execution │ +│ • Response serialization │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ +HTTP Response +``` + +## Application Factory + +The application is created using a factory pattern in `api/main.py`: + +```python +def create_application() -> FastAPI: + """Create and configure FastAPI application.""" + application = FastAPI( + title="Rendiff API", + description="Production-grade media processing API powered by FFmpeg", + version=settings.VERSION, + docs_url="/docs" if settings.DEBUG else None, + redoc_url="/redoc" if settings.DEBUG else None, + openapi_url="/openapi.json" if settings.DEBUG else None, + lifespan=lifespan, + ) + + # Configure middleware (order matters!) + _configure_middleware(application) + + # Register routers + _register_routers(application) + + # Add exception handlers + _register_exception_handlers(application) + + return application +``` + +### Lifespan Management + +```python +@asynccontextmanager +async def lifespan(app: FastAPI): + """Application lifespan manager.""" + # Startup + logger.info("Starting Rendiff API", version=settings.VERSION) + await initialize_database() + await initialize_redis() + await initialize_storage() + + yield # Application runs + + # Shutdown + logger.info("Shutting down Rendiff API") + await close_database() + await close_redis() +``` + +## Middleware Stack + +### Security Headers Middleware + +Adds security headers to all responses: + +```python +class SecurityHeadersMiddleware(BaseHTTPMiddleware): + SECURITY_HEADERS = { + "X-Content-Type-Options": "nosniff", + "X-Frame-Options": "DENY", + "X-XSS-Protection": "1; mode=block", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Content-Security-Policy": "default-src 'self'", + "Referrer-Policy": "strict-origin-when-cross-origin", + "Permissions-Policy": "geolocation=(), microphone=(), camera=()", + } + + async def dispatch(self, request: Request, call_next): + response = await call_next(request) + for header, value in self.SECURITY_HEADERS.items(): + response.headers[header] = value + return response +``` + +### Rate Limit Middleware + +Implements tiered rate limiting: + +```python +class RateLimitMiddleware(BaseHTTPMiddleware): + def __init__(self, app, calls: int = 100, period: int = 3600): + super().__init__(app) + self.calls = calls + self.period = period + self.redis = None + + async def dispatch(self, request: Request, call_next): + # Extract client identifier + client_id = self._get_client_id(request) + + # Check rate limit + key = f"rate_limit:{client_id}" + current = await self.redis.incr(key) + + if current == 1: + await self.redis.expire(key, self.period) + + if current > self._get_limit(client_id): + return JSONResponse( + status_code=429, + content={"error": "Rate limit exceeded"}, + headers={"Retry-After": str(self.period)} + ) + + response = await call_next(request) + + # Add rate limit headers + response.headers["X-RateLimit-Limit"] = str(self._get_limit(client_id)) + response.headers["X-RateLimit-Remaining"] = str(max(0, self._get_limit(client_id) - current)) + + return response +``` + +### Rate Limit Tiers + +| Tier | Identifier Pattern | Requests/Hour | +|------|-------------------|---------------| +| Basic | `basic_*` | 500 | +| Pro | `pro_*` | 2,000 | +| Enterprise | `ent_*` | 10,000 | +| Internal | `internal_*` | Unlimited | + +## Dependency Injection + +### Database Session + +```python +async def get_db() -> AsyncGenerator[AsyncSession, None]: + """Provide database session with automatic cleanup.""" + async with async_session_maker() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + finally: + await session.close() +``` + +### API Key Validation + +```python +async def get_api_key( + request: Request, + db: AsyncSession = Depends(get_db), +) -> Optional[APIKey]: + """Validate and return API key from request.""" + if not settings.ENABLE_API_KEYS: + return None + + api_key_header = request.headers.get("X-API-Key") + if not api_key_header: + raise HTTPException( + status_code=401, + detail="API key required" + ) + + # Constant-time comparison to prevent timing attacks + api_key = await validate_api_key(db, api_key_header) + if not api_key: + # Add fixed delay to prevent timing attacks + await asyncio.sleep(0.1) + raise HTTPException( + status_code=401, + detail="Invalid API key" + ) + + return api_key +``` + +### Storage Service + +```python +async def get_storage_service() -> StorageService: + """Provide storage service with configured backends.""" + return StorageService(settings.STORAGE_CONFIG) +``` + +## Router Structure + +### Router Registration + +```python +def _register_routers(app: FastAPI) -> None: + """Register all API routers.""" + app.include_router( + health_router, + prefix="/api/v1", + tags=["Health"] + ) + app.include_router( + jobs_router, + prefix="/api/v1/jobs", + tags=["Jobs"] + ) + app.include_router( + convert_router, + prefix="/api/v1", + tags=["Processing"] + ) + if settings.ENABLE_ADMIN_ENDPOINTS: + app.include_router( + admin_router, + prefix="/api/v1/admin", + tags=["Admin"] + ) +``` + +### Convert Router Example + +```python +# api/routers/convert.py + +router = APIRouter() + +@router.post( + "/convert", + response_model=JobResponse, + status_code=201, +) +async def create_conversion_job( + request: ConversionRequest, + db: AsyncSession = Depends(get_db), + api_key: APIKey = Depends(get_api_key), + storage: StorageService = Depends(get_storage_service), +) -> JobResponse: + """Create a new media conversion job.""" + + # 1. Validate input path + input_backend, input_path = await validate_input_path( + request.input_path, storage + ) + + # 2. Validate output path + output_backend, output_path = await validate_output_path( + request.output_path, storage + ) + + # 3. Validate operations + validated_ops = validate_operations(request.operations or []) + + # 4. Create job record + job = Job( + input_path=request.input_path, + output_path=request.output_path, + operations=validated_ops, + options=request.options, + api_key=api_key.id if api_key else None, + webhook_url=validate_webhook_url(request.webhook_url), + ) + + db.add(job) + await db.flush() # Get ID before commit + + # 5. Queue job for processing + celery_app.send_task( + "worker.tasks.process_job", + args=[str(job.id)] + ) + + await db.commit() + + return JobResponse.from_orm(job) +``` + +## Request Validation + +### Pydantic Models + +```python +# api/schemas/conversion.py + +class ConversionRequest(BaseModel): + """Request schema for conversion endpoint.""" + + input_path: str = Field( + ..., + description="Source file path (storage URI)", + example="local:///storage/input.mp4" + ) + output_path: str = Field( + ..., + description="Destination file path", + example="local:///storage/output.mp4" + ) + operations: Optional[List[OperationSchema]] = Field( + default=[], + description="Processing operations to apply" + ) + options: Optional[ProcessingOptions] = Field( + default=None, + description="Global processing options" + ) + webhook_url: Optional[str] = Field( + default=None, + description="URL for completion notification" + ) + + @validator("input_path", "output_path") + def validate_storage_uri(cls, v): + """Validate storage URI format.""" + if not re.match(r"^(local|s3|azure|gcs)://", v): + raise ValueError("Invalid storage URI format") + return v + + class Config: + schema_extra = { + "example": { + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [ + {"type": "transcode", "video_codec": "vp9"} + ] + } + } +``` + +### Custom Validators + +```python +# api/utils/validators.py + +def validate_secure_path(path: str, base_paths: set = None) -> str: + """ + Validate file path to prevent directory traversal. + + Security measures: + 1. Null byte detection + 2. Dangerous character blocking + 3. Path canonicalization + 4. Base path enforcement + """ + if not path: + raise SecurityError("Path cannot be empty") + + # Check for null bytes + if "\x00" in path: + raise SecurityError("Null byte detected in path") + + # Check for dangerous characters + dangerous_chars = ["|", ";", "&", "$", "`", "<", ">", '"', "'"] + for char in dangerous_chars: + if char in path: + raise SecurityError(f"Dangerous character: {char}") + + # Canonicalize path + canonical = os.path.realpath(os.path.abspath(path)) + + # Check for traversal after canonicalization + if ".." in canonical: + raise SecurityError("Directory traversal detected") + + # Verify against allowed base paths + if base_paths: + allowed = any( + canonical.startswith(os.path.realpath(base) + os.sep) + for base in base_paths + ) + if not allowed: + raise SecurityError("Path outside allowed directories") + + return canonical +``` + +## Error Handling + +### Exception Handlers + +```python +# api/utils/error_handlers.py + +async def rendiff_exception_handler( + request: Request, + exc: RendiffError +) -> JSONResponse: + """Handle Rendiff-specific exceptions.""" + logger.error( + "Application error", + error_code=exc.code, + error_message=exc.message, + path=request.url.path, + ) + + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": exc.code, + "message": exc.message, + "type": type(exc).__name__, + } + } + ) + +async def general_exception_handler( + request: Request, + exc: Exception +) -> JSONResponse: + """Handle unexpected exceptions.""" + logger.error( + "Unhandled exception", + exc_type=type(exc).__name__, + exc_message=str(exc), + traceback=traceback.format_exc(), + ) + + # Don't expose internal details in production + message = "An internal error occurred" + if settings.DEBUG: + message = str(exc) + + return JSONResponse( + status_code=500, + content={ + "error": { + "code": "INTERNAL_ERROR", + "message": message, + } + } + ) +``` + +### Exception Hierarchy + +```python +class RendiffError(Exception): + """Base exception for all Rendiff errors.""" + + def __init__(self, message: str, code: str, status_code: int = 500): + self.message = message + self.code = code + self.status_code = status_code + +class ValidationError(RendiffError): + """Input validation errors (400).""" + + def __init__(self, message: str, field: str = None): + super().__init__(message, "VALIDATION_ERROR", 400) + self.field = field + +class AuthenticationError(RendiffError): + """Authentication errors (401).""" + + def __init__(self, message: str = "Authentication required"): + super().__init__(message, "AUTH_ERROR", 401) + +class AuthorizationError(RendiffError): + """Authorization errors (403).""" + + def __init__(self, message: str = "Insufficient permissions"): + super().__init__(message, "AUTHZ_ERROR", 403) + +class RateLimitError(RendiffError): + """Rate limit errors (429).""" + + def __init__(self, message: str = "Rate limit exceeded"): + super().__init__(message, "RATE_LIMIT_ERROR", 429) + +class ProcessingError(RendiffError): + """Media processing errors (500).""" + + def __init__(self, message: str, job_id: str = None): + super().__init__(message, "PROCESSING_ERROR", 500) + self.job_id = job_id + +class StorageError(RendiffError): + """Storage backend errors (500).""" + + def __init__(self, message: str, backend: str = None): + super().__init__(message, "STORAGE_ERROR", 500) + self.backend = backend +``` + +## Response Serialization + +### Response Models + +```python +class JobResponse(BaseModel): + """Job response schema.""" + + id: UUID + status: JobStatus + progress: float + input_path: str + output_path: str + created_at: datetime + started_at: Optional[datetime] + completed_at: Optional[datetime] + error_message: Optional[str] + + class Config: + from_attributes = True # Enable ORM mode + +class PaginatedResponse(BaseModel, Generic[T]): + """Generic paginated response.""" + + items: List[T] + total: int + page: int + page_size: int + has_next: bool + has_prev: bool +``` + +## Database Operations + +### Async Session Usage + +```python +async def get_job_by_id( + db: AsyncSession, + job_id: UUID +) -> Optional[Job]: + """Fetch job by ID.""" + result = await db.execute( + select(Job).where(Job.id == job_id) + ) + return result.scalar_one_or_none() + +async def list_jobs( + db: AsyncSession, + api_key_id: UUID, + status: Optional[JobStatus] = None, + page: int = 1, + page_size: int = 20, +) -> Tuple[List[Job], int]: + """List jobs with pagination.""" + query = select(Job).where(Job.api_key == api_key_id) + + if status: + query = query.where(Job.status == status) + + # Count total + count_query = select(func.count()).select_from(query.subquery()) + total = await db.scalar(count_query) + + # Fetch page + query = query.order_by(Job.created_at.desc()) + query = query.offset((page - 1) * page_size).limit(page_size) + + result = await db.execute(query) + jobs = result.scalars().all() + + return jobs, total +``` + +## Background Tasks + +### Celery Integration + +```python +from worker.celery_app import celery_app + +async def queue_processing_job(job_id: str, priority: str = "normal"): + """Queue job for background processing.""" + queue = { + "low": "rendiff.low", + "normal": "rendiff.default", + "high": "rendiff.high", + }.get(priority, "rendiff.default") + + celery_app.send_task( + "worker.tasks.process_job", + args=[job_id], + queue=queue, + ) +``` + +## Health Checks + +```python +@router.get("/health") +async def health_check( + db: AsyncSession = Depends(get_db), +) -> HealthResponse: + """Comprehensive health check.""" + checks = {} + + # Database check + try: + await db.execute(text("SELECT 1")) + checks["database"] = {"status": "healthy"} + except Exception as e: + checks["database"] = {"status": "unhealthy", "error": str(e)} + + # Redis check + try: + await redis.ping() + checks["redis"] = {"status": "healthy"} + except Exception as e: + checks["redis"] = {"status": "unhealthy", "error": str(e)} + + # FFmpeg check + try: + result = subprocess.run( + ["ffmpeg", "-version"], + capture_output=True, + timeout=5 + ) + checks["ffmpeg"] = { + "status": "healthy" if result.returncode == 0 else "unhealthy" + } + except Exception as e: + checks["ffmpeg"] = {"status": "unhealthy", "error": str(e)} + + overall = all(c["status"] == "healthy" for c in checks.values()) + + return HealthResponse( + status="healthy" if overall else "degraded", + checks=checks, + version=settings.VERSION, + ) +``` + +## Metrics + +### Prometheus Metrics + +```python +from prometheus_client import Counter, Histogram, Gauge + +# Request metrics +REQUEST_COUNT = Counter( + "rendiff_requests_total", + "Total HTTP requests", + ["method", "endpoint", "status"] +) + +REQUEST_LATENCY = Histogram( + "rendiff_request_duration_seconds", + "Request latency", + ["method", "endpoint"] +) + +# Job metrics +JOBS_TOTAL = Counter( + "rendiff_jobs_total", + "Total jobs created", + ["status"] +) + +JOBS_ACTIVE = Gauge( + "rendiff_jobs_active", + "Currently processing jobs" +) + +JOB_DURATION = Histogram( + "rendiff_job_duration_seconds", + "Job processing duration", + ["type"], + buckets=[1, 5, 10, 30, 60, 300, 600, 1800, 3600] +) +``` + +## Configuration + +### Settings Class + +```python +class Settings(BaseSettings): + """Application configuration.""" + + # Application + VERSION: str = "1.0.0" + DEBUG: bool = False + + # API + API_HOST: str = "0.0.0.0" + API_PORT: int = 8000 + API_WORKERS: int = 4 + + # Database + DATABASE_URL: str = "postgresql+asyncpg://..." + DATABASE_POOL_SIZE: int = 20 + + # Redis + REDIS_URL: str = "redis://localhost:6379/0" + + # Security + ENABLE_API_KEYS: bool = True + RATE_LIMIT_CALLS: int = 100 + RATE_LIMIT_PERIOD: int = 3600 + + model_config = SettingsConfigDict( + env_file=".env", + case_sensitive=False, + ) + +@lru_cache() +def get_settings() -> Settings: + """Get cached settings instance.""" + return Settings() + +settings = get_settings() +``` diff --git a/docs/developer/ARCHITECTURE.md b/docs/developer/ARCHITECTURE.md new file mode 100644 index 0000000..92f694a --- /dev/null +++ b/docs/developer/ARCHITECTURE.md @@ -0,0 +1,341 @@ +# Rendiff Architecture + +This document describes the system architecture of Rendiff, including component design, data flow, and integration patterns. + +## System Overview + +Rendiff follows a distributed microservices architecture optimized for high-throughput media processing workloads. + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ CLIENTS │ +│ (Web Apps, Mobile Apps, CLI Tools) │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ TRAEFIK (Reverse Proxy) │ +│ • SSL Termination • Rate Limiting • Load Balancing │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ┌───────────────┼───────────────┐ + ▼ ▼ ▼ + ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ + │ API #1 │ │ API #2 │ │ API #N │ + │ (FastAPI) │ │ (FastAPI) │ │ (FastAPI) │ + └──────┬──────┘ └──────┬──────┘ └──────┬──────┘ + │ │ │ + └───────────────┼───────────────┘ + │ + ┌───────────────────────┼───────────────────────┐ + ▼ ▼ ▼ + ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ + │ PostgreSQL │ │ Redis │ │ Storage │ + │ (Primary │ │ (Queue + │ │ (Local/S3/ │ + │ Database) │ │ Cache) │ │ Azure) │ + └─────────────┘ └──────┬──────┘ └─────────────┘ + │ + ┌──────────────┼──────────────┐ + ▼ ▼ ▼ + ┌─────────────┐┌─────────────┐┌─────────────┐ + │ Worker #1 ││ Worker #2 ││ Worker #N │ + │ (Celery + ││ (Celery + ││ (Celery + │ + │ FFmpeg) ││ FFmpeg) ││ FFmpeg) │ + └─────────────┘└─────────────┘└─────────────┘ +``` + +## Core Components + +### 1. API Layer (FastAPI) + +The API layer handles all HTTP requests and serves as the system's entry point. + +**Responsibilities:** +- Request validation and authentication +- Job creation and status queries +- Health checks and metrics exposure +- WebSocket connections for real-time updates + +**Key Files:** +``` +api/ +├── main.py # Application factory, middleware setup +├── config.py # Pydantic settings management +├── dependencies.py # Dependency injection (DB, Redis, Auth) +├── routers/ +│ ├── convert.py # /api/v1/convert endpoints +│ ├── jobs.py # /api/v1/jobs endpoints +│ ├── health.py # /api/v1/health endpoints +│ └── admin.py # /api/v1/admin endpoints +├── models/ +│ ├── job.py # Job SQLAlchemy model +│ ├── api_key.py # API key model +│ └── database.py # Database connection setup +├── services/ +│ ├── job_service.py # Job business logic +│ └── storage.py # Storage abstraction +└── middleware/ + └── security.py # Security middleware stack +``` + +**Request Flow:** +``` +Request → Traefik → Security Middleware → Rate Limiter → +Authentication → Validation → Router → Service → Database → Response +``` + +### 2. Worker Layer (Celery) + +Workers process media jobs asynchronously using FFmpeg. + +**Responsibilities:** +- Consume jobs from Redis queue +- Execute FFmpeg commands +- Report progress updates +- Handle retries and failures +- Send webhook notifications + +**Key Files:** +``` +worker/ +├── tasks.py # Celery task definitions +├── celery_app.py # Celery configuration +├── processors/ +│ ├── video.py # Video processing logic +│ ├── audio.py # Audio processing logic +│ ├── analysis.py # Quality analysis (VMAF, PSNR) +│ └── streaming.py # HLS/DASH generation +└── utils/ + ├── ffmpeg.py # FFmpeg command builder + └── progress.py # Progress tracking +``` + +**Task Flow:** +``` +Job Created → Redis Queue → Worker Claims → Download Input → +FFmpeg Process → Upload Output → Update Database → Webhook +``` + +### 3. Database Layer (PostgreSQL) + +PostgreSQL stores all persistent application data. + +**Tables:** +| Table | Purpose | +|-------|---------| +| `jobs` | Processing job records | +| `api_keys` | Authentication credentials | +| `job_logs` | Detailed job event logs | +| `system_metrics` | Performance metrics | +| `storage_usage` | Storage tracking | + +**Key Design Decisions:** +- UUID primary keys for distributed generation +- JSONB columns for flexible metadata +- Partial indexes for active job queries +- GIN indexes for JSON search + +### 4. Queue Layer (Redis) + +Redis serves dual purposes: message broker and cache. + +**Usage:** +- **Celery Broker:** Job queue management +- **Rate Limiting:** Request counting per API key +- **Distributed Locks:** Preventing duplicate processing +- **Cache:** Temporary data storage +- **Progress Tracking:** Real-time job status + +**Key Patterns:** +```python +# Rate limiting key structure +rate_limit:{api_key}:{endpoint} → count + +# Job progress key structure +job:progress:{job_id} → {stage, percent, eta} + +# Distributed lock +lock:job:{job_id} → worker_id (with TTL) +``` + +### 5. Storage Layer + +Abstraction layer supporting multiple storage backends. + +**Supported Backends:** +| Backend | Use Case | +|---------|----------| +| Local | Development, single-node deployments | +| S3 | AWS production deployments | +| Azure Blob | Azure production deployments | +| GCS | Google Cloud deployments | +| MinIO | Self-hosted S3-compatible storage | + +**Storage Flow:** +``` +Input URL → Parse Backend → Download to Temp → +Process with FFmpeg → Upload to Output Backend → Cleanup Temp +``` + +## Data Flow + +### Job Processing Flow + +``` +1. Client submits job via POST /api/v1/convert + │ +2. API validates request and creates job record + │ +3. Job enqueued to Redis (Celery task) + │ +4. Worker claims job from queue + │ +5. Worker downloads input file to temp storage + │ +6. FFmpeg processes the file (with progress updates) + │ +7. Worker uploads output file to destination + │ +8. Database updated with completion status + │ +9. Webhook sent to client (if configured) +``` + +### Authentication Flow + +``` +1. Request received with X-API-Key header + │ +2. API key extracted and validated + │ +3. Key hash compared (constant-time) + │ +4. Rate limits checked against Redis + │ +5. Permissions validated for endpoint + │ +6. Request processed or rejected +``` + +## Scalability Design + +### Horizontal Scaling + +| Component | Scaling Strategy | +|-----------|-----------------| +| API | Add instances behind load balancer | +| Workers | Add worker containers/pods | +| Redis | Redis Cluster or Sentinel | +| PostgreSQL | Read replicas, connection pooling | +| Storage | Cloud storage (infinite scale) | + +### Resource Limits + +```yaml +# Per-job limits +MAX_INPUT_SIZE: 10GB +MAX_RESOLUTION: 7680x4320 (8K) +MAX_BITRATE: 100Mbps +MAX_DURATION: 24 hours +MAX_OPERATIONS: 50 per job + +# Per-worker limits +FFMPEG_THREADS: Auto (CPU count) +MEMORY_LIMIT: Configurable +TEMP_STORAGE: Cleaned per job +``` + +## Security Architecture + +### Defense in Depth + +``` +Layer 1: Network +├── Traefik rate limiting +├── IP whitelisting (optional) +└── TLS 1.2+ encryption + +Layer 2: Authentication +├── API key validation +├── Timing attack protection +└── Key rotation support + +Layer 3: Input Validation +├── Path traversal prevention +├── Command injection blocking +├── File type validation +└── Size limits + +Layer 4: Processing +├── Sandboxed FFmpeg execution +├── Resource limits +└── Timeout enforcement + +Layer 5: Output +├── Error message sanitization +├── SSRF prevention (webhooks) +└── Audit logging +``` + +## Monitoring Architecture + +``` +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ API │────▶│ Prometheus │────▶│ Grafana │ +│ /metrics │ │ (Scraper) │ │ (Dashboard) │ +└─────────────┘ └─────────────┘ └─────────────┘ + │ + ▼ +┌─────────────┐ +│ Alerting │ +│ Rules │ +└─────────────┘ +``` + +**Key Metrics:** +- `rendiff_jobs_total` - Job count by status +- `rendiff_job_duration_seconds` - Processing time histogram +- `rendiff_api_requests_total` - API request count +- `rendiff_storage_bytes` - Storage usage +- `rendiff_worker_active` - Active worker count + +## Configuration Management + +### Environment-Based Configuration + +``` +Production: .env.production +Staging: .env.staging +Development: .env.development +Testing: .env.test +``` + +### Configuration Hierarchy + +``` +1. Environment variables (highest priority) +2. .env file +3. Default values in config.py (lowest priority) +``` + +## Deployment Patterns + +### Single Node (Development) + +``` +docker compose up -d +``` + +### Production (Docker Compose) + +``` +docker compose -f compose.prod.yml up -d +``` + +### Kubernetes + +``` +kubectl apply -f k8s/ +``` + +See the [deployment guide](../user-manual/DEPLOYMENT.md) for detailed instructions. diff --git a/docs/developer/CONTRIBUTING.md b/docs/developer/CONTRIBUTING.md new file mode 100644 index 0000000..2e245c4 --- /dev/null +++ b/docs/developer/CONTRIBUTING.md @@ -0,0 +1,450 @@ +# Contributing to Rendiff + +Thank you for your interest in contributing to Rendiff! This guide covers everything you need to know to contribute effectively. + +## Code of Conduct + +By participating in this project, you agree to maintain a respectful and inclusive environment for all contributors. + +## How to Contribute + +### Reporting Issues + +Before creating an issue: + +1. **Search existing issues** to avoid duplicates +2. **Use the issue templates** when available +3. **Provide complete information:** + - Rendiff version + - Operating system + - Steps to reproduce + - Expected vs actual behavior + - Relevant logs or screenshots + +### Feature Requests + +We welcome feature suggestions! When proposing new features: + +1. Explain the use case clearly +2. Describe the expected behavior +3. Consider backward compatibility +4. Be open to discussion and alternatives + +### Pull Requests + +#### Before Starting + +1. Check for existing PRs addressing the same issue +2. For significant changes, open an issue first to discuss +3. Ensure you can sign off your commits (DCO) + +#### Development Workflow + +```bash +# 1. Fork the repository on GitHub + +# 2. Clone your fork +git clone https://github.com/YOUR_USERNAME/rendiff-dev.git +cd rendiff-dev + +# 3. Add upstream remote +git remote add upstream https://github.com/rendiffdev/rendiff-dev.git + +# 4. Create a feature branch +git checkout -b feature/your-feature-name + +# 5. Make your changes +# ... edit files ... + +# 6. Run tests and quality checks +pytest tests/ -v +black api/ worker/ tests/ +flake8 api/ worker/ tests/ +mypy api/ worker/ + +# 7. Commit your changes +git add . +git commit -m "feat(api): add new endpoint for batch processing" + +# 8. Push to your fork +git push origin feature/your-feature-name + +# 9. Create Pull Request on GitHub +``` + +#### PR Requirements + +- [ ] All tests pass +- [ ] Code follows project style guidelines +- [ ] New code has appropriate test coverage +- [ ] Documentation updated if needed +- [ ] Commit messages follow conventions +- [ ] PR description explains changes clearly + +## Coding Standards + +### Python Style Guide + +We follow PEP 8 with some project-specific conventions: + +```python +# Good: Clear, descriptive names +async def create_conversion_job( + input_path: str, + output_path: str, + options: ConversionOptions, +) -> Job: + """ + Create a new media conversion job. + + Args: + input_path: Source file path (storage URI format) + output_path: Destination file path + options: Conversion parameters + + Returns: + Created Job instance + + Raises: + ValidationError: If paths are invalid + StorageError: If storage backend unavailable + """ + # Implementation... + +# Bad: Unclear names, no type hints +async def create(p1, p2, opts): + # No docstring... +``` + +### Type Hints + +Always use type hints for function signatures: + +```python +from typing import List, Optional, Dict, Any + +def process_operations( + operations: List[Dict[str, Any]], + options: Optional[ProcessingOptions] = None, +) -> ProcessingResult: + ... +``` + +### Async/Await + +Use async functions for I/O operations: + +```python +# Good: Async for I/O +async def fetch_job(job_id: str) -> Optional[Job]: + async with get_session() as session: + return await session.get(Job, job_id) + +# Bad: Blocking I/O in async context +async def fetch_job_bad(job_id: str) -> Optional[Job]: + with get_sync_session() as session: # Blocks! + return session.get(Job, job_id) +``` + +### Error Handling + +Use specific exceptions and proper error messages: + +```python +from api.utils.error_handlers import ValidationError, StorageError + +# Good: Specific exception with context +if not input_path.startswith(('local://', 's3://')): + raise ValidationError( + f"Invalid storage URI format: {input_path}", + field="input_path" + ) + +# Bad: Generic exception +if not valid: + raise Exception("error") +``` + +### Imports + +Organize imports in this order: + +```python +# 1. Standard library +import os +from pathlib import Path +from typing import List, Optional + +# 2. Third-party packages +from fastapi import APIRouter, Depends, HTTPException +from sqlalchemy.ext.asyncio import AsyncSession +import structlog + +# 3. Local imports +from api.config import settings +from api.models.job import Job +from api.services.storage import StorageService +``` + +## Commit Message Conventions + +Follow the Conventional Commits specification: + +``` +type(scope): subject + +body (optional) + +footer (optional) +``` + +### Types + +| Type | Description | +|------|-------------| +| `feat` | New feature | +| `fix` | Bug fix | +| `docs` | Documentation only | +| `style` | Code style (formatting, semicolons) | +| `refactor` | Code change that neither fixes nor adds | +| `perf` | Performance improvement | +| `test` | Adding or updating tests | +| `chore` | Build process, dependencies | +| `ci` | CI/CD configuration | + +### Scopes + +| Scope | Description | +|-------|-------------| +| `api` | API layer changes | +| `worker` | Worker/task changes | +| `storage` | Storage backend changes | +| `db` | Database/models changes | +| `config` | Configuration changes | +| `docker` | Docker/deployment changes | +| `deps` | Dependency updates | + +### Examples + +```bash +# Feature +git commit -m "feat(api): add batch processing endpoint" + +# Bug fix +git commit -m "fix(worker): handle timeout in FFmpeg processing" + +# Documentation +git commit -m "docs(api): update authentication examples" + +# Breaking change +git commit -m "feat(api)!: change job status enum values + +BREAKING CHANGE: Job status 'in_progress' renamed to 'processing'" +``` + +## Testing Guidelines + +### Test Structure + +``` +tests/ +├── conftest.py # Shared fixtures +├── test_health.py # Health check tests +├── test_jobs.py # Job endpoint tests +├── test_models.py # Model unit tests +├── test_services.py # Service layer tests +├── test_integration.py # Integration tests +├── test_security.py # Security tests +└── test_performance.py # Performance tests +``` + +### Writing Tests + +```python +import pytest +from fastapi.testclient import TestClient + +class TestJobEndpoints: + """Tests for job management endpoints.""" + + def test_create_job_success(self, client: TestClient, auth_headers: dict): + """Test successful job creation.""" + response = client.post( + "/api/v1/convert", + json={ + "input_path": "local:///storage/test.mp4", + "output_path": "local:///storage/output.mp4", + }, + headers=auth_headers, + ) + + assert response.status_code == 201 + data = response.json() + assert "id" in data + assert data["status"] == "pending" + + def test_create_job_invalid_path(self, client: TestClient, auth_headers: dict): + """Test job creation with invalid path.""" + response = client.post( + "/api/v1/convert", + json={ + "input_path": "../../../etc/passwd", # Path traversal attempt + "output_path": "local:///storage/output.mp4", + }, + headers=auth_headers, + ) + + assert response.status_code == 400 + assert "traversal" in response.json()["detail"].lower() +``` + +### Test Coverage + +- Aim for >80% code coverage +- All new features must have tests +- Bug fixes should include regression tests + +```bash +# Run with coverage +pytest tests/ --cov=api --cov=worker --cov-report=html + +# View report +open htmlcov/index.html +``` + +## Documentation Guidelines + +### Code Documentation + +Every public function needs a docstring: + +```python +async def transcode_video( + input_path: str, + output_path: str, + codec: str = "h264", + preset: str = "medium", +) -> TranscodeResult: + """ + Transcode video file using FFmpeg. + + This function handles the transcoding of video files to different + codecs and formats. It supports hardware acceleration when available. + + Args: + input_path: Source video file path (storage URI) + output_path: Destination file path + codec: Video codec (h264, h265, vp9, av1). Default: h264 + preset: Encoding preset (ultrafast to veryslow). Default: medium + + Returns: + TranscodeResult with output path and processing metrics + + Raises: + ValidationError: If codec is not supported + ProcessingError: If FFmpeg fails + StorageError: If file operations fail + + Example: + >>> result = await transcode_video( + ... "local:///input.mp4", + ... "local:///output.webm", + ... codec="vp9" + ... ) + >>> print(result.output_path) + local:///output.webm + """ +``` + +### API Documentation + +Update OpenAPI documentation for endpoint changes: + +```python +@router.post( + "/convert", + response_model=JobResponse, + status_code=201, + summary="Create conversion job", + description=""" + Create a new media conversion job. + + The job will be queued for processing by a worker. Use the returned + job ID to track progress via the `/jobs/{id}` endpoint. + + **Supported formats:** + - Video: MP4, WebM, MKV, AVI, MOV + - Audio: MP3, AAC, FLAC, WAV + + **Rate limits:** 200 requests/hour per API key + """, + responses={ + 201: {"description": "Job created successfully"}, + 400: {"description": "Invalid request parameters"}, + 401: {"description": "Missing or invalid API key"}, + 429: {"description": "Rate limit exceeded"}, + }, +) +async def create_conversion_job(...): + ... +``` + +## Security Guidelines + +### Reporting Security Issues + +**Do not** create public GitHub issues for security vulnerabilities. + +Instead, report security issues to: security@rendiff.dev + +### Security Best Practices + +1. **Input Validation:** Always validate and sanitize user input +2. **Path Security:** Use the validators module for file paths +3. **SQL Injection:** Use SQLAlchemy ORM, never raw SQL with user input +4. **Secrets:** Never commit secrets, use environment variables +5. **Dependencies:** Keep dependencies updated, run security scans + +```python +# Good: Use validators +from api.utils.validators import validate_secure_path + +path = validate_secure_path(user_input) + +# Bad: Direct path usage +path = user_input # Dangerous! +``` + +## Review Process + +### What Reviewers Look For + +1. **Correctness:** Does the code work as intended? +2. **Security:** Are there any security concerns? +3. **Performance:** Any obvious performance issues? +4. **Maintainability:** Is the code readable and maintainable? +5. **Testing:** Are there adequate tests? +6. **Documentation:** Is documentation updated? + +### Addressing Review Feedback + +- Respond to all comments +- Push fixes as new commits (easier to review) +- Request re-review when ready +- Squash commits before merge if requested + +## Getting Help + +- **Questions:** Open a GitHub Discussion +- **Bugs:** Create a GitHub Issue +- **Security:** Email security@rendiff.dev +- **Chat:** Join our Discord server + +## Recognition + +Contributors are recognized in: +- CONTRIBUTORS.md file +- Release notes +- Project documentation + +Thank you for contributing to Rendiff! diff --git a/docs/developer/DEVELOPMENT_SETUP.md b/docs/developer/DEVELOPMENT_SETUP.md new file mode 100644 index 0000000..a3c7444 --- /dev/null +++ b/docs/developer/DEVELOPMENT_SETUP.md @@ -0,0 +1,431 @@ +# Development Setup Guide + +This guide walks you through setting up a local development environment for Rendiff. + +## Prerequisites + +### Required Software + +| Software | Minimum Version | Purpose | +|----------|----------------|---------| +| Docker | 20.10+ | Container runtime | +| Docker Compose | 2.0+ | Multi-container orchestration | +| Python | 3.12+ | API and worker runtime | +| Git | 2.30+ | Version control | + +### Optional (for local development without Docker) + +| Software | Version | Purpose | +|----------|---------|---------| +| FFmpeg | 6.0+ | Media processing | +| PostgreSQL | 14+ | Database | +| Redis | 7.0+ | Queue and cache | + +### Hardware Recommendations + +| Environment | CPU | RAM | Storage | +|-------------|-----|-----|---------| +| Development | 4 cores | 8GB | 50GB SSD | +| Testing | 8 cores | 16GB | 100GB SSD | + +## Quick Start (Docker) + +The fastest way to get started: + +```bash +# Clone the repository +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev + +# Start development environment +./setup.sh --development + +# Verify it's running +curl http://localhost:8000/api/v1/health +``` + +The development setup includes: +- SQLite database (no PostgreSQL needed) +- Redis for job queue +- Hot-reload enabled +- Debug mode active +- No authentication required + +## Manual Setup + +### 1. Clone and Configure + +```bash +# Clone repository +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev + +# Create Python virtual environment +python3.12 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate + +# Install dependencies +pip install -r requirements.txt + +# Copy environment template +cp .env.example .env +``` + +### 2. Configure Environment + +Edit `.env` for development: + +```bash +# Development settings +DEBUG=true +TESTING=false + +# API Configuration +API_HOST=0.0.0.0 +API_PORT=8000 +API_LOG_LEVEL=debug +API_WORKERS=1 + +# Database (SQLite for simplicity) +DATABASE_URL=sqlite+aiosqlite:///data/rendiff.db + +# Redis (if running locally) +REDIS_URL=redis://localhost:6379/0 + +# Disable authentication for development +ENABLE_API_KEYS=false + +# Storage +STORAGE_PATH=./storage +TEMP_PATH=/tmp/rendiff +``` + +### 3. Start Dependencies + +Using Docker (recommended): + +```bash +# Start only Redis +docker run -d --name rendiff-redis -p 6379:6379 redis:7.2-alpine +``` + +Or install locally: + +```bash +# macOS +brew install redis +brew services start redis + +# Ubuntu/Debian +sudo apt install redis-server +sudo systemctl start redis +``` + +### 4. Initialize Database + +```bash +# Create data directory +mkdir -p data + +# Run migrations +alembic upgrade head +``` + +### 5. Start the API + +```bash +# Development mode with hot reload +uvicorn api.main:app --reload --host 0.0.0.0 --port 8000 +``` + +### 6. Start a Worker (Optional) + +In a separate terminal: + +```bash +source venv/bin/activate +celery -A worker.celery_app worker --loglevel=info +``` + +## Docker Development Environment + +### Full Stack Development + +```bash +# Start all services with development overrides +docker compose -f compose.yml -f compose.override.yml up -d + +# View logs +docker compose logs -f api + +# Rebuild after code changes +docker compose build api +docker compose up -d api +``` + +### Service-Specific Commands + +```bash +# Start only specific services +docker compose up -d postgres redis + +# Run API locally while using Docker services +export DATABASE_URL=postgresql://rendiff_user:defaultpassword@localhost:5432/rendiff +export REDIS_URL=redis://localhost:6379/0 +uvicorn api.main:app --reload +``` + +## IDE Configuration + +### VS Code + +Recommended extensions: +- Python (Microsoft) +- Pylance +- Docker +- GitLens +- REST Client + +`.vscode/settings.json`: +```json +{ + "python.defaultInterpreterPath": "${workspaceFolder}/venv/bin/python", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.formatting.provider": "black", + "python.testing.pytestEnabled": true, + "python.testing.pytestArgs": ["tests/"], + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } +} +``` + +### PyCharm + +1. Set Python interpreter to `venv/bin/python` +2. Mark `api/` and `worker/` as Sources Root +3. Configure pytest as test runner +4. Enable Black formatter + +## Running Tests + +```bash +# Run all tests +pytest tests/ -v + +# Run with coverage +pytest tests/ --cov=api --cov=worker --cov-report=html + +# Run specific test file +pytest tests/test_jobs.py -v + +# Run tests matching pattern +pytest tests/ -k "test_convert" -v + +# Run only fast tests (no integration) +pytest tests/ -m "not integration" -v +``` + +## Code Quality Tools + +### Formatting + +```bash +# Format code with Black +black api/ worker/ tests/ + +# Check without modifying +black api/ worker/ tests/ --check +``` + +### Linting + +```bash +# Run flake8 +flake8 api/ worker/ tests/ + +# Run with specific config +flake8 --config=.flake8 +``` + +### Type Checking + +```bash +# Run mypy +mypy api/ worker/ + +# Strict mode +mypy api/ worker/ --strict +``` + +### Import Sorting + +```bash +# Sort imports +isort api/ worker/ tests/ + +# Check without modifying +isort api/ worker/ tests/ --check +``` + +### Pre-commit Hooks + +```bash +# Install pre-commit +pip install pre-commit + +# Install hooks +pre-commit install + +# Run manually +pre-commit run --all-files +``` + +## Database Management + +### Migrations + +```bash +# Create new migration +alembic revision --autogenerate -m "Add new column" + +# Apply migrations +alembic upgrade head + +# Rollback one migration +alembic downgrade -1 + +# View migration history +alembic history + +# View current revision +alembic current +``` + +### Database Shell + +```bash +# SQLite +sqlite3 data/rendiff.db + +# PostgreSQL (Docker) +docker compose exec postgres psql -U rendiff_user -d rendiff +``` + +## Debugging + +### API Debugging + +```python +# Add to code for breakpoints +import pdb; pdb.set_trace() + +# Or use debugpy for VS Code +import debugpy +debugpy.listen(5678) +debugpy.wait_for_client() +``` + +### Worker Debugging + +```bash +# Run worker with verbose logging +celery -A worker.celery_app worker --loglevel=debug + +# Run single task synchronously +python -c "from worker.tasks import process_job; process_job('job-id')" +``` + +### Request Debugging + +```bash +# View all requests in real-time +docker compose logs -f api | grep -E "(GET|POST|PUT|DELETE)" + +# Test endpoint with curl +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{"input_path": "local:///storage/test.mp4", "output_path": "local:///storage/out.mp4"}' +``` + +## Common Development Tasks + +### Adding a New Endpoint + +1. Create router in `api/routers/` +2. Add route handler functions +3. Register router in `api/main.py` +4. Add tests in `tests/` +5. Update API documentation + +### Adding a New Model + +1. Create model in `api/models/` +2. Create Alembic migration +3. Add repository/service methods +4. Add tests + +### Adding a New Worker Task + +1. Define task in `worker/tasks.py` +2. Implement processor in `worker/processors/` +3. Add task tests +4. Update documentation + +## Troubleshooting Development Issues + +### Port Already in Use + +```bash +# Find process using port +lsof -i :8000 + +# Kill process +kill -9 +``` + +### Database Connection Issues + +```bash +# Check if PostgreSQL is running +docker compose ps postgres + +# View PostgreSQL logs +docker compose logs postgres + +# Reset database +docker compose down -v +docker compose up -d postgres +alembic upgrade head +``` + +### Redis Connection Issues + +```bash +# Test Redis connection +redis-cli ping + +# Check Redis logs +docker compose logs redis +``` + +### FFmpeg Not Found + +```bash +# Check FFmpeg installation +ffmpeg -version + +# Install on macOS +brew install ffmpeg + +# Install on Ubuntu +sudo apt install ffmpeg +``` + +## Next Steps + +- Read the [Architecture Guide](./ARCHITECTURE.md) +- Review the [Contributing Guide](./CONTRIBUTING.md) +- Explore [API Internals](./API_INTERNALS.md) diff --git a/docs/developer/README.md b/docs/developer/README.md new file mode 100644 index 0000000..1542a5a --- /dev/null +++ b/docs/developer/README.md @@ -0,0 +1,91 @@ +# Rendiff Developer Documentation + +Welcome to the Rendiff developer documentation. This guide is intended for developers who want to contribute to, extend, or deeply understand the Rendiff codebase. + +> **About Rendiff:** Rendiff is a production-grade REST API layer built on top of [FFmpeg](https://ffmpeg.org/), providing enterprise-ready media processing capabilities. + +## Documentation Index + +| Document | Description | +|----------|-------------| +| [Architecture](./ARCHITECTURE.md) | System design, components, and data flow | +| [Development Setup](./DEVELOPMENT_SETUP.md) | Local environment setup and tooling | +| [Contributing Guide](./CONTRIBUTING.md) | Code standards, PR process, and guidelines | +| [API Internals](./API_INTERNALS.md) | Deep dive into API implementation | +| [Worker System](./WORKER_SYSTEM.md) | Celery workers and job processing | +| [Security Implementation](./SECURITY_IMPLEMENTATION.md) | Security architecture and practices | +| [Testing Guide](./TESTING.md) | Test strategies and running tests | +| [Database Schema](./DATABASE_SCHEMA.md) | Data models and migrations | + +## Quick Links + +- **Repository:** [github.com/rendiffdev/rendiff-dev](https://github.com/rendiffdev/rendiff-dev) +- **User Manual:** [../user-manual/](../user-manual/README.md) +- **API Reference:** [../user-manual/API_REFERENCE.md](../user-manual/API_REFERENCE.md) + +## Technology Stack + +| Layer | Technology | Purpose | +|-------|------------|---------| +| API Framework | FastAPI 0.115+ | Async REST API with OpenAPI | +| Media Processing | FFmpeg 6.0+ | Video/audio encoding, transcoding | +| Task Queue | Celery 5.4+ | Distributed job processing | +| Message Broker | Redis 7.0+ / Valkey | Task queue and caching | +| Database | PostgreSQL 14+ | Primary data store | +| ORM | SQLAlchemy 2.0+ | Async database operations | +| Monitoring | Prometheus + Grafana | Metrics and dashboards | +| Reverse Proxy | Traefik 3.x | Load balancing, SSL termination | + +## Getting Started as a Developer + +```bash +# Clone the repository +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev + +# Start development environment +./setup.sh --development + +# Run tests +pytest tests/ -v + +# Check code quality +black api/ worker/ tests/ +flake8 api/ worker/ tests/ +mypy api/ worker/ +``` + +See [Development Setup](./DEVELOPMENT_SETUP.md) for detailed instructions. + +## Project Structure + +``` +rendiff-dev/ +├── api/ # FastAPI application +│ ├── main.py # Application entry point +│ ├── config.py # Configuration management +│ ├── dependencies.py # Dependency injection +│ ├── models/ # SQLAlchemy models +│ ├── routers/ # API route handlers +│ ├── services/ # Business logic +│ ├── middleware/ # Request/response middleware +│ └── utils/ # Utility functions +├── worker/ # Celery worker application +│ ├── tasks.py # Task definitions +│ ├── processors/ # Media processing logic +│ └── utils/ # Worker utilities +├── storage/ # Storage backend implementations +├── docker/ # Docker configurations +├── config/ # Application configs +├── tests/ # Test suite +├── docs/ # Documentation +│ ├── developer/ # Developer docs (you are here) +│ └── user-manual/ # End-user documentation +├── scripts/ # Utility scripts +└── monitoring/ # Prometheus/Grafana configs +``` + +## Support + +- **Issues:** [GitHub Issues](https://github.com/rendiffdev/rendiff-dev/issues) +- **Discussions:** [GitHub Discussions](https://github.com/rendiffdev/rendiff-dev/discussions) diff --git a/docs/user-manual/API_REFERENCE.md b/docs/user-manual/API_REFERENCE.md new file mode 100644 index 0000000..f655d69 --- /dev/null +++ b/docs/user-manual/API_REFERENCE.md @@ -0,0 +1,678 @@ +# Rendiff API Reference + +Complete reference documentation for all Rendiff API endpoints. + +## Base URL + +``` +http://localhost:8000/api/v1 +``` + +For production deployments with HTTPS: + +``` +https://your-domain.com/api/v1 +``` + +## Authentication + +All endpoints (except `/health`) require authentication when `ENABLE_API_KEYS=true`. + +Include your API key in the `X-API-Key` header: + +```bash +curl -H "X-API-Key: sk-your-api-key" https://api.example.com/api/v1/jobs +``` + +## Response Format + +All responses are JSON with consistent structure: + +### Success Response + +```json +{ + "id": "uuid", + "status": "string", + "data": {} +} +``` + +### Error Response + +```json +{ + "error": { + "code": "ERROR_CODE", + "message": "Human readable message", + "details": {} + } +} +``` + +## Rate Limits + +| Tier | Requests/Hour | Burst | +|------|---------------|-------| +| Basic | 500 | 50 | +| Pro | 2,000 | 200 | +| Enterprise | 10,000 | 1,000 | + +Rate limit headers are included in responses: + +``` +X-RateLimit-Limit: 1000 +X-RateLimit-Remaining: 999 +X-RateLimit-Reset: 1642000000 +``` + +--- + +## Health Endpoints + +### GET /health + +Check API and dependency health status. + +**Authentication:** Not required + +**Response:** + +```json +{ + "status": "healthy", + "version": "1.0.0", + "checks": { + "database": { + "status": "healthy", + "latency_ms": 2 + }, + "redis": { + "status": "healthy", + "latency_ms": 1 + }, + "ffmpeg": { + "status": "healthy", + "version": "6.1" + }, + "storage": { + "status": "healthy", + "backends": ["local", "s3"] + } + } +} +``` + +**Status Values:** +- `healthy` - All systems operational +- `degraded` - Some systems have issues +- `unhealthy` - Critical systems failing + +--- + +## Conversion Endpoints + +### POST /convert + +Create a new media conversion job. + +**Request Body:** + +```json +{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [ + { + "type": "transcode", + "video_codec": "vp9", + "audio_codec": "opus", + "crf": 23 + } + ], + "options": { + "priority": "normal", + "metadata": { + "title": "My Video" + } + }, + "webhook_url": "https://example.com/webhook" +} +``` + +**Parameters:** + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `input_path` | string | Yes | Source file URI | +| `output_path` | string | Yes | Destination file URI | +| `operations` | array | No | Processing operations | +| `options` | object | No | Global options | +| `webhook_url` | string | No | Notification URL | + +**Response (201 Created):** + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "pending", + "progress": 0.0, + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "created_at": "2025-01-15T10:30:00Z" +} +``` + +**Error Responses:** + +| Code | Description | +|------|-------------| +| 400 | Invalid request (validation error) | +| 401 | Missing or invalid API key | +| 429 | Rate limit exceeded | +| 500 | Internal server error | + +--- + +## Operations Reference + +### Transcode Operation + +Convert video/audio codecs and adjust quality. + +```json +{ + "type": "transcode", + "video_codec": "h264", + "audio_codec": "aac", + "width": 1920, + "height": 1080, + "crf": 23, + "preset": "medium", + "video_bitrate": "5M", + "audio_bitrate": "128k", + "fps": 30 +} +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `video_codec` | string | - | h264, h265, vp9, av1, copy | +| `audio_codec` | string | - | aac, mp3, opus, flac, copy | +| `width` | integer | - | Output width (must be even) | +| `height` | integer | - | Output height (must be even) | +| `crf` | integer | 23 | Quality (0-51, lower=better) | +| `preset` | string | medium | ultrafast to veryslow | +| `video_bitrate` | string | - | e.g., "5M", "2500k" | +| `audio_bitrate` | string | - | e.g., "128k", "320k" | +| `fps` | number | - | Frame rate (1-120) | + +### Trim Operation + +Extract a portion of the media file. + +```json +{ + "type": "trim", + "start": 60, + "duration": 30 +} +``` + +or + +```json +{ + "type": "trim", + "start": "1:00", + "end": "1:30" +} +``` + +| Parameter | Type | Description | +|-----------|------|-------------| +| `start` | number/string | Start time in seconds or HH:MM:SS | +| `duration` | number/string | Duration (if not using end) | +| `end` | number/string | End time (if not using duration) | + +### Filter Operation + +Apply video/audio filters. + +```json +{ + "type": "filter", + "name": "denoise", + "params": { + "strength": "medium" + } +} +``` + +**Available Filters:** + +| Filter | Description | Parameters | +|--------|-------------|------------| +| `denoise` | Reduce noise | `strength`: low/medium/high | +| `deinterlace` | Remove interlacing | - | +| `stabilize` | Stabilize shaky video | `strength`: 0.0-1.0 | +| `sharpen` | Sharpen video | `amount`: 0.0-2.0 | +| `blur` | Blur video | `radius`: 1-20 | +| `brightness` | Adjust brightness | `value`: -1.0 to 1.0 | +| `contrast` | Adjust contrast | `value`: 0.0 to 2.0 | +| `saturation` | Adjust saturation | `value`: 0.0 to 3.0 | + +### Watermark Operation + +Add image overlay to video. + +```json +{ + "type": "watermark", + "image": "local:///storage/logo.png", + "position": "bottom-right", + "opacity": 0.8, + "scale": 0.1 +} +``` + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `image` | string | - | Watermark image URI | +| `position` | string | bottom-right | top-left, top-right, bottom-left, bottom-right, center | +| `opacity` | number | 0.8 | Opacity (0.0-1.0) | +| `scale` | number | 0.1 | Scale relative to video | + +--- + +## Streaming Endpoints + +### POST /stream + +Create HLS or DASH streaming output. + +**Request Body:** + +```json +{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/stream/", + "options": { + "format": "hls", + "segment_duration": 6, + "variants": [ + {"height": 360, "bitrate": "800k"}, + {"height": 480, "bitrate": "1400k"}, + {"height": 720, "bitrate": "2800k"}, + {"height": 1080, "bitrate": "5000k"} + ] + } +} +``` + +**Parameters:** + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `format` | string | hls | hls or dash | +| `segment_duration` | integer | 6 | Segment length in seconds | +| `variants` | array | - | Quality variants | + +**Variant Parameters:** + +| Field | Type | Description | +|-------|------|-------------| +| `height` | integer | Output height | +| `bitrate` | string | Target bitrate | +| `width` | integer | Output width (optional) | + +--- + +## Analysis Endpoints + +### POST /analyze + +Analyze video quality metrics. + +**Request Body:** + +```json +{ + "reference_path": "local:///storage/original.mp4", + "distorted_path": "local:///storage/compressed.mp4", + "metrics": ["vmaf", "psnr", "ssim"] +} +``` + +**Response:** + +```json +{ + "id": "job-id", + "status": "completed", + "results": { + "vmaf": { + "score": 95.234, + "min": 89.5, + "max": 98.7 + }, + "psnr": { + "y": 42.5, + "u": 48.2, + "v": 49.1 + }, + "ssim": { + "score": 0.987 + } + } +} +``` + +--- + +## Job Management Endpoints + +### GET /jobs + +List all jobs for the authenticated API key. + +**Query Parameters:** + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `status` | string | - | Filter by status | +| `page` | integer | 1 | Page number | +| `page_size` | integer | 20 | Items per page (max 100) | + +**Response:** + +```json +{ + "items": [ + { + "id": "job-id-1", + "status": "completed", + "progress": 100.0, + "created_at": "2025-01-15T10:30:00Z" + } + ], + "total": 42, + "page": 1, + "page_size": 20, + "has_next": true, + "has_prev": false +} +``` + +### GET /jobs/{id} + +Get detailed job status. + +**Response:** + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "processing", + "progress": 67.5, + "stage": "encoding", + "fps": 45.2, + "eta_seconds": 120, + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [...], + "created_at": "2025-01-15T10:30:00Z", + "started_at": "2025-01-15T10:30:05Z", + "worker_id": "worker-1" +} +``` + +### DELETE /jobs/{id} + +Cancel a pending or processing job. + +**Response (200 OK):** + +```json +{ + "id": "job-id", + "status": "cancelled", + "message": "Job cancelled successfully" +} +``` + +**Error Responses:** + +| Code | Description | +|------|-------------| +| 404 | Job not found | +| 409 | Job already completed/failed | + +--- + +## Admin Endpoints + +> Requires admin API key (configured via `ADMIN_API_KEYS`) + +### POST /admin/api-keys + +Create a new API key. + +**Request:** + +```json +{ + "name": "My Application", + "description": "Production API key", + "rate_limit": 2000, + "expires_in_days": 365 +} +``` + +**Response:** + +```json +{ + "id": "key-id", + "key": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "name": "My Application", + "created_at": "2025-01-15T10:30:00Z", + "expires_at": "2026-01-15T10:30:00Z" +} +``` + +### GET /admin/api-keys + +List all API keys. + +### DELETE /admin/api-keys/{id} + +Revoke an API key. + +### GET /admin/stats + +Get system statistics. + +**Response:** + +```json +{ + "jobs": { + "total": 10000, + "pending": 5, + "processing": 3, + "completed": 9800, + "failed": 192 + }, + "workers": { + "active": 4, + "idle": 2 + }, + "storage": { + "used_bytes": 1099511627776, + "backends": { + "local": {"files": 500, "bytes": 549755813888}, + "s3": {"files": 200, "bytes": 549755813888} + } + } +} +``` + +--- + +## Error Codes + +| Code | HTTP Status | Description | +|------|-------------|-------------| +| `VALIDATION_ERROR` | 400 | Invalid request parameters | +| `AUTH_ERROR` | 401 | Authentication failed | +| `AUTHZ_ERROR` | 403 | Insufficient permissions | +| `NOT_FOUND` | 404 | Resource not found | +| `RATE_LIMIT_ERROR` | 429 | Too many requests | +| `PROCESSING_ERROR` | 500 | Processing failed | +| `STORAGE_ERROR` | 500 | Storage operation failed | +| `INTERNAL_ERROR` | 500 | Unexpected error | + +--- + +## Webhooks + +When a job completes, Rendiff sends a POST request to your webhook URL. + +### Completion Webhook + +```json +{ + "event": "job.completed", + "timestamp": "2025-01-15T10:35:00Z", + "job_id": "550e8400-e29b-41d4-a716-446655440000", + "status": "completed", + "output_path": "local:///storage/output.webm", + "processing_time": 300.5, + "metrics": { + "input_size": 104857600, + "output_size": 52428800, + "compression_ratio": 2.0 + } +} +``` + +### Failure Webhook + +```json +{ + "event": "job.failed", + "timestamp": "2025-01-15T10:35:00Z", + "job_id": "550e8400-e29b-41d4-a716-446655440000", + "status": "failed", + "error": "Processing failed" +} +``` + +--- + +## SDK Examples + +### Python + +```python +import requests + +API_URL = "http://localhost:8000/api/v1" +API_KEY = "sk-your-api-key" + +headers = { + "X-API-Key": API_KEY, + "Content-Type": "application/json" +} + +# Create job +response = requests.post( + f"{API_URL}/convert", + headers=headers, + json={ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [ + {"type": "transcode", "video_codec": "vp9"} + ] + } +) + +job = response.json() +print(f"Job created: {job['id']}") + +# Poll for completion +import time +while True: + status = requests.get( + f"{API_URL}/jobs/{job['id']}", + headers=headers + ).json() + + if status["status"] in ["completed", "failed"]: + break + + print(f"Progress: {status['progress']}%") + time.sleep(5) +``` + +### JavaScript + +```javascript +const API_URL = 'http://localhost:8000/api/v1'; +const API_KEY = 'sk-your-api-key'; + +async function convertVideo() { + // Create job + const response = await fetch(`${API_URL}/convert`, { + method: 'POST', + headers: { + 'X-API-Key': API_KEY, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + input_path: 'local:///storage/input.mp4', + output_path: 'local:///storage/output.webm', + operations: [ + { type: 'transcode', video_codec: 'vp9' } + ] + }) + }); + + const job = await response.json(); + console.log(`Job created: ${job.id}`); + + // Poll for completion + while (true) { + const status = await fetch(`${API_URL}/jobs/${job.id}`, { + headers: { 'X-API-Key': API_KEY } + }).then(r => r.json()); + + if (['completed', 'failed'].includes(status.status)) { + return status; + } + + console.log(`Progress: ${status.progress}%`); + await new Promise(r => setTimeout(r, 5000)); + } +} +``` + +### cURL + +```bash +# Create conversion job +curl -X POST http://localhost:8000/api/v1/convert \ + -H "X-API-Key: sk-your-api-key" \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [{"type": "transcode", "video_codec": "vp9"}] + }' + +# Check job status +curl http://localhost:8000/api/v1/jobs/JOB_ID \ + -H "X-API-Key: sk-your-api-key" +``` diff --git a/docs/user-manual/CONFIGURATION.md b/docs/user-manual/CONFIGURATION.md new file mode 100644 index 0000000..5c62208 --- /dev/null +++ b/docs/user-manual/CONFIGURATION.md @@ -0,0 +1,503 @@ +# Configuration Guide + +This guide covers all configuration options for Rendiff, including environment variables, storage backends, and performance tuning. + +## Configuration Methods + +Rendiff can be configured using: + +1. **Environment variables** (highest priority) +2. **`.env` file** in the project root +3. **Default values** (lowest priority) + +## Quick Start Configuration + +Create a `.env` file based on the template: + +```bash +cp .env.example .env +``` + +Edit the file with your settings: + +```bash +# Minimal production configuration +DATABASE_URL=postgresql://rendiff_user:secure_password@postgres:5432/rendiff +REDIS_URL=redis://redis:6379/0 +ENABLE_API_KEYS=true +ADMIN_API_KEYS=your-admin-key-here +``` + +--- + +## Core Settings + +### Application + +| Variable | Default | Description | +|----------|---------|-------------| +| `DEBUG` | `false` | Enable debug mode (shows docs, detailed errors) | +| `TESTING` | `false` | Enable test mode | +| `VERSION` | `1.0.0` | Application version | + +### API Server + +| Variable | Default | Description | +|----------|---------|-------------| +| `API_HOST` | `0.0.0.0` | Listen address | +| `API_PORT` | `8000` | Listen port | +| `API_WORKERS` | `4` | Number of uvicorn workers | +| `API_LOG_LEVEL` | `info` | Log level (debug, info, warning, error) | +| `API_RELOAD` | `false` | Enable auto-reload (development) | + +**Example:** + +```bash +API_HOST=0.0.0.0 +API_PORT=8000 +API_WORKERS=8 +API_LOG_LEVEL=info +``` + +--- + +## Database Configuration + +### PostgreSQL (Recommended) + +| Variable | Default | Description | +|----------|---------|-------------| +| `DATABASE_URL` | - | PostgreSQL connection string | +| `DATABASE_POOL_SIZE` | `20` | Connection pool size | +| `DATABASE_MAX_OVERFLOW` | `40` | Max overflow connections | +| `DATABASE_POOL_TIMEOUT` | `30` | Connection timeout (seconds) | + +**Connection String Format:** + +``` +postgresql://username:password@host:port/database +``` + +**Example:** + +```bash +DATABASE_URL=postgresql://rendiff_user:secure_password@postgres:5432/rendiff +DATABASE_POOL_SIZE=20 +DATABASE_MAX_OVERFLOW=40 +``` + +### SQLite (Development Only) + +```bash +DATABASE_URL=sqlite+aiosqlite:///data/rendiff.db +``` + +> **Warning:** SQLite does not support concurrent writes. Use PostgreSQL for production. + +--- + +## Redis Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `REDIS_URL` | `redis://localhost:6379/0` | Redis connection URL | +| `REDIS_MAX_CONNECTIONS` | `100` | Maximum connections | + +**URL Format:** + +``` +redis://[:password@]host:port/db +``` + +**Examples:** + +```bash +# Without password +REDIS_URL=redis://redis:6379/0 + +# With password +REDIS_URL=redis://:your_password@redis:6379/0 + +# Redis Cluster +REDIS_URL=redis://node1:6379,node2:6379,node3:6379/0 +``` + +--- + +## Security Configuration + +### API Key Authentication + +| Variable | Default | Description | +|----------|---------|-------------| +| `ENABLE_API_KEYS` | `true` | Require API keys | +| `ADMIN_API_KEYS` | - | Comma-separated admin keys | +| `API_KEY_LENGTH` | `32` | Generated key length | +| `API_KEY_EXPIRY_DAYS` | `365` | Default key expiration | + +**Example:** + +```bash +ENABLE_API_KEYS=true +ADMIN_API_KEYS=sk-admin-key-1,sk-admin-key-2 +``` + +### Rate Limiting + +| Variable | Default | Description | +|----------|---------|-------------| +| `RATE_LIMIT_ENABLED` | `true` | Enable rate limiting | +| `RATE_LIMIT_CALLS` | `100` | Requests per period | +| `RATE_LIMIT_PERIOD` | `3600` | Period in seconds | +| `CONVERT_RATE_LIMIT` | `200` | /convert endpoint limit | +| `ANALYZE_RATE_LIMIT` | `100` | /analyze endpoint limit | +| `STREAM_RATE_LIMIT` | `50` | /stream endpoint limit | + +**Example:** + +```bash +RATE_LIMIT_ENABLED=true +RATE_LIMIT_CALLS=1000 +RATE_LIMIT_PERIOD=3600 +``` + +### CORS and Security Headers + +| Variable | Default | Description | +|----------|---------|-------------| +| `API_CORS_ORIGINS` | `*` | Allowed origins (comma-separated) | +| `API_TRUSTED_HOSTS` | `*` | Trusted hosts | +| `ENABLE_SECURITY_HEADERS` | `true` | Add security headers | + +**Example:** + +```bash +# Restrict to specific origins +API_CORS_ORIGINS=https://app.example.com,https://admin.example.com + +# Multiple trusted hosts +API_TRUSTED_HOSTS=api.example.com,*.example.com +``` + +### IP Whitelisting + +| Variable | Default | Description | +|----------|---------|-------------| +| `ENABLE_IP_WHITELIST` | `false` | Enable IP filtering | +| `IP_WHITELIST` | - | Allowed IPs/CIDRs | + +**Example:** + +```bash +ENABLE_IP_WHITELIST=true +IP_WHITELIST=10.0.0.0/8,192.168.0.0/16,203.0.113.0/24 +``` + +--- + +## Storage Configuration + +### Storage Paths + +| Variable | Default | Description | +|----------|---------|-------------| +| `STORAGE_CONFIG` | `/app/config/storage.yml` | Storage config file | +| `STORAGE_PATH` | `./storage` | Default storage directory | +| `TEMP_PATH` | `/tmp/rendiff` | Temporary files directory | + +### Storage Configuration File + +Create `config/storage.yml`: + +```yaml +# Default backend for URI without prefix +default_backend: local + +backends: + # Local filesystem + local: + type: local + base_path: /storage + + # AWS S3 + s3: + type: s3 + bucket: my-rendiff-bucket + region: us-east-1 + access_key: ${AWS_ACCESS_KEY_ID} + secret_key: ${AWS_SECRET_ACCESS_KEY} + + # Azure Blob Storage + azure: + type: azure + container: rendiff-container + connection_string: ${AZURE_STORAGE_CONNECTION_STRING} + + # Google Cloud Storage + gcs: + type: gcs + bucket: my-rendiff-bucket + credentials_file: /app/credentials/gcs.json + +policies: + # Allowed backends for input files + input_backends: + - local + - s3 + - azure + - gcs + + # Allowed backends for output files + output_backends: + - local + - s3 + - azure +``` + +### Cloud Storage Environment Variables + +**AWS S3:** + +```bash +AWS_ACCESS_KEY_ID=AKIA... +AWS_SECRET_ACCESS_KEY=... +AWS_DEFAULT_REGION=us-east-1 +S3_BUCKET=my-bucket +``` + +**Azure Blob:** + +```bash +AZURE_STORAGE_CONNECTION_STRING=DefaultEndpointsProtocol=https;AccountName=... +AZURE_CONTAINER_NAME=rendiff-container +``` + +**Google Cloud:** + +```bash +GOOGLE_APPLICATION_CREDENTIALS=/app/credentials/gcs.json +GCS_BUCKET=my-bucket +``` + +--- + +## Processing Configuration + +### Worker Settings + +| Variable | Default | Description | +|----------|---------|-------------| +| `WORKER_CONCURRENCY` | `4` | Concurrent jobs per worker | +| `MAX_CONCURRENT_JOBS` | `10` | Total concurrent jobs | +| `JOB_TIMEOUT` | `3600` | Maximum job duration (seconds) | +| `JOB_RETRY_MAX` | `3` | Maximum retry attempts | + +**Example:** + +```bash +WORKER_CONCURRENCY=4 +MAX_CONCURRENT_JOBS=20 +JOB_TIMEOUT=7200 +``` + +### FFmpeg Settings + +| Variable | Default | Description | +|----------|---------|-------------| +| `FFMPEG_PATH` | `/usr/bin/ffmpeg` | FFmpeg binary path | +| `FFPROBE_PATH` | `/usr/bin/ffprobe` | FFprobe binary path | +| `FFMPEG_THREADS` | `0` | Threads (0=auto) | +| `FFMPEG_HARDWARE_ACCELERATION` | `auto` | Hardware acceleration | + +**Hardware Acceleration Options:** + +| Value | Description | +|-------|-------------| +| `auto` | Auto-detect available hardware | +| `none` | Disable hardware acceleration | +| `nvenc` | NVIDIA NVENC | +| `qsv` | Intel Quick Sync | +| `vaapi` | VA-API (Linux) | +| `videotoolbox` | macOS VideoToolbox | + +**Example:** + +```bash +FFMPEG_THREADS=0 +FFMPEG_HARDWARE_ACCELERATION=nvenc +ENABLE_GPU_WORKERS=true +``` + +### Resource Limits + +| Variable | Default | Description | +|----------|---------|-------------| +| `MAX_FILE_SIZE` | `10737418240` | Max upload size (10GB) | +| `MAX_RESOLUTION` | `7680x4320` | Max output resolution | +| `MAX_BITRATE` | `100M` | Max output bitrate | +| `MAX_OPERATIONS` | `50` | Max operations per job | + +**Example:** + +```bash +MAX_FILE_SIZE=21474836480 # 20GB +MAX_RESOLUTION=3840x2160 # 4K +MAX_BITRATE=50M +``` + +--- + +## Monitoring Configuration + +### Prometheus Metrics + +| Variable | Default | Description | +|----------|---------|-------------| +| `ENABLE_METRICS` | `true` | Enable /metrics endpoint | +| `METRICS_PORT` | `9090` | Prometheus port | + +### Logging + +| Variable | Default | Description | +|----------|---------|-------------| +| `LOG_LEVEL` | `INFO` | Log level | +| `LOG_FORMAT` | `json` | Log format (json, text) | +| `LOG_FILE` | - | Log file path (optional) | + +**Example:** + +```bash +LOG_LEVEL=INFO +LOG_FORMAT=json +``` + +### Health Checks + +| Variable | Default | Description | +|----------|---------|-------------| +| `HEALTH_CHECK_INTERVAL` | `30` | Check interval (seconds) | +| `HEALTH_CHECK_TIMEOUT` | `10` | Check timeout (seconds) | + +--- + +## Docker Compose Configuration + +### Environment File + +The Docker Compose files read from `.env`: + +```bash +# .env +COMPOSE_PROJECT_NAME=rendiff +POSTGRES_PASSWORD=secure_password +GRAFANA_PASSWORD=admin_password +``` + +### Volume Configuration + +```bash +# Data persistence paths +POSTGRES_DATA_PATH=./data/postgres +REDIS_DATA_PATH=./data/redis +STORAGE_PATH=./storage +``` + +### Network Configuration + +```bash +# Docker network +DOCKER_NETWORK=rendiff-network +``` + +--- + +## Production Configuration Example + +Complete production `.env` file: + +```bash +# ============================================================================= +# RENDIFF PRODUCTION CONFIGURATION +# ============================================================================= + +# Application +DEBUG=false +API_LOG_LEVEL=info +API_WORKERS=8 + +# Database +DATABASE_URL=postgresql://rendiff_user:CHANGE_THIS_PASSWORD@postgres:5432/rendiff +DATABASE_POOL_SIZE=30 +DATABASE_MAX_OVERFLOW=50 + +# Redis +REDIS_URL=redis://:REDIS_PASSWORD@redis:6379/0 +REDIS_MAX_CONNECTIONS=200 + +# Security +ENABLE_API_KEYS=true +ADMIN_API_KEYS=sk-your-secure-admin-key +API_CORS_ORIGINS=https://app.yourdomain.com +RATE_LIMIT_CALLS=2000 +RATE_LIMIT_PERIOD=3600 + +# Storage +STORAGE_PATH=/data/storage +TEMP_PATH=/tmp/rendiff + +# Workers +WORKER_CONCURRENCY=4 +MAX_CONCURRENT_JOBS=20 +JOB_TIMEOUT=3600 + +# FFmpeg +FFMPEG_THREADS=0 +FFMPEG_HARDWARE_ACCELERATION=auto + +# Monitoring +ENABLE_METRICS=true +LOG_LEVEL=INFO +LOG_FORMAT=json + +# Docker +COMPOSE_PROJECT_NAME=rendiff +POSTGRES_PASSWORD=CHANGE_THIS_PASSWORD +GRAFANA_PASSWORD=CHANGE_THIS_PASSWORD +``` + +--- + +## Configuration Validation + +Rendiff validates configuration on startup. Check logs for warnings: + +```bash +docker compose logs api | grep -i "config\|warn\|error" +``` + +Common validation warnings: + +| Warning | Cause | Solution | +|---------|-------|----------| +| `DEBUG mode enabled` | DEBUG=true | Set DEBUG=false for production | +| `Default password detected` | Using default DB password | Change POSTGRES_PASSWORD | +| `CORS wildcard` | API_CORS_ORIGINS=* | Set specific origins | +| `API keys disabled` | ENABLE_API_KEYS=false | Enable for production | + +--- + +## Environment-Specific Files + +For different environments, create separate files: + +``` +.env.development +.env.staging +.env.production +``` + +Load with Docker Compose: + +```bash +docker compose --env-file .env.production up -d +``` diff --git a/docs/user-manual/GETTING_STARTED.md b/docs/user-manual/GETTING_STARTED.md new file mode 100644 index 0000000..0a80a0d --- /dev/null +++ b/docs/user-manual/GETTING_STARTED.md @@ -0,0 +1,432 @@ +# Getting Started with Rendiff + +This guide walks you through deploying Rendiff and making your first API calls. + +## Prerequisites + +Before starting, ensure you have: + +- **Docker** 20.10 or later +- **Docker Compose** 2.0 or later +- **curl** or similar HTTP client (for testing) + +Verify Docker is installed: + +```bash +docker --version +# Docker version 24.x.x + +docker compose version +# Docker Compose version v2.x.x +``` + +## Step 1: Deploy Rendiff + +### Option A: Quick Development Setup + +For local development and testing: + +```bash +# Clone the repository +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev + +# Start with development configuration +./setup.sh --development +``` + +This starts Rendiff with: +- SQLite database (no PostgreSQL needed) +- Single API worker +- Debug mode enabled +- No authentication required + +### Option B: Production Setup + +For production deployments: + +```bash +# Clone the repository +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev + +# Start with production configuration +docker compose -f compose.prod.yml up -d +``` + +This includes: +- PostgreSQL database +- Redis queue +- Multiple workers +- Monitoring (Prometheus/Grafana) +- SSL via Traefik + +### Verify Deployment + +Check that all services are running: + +```bash +# Check service status +docker compose ps + +# Check API health +curl http://localhost:8000/api/v1/health +``` + +Expected response: + +```json +{ + "status": "healthy", + "version": "1.0.0", + "checks": { + "database": {"status": "healthy"}, + "redis": {"status": "healthy"}, + "ffmpeg": {"status": "healthy"} + } +} +``` + +## Step 2: Understand Storage URIs + +Rendiff uses storage URIs to specify file locations: + +| URI Format | Description | Example | +|------------|-------------|---------| +| `local://` | Local filesystem | `local:///storage/video.mp4` | +| `s3://` | Amazon S3 | `s3://bucket/path/video.mp4` | +| `azure://` | Azure Blob | `azure://container/video.mp4` | +| `gcs://` | Google Cloud | `gcs://bucket/video.mp4` | + +For this guide, we'll use local storage. + +### Prepare a Test File + +Place a test video file in the storage directory: + +```bash +# Create storage directory +mkdir -p storage + +# Copy a test file (or download a sample) +cp /path/to/your/video.mp4 storage/input.mp4 + +# Verify the file is accessible +ls -la storage/ +``` + +## Step 3: Create Your First Job + +### Basic Conversion + +Convert a video to WebM format: + +```bash +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [ + { + "type": "transcode", + "video_codec": "vp9", + "audio_codec": "opus" + } + ] + }' +``` + +Response: + +```json +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "pending", + "progress": 0.0, + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "created_at": "2025-01-15T10:30:00Z" +} +``` + +Save the job `id` for the next step. + +### Check Job Status + +Poll the job status until completion: + +```bash +# Replace with your job ID +JOB_ID="550e8400-e29b-41d4-a716-446655440000" + +curl http://localhost:8000/api/v1/jobs/$JOB_ID +``` + +Responses during processing: + +```json +// Processing +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "processing", + "progress": 45.5, + "stage": "encoding" +} + +// Completed +{ + "id": "550e8400-e29b-41d4-a716-446655440000", + "status": "completed", + "progress": 100.0, + "completed_at": "2025-01-15T10:32:45Z", + "processing_time": 165.3 +} +``` + +### Verify Output + +Check that the output file was created: + +```bash +ls -la storage/output.webm +``` + +## Step 4: Common Operations + +### Trim a Video + +Extract a 30-second clip starting at 1 minute: + +```bash +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/clip.mp4", + "operations": [ + { + "type": "trim", + "start": 60, + "duration": 30 + } + ] + }' +``` + +### Change Resolution + +Resize video to 720p: + +```bash +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/720p.mp4", + "operations": [ + { + "type": "transcode", + "video_codec": "h264", + "width": 1280, + "height": 720 + } + ] + }' +``` + +### Adjust Quality + +Use CRF (Constant Rate Factor) for quality control: + +```bash +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/highquality.mp4", + "operations": [ + { + "type": "transcode", + "video_codec": "h264", + "crf": 18, + "preset": "slow" + } + ] + }' +``` + +CRF values: 0 (lossless) to 51 (worst). Recommended: 18-23. + +### Generate HLS Stream + +Create adaptive streaming output: + +```bash +curl -X POST http://localhost:8000/api/v1/stream \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/stream/", + "options": { + "format": "hls", + "variants": [ + {"height": 360, "bitrate": "800k"}, + {"height": 480, "bitrate": "1400k"}, + {"height": 720, "bitrate": "2800k"}, + {"height": 1080, "bitrate": "5000k"} + ] + } + }' +``` + +### Analyze Video Quality + +Measure VMAF, PSNR, and SSIM: + +```bash +curl -X POST http://localhost:8000/api/v1/analyze \ + -H "Content-Type: application/json" \ + -d '{ + "reference_path": "local:///storage/original.mp4", + "distorted_path": "local:///storage/compressed.mp4", + "metrics": ["vmaf", "psnr", "ssim"] + }' +``` + +## Step 5: Use Webhooks (Optional) + +Receive notifications when jobs complete: + +```bash +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.mp4", + "webhook_url": "https://your-server.com/webhook", + "operations": [ + {"type": "transcode", "video_codec": "h264"} + ] + }' +``` + +Webhook payload on completion: + +```json +{ + "event": "job.completed", + "job_id": "550e8400-e29b-41d4-a716-446655440000", + "status": "completed", + "output_path": "local:///storage/output.mp4", + "processing_time": 120.5, + "metrics": { + "vmaf": 95.2 + } +} +``` + +## Step 6: Enable Authentication (Production) + +For production, enable API key authentication: + +### Generate an API Key + +```bash +# Using the admin endpoint (requires ADMIN_API_KEYS environment variable) +curl -X POST http://localhost:8000/api/v1/admin/api-keys \ + -H "X-API-Key: your-admin-key" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "My Application", + "rate_limit": 1000 + }' +``` + +Response: + +```json +{ + "id": "key-id", + "key": "sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "name": "My Application" +} +``` + +**Important:** Save the `key` value immediately. It won't be shown again. + +### Use the API Key + +Include the key in all requests: + +```bash +curl -X POST http://localhost:8000/api/v1/convert \ + -H "X-API-Key: sk-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" \ + -H "Content-Type: application/json" \ + -d '{...}' +``` + +## Step 7: Monitor Jobs + +### List All Jobs + +```bash +curl http://localhost:8000/api/v1/jobs +``` + +### Filter by Status + +```bash +# Get only processing jobs +curl "http://localhost:8000/api/v1/jobs?status=processing" + +# Get failed jobs +curl "http://localhost:8000/api/v1/jobs?status=failed" +``` + +### Cancel a Job + +```bash +curl -X DELETE http://localhost:8000/api/v1/jobs/JOB_ID +``` + +## Next Steps + +Now that you have Rendiff running: + +1. **Read the [API Reference](./API_REFERENCE.md)** for complete endpoint documentation +2. **Configure [Storage Backends](./STORAGE.md)** for cloud storage (S3, Azure, GCS) +3. **Set up [Authentication](./AUTHENTICATION.md)** for production +4. **Review [Configuration](./CONFIGURATION.md)** for tuning options +5. **Check [Troubleshooting](./TROUBLESHOOTING.md)** if you encounter issues + +## Quick Reference + +### Job Statuses + +| Status | Description | +|--------|-------------| +| `pending` | Job created, waiting for worker | +| `processing` | Currently being processed | +| `completed` | Successfully finished | +| `failed` | Processing failed | +| `cancelled` | Cancelled by user | + +### Video Codecs + +| Codec | Description | Use Case | +|-------|-------------|----------| +| `h264` | H.264/AVC | Broad compatibility | +| `h265` | H.265/HEVC | Better compression | +| `vp9` | VP9 | Web (WebM) | +| `av1` | AV1 | Best compression | + +### Presets (Speed vs Quality) + +| Preset | Speed | Quality | Use Case | +|--------|-------|---------|----------| +| `ultrafast` | Fastest | Lowest | Testing | +| `fast` | Fast | Low | Real-time | +| `medium` | Balanced | Medium | Default | +| `slow` | Slow | High | Quality | +| `veryslow` | Slowest | Highest | Archival | diff --git a/docs/user-manual/README.md b/docs/user-manual/README.md new file mode 100644 index 0000000..f3f7d4e --- /dev/null +++ b/docs/user-manual/README.md @@ -0,0 +1,167 @@ +# Rendiff User Manual + +Welcome to the Rendiff User Manual. This guide provides everything you need to deploy, configure, and use Rendiff for your media processing needs. + +> **About Rendiff:** Rendiff is a production-ready REST API for media processing, powered by [FFmpeg](https://ffmpeg.org/). It enables you to transcode, analyze, and stream video/audio files through a simple HTTP interface. + +## Documentation Index + +| Document | Description | +|----------|-------------| +| [Getting Started](./GETTING_STARTED.md) | Quick start guide and first API call | +| [Installation](./INSTALLATION.md) | Deployment options and setup | +| [Configuration](./CONFIGURATION.md) | Environment variables and settings | +| [API Reference](./API_REFERENCE.md) | Complete endpoint documentation | +| [Authentication](./AUTHENTICATION.md) | API keys and security | +| [Storage Backends](./STORAGE.md) | File storage configuration | +| [Webhooks](./WEBHOOKS.md) | Event notifications | +| [Troubleshooting](./TROUBLESHOOTING.md) | Common issues and solutions | + +## Quick Overview + +### What Can Rendiff Do? + +| Feature | Description | +|---------|-------------| +| **Transcode** | Convert between video/audio formats (H.264, H.265, VP9, AV1) | +| **Quality Analysis** | Measure VMAF, PSNR, SSIM scores | +| **Streaming** | Generate HLS and DASH adaptive streams | +| **Trim/Cut** | Extract segments from media files | +| **Filters** | Apply denoise, sharpen, stabilize effects | +| **Batch Processing** | Process multiple files concurrently | + +### Supported Formats + +**Video Input:** MP4, AVI, MOV, MKV, WebM, FLV, WMV, MPEG, TS +**Video Output:** MP4, WebM, MKV, HLS, DASH +**Audio Input:** MP3, WAV, FLAC, AAC, OGG, WMA, M4A +**Audio Output:** MP3, AAC, FLAC, WAV, OGG + +### Architecture Overview + +``` +Your Application ──► Rendiff API ──► FFmpeg ──► Output Files + │ │ + │ ▼ + │ Job Queue (Redis) + │ │ + │ ▼ + │ Worker (Processing) + │ │ + ◄────────────────┘ + Webhook Notification +``` + +## Getting Started in 5 Minutes + +### 1. Deploy Rendiff + +```bash +# Clone the repository +git clone https://github.com/rendiffdev/rendiff-dev.git +cd rendiff-dev + +# Start all services +docker compose up -d + +# Verify it's running +curl http://localhost:8000/api/v1/health +``` + +### 2. Create Your First Job + +```bash +# Submit a conversion job +curl -X POST http://localhost:8000/api/v1/convert \ + -H "Content-Type: application/json" \ + -d '{ + "input_path": "local:///storage/input.mp4", + "output_path": "local:///storage/output.webm", + "operations": [ + {"type": "transcode", "video_codec": "vp9", "audio_codec": "opus"} + ] + }' + +# Response: +# {"id": "550e8400-e29b-41d4-a716-446655440000", "status": "pending", ...} +``` + +### 3. Check Job Status + +```bash +curl http://localhost:8000/api/v1/jobs/550e8400-e29b-41d4-a716-446655440000 + +# Response: +# {"id": "...", "status": "completed", "progress": 100.0, ...} +``` + +See [Getting Started](./GETTING_STARTED.md) for a complete walkthrough. + +## System Requirements + +### Minimum Requirements + +| Resource | Requirement | +|----------|-------------| +| CPU | 4 cores | +| RAM | 8 GB | +| Storage | 50 GB SSD | +| Docker | 20.10+ | + +### Recommended for Production + +| Resource | Recommendation | +|----------|---------------| +| CPU | 8+ cores (16+ for 4K) | +| RAM | 32 GB (64 GB for 4K/8K) | +| GPU | NVIDIA RTX/Quadro (optional) | +| Storage | 500 GB+ NVMe SSD | +| Network | 1 Gbps+ | + +## API Endpoints + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/api/v1/health` | GET | Health check | +| `/api/v1/convert` | POST | Create conversion job | +| `/api/v1/analyze` | POST | Create analysis job | +| `/api/v1/stream` | POST | Create streaming job | +| `/api/v1/jobs` | GET | List jobs | +| `/api/v1/jobs/{id}` | GET | Get job status | +| `/api/v1/jobs/{id}` | DELETE | Cancel job | + +See [API Reference](./API_REFERENCE.md) for complete documentation. + +## Configuration Quick Reference + +### Essential Environment Variables + +```bash +# Database +DATABASE_URL=postgresql://user:pass@host:5432/rendiff + +# Redis +REDIS_URL=redis://host:6379/0 + +# Security +ENABLE_API_KEYS=true +ADMIN_API_KEYS=your-admin-key + +# Processing +MAX_CONCURRENT_JOBS=10 +WORKER_CONCURRENCY=4 +``` + +See [Configuration](./CONFIGURATION.md) for all options. + +## Support + +- **Documentation:** You're reading it! +- **Issues:** [GitHub Issues](https://github.com/rendiffdev/rendiff-dev/issues) +- **Discussions:** [GitHub Discussions](https://github.com/rendiffdev/rendiff-dev/discussions) + +## License + +Rendiff is licensed under the MIT License. + +**FFmpeg Notice:** Rendiff uses FFmpeg for media processing. FFmpeg is licensed under LGPL/GPL. See [ffmpeg.org/legal.html](https://ffmpeg.org/legal.html) for details. diff --git a/docs/user-manual/TROUBLESHOOTING.md b/docs/user-manual/TROUBLESHOOTING.md new file mode 100644 index 0000000..7858b5b --- /dev/null +++ b/docs/user-manual/TROUBLESHOOTING.md @@ -0,0 +1,502 @@ +# Troubleshooting Guide + +This guide helps you diagnose and resolve common issues with Rendiff. + +## Quick Diagnostics + +### Check System Health + +```bash +# API health check +curl http://localhost:8000/api/v1/health + +# Docker service status +docker compose ps + +# View recent logs +docker compose logs --tail=100 +``` + +### Expected Health Response + +```json +{ + "status": "healthy", + "checks": { + "database": {"status": "healthy"}, + "redis": {"status": "healthy"}, + "ffmpeg": {"status": "healthy"} + } +} +``` + +--- + +## Common Issues + +### 1. API Not Responding + +**Symptoms:** +- Connection refused on port 8000 +- Timeout errors + +**Diagnosis:** + +```bash +# Check if container is running +docker compose ps api + +# Check API logs +docker compose logs api --tail=50 + +# Check if port is listening +netstat -tlnp | grep 8000 +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| Container not started | `docker compose up -d api` | +| Port conflict | Change `API_PORT` in `.env` | +| Startup crash | Check logs for errors | +| Health check failing | Check database/Redis connectivity | + +### 2. Database Connection Failed + +**Symptoms:** +- Health check shows database unhealthy +- `connection refused` errors in logs + +**Diagnosis:** + +```bash +# Check PostgreSQL status +docker compose ps postgres + +# Test database connection +docker compose exec postgres psql -U rendiff_user -d rendiff -c "SELECT 1" + +# Check connection URL +echo $DATABASE_URL +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| PostgreSQL not running | `docker compose up -d postgres` | +| Wrong credentials | Verify `DATABASE_URL` in `.env` | +| Database not created | Run migrations: `alembic upgrade head` | +| Network issue | Check Docker network connectivity | + +**Reset Database:** + +```bash +docker compose down -v +docker compose up -d postgres +sleep 10 +docker compose exec api alembic upgrade head +docker compose up -d +``` + +### 3. Redis Connection Failed + +**Symptoms:** +- Jobs stuck in pending state +- Rate limiting not working +- `Connection refused` errors + +**Diagnosis:** + +```bash +# Check Redis status +docker compose ps redis + +# Test Redis connection +docker compose exec redis redis-cli ping + +# Check Redis URL +echo $REDIS_URL +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| Redis not running | `docker compose up -d redis` | +| Wrong password | Check `REDIS_URL` password | +| Memory exhausted | Increase Redis memory or clear cache | +| Network issue | Verify Docker network | + +### 4. Jobs Stuck in Pending + +**Symptoms:** +- Jobs created but never start processing +- Progress stays at 0% + +**Diagnosis:** + +```bash +# Check worker status +docker compose ps worker + +# Check worker logs +docker compose logs worker --tail=50 + +# Check Celery queue +docker compose exec redis redis-cli LLEN celery +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| No workers running | `docker compose up -d worker` | +| Worker crashed | Check logs, restart: `docker compose restart worker` | +| Queue overloaded | Scale workers: `docker compose up -d --scale worker=4` | +| Redis connection lost | Restart Redis and workers | + +### 5. FFmpeg Processing Errors + +**Symptoms:** +- Jobs fail with processing errors +- Specific codecs/formats not working + +**Diagnosis:** + +```bash +# Check FFmpeg version +docker compose exec api ffmpeg -version + +# Check FFmpeg codecs +docker compose exec api ffmpeg -codecs | grep -E "(h264|h265|vp9|aac)" + +# Test FFmpeg manually +docker compose exec worker ffmpeg -i /storage/input.mp4 -c:v libx264 /tmp/test.mp4 +``` + +**Common FFmpeg Errors:** + +| Error | Cause | Solution | +|-------|-------|----------| +| `Unknown encoder` | Codec not available | Check codec name, use supported codec | +| `Invalid data found` | Corrupt input file | Verify input file integrity | +| `No such file` | File not found | Check file path and permissions | +| `Permission denied` | File permission issue | Fix file permissions | +| `Out of memory` | Insufficient RAM | Reduce resolution or bitrate | + +### 6. Authentication Errors + +**Symptoms:** +- 401 Unauthorized responses +- API key not accepted + +**Diagnosis:** + +```bash +# Test with API key +curl -H "X-API-Key: your-key" http://localhost:8000/api/v1/jobs + +# Check if API keys enabled +echo $ENABLE_API_KEYS + +# List API keys (admin) +curl -H "X-API-Key: admin-key" http://localhost:8000/api/v1/admin/api-keys +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| API keys disabled | Set `ENABLE_API_KEYS=true` | +| Invalid key format | Regenerate API key | +| Key expired | Create new key | +| Wrong header | Use `X-API-Key` header | + +### 7. Rate Limit Exceeded + +**Symptoms:** +- 429 Too Many Requests +- `Rate limit exceeded` error + +**Diagnosis:** + +```bash +# Check rate limit headers +curl -I -H "X-API-Key: key" http://localhost:8000/api/v1/jobs + +# Headers show: +# X-RateLimit-Limit: 100 +# X-RateLimit-Remaining: 0 +# X-RateLimit-Reset: 1642000000 +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| Too many requests | Wait for rate limit reset | +| Low limit configured | Increase `RATE_LIMIT_CALLS` | +| Need higher tier | Upgrade API key tier | + +### 8. Storage Errors + +**Symptoms:** +- File not found errors +- Upload/download failures +- Permission denied + +**Diagnosis:** + +```bash +# Check storage directory +ls -la /storage/ + +# Check file permissions +docker compose exec api ls -la /storage/ + +# Test write access +docker compose exec api touch /storage/test.txt +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| Directory doesn't exist | Create: `mkdir -p storage` | +| Permission denied | Fix ownership: `chown -R 1000:1000 storage` | +| Wrong path | Verify `STORAGE_PATH` | +| Disk full | Free up disk space | + +### 9. Webhook Failures + +**Symptoms:** +- Webhooks not received +- Jobs complete but no notification + +**Diagnosis:** + +```bash +# Check webhook URL in job +curl http://localhost:8000/api/v1/jobs/JOB_ID + +# Check worker logs for webhook errors +docker compose logs worker | grep -i webhook +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| Invalid URL | Verify webhook URL format | +| Network blocked | Check firewall, ensure URL reachable | +| SSRF protection | Don't use internal IPs (10.x, 192.168.x) | +| Server error | Check your webhook endpoint | + +### 10. Out of Memory + +**Symptoms:** +- Container killed (OOMKilled) +- Processing very slow +- System unresponsive + +**Diagnosis:** + +```bash +# Check container memory usage +docker stats + +# Check OOM kills +docker inspect api | grep -i oom +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| High resolution video | Reduce `MAX_RESOLUTION` | +| Too many workers | Reduce `WORKER_CONCURRENCY` | +| Memory leak | Restart workers regularly | +| Insufficient RAM | Add more RAM or scale horizontally | + +--- + +## Log Analysis + +### Viewing Logs + +```bash +# All services +docker compose logs + +# Specific service +docker compose logs api +docker compose logs worker + +# Follow logs +docker compose logs -f api + +# Last N lines +docker compose logs --tail=100 api + +# Filter by time +docker compose logs --since="1h" api +``` + +### Common Log Patterns + +**Successful Job:** + +``` +INFO Job created: 550e8400-e29b-41d4-a716-446655440000 +INFO Job started: 550e8400-e29b-41d4-a716-446655440000 +INFO Processing: 25% complete +INFO Processing: 50% complete +INFO Processing: 75% complete +INFO Job completed: 550e8400-e29b-41d4-a716-446655440000 +``` + +**Failed Job:** + +``` +INFO Job created: 550e8400-e29b-41d4-a716-446655440000 +INFO Job started: 550e8400-e29b-41d4-a716-446655440000 +ERROR Processing failed: Invalid input file +ERROR Job failed: 550e8400-e29b-41d4-a716-446655440000 +``` + +--- + +## Performance Issues + +### Slow Processing + +**Diagnosis:** + +```bash +# Check system resources +docker stats + +# Check job metrics +curl http://localhost:8000/api/v1/admin/stats + +# Check FFmpeg process +docker compose exec worker ps aux | grep ffmpeg +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| CPU bottleneck | Enable hardware acceleration | +| Disk I/O slow | Use SSD storage | +| Network slow | Use local storage for processing | +| Wrong preset | Use faster preset (e.g., `fast`) | + +### Hardware Acceleration Not Working + +**Diagnosis:** + +```bash +# Check GPU availability +docker compose exec worker nvidia-smi + +# Check FFmpeg NVENC support +docker compose exec worker ffmpeg -encoders | grep nvenc +``` + +**Solutions:** + +| Cause | Solution | +|-------|----------| +| No GPU | Use CPU encoding | +| Driver missing | Install NVIDIA drivers | +| Docker GPU access | Use `--gpus all` in Docker | +| Wrong FFmpeg build | Use GPU-enabled image | + +--- + +## Recovery Procedures + +### Full System Reset + +```bash +# Stop all services +docker compose down + +# Remove all data (WARNING: destroys all data) +docker compose down -v + +# Restart fresh +docker compose up -d + +# Run migrations +docker compose exec api alembic upgrade head +``` + +### Recover Stuck Jobs + +```bash +# Find stuck jobs +curl "http://localhost:8000/api/v1/jobs?status=processing" + +# Cancel stuck jobs via API +curl -X DELETE http://localhost:8000/api/v1/jobs/JOB_ID + +# Or reset via database +docker compose exec postgres psql -U rendiff_user -d rendiff \ + -c "UPDATE jobs SET status='failed', error_message='Manual reset' WHERE status='processing'" +``` + +### Clear Queue + +```bash +# Clear Celery queue +docker compose exec redis redis-cli FLUSHDB + +# Restart workers +docker compose restart worker +``` + +--- + +## Getting Help + +### Collect Diagnostic Information + +Before reporting an issue, collect: + +```bash +# System info +docker version +docker compose version +uname -a + +# Service status +docker compose ps + +# Recent logs +docker compose logs --tail=200 > logs.txt + +# Health check +curl http://localhost:8000/api/v1/health > health.json + +# Configuration (remove secrets!) +cat .env | grep -v PASSWORD | grep -v KEY > config.txt +``` + +### Report an Issue + +1. Search [existing issues](https://github.com/rendiffdev/rendiff-dev/issues) +2. Create a new issue with: + - Description of the problem + - Steps to reproduce + - Expected vs actual behavior + - Diagnostic information (above) + - Relevant log excerpts + +### Community Support + +- **GitHub Issues:** Bug reports and feature requests +- **GitHub Discussions:** Questions and community help diff --git a/setup.sh b/setup.sh index c0daadf..0f579f4 100755 --- a/setup.sh +++ b/setup.sh @@ -1,19 +1,20 @@ #!/bin/bash -# Rendiff FFmpeg API - Simple Setup Script +# Rendiff - Simple Setup Script +# A REST API layer powered by FFmpeg for media processing set -e PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$PROJECT_DIR" -echo "🚀 FFmpeg API Setup" -echo "==================" +echo "🚀 Rendiff Setup (Powered by FFmpeg)" +echo "======================================" # Function to show usage show_usage() { echo "Usage: $0 [OPTION]" echo "" - echo "🚀 FFmpeg API - Production-Ready Setup Script" + echo "🚀 Rendiff - Production-Ready Setup Script (Powered by FFmpeg)" echo "" echo "Deployment Options:" echo " --development 🛠️ Fast local development (SQLite, debug mode, no auth)" @@ -76,7 +77,7 @@ API_LOG_LEVEL=debug API_WORKERS=1 # Database (SQLite for simplicity) -DATABASE_URL=sqlite+aiosqlite:///data/ffmpeg_api.db +DATABASE_URL=sqlite+aiosqlite:///data/rendiff.db # Queue (Redis) REDIS_URL=redis://redis:6379/0 diff --git a/worker/tasks.py b/worker/tasks.py index 8babde0..76f8b89 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -6,11 +6,13 @@ import os import tempfile from datetime import datetime +from functools import lru_cache from pathlib import Path from typing import Dict, Any, Optional # Import removed - using internal FFmpeg wrapper instead import structlog +import yaml from celery import Task, current_task from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -24,6 +26,17 @@ logger = structlog.get_logger() + +@lru_cache(maxsize=1) +def get_storage_config() -> Dict[str, Any]: + """Load and cache storage configuration. + + Reads the YAML config file once and caches it to avoid + synchronous I/O in async contexts. + """ + with open(settings.STORAGE_CONFIG, 'r') as f: + return yaml.safe_load(f) + # Database setup for worker # Configure engine based on database type if "sqlite" in settings.DATABASE_URL: @@ -166,13 +179,13 @@ def process_job(job_id: str) -> Dict[str, Any]: error_msg = "Processing timeout" else: error_msg = "Processing failed" - - send_webhook(job.webhook_url, "error", { + + asyncio.run(send_webhook(job.webhook_url, "error", { "job_id": str(job.id), "status": "failed", "error": error_msg, # Sanitized error - }) - + })) + raise finally: db.close() @@ -184,12 +197,10 @@ async def process_job_async(job: Job, progress: ProgressTracker) -> Dict[str, An """ import contextlib import shutil - - # Load storage configuration - with open(settings.STORAGE_CONFIG, 'r') as f: - import yaml - storage_config = yaml.safe_load(f) - + + # Load storage configuration (cached) + storage_config = get_storage_config() + # Parse input/output paths input_backend_name, input_path = parse_storage_path(job.input_path) output_backend_name, output_path = parse_storage_path(job.output_path) @@ -334,15 +345,16 @@ def create_streaming(job_id: str) -> Dict[str, Any]: job.processing_time = (job.completed_at - job.started_at).total_seconds() db.commit() - + # Send webhook - send_webhook(job.webhook_url, "complete", { - "job_id": str(job.id), - "status": "completed", - "output_path": job.output_path, - "streaming_info": result.get("streaming_info", {}), - }) - + if job.webhook_url: + asyncio.run(send_webhook(job.webhook_url, "complete", { + "job_id": str(job.id), + "status": "completed", + "output_path": job.output_path, + "streaming_info": result.get("streaming_info", {}), + })) + logger.info(f"Streaming job completed: {job_id}") return result @@ -366,13 +378,13 @@ def create_streaming(job_id: str) -> Dict[str, Any]: error_msg = "Processing timeout" else: error_msg = "Processing failed" - - send_webhook(job.webhook_url, "error", { + + asyncio.run(send_webhook(job.webhook_url, "error", { "job_id": str(job.id), "status": "failed", "error": error_msg, # Sanitized error - }) - + })) + raise finally: db.close() @@ -383,12 +395,10 @@ async def process_streaming_async(job: Job, progress: ProgressTracker) -> Dict[s Async streaming processing logic. """ from worker.processors.streaming import StreamingProcessor - - # Load storage configuration - with open(settings.STORAGE_CONFIG, 'r') as f: - import yaml - storage_config = yaml.safe_load(f) - + + # Load storage configuration (cached) + storage_config = get_storage_config() + # Parse input/output paths input_backend_name, input_path = parse_storage_path(job.input_path) output_backend_name, output_path = parse_storage_path(job.output_path)