diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..1b752ac --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,156 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +Vagrant.configure("2") do |config| + config.vm.define "dev" do |vm| + vm.vm.box = "debian/bookworm64" + vm.vm.hostname = "crowdsec-spoa-test" + vm.vm.network "private_network", ip: "192.168.56.10" + vm.vm.network "forwarded_port", guest: 9090, host: 9090 + + vm.vm.provider "libvirt" do |lv| + lv.memory = "4096" + lv.cpus = 2 + end + + vm.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__exclude: [".git/", "node_modules/", "*.log"] + + vm.vm.provision "shell", inline: <<-SHELL + set -e + + # Update system and install base packages + apt-get update && apt-get upgrade -y + apt-get install -y tcpdump vim curl wget git build-essential ca-certificates \ + gnupg lsb-release apt-transport-https software-properties-common nginx unzip + + # Install HAProxy 3.1 + curl -fsSL https://haproxy.debian.net/haproxy-archive-keyring.gpg \ + --create-dirs --output /etc/apt/keyrings/haproxy-archive-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/haproxy-archive-keyring.gpg]" \ + https://haproxy.debian.net bookworm-backports-3.1 main > /etc/apt/sources.list.d/haproxy.list + apt-get update && apt-get install -y haproxy=3.1.* + + # Install Go 1.25.2 + GO_VERSION="1.25.2" + wget -qO- "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" | tar -xzC /usr/local + echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile + echo 'export PATH=$PATH:/usr/local/go/bin' >> /home/vagrant/.bashrc + + # Install CrowdSec + curl -s https://install.crowdsec.net | sh + apt-get install -y crowdsec + + # Install Nuclei for AppSec testing + NUCLEI_VERSION="3.1.7" + wget -qO /tmp/nuclei.zip "https://github.com/projectdiscovery/nuclei/releases/download/v${NUCLEI_VERSION}/nuclei_${NUCLEI_VERSION}_linux_amd64.zip" + unzip -q /tmp/nuclei.zip -d /tmp && mv /tmp/nuclei /usr/local/bin/nuclei && chmod +x /usr/local/bin/nuclei + rm -f /tmp/nuclei.zip + nuclei -update-templates -silent 2>/dev/null || true + + # Clone CrowdSec Hub + git clone -q https://github.com/crowdsecurity/hub.git /opt/hub || true + + # Create user and directories + groupadd -r crowdsec-spoa 2>/dev/null || true + useradd -r -g crowdsec-spoa -d /opt/crowdsec-spoa-bouncer -s /bin/false crowdsec-spoa 2>/dev/null || true + mkdir -p /opt/crowdsec-spoa-bouncer /etc/crowdsec/bouncers /var/log/crowdsec-spoa-bouncer \ + /run/crowdsec-spoa /usr/lib/crowdsec-haproxy-spoa-bouncer/lua /var/lib/crowdsec-haproxy-spoa-bouncer/html + chown -R crowdsec-spoa:crowdsec-spoa /opt/crowdsec-spoa-bouncer /var/log/crowdsec-spoa-bouncer /run/crowdsec-spoa + + # Copy Lua scripts and templates + mkdir -p /usr/lib/crowdsec-haproxy-spoa-bouncer/lua /var/lib/crowdsec-haproxy-spoa-bouncer/html + cp /vagrant/lua/*.lua /usr/lib/crowdsec-haproxy-spoa-bouncer/lua/ 2>/dev/null || true + cp /vagrant/templates/*.html /var/lib/crowdsec-haproxy-spoa-bouncer/html/ 2>/dev/null || true + chmod 644 /usr/lib/crowdsec-haproxy-spoa-bouncer/lua/*.lua 2>/dev/null || true + chmod 644 /var/lib/crowdsec-haproxy-spoa-bouncer/html/*.html 2>/dev/null || true + + # Configure nginx + cat > /etc/nginx/sites-available/default << 'EOF' +server { + listen 4444 default_server; + listen [::]:4444 default_server; + + root /var/www/html; + index index.html index.htm index.nginx-debian.html; + + server_name _; + + location / { + try_files $uri $uri/ =404; + } +} +EOF + + # Copy and configure HAProxy + cp /vagrant/config/haproxy.cfg /etc/haproxy/haproxy.cfg 2>/dev/null || true + cp /vagrant/config/crowdsec.cfg /etc/haproxy/crowdsec.cfg 2>/dev/null || true + # Update server addresses and remove the second SPOA server (port 9001 doesn't exist) + sed -i 's/whoami:2020/127.0.0.1:4444/g; s/spoa:9000/127.0.0.1:9000/g; /server s3 spoa:9001/d' \ + /etc/haproxy/haproxy.cfg 2>/dev/null || true + # Increase SPOA processing timeout to accommodate AppSec calls (AppSec has 5s timeout) + sed -i 's/timeout\s\+processing\s\+500ms/timeout processing 6s/' \ + /etc/haproxy/crowdsec.cfg 2>/dev/null || true + + # Copy and configure bouncer + cp /vagrant/config/crowdsec-spoa-bouncer.yaml.local /etc/crowdsec/bouncers/crowdsec-spoa-bouncer.yaml 2>/dev/null || true + # Update URLs (match with or without trailing slash) and API key + sed -i 's|http://crowdsec:8080|http://127.0.0.1:8080|g; s|http://crowdsec:7422|http://127.0.0.1:4241|g; s|api_key:.*|api_key: this_is_a_bad_password|g' \ + /etc/crowdsec/bouncers/crowdsec-spoa-bouncer.yaml 2>/dev/null || true + + # Configure AppSec before starting CrowdSec + # Install AppSec collections first + cscli collections install crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules || true + + # Configure AppSec acquisition + mkdir -p /etc/crowdsec/acquis.d + cat > /etc/crowdsec/acquis.d/appsec.yaml << 'EOF' +appsec_config: crowdsecurity/appsec-default +labels: + type: appsec +listen_addr: 0.0.0.0:4241 +source: appsec +EOF + + # Now start all services with CrowdSec properly configured + systemctl enable --now nginx haproxy crowdsec + sleep 5 + cscli bouncers add crowdsec-spoa-bouncer --key this_is_a_bad_password 2>/dev/null || true + SHELL + + vm.vm.provision "shell", run: "always", inline: <<-SHELL + set -e + export PATH=$PATH:/usr/local/go/bin + + # Build SPOA bouncer + if [ -f "/vagrant/main.go" ]; then + cd /vagrant + if go build -ldflags="-s -w" -o /opt/crowdsec-spoa-bouncer/crowdsec-spoa-bouncer .; then + chown crowdsec-spoa:crowdsec-spoa /opt/crowdsec-spoa-bouncer/crowdsec-spoa-bouncer + chmod +x /opt/crowdsec-spoa-bouncer/crowdsec-spoa-bouncer + + # Install systemd service + cp /vagrant/config/crowdsec-spoa-bouncer.service /etc/systemd/system/crowdsec-spoa-bouncer.service + sed -i 's|${BIN}|/opt/crowdsec-spoa-bouncer/crowdsec-spoa-bouncer|g; s|${CFG}|/etc/crowdsec/bouncers|g' \ + /etc/systemd/system/crowdsec-spoa-bouncer.service + sed -i 's|Type=notify|Type=simple|g; /ExecStartPre=/d' \ + /etc/systemd/system/crowdsec-spoa-bouncer.service + + systemctl daemon-reload + systemctl enable --now crowdsec-spoa-bouncer + fi + fi + + # Restart services in order + systemctl restart nginx + sleep 2 + systemctl restart crowdsec-spoa-bouncer 2>/dev/null || true + sleep 3 + systemctl restart haproxy + + # Verify services + for svc in nginx crowdsec-spoa-bouncer haproxy; do + systemctl is-active --quiet $svc && echo "✅ $svc: running" || echo "❌ $svc: failed" + done + SHELL + end +end diff --git a/cmd/root.go b/cmd/root.go index ecc5857..e28bfcb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -162,6 +162,9 @@ func Execute() error { // Create a base logger for the host manager hostManagerLogger := log.WithField("component", "host_manager") HostManager := host.NewManager(hostManagerLogger) + + // Set global AppSec configuration (uses same API key as LAPI) + HostManager.SetGlobalAppSecConfig(config.AppSecURL, config.APIKey) // Create and initialize global session manager (single GC goroutine for all hosts) globalSessions := &session.Sessions{ diff --git a/config/haproxy-upstreamproxy.cfg b/config/haproxy-upstreamproxy.cfg index 7aaf4df..585c934 100644 --- a/config/haproxy-upstreamproxy.cfg +++ b/config/haproxy-upstreamproxy.cfg @@ -12,6 +12,7 @@ global defaults log global option httplog + option http-buffer-request timeout client 1m timeout server 1m timeout connect 10s @@ -68,4 +69,3 @@ backend crowdsec-spoa mode tcp balance roundrobin server s2 spoa:9000 - server s3 spoa:9001 diff --git a/config/haproxy.cfg b/config/haproxy.cfg index 79b5321..31fcddc 100644 --- a/config/haproxy.cfg +++ b/config/haproxy.cfg @@ -9,6 +9,7 @@ global defaults log global option httplog + option http-buffer-request timeout client 1m timeout server 1m timeout connect 10s @@ -58,4 +59,3 @@ backend crowdsec-spoa mode tcp balance roundrobin server s2 spoa:9000 - server s3 spoa:9001 diff --git a/docker-compose.proxy-test.yaml b/docker-compose.proxy-test.yaml index b2963e4..792deb6 100644 --- a/docker-compose.proxy-test.yaml +++ b/docker-compose.proxy-test.yaml @@ -63,8 +63,10 @@ services: - BOUNCER_KEY_SPOA=+4iYgItcalc9+0tWrvrj9R6Wded/W1IRwRtNmcWR9Ws - DISABLE_ONLINE_API=true - CROWDSEC_BYPASS_DB_VOLUME_CHECK=true + - COLLECTIONS=crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules volumes: - geodb:/staging/var/lib/crowdsec/data/ + - ./docker/crowdsec/acquisitions/appsec.yaml:/etc/crowdsec/acquis.d/appsec.yaml networks: - crowdsec diff --git a/docker-compose.yaml b/docker-compose.yaml index 29af5b5..c1da824 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -55,8 +55,10 @@ services: - BOUNCER_KEY_SPOA=+4iYgItcalc9+0tWrvrj9R6Wded/W1IRwRtNmcWR9Ws - DISABLE_ONLINE_API=true - CROWDSEC_BYPASS_DB_VOLUME_CHECK=true + - COLLECTIONS=crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules volumes: - geodb:/staging/var/lib/crowdsec/data/ + - ./docker/crowdsec/acquisitions/appsec.yaml:/etc/crowdsec/acquis.d/appsec.yaml networks: - crowdsec diff --git a/docker/crowdsec/README.md b/docker/crowdsec/README.md new file mode 100644 index 0000000..17bac01 --- /dev/null +++ b/docker/crowdsec/README.md @@ -0,0 +1,29 @@ +# CrowdSec Configuration for Docker Compose + +This directory contains CrowdSec configuration files for testing the HAProxy SPOA bouncer with AppSec. + +## Collections + +The following collections are required for AppSec functionality: + +- `crowdsecurity/appsec-virtual-patching` - Protection against known vulnerabilities +- `crowdsecurity/appsec-generic-rules` - Generic attack vector detection + +These collections are automatically installed when the CrowdSec container starts via the `COLLECTIONS` environment variable in `docker-compose.yaml`. + +## Acquisitions + +The `acquisitions/appsec.yaml` file configures the AppSec Component to listen on port 7422 for HTTP request validation. + +## Manual Collection Installation + +If you need to manually install collections (e.g., when persisting `/etc/crowdsec/` on the host): + +```bash +docker exec -it crowdsec cscli collections install crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules +``` + +## AppSec Component + +The AppSec Component listens on `0.0.0.0:7422` and can be accessed from other containers in the Docker network at `crowdsec:7422`. + diff --git a/docker/crowdsec/acquisitions/appsec.yaml b/docker/crowdsec/acquisitions/appsec.yaml new file mode 100644 index 0000000..b396f17 --- /dev/null +++ b/docker/crowdsec/acquisitions/appsec.yaml @@ -0,0 +1,6 @@ +appsec_config: crowdsecurity/appsec-default +labels: + type: appsec +listen_addr: 0.0.0.0:7422 +source: appsec + diff --git a/internal/appsec/root.go b/internal/appsec/root.go index 71398bb..2e20241 100644 --- a/internal/appsec/root.go +++ b/internal/appsec/root.go @@ -1,19 +1,225 @@ package appsec import ( + "bytes" + "context" + "fmt" + "io" + "maps" + "net/http" + "strings" + "time" + + "github.com/crowdsecurity/crowdsec-spoa/internal/remediation" log "github.com/sirupsen/logrus" ) +// AppSecRequest represents the HTTP request data to be validated by AppSec +type AppSecRequest struct { + Host string + Method string + URL string + RemoteIP string + UserAgent string + Version string + Headers http.Header + Body []byte +} + +// AppSecClient handles HTTP communication with the AppSec engine +type AppSecClient struct { + HTTPClient *http.Client + APIKey string + URL string + logger *log.Entry +} + +// AppSec represents the AppSec configuration for a host type AppSec struct { - AlwaysSend bool `yaml:"always_send"` + AlwaysSend bool `yaml:"always_send"` + URL string `yaml:"url,omitempty"` // Host-specific AppSec URL (overrides global) + APIKey string `yaml:"api_key,omitempty"` // Host-specific API key (overrides global) + Client *AppSecClient logger *log.Entry `yaml:"-"` } -func (a *AppSec) Init(logger *log.Entry) error { +func (a *AppSec) Init(logger *log.Entry, globalURL, globalAPIKey string) error { a.InitLogger(logger) + + // Use host-specific config if provided, otherwise fall back to global + url := a.URL + if url == "" { + url = globalURL + } + + apiKey := a.APIKey + if apiKey == "" { + apiKey = globalAPIKey + } + + // Only create client if URL is configured + if url != "" { + // Configure transport with keep-alive enabled and optimized connection pooling + transport := &http.Transport{ + MaxIdleConns: 100, // Total idle connections across all hosts + MaxIdleConnsPerHost: 10, // Idle connections per host (default is 2) + IdleConnTimeout: 90 * time.Second, // How long idle connections are kept + DisableKeepAlives: false, // Enable keep-alive (default) + } + + a.Client = &AppSecClient{ + HTTPClient: &http.Client{ + Timeout: 5 * time.Second, + Transport: transport, + }, + APIKey: apiKey, + URL: url, + logger: a.logger, + } + + a.logger.WithFields(log.Fields{ + "url": url, + "api_key_set": apiKey != "", + }).Debug("AppSec client initialized") + } else { + a.logger.Debug("AppSec URL not configured, AppSec validation will be skipped") + } + return nil } +func (a *AppSec) IsValid() bool { + return a.Client != nil && a.Client.URL != "" +} + func (a *AppSec) InitLogger(logger *log.Entry) { a.logger = logger.WithField("type", "appsec") } + +// ValidateRequest sends the HTTP request to the AppSec engine and returns the remediation +func (a *AppSec) ValidateRequest(ctx context.Context, req *AppSecRequest) (remediation.Remediation, error) { + if a.Client == nil { + a.logger.Debug("AppSec client not initialized, allowing request") + return remediation.Allow, nil + } + + if a.Client.URL == "" { + a.logger.Debug("AppSec URL not configured, allowing request") + return remediation.Allow, nil + } + + // Create HTTP request to AppSec engine + httpReq, err := a.createAppSecRequest(req) + if err != nil { + a.logger.Errorf("Failed to create AppSec request: %v", err) + return remediation.Allow, err + } + + // Send request to AppSec engine + resp, err := a.Client.HTTPClient.Do(httpReq.WithContext(ctx)) + if err != nil { + a.logger.Errorf("Failed to send request to AppSec engine: %v", err) + return remediation.Allow, err + } + defer resp.Body.Close() + + // Ensure response body is fully read for proper connection reuse + // This allows the connection to be reused via keep-alive + _, _ = io.Copy(io.Discard, resp.Body) + + // Process response based on HTTP status code + return a.processAppSecResponse(resp) +} + +func (a *AppSec) createAppSecRequest(req *AppSecRequest) (*http.Request, error) { + var httpReq *http.Request + var err error + + // Determine HTTP method based on whether there's a body + if len(req.Body) > 0 { + httpReq, err = http.NewRequest(http.MethodPost, a.Client.URL, bytes.NewReader(req.Body)) + } else { + httpReq, err = http.NewRequest(http.MethodGet, a.Client.URL, http.NoBody) + } + + if err != nil { + return nil, err + } + + // Copy original headers + if req.Headers != nil { + httpReq.Header = maps.Clone(req.Headers) + } + + // Now override with our trusted CrowdSec headers + httpReq.Header.Set("X-Crowdsec-Appsec-Ip", req.RemoteIP) + httpReq.Header.Set("X-Crowdsec-Appsec-Uri", req.URL) + httpReq.Header.Set("X-Crowdsec-Appsec-Host", req.Host) + httpReq.Header.Set("X-Crowdsec-Appsec-Verb", req.Method) + + // Ensure we have a valid API key + if a.Client.APIKey == "" { + a.logger.Error("AppSec API key is empty") + return nil, fmt.Errorf("AppSec API key is not configured") + } + + httpReq.Header.Set("X-Crowdsec-Appsec-Api-Key", a.Client.APIKey) + httpReq.Header.Set("X-Crowdsec-Appsec-User-Agent", req.UserAgent) + + // Debug logging to see what we're actually sending (log after all headers are set) + a.logger.WithFields(log.Fields{ + "host": req.Host, + "method": req.Method, + "url": req.URL, + "remote_ip": req.RemoteIP, + "user_agent": req.UserAgent, + }).Debug("Created AppSec request with headers") + + // Set HTTP version from the request (set by HAProxy SPOE) + httpVersion := "11" // Default to HTTP/1.1 + if req.Version != "" { + // Convert version format from HAProxy (e.g., "1.1", "2.0") to our format (e.g., "11", "20") + switch req.Version { + case "1.0": + httpVersion = "10" + case "1.1": + httpVersion = "11" + case "2.0": + httpVersion = "20" + case "3.0": + httpVersion = "30" + default: + // For any other version, just strip the dots + httpVersion = strings.ReplaceAll(req.Version, ".", "") + } + } + httpReq.Header.Set("X-Crowdsec-Appsec-Http-Version", httpVersion) + + return httpReq, nil +} + +func (a *AppSec) processAppSecResponse(resp *http.Response) (remediation.Remediation, error) { + switch resp.StatusCode { + case http.StatusOK: + // Request allowed + return remediation.Allow, nil + + case http.StatusForbidden: + // Request blocked - return ban remediation + return remediation.Ban, nil + + case http.StatusUnauthorized: + // Authentication failed + a.logger.Error("AppSec authentication failed - check API key") + return remediation.Allow, fmt.Errorf("AppSec authentication failed") + + case http.StatusInternalServerError: + // AppSec engine error + a.logger.Error("AppSec engine error") + return remediation.Allow, fmt.Errorf("AppSec engine error") + + default: + a.logger.Warnf("Unexpected AppSec response code: %d", resp.StatusCode) + return remediation.Allow, fmt.Errorf("unexpected AppSec response code: %d", resp.StatusCode) + } +} diff --git a/pkg/cfg/config.go b/pkg/cfg/config.go index a98b041..0268265 100644 --- a/pkg/cfg/config.go +++ b/pkg/cfg/config.go @@ -26,6 +26,8 @@ type BouncerConfig struct { ListenTCP string `yaml:"listen_tcp"` ListenUnix string `yaml:"listen_unix"` PrometheusConfig PrometheusConfig `yaml:"prometheus"` + APIKey string `yaml:"api_key"` // LAPI API key (also used for AppSec) + AppSecURL string `yaml:"appsec_url,omitempty"` // Global AppSec URL } // MergedConfig() returns the byte content of the patched configuration file (with .yaml.local). diff --git a/pkg/host/root.go b/pkg/host/root.go index 0d97d3b..09a7298 100644 --- a/pkg/host/root.go +++ b/pkg/host/root.go @@ -39,10 +39,12 @@ type Host struct { } type Manager struct { - Hosts []*Host - Chan chan HostOp - Logger *log.Entry - cache map[string]*Host + Hosts []*Host + Chan chan HostOp + Logger *log.Entry + cache map[string]*Host + GlobalAppSecURL string // Global AppSec URL (can be overridden per-host) + GlobalAppSecAPIKey string // Global AppSec API key (can be overridden per-host) sync.RWMutex } @@ -73,6 +75,14 @@ func NewManager(l *log.Entry) *Manager { } } +// SetGlobalAppSecConfig sets the global AppSec configuration for the host manager +func (h *Manager) SetGlobalAppSecConfig(appSecURL, apiKey string) { + h.Lock() + defer h.Unlock() + h.GlobalAppSecURL = appSecURL + h.GlobalAppSecAPIKey = apiKey +} + func (h *Manager) MatchFirstHost(toMatch string) *Host { h.RLock() defer h.RUnlock() @@ -246,7 +256,7 @@ func (h *Manager) addHost(host *Host) { if err := host.Ban.Init(host.logger); err != nil { host.logger.Error(err) } - if err := host.AppSec.Init(host.logger); err != nil { + if err := host.AppSec.Init(host.logger, h.GlobalAppSecURL, h.GlobalAppSecAPIKey); err != nil { host.logger.Error(err) } h.Hosts = append(h.Hosts, host) diff --git a/pkg/spoa/root.go b/pkg/spoa/root.go index 8e2f3d0..5dd73f5 100644 --- a/pkg/spoa/root.go +++ b/pkg/spoa/root.go @@ -10,7 +10,9 @@ import ( "strings" "sync" "syscall" + "time" + "github.com/crowdsecurity/crowdsec-spoa/internal/appsec" "github.com/crowdsecurity/crowdsec-spoa/internal/geo" "github.com/crowdsecurity/crowdsec-spoa/internal/remediation" "github.com/crowdsecurity/crowdsec-spoa/internal/remediation/captcha" @@ -314,92 +316,128 @@ func (s *Spoa) handleHTTPRequest(req *request.Request, mes *message.Message) { return // Cannot proceed without unset cookie } - s.logger.WithFields(log.Fields{ - "host": matchedHost.Host, - }).Debug("Allow decision but captcha cookie present, will clear cookie") req.Actions.SetVar(action.ScopeTransaction, "captcha_cookie", unsetCookie.String()) // Note: We deliberately don't set captcha_status here } - // Parse HTTP data for AppSec processing - httpData = parseHTTPData(s.logger, mes) case remediation.Ban: //Handle ban matchedHost.Ban.InjectKeyValues(&req.Actions) - // Parse HTTP data for AppSec processing - httpData = parseHTTPData(s.logger, mes) case remediation.Captcha: + // Handle captcha r, httpData = s.handleCaptchaRemediation(req, mes, matchedHost) - // If remediation changed to fallback, return early - // If it became Allow, continue for AppSec processing - if r != remediation.Captcha && r != remediation.Allow { - return - } } // If remediation is ban/captcha we dont need to create a request to send to appsec unless always send is on if r > remediation.Unknown && !matchedHost.AppSec.AlwaysSend { return } - // !TODO APPSEC STUFF - httpData contains parsed URL, Method, Body, Headers for reuse - _ = httpData // Reserved for AppSec implementation - // request, err := http.NewRequest(httpData.Method, httpData.URL, strings.NewReader(httpData.Body)) - // if err != nil { - // log.Printf("failed to create request: %v", err) - // return - // } - // request.Header = httpData.Headers -} + if !matchedHost.AppSec.IsValid() { + return + } -// parseHTTPData extracts HTTP request data from the message for reuse in AppSec processing -// -//nolint:unparam // httpData will be used when AppSec is implemented -func parseHTTPData(logger *log.Entry, mes *message.Message) HTTPRequestData { - var httpData HTTPRequestData + // Only parse HTTP data if it hasn't been parsed yet (i.e., for Allow/Ban cases) + // For Captcha, httpData is already populated by handleCaptchaRemediation + if httpData.Headers == nil { + httpData = parseHTTPData(mes) + } - url, err := readKeyFromMessage[string](mes, "url") + // AppSec validation - reuse httpData already parsed above + srcIPPtr, err := readKeyFromMessage[netip.Addr](mes, "src-ip") if err != nil { - logger.WithFields(log.Fields{ - "error": err, - "key": "url", - }).Debug("failed to read url from message for AppSec processing - ensure HAProxy is sending the 'url' variable in crowdsec-http message") + s.logger.WithError(err).Warn("AppSec validation failed: failed to read src-ip") + return + } + srcIP := "" + if srcIPPtr != nil { + srcIP = srcIPPtr.String() } - httpData.URL = url - method, err := readKeyFromMessage[string](mes, "method") + // Get optional fields - extract User-Agent from headers since it's already sent via headers=req.hdrs + userAgent := "" + if httpData.Headers != nil { + userAgent = httpData.Headers.Get("User-Agent") + } + version, _ := readKeyFromMessage[string](mes, "version") + + // Build AppSec request from already-parsed httpData + appSecReq := &appsec.AppSecRequest{ + Host: *hoststring, + Method: "", + URL: "", + RemoteIP: srcIP, + UserAgent: userAgent, + Version: "", + Headers: httpData.Headers, + Body: nil, + } + + if httpData.Method != nil { + appSecReq.Method = *httpData.Method + } + if httpData.URL != nil { + appSecReq.URL = *httpData.URL + } + if version != nil { + appSecReq.Version = *version + } + if httpData.Body != nil { + appSecReq.Body = *httpData.Body + } + + // Validate with AppSec - update r directly (defer will handle setting it) + // Use a timeout context that's slightly less than HAProxy's processing timeout (6s) to avoid connection resets + appSecCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + appSecRemediation, err := matchedHost.AppSec.ValidateRequest(appSecCtx, appSecReq) if err != nil { - logger.WithFields(log.Fields{ - "error": err, - "key": "method", - }).Debug("failed to read method from message for AppSec processing - ensure HAProxy is sending the 'method' variable in crowdsec-http message") + s.logger.WithError(err).Warn("AppSec validation failed, using original remediation") + return + } + + // Track metrics if AppSec returns a more restrictive remediation than Allow + if appSecRemediation > remediation.Allow && srcIPPtr != nil && srcIPPtr.IsValid() { + ipTypeLabel := "ipv4" + if srcIPPtr.Is6() { + ipTypeLabel = "ipv6" + } + + metrics.TotalBlockedRequests.With(prometheus.Labels{ + "ip_type": ipTypeLabel, + "origin": "appsec", + "remediation": appSecRemediation.String(), + }).Inc() + } + + // If AppSec returns a more restrictive remediation, use it (defer will handle setting it) + if appSecRemediation > r { + r = appSecRemediation + // If AppSec returns ban, inject ban values + if r == remediation.Ban { + matchedHost.Ban.InjectKeyValues(&req.Actions) + } } +} + +// parseHTTPData extracts HTTP request data from the message for reuse in AppSec processing +func parseHTTPData(mes *message.Message) HTTPRequestData { + var httpData HTTPRequestData + + url, _ := readKeyFromMessage[string](mes, "url") + httpData.URL = url + + method, _ := readKeyFromMessage[string](mes, "method") httpData.Method = method headersType, err := readKeyFromMessage[string](mes, "headers") - if err != nil { - logger.WithFields(log.Fields{ - "error": err, - "key": "headers", - }).Debug("failed to read headers from message for AppSec processing - ensure HAProxy is sending the 'headers' variable in crowdsec-http message") - } else if headersType != nil { + if err == nil && headersType != nil { headers, parseErr := readHeaders(*headersType) - if parseErr != nil { - logger.WithFields(log.Fields{ - "error": parseErr, - "key": "headers", - }).Debug("failed to parse headers from message for AppSec processing") - } else { + if parseErr == nil { httpData.Headers = headers } } - body, err := readKeyFromMessage[[]byte](mes, "body") - if err != nil { - logger.WithFields(log.Fields{ - "error": err, - "key": "body", - }).Debug("failed to read body from message for AppSec processing - ensure HAProxy is sending the 'body' variable in crowdsec-http message") - } + body, _ := readKeyFromMessage[[]byte](mes, "body") httpData.Body = body return httpData @@ -617,7 +655,6 @@ func (s *Spoa) handleCaptchaRemediation(req *request.Request, mes *message.Messa if captchaStatus == captcha.Valid { storedURL := ses.Get(session.URI) if storedURL != nil && storedURL != "" { - s.logger.Debug("redirecting to: ", storedURL) req.Actions.SetVar(action.ScopeTransaction, "redirect", storedURL) // Delete the URI from the session so we dont redirect loop ses.Delete(session.URI) @@ -801,20 +838,41 @@ func readKeyFromMessage[T string | net.IP | netip.Addr | bool | []byte](msg *mes func readHeaders(headers string) (http.Header, error) { h := http.Header{} - hs := strings.Split(headers, "\r\n") + + // Normalize line endings: replace \r\n with \n first, then split by \n + // HAProxy's req.hdrs can send headers with either \r\n or \n separators + normalized := strings.ReplaceAll(headers, "\r\n", "\n") + hs := strings.Split(normalized, "\n") if len(hs) == 0 { return nil, fmt.Errorf("no headers found") } - for _, header := range hs { + for i, header := range hs { + header = strings.TrimSpace(header) if header == "" { continue } + // Skip the HTTP request line if present (first line that looks like "METHOD PATH HTTP/VERSION") + // HAProxy's req.hdrs may include the request line at the beginning + if i == 0 { + // Check if this looks like an HTTP request line (starts with HTTP method) + parts := strings.Fields(header) + if len(parts) >= 3 { + // Check if third part looks like HTTP version (e.g., "HTTP/1.1") + // Format: "METHOD PATH HTTP/VERSION" + if strings.HasPrefix(parts[2], "HTTP/") { + // This is the request line, skip it + continue + } + } + } + kv := strings.SplitN(header, ":", 2) if len(kv) != 2 { - return nil, fmt.Errorf("invalid header: %q", header) + // Skip lines without colon (might be continuation or malformed) + continue } h.Add(strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1]))