Merge pull request #150 from projectdiscovery/ztls-fallback-fastdialer

use fastdialer for ztls fallback

README.md

@@ -1,88 +1,25 @@
 # rawhttp
 
+[![License](https://img.shields.io/github/license/projectdiscovery/rawhttp)](LICENSE.md)
+![Go version](https://img.shields.io/github/go-mod/go-version/projectdiscovery/rawhttp?filename=go.mod)
+[![Release](https://img.shields.io/github/release/projectdiscovery/rawhttp)](https://github.com/projectdiscovery/rawhttp/releases/)
+[![Checks](https://github.com/projectdiscovery/rawhttp/actions/workflows/build_test.yaml/badge.svg)](https://github.com/projectdiscovery/rawhttp/actions/workflows/build_test.yaml)
+[![GoDoc](https://img.shields.io/badge/go-reference-blue)](https://pkg.go.dev/github.com/projectdiscovery/rawhttp)
+
 rawhttp is a Go package for making HTTP requests in a raw way.
 
 
 - Forked and adapted from [https://github.com/gorilla/http](https://github.com/gorilla/http) and [https://github.com/valyala/fasthttp](https://github.com/valyala/fasthttp)
 - The original idea is inspired by [@tomnomnom/rawhttp](https://github.com/tomnomnom/rawhttp) work
 
 
-### ZTLS fallback support
-
-### ZTLS Fallback
-
-`rawhttp` by default fallbacks to using zcrypto when there is an error in TLS handshake (ex: ` insufficient security level` etc ). This is done to support older TLS versions and ciphers. This can be disabled by setting `rawhttp.DisableZtlsFallback` to `true` or by using `DISABLE_ZTLS_FALLBACK` environment variable. when falling back to ztls, `ChromeCiphers` are used
-
-
-
-# Example
-
-First you need to declare a `server`
-
-```go
-...
-...
-
-func headers(w http.ResponseWriter, req *http.Request) {
-	for name, headers := range req.Header {
-		for _, h := range headers {
-			fmt.Fprintf(w, "%v: %v\n", name, h)
-		}
-	}
-}
-
-func main() {
-	http.HandleFunc("/headers", headers)
-	if err := http.ListenAndServe(":10000", nil); err != nil {
-		gologger.Fatal().Msgf("Could not listen and serve: %s\n", err)
-	}
-}
-```
-
-```
-go run server.go
-```
-
-Second you need to start the client
-
-```go
-func main() {
-    host := "127.0.0.1:10000"
-	swg := sizedwaitgroup.New(25)
-	pipeOptions := rawhttp.DefaultPipelineOptions
-	pipeOptions.Host = host
-	pipeOptions.MaxConnections = 1
-	pipeclient := rawhttp.NewPipelineClient(pipeOptions)
-	for i := 0; i < 50; i++ {
-		swg.Add()
-		go func(swg *sizedwaitgroup.SizedWaitGroup) {
-			defer swg.Done()
-			req, err := http.NewRequest("GET", host + "/headers", nil)
-			if err != nil {
-				log.Printf("Error sending request to API endpoint. %+v", err)
-				return
-			}
-			req.Host = host
-			req.Header.Set("Host", host)
-			resp, err := pipeclient.Do(req)
-			if err != nil {
-				log.Printf("Error sending request to API endpoint. %+v", err)
-				return
-			}
-			log.Printf("%+v\n", resp)
-			_ = resp
-		}(&swg)
-	}
-
-	swg.Wait()
+# Library Usage
 
-}
-```
+A simple example to get started with rawhttp is available at [examples](./example/simple/main.go). For documentation, please refer [godoc](https://pkg.go.dev/github.com/projectdiscovery/rawhttp)
 
-```
-go run client.go
-```
+## Note
 
+rawhttp internally uses [fastdialer](https://github.com/projectdiscovery/fastdialer) to dial connections and fastdialer has a disk cache for DNS lookups. While using rawhttp `.Close()` method should be called at end of the program to remove temporary files created by fastdialer.
 
 # License
 

client.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer"
+	"github.com/projectdiscovery/gologger"
 	retryablehttp "github.com/projectdiscovery/retryablehttp-go"
 	urlutil "github.com/projectdiscovery/utils/url"
 )
@@ -33,6 +35,13 @@ func NewClient(options *Options) *Client {
 		dialer:  new(dialer),
 		Options: options,
 	}
+	if options.FastDialer == nil {
+		var err error
+		options.FastDialer, err = fastdialer.NewDialer(fastdialer.DefaultOptions)
+		if err != nil {
+			gologger.Error().Msgf("Could not create fast dialer: %s\n", err)
+		}
+	}
 	return client
 }
 
@@ -91,6 +100,13 @@ func (c *Client) DoRawWithOptions(method, url, uripath string, headers map[strin
 	return c.do(method, url, uripath, headers, body, redirectstatus, options)
 }
 
+// Close closes client and any resources it holds
+func (c *Client) Close() {
+	if c.Options.FastDialer != nil {
+		c.Options.FastDialer.Close()
+	}
+}
+
 func (c *Client) getConn(protocol, host string, options *Options) (Conn, error) {
 	if options.Proxy != "" {
 		return c.dialer.DialWithProxy(protocol, host, c.Options.Proxy, c.Options.ProxyDialTimeout, options)

conn.go

@@ -3,25 +3,19 @@ package rawhttp
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/url"
-	"os"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/rawhttp/client"
 	"github.com/projectdiscovery/rawhttp/proxy"
-	ztls "github.com/zmap/zcrypto/tls"
 )
 
-// DisableZtlsFallback disables ztls fallback when tls handshake fails
-// can also be set using the environment variable DISABLE_ZTLS_FALLBACK
-var DisableZtlsFallback = false
-
 // Dialer can dial a remote HTTP server.
 type Dialer interface {
 	// Dial dials a remote http server returning a Conn.
@@ -123,28 +117,27 @@ func clientDial(protocol, addr string, timeout time.Duration, options *Options)
 		tlsConfig.ServerName = options.SNI
 	}
 
-	// currently fastdialer tls dial and ztls fallback are mutually exclusive
-	// TODO: add support for fallback in fastDialer.DialZTLS()
-	if options.FastDialer != nil {
-		return options.FastDialer.DialTLSWithConfig(ctx, "tcp", addr, tlsConfig)
-
+	if options.FastDialer == nil {
+		// always use fastdialer tls dial if available
+		opts := fastdialer.DefaultOptions
+		if timeout > 0 {
+			opts.DialerTimeout = timeout
+		}
+		var err error
+		options.FastDialer, err = fastdialer.NewDialer(opts)
+		// use net.Dialer if fastdialer tls dial is not available
+		if err != nil {
+			var dialer *net.Dialer
+			if timeout > 0 {
+				dialer = &net.Dialer{Timeout: timeout}
+			} else {
+				dialer = &net.Dialer{Timeout: 8 * time.Second} // should be more than enough
+			}
+			return tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
+		}
 	}
 
-	var dialer *net.Dialer
-	if timeout > 0 {
-		dialer = &net.Dialer{Timeout: timeout}
-	} else {
-		dialer = &net.Dialer{Timeout: 8 * time.Second} // should be more than enough
-	}
-	tlsConn, err := tls.DialWithDialer(dialer, "tcp", addr, tlsConfig)
-	if err != nil && !DisableZtlsFallback && !errors.Is(err, os.ErrDeadlineExceeded) {
-		return ztls.DialWithDialer(dialer, "tcp", addr, &ztls.Config{
-			CipherSuites:       ztls.ChromeCiphers,
-			ServerName:         tlsConfig.ServerName,
-			InsecureSkipVerify: true,
-		})
-	}
-	return tlsConn, err
+	return options.FastDialer.DialTLS(ctx, "tcp", addr)
 }
 
 // TlsHandshake tls handshake on a plain connection
@@ -171,18 +164,6 @@ func TlsHandshake(conn net.Conn, addr string, timeout time.Duration) (net.Conn,
 		ServerName:         hostname,
 	})
 	if err := tlsConn.HandshakeContext(ctx); err != nil {
-		if !errors.Is(err, os.ErrDeadlineExceeded) && !DisableZtlsFallback {
-			// fallback to ztls
-			ztlsConn := ztls.Client(conn, &ztls.Config{
-				InsecureSkipVerify: true,
-				ServerName:         hostname,
-				CipherSuites:       ztls.ChromeCiphers,
-			})
-			if err := ztlsConn.Handshake(); err != nil {
-				return nil, err
-			}
-			return ztlsConn, nil
-		}
 		return nil, err
 	}
 	return tlsConn, nil
@@ -211,10 +192,3 @@ func (c *conn) Release() {
 	addr := c.Conn.RemoteAddr().String()
 	c.dialer.conns[addr] = append(c.dialer.conns[addr], c)
 }
-
-func init() {
-	value := os.Getenv("DISABLE_ZTLS_FALLBACK")
-	if strings.EqualFold(value, "true") {
-		DisableZtlsFallback = true
-	}
-}

go.mod

@@ -4,14 +4,13 @@ go 1.18
 
 require (
 	github.com/julienschmidt/httprouter v1.3.0
-	github.com/projectdiscovery/fastdialer v0.0.32
+	github.com/projectdiscovery/fastdialer v0.0.34
 	github.com/projectdiscovery/gologger v1.1.11
 	github.com/projectdiscovery/retryablehttp-go v1.0.18
 	github.com/projectdiscovery/stringsutil v0.0.2
 	github.com/projectdiscovery/utils v0.0.41
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/stretchr/testify v1.8.4
-	github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be
 	golang.org/x/net v0.12.0
 )
 
@@ -61,6 +60,7 @@ require (
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yl2chen/cidranger v1.0.2 // indirect
 	github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521 // indirect
+	github.com/zmap/zcrypto v0.0.0-20220803033029-557f3e4940be // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.11.0 // indirect

go.sum

@@ -82,8 +82,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
-github.com/projectdiscovery/fastdialer v0.0.32 h1:2sMAXLUcdyHMmXh46PkoRRewBBjZBMiraawSHDT/fjs=
-github.com/projectdiscovery/fastdialer v0.0.32/go.mod h1:ttLvt0xnpNQAStYYQ6ElIBHfSXHuPEiXBkLH/OLbYlc=
+github.com/projectdiscovery/fastdialer v0.0.34 h1:7czrRJJ4f5lYSKsEvBvm8a9DwDeg5MNaANgfDnt0nGo=
+github.com/projectdiscovery/fastdialer v0.0.34/go.mod h1:8Xw7r4kiHO1C1/wTnMrwUwQG6KIKCaPoeT5XLoJptMo=
 github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8=
 github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY=
 github.com/projectdiscovery/hmap v0.0.13 h1:8v5j99Pz0S7V1YrTeWp7xtr1yNOffKQ/KusHZfB+mrI=

proxy/http.go

@@ -1,13 +1,15 @@
 package proxy
 
 import (
+	"context"
 	"encoding/base64"
 	"fmt"
 	"net"
 	"net/url"
 	"strings"
 	"time"
 
+	"github.com/projectdiscovery/fastdialer/fastdialer"
 	"github.com/projectdiscovery/rawhttp/client"
 )
 
@@ -31,11 +33,17 @@ func HTTPDialer(proxyAddr string, timeout time.Duration) DialFunc {
 			auth = base64.StdEncoding.EncodeToString([]byte(split[0]))
 			proxyAddr = split[1]
 		}
-		if timeout == 0 {
-			netConn, err = net.Dial("tcp", u.Host)
+		fd, err := fastdialer.NewDialer(fastdialer.DefaultOptions)
+		if err != nil {
+			if timeout == 0 {
+				netConn, err = net.Dial("tcp", u.Host)
+			} else {
+				netConn, err = net.DialTimeout("tcp", u.Host, timeout)
+			}
 		} else {
-			netConn, err = net.DialTimeout("tcp", u.Host, timeout)
+			netConn, err = fd.Dial(context.TODO(), "tcp", u.Host)
 		}
+
 		if err != nil {
 			return nil, err
 		}