diff --git a/README.md b/README.md index 6a07d06..84c80c0 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ $ gau -h |`--providers`| list of providers to use (wayback,commoncrawl,otx,urlscan) | gau --providers wayback| |`--proxy`| http proxy to use (socks5:// or http:// | gau --proxy http://proxy.example.com:8080 | |`--retries`| retries for HTTP client | gau --retries 10 | +|`--timeout`| timeout (in seconds) for HTTP client | gau --timeout 60 | |`--subs`| include subdomains of target domain | gau example.com --subs | |`--threads`| number of workers to spawn | gau example.com --threads | |`--to`| fetch urls to date (format: YYYYMM) | gau example.com --to 202101 | diff --git a/pkg/httpclient/client.go b/pkg/httpclient/client.go index 408a849..d9f2cd1 100644 --- a/pkg/httpclient/client.go +++ b/pkg/httpclient/client.go @@ -15,12 +15,12 @@ type Header struct { Value string } -func MakeRequest(c *fasthttp.Client, url string, maxRetries int, headers ...Header) ([]byte, error) { +func MakeRequest(c *fasthttp.Client, url string, maxRetries uint, timeout uint, headers ...Header) ([]byte, error) { var ( req *fasthttp.Request resp *fasthttp.Response ) - retries := maxRetries + retries := int(maxRetries) for i := retries; i >= 0; i-- { req = fasthttp.AcquireRequest() defer fasthttp.ReleaseRequest(req) @@ -35,7 +35,7 @@ func MakeRequest(c *fasthttp.Client, url string, maxRetries int, headers ...Head resp = fasthttp.AcquireResponse() defer fasthttp.ReleaseResponse(resp) - if err := c.DoTimeout(req, resp, time.Second*45); err != nil { + if err := c.DoTimeout(req, resp, time.Second*time.Duration(timeout)); err != nil { fasthttp.ReleaseRequest(req) if retries == 0 { return nil, err diff --git a/pkg/providers/commoncrawl/commoncrawl.go b/pkg/providers/commoncrawl/commoncrawl.go index ccc324e..920a71b 100644 --- a/pkg/providers/commoncrawl/commoncrawl.go +++ b/pkg/providers/commoncrawl/commoncrawl.go @@ -31,7 +31,7 @@ type Client struct { func New(c *providers.Config, filters providers.Filters) (*Client, error) { client := &Client{config: c, filters: filters} // Fetch the list of available CommonCrawl Api URLs. - resp, err := httpclient.MakeRequest(c.Client, "http://index.commoncrawl.org/collinfo.json", int(c.MaxRetries)) + resp, err := httpclient.MakeRequest(c.Client, "http://index.commoncrawl.org/collinfo.json", c.MaxRetries, c.Timeout) if err != nil { return nil, err } @@ -78,7 +78,7 @@ paginate: logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain) } apiURL := c.formatURL(domain, page) - resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries)) + resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout) if err != nil { return fmt.Errorf("failed to fetch commoncrawl(%d): %s", page, err) } @@ -114,7 +114,7 @@ func (c *Client) formatURL(domain string, page uint) string { func (c *Client) getPagination(domain string) (paginationResult, error) { url := fmt.Sprintf("%s&showNumPages=true", c.formatURL(domain, 0)) - resp, err := httpclient.MakeRequest(c.config.Client, url, int(c.config.MaxRetries)) + resp, err := httpclient.MakeRequest(c.config.Client, url, c.config.MaxRetries, c.config.Timeout) if err != nil { return paginationResult{}, err } diff --git a/pkg/providers/otx/otx.go b/pkg/providers/otx/otx.go index 397ed01..6a32359 100644 --- a/pkg/providers/otx/otx.go +++ b/pkg/providers/otx/otx.go @@ -56,7 +56,7 @@ paginate: logrus.WithFields(logrus.Fields{"provider": Name, "page": page - 1}).Infof("fetching %s", domain) } apiURL := c.formatURL(domain, page) - resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries)) + resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout) if err != nil { return fmt.Errorf("failed to fetch alienvault(%d): %s", page, err) } diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index 4324413..a826597 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -5,7 +5,7 @@ import ( "github.com/valyala/fasthttp" ) -const Version = `2.0.9` +const Version = `2.1.0` // Provider is a generic interface for all archive fetchers type Provider interface { @@ -20,6 +20,7 @@ type URLScan struct { type Config struct { Threads uint + Timeout uint Verbose bool MaxRetries uint IncludeSubdomains bool diff --git a/pkg/providers/urlscan/urlscan.go b/pkg/providers/urlscan/urlscan.go index af21c85..0a102cc 100644 --- a/pkg/providers/urlscan/urlscan.go +++ b/pkg/providers/urlscan/urlscan.go @@ -51,7 +51,7 @@ paginate: logrus.WithFields(logrus.Fields{"provider": Name, "page": page}).Infof("fetching %s", domain) } apiURL := c.formatURL(domain, searchAfter) - resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries), header) + resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout, header) if err != nil { return fmt.Errorf("failed to fetch urlscan: %s", err) } diff --git a/pkg/providers/wayback/wayback.go b/pkg/providers/wayback/wayback.go index fd5de2a..3b73c79 100644 --- a/pkg/providers/wayback/wayback.go +++ b/pkg/providers/wayback/wayback.go @@ -53,7 +53,7 @@ func (c *Client) Fetch(ctx context.Context, domain string, results chan string) } apiURL := c.formatURL(domain, page) // make HTTP request - resp, err := httpclient.MakeRequest(c.config.Client, apiURL, int(c.config.MaxRetries)) + resp, err := httpclient.MakeRequest(c.config.Client, apiURL, c.config.MaxRetries, c.config.Timeout) if err != nil { return fmt.Errorf("failed to fetch wayback results page %d: %s", page, err) } @@ -96,7 +96,8 @@ func (c *Client) formatURL(domain string, page uint) string { // getPagination returns the number of pages for Wayback func (c *Client) getPagination(domain string) (uint, error) { url := fmt.Sprintf("%s&showNumPages=true", c.formatURL(domain, 0)) - resp, err := httpclient.MakeRequest(c.config.Client, url, int(c.config.MaxRetries)) + resp, err := httpclient.MakeRequest(c.config.Client, url, c.config.MaxRetries, c.config.Timeout) + if err != nil { return 0, err } diff --git a/runner/flags/flags.go b/runner/flags/flags.go index 0d1eeec..b3bbc21 100644 --- a/runner/flags/flags.go +++ b/runner/flags/flags.go @@ -26,6 +26,7 @@ type Config struct { Filters providers.Filters `mapstructure:"filters"` Proxy string `mapstructure:"proxy"` Threads uint `mapstructure:"threads"` + Timeout uint `mapstructure:"timeout"` Verbose bool `mapstructure:"verbose"` MaxRetries uint `mapstructure:"retries"` IncludeSubdomains bool `mapstructure:"subdomains"` @@ -58,6 +59,7 @@ func (c *Config) ProviderConfig() (*providers.Config, error) { pc := &providers.Config{ Threads: c.Threads, + Timeout: c.Timeout, Verbose: c.Verbose, MaxRetries: c.MaxRetries, IncludeSubdomains: c.IncludeSubdomains, @@ -94,7 +96,8 @@ func New() *Options { v := viper.New() pflag.String("o", "", "filename to write results to") - pflag.Uint("threads", 1, "number of workers to spawn, default: 1") + pflag.Uint("threads", 1, "number of workers to spawn") + pflag.Uint("timeout", 45, "timeout (in seconds) for HTTP client") pflag.Uint("retries", 0, "retries for HTTP client") pflag.String("proxy", "", "http proxy to use") pflag.StringSlice("blacklist", []string{}, "list of extensions to skip")