Update v0.4

README.md

@@ -11,7 +11,7 @@ GO111MODULE=on go get github.com/j3ssie/goverview
 ## Example Commands
 
 ```shell
-goverview - Overview about list of URLs - beta v0.2.2 by @j3ssiejjj
+goverview - Overview about list of URLs - beta v0.4 by @j3ssiejjj
 
 Usage:
   goverview [command]
@@ -31,14 +31,16 @@ Flags:
   -I, --inputFile string    Custom headers (e.g: -H 'Referer: {{.BaseURL}}') (Multiple -H flags are accepted)
   -i, --inputs strings      Custom headers (e.g: -H 'Referer: {{.BaseURL}}') (Multiple -H flags are accepted)
   -j, --json                Output as JSON
-  -l, --level int           Set level to calculate CheckSum
+  -l, --level int           Set level to calculate CheckSum (default: 0)
   -N, --no-output           No output
   -o, --output string       Output Directory (default "out")
+  -P, --proxy string        Proxy to send http request
   -L, --redirect            Allow redirect
       --retry int           Number of retry
   -R, --save-redirect       Save redirect URL to overview file too
   -S, --screenshot string   Summary File for Screenshot (default 'out/screenshot-summary.txt')
       --sortTag             Sort HTML tag before do checksum
+  -a, --tech string         Technology File (default "technologies.json")
   -t, --threads int         Set the threads level for do screenshot (default 5)
       --timeout int         HTTP timeout (default 15)
   -v, --verbose             Verbose output
@@ -61,16 +63,20 @@ Examples:
   # Get summary content and store raw response without screenshot
   cat http_lists.txt | goverview probe -c 20 -M --json
 
-  # Only do screenshot
-  cat list_of_urls.txt | goverview --skip-probe
+  # Pass all urls to proxy with real browser
+  cat list_of_urls.txt | goverview screen --proxy http://127.0.0.1:8080
 
-  # Do screnshot
+  # Do screenshot and store JSON Output
   cat http_lists.txt | goverview screen -c 5 --json
 
   # Do screnshot based on success HTTP site
-  cat overview/target.com-http-overview.txt | jq -r '. | select(.status=="200") | .url' | goverview screen -c 5 -o overview -S overview/target.com-screen.txt
+  cat overview/target.com-http-overview.txt | jq -r '. | select(.status=="200") | .url' | goverview screen -c 5 -o overview -S overview/target.com-
 ```
 
 ## License
 
 `goverview` is made with ♥ by [@j3ssiejjj](https://twitter.com/j3ssiejjj) and it is released under the MIT license.
+
+## Donation
+
+[![paypal](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif)](https://paypal.me/j3ssiejjj)

cmd/root.go

@@ -51,6 +51,7 @@ func init() {
 	RootCmd.PersistentFlags().BoolVarP(&options.SaveRedirectURL, "save-redirect", "R", false, "Save redirect URL to overview file too")
 	RootCmd.PersistentFlags().IntVar(&options.Timeout, "timeout", 15, "HTTP timeout")
 	RootCmd.PersistentFlags().IntVar(&options.Retry, "retry", 0, "Number of retry")
+	RootCmd.PersistentFlags().StringVarP(&options.Proxy, "proxy", "P", "", "Proxy to send http request")
 	RootCmd.PersistentFlags().StringSliceVarP(&options.Headers, "headers", "H", []string{}, "Custom headers (e.g: -H 'Referer: {{.BaseURL}}') (Multiple -H flags are accepted)")
 
 	RootCmd.PersistentFlags().BoolVarP(&options.Verbose, "verbose", "v", false, "Verbose output")
@@ -85,12 +86,7 @@ func initConfig() {
 					urls = append(urls, url)
 				}
 			}
-			//
-			//// store stdin as a temp file
-			//urlFile := path.Join(options.TmpDir, fmt.Sprintf("raw-%s", utils.RandomString(8)))
-			//os.MkdirAll(options.TmpDir, 0755)
-			//utils.DebugF("Write stdin data to: %v", urlFile)
-			//utils.WriteToFile(urlFile, strings.Join(urls, "\n"))
+
 		}
 	}
 	inputs = urls
@@ -112,9 +108,9 @@ func HelpMessage(cmd *cobra.Command, _ []string) {
 	h += "  cat http_lists.txt | goverview probe -N -c 50 | tee only-overview.txt\n\n"
 	h += "  # Get summary content and store raw response without screenshot \n"
 	h += "  cat http_lists.txt | goverview probe -c 20 -M --json\n\n"
-	h += "  # Only do screenshot \n"
-	h += "  cat list_of_urls.txt | goverview --skip-probe \n\n"
-	h += "  # Do screnshot \n"
+	h += "  # Pass all urls to proxy with real browser\n"
+	h += "  cat list_of_urls.txt | goverview screen --proxy http://127.0.0.1:8080 \n\n"
+	h += "  # Do screenshot and store JSON Output\n"
 	h += "  cat http_lists.txt | goverview screen -c 5 --json\n\n"
 	h += "  # Do screnshot based on success HTTP site \n"
 	h += "  cat overview/target.com-http-overview.txt | jq -r '. | select(.status==\"200\") | .url' | goverview screen -c 5 -o overview -S overview/target.com-screen.txt\n\n"

core/request.go

@@ -58,6 +58,10 @@ func BuildClient(options libs.Options) *resty.Client {
 		TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
 	})
 
+	if options.Proxy != "" {
+		client.SetProxy(options.Proxy)
+	}
+
 	client.SetHeaders(headers)
 	client.SetCloseConnection(true)
 	if options.Retry > 0 {

core/screenshot.go

@@ -81,6 +81,10 @@ func DoScreenshot(options libs.Options, raw string) string {
 		chromedp.Flag("no-default-browser-check", true),
 	)
 
+	if options.Proxy != "" {
+		opts = append(opts, chromedp.ProxyServer(options.Proxy))
+	}
+
 	// create context
 	allocCtx, bcancel := chromedp.NewExecAllocator(context.Background(), opts...)
 	defer bcancel()

libs/options.go

@@ -16,6 +16,7 @@ type Options struct {
 	Headers         []string
 	Inputs          []string
 	InputFile       string
+	Proxy       string
 	Timeout         int
 	Retry           int
 	Level           int

libs/version.go

@@ -2,7 +2,7 @@ package libs
 
 const (
 	// VERSION current goverview version
-	VERSION = "beta v0.3.2"
+	VERSION = "beta v0.4"
 	// AUTHOR author of this
 	AUTHOR = "@j3ssiejjj"
 )