feat: rule improvement & optional output

riza · Jun 26, 2022 · ed206af · ed206af
1 parent 76752fc
commit ed206af
Show file tree

Hide file tree

Showing 10 changed files with 79 additions and 111 deletions.
diff --git a/README.md b/README.md
@@ -26,14 +26,14 @@ go install -v github.com/riza/linx/cmd/linx@latest
 # Usage
 
 ```sh
-linx --target=https://rizasabuncu.com/assets/admin_acces.js
+linx --target=https://rizasabuncu.com/assets/admin_acces.js --output=admin_access_result.html
 ```
 
 # TODOs
 
-* [x] HTML output support 
+* [x] HTML output support
 * [ ] JSON output support
 * [ ] Custom cookie support
-* [ ] Rule improvement & blacklist support
+* [x] Rule improvement & blacklist support
 * [ ] Support parallel scan multiple files
 * [ ] ...
diff --git a/cmd/linx/linx.go b/cmd/linx/linx.go
@@ -7,7 +7,7 @@ import (
 	"github.com/riza/linx/pkg/logger"
 )
 
-const Version = "v0.0.2"
+const Version = "v0.0.4"
 
 func main() {
 	banner.Show(Version)
@@ -17,7 +17,7 @@ func main() {
 		logger.Get().Fatal(err)
 	}
 
-	scanner := scanner.NewScanner(opts.Target)
+	scanner := scanner.NewScanner(opts)
 	err = scanner.Run()
 	if err != nil {
 		logger.Get().Error(err)

diff --git a/internal/options/options.go b/internal/options/options.go
@@ -10,6 +10,7 @@ import (
 
 type Options struct {
 	Target string
+	Output string
 	Debug  bool
 }
 
@@ -28,6 +29,7 @@ func Get() *Options {
 func (o *Options) Parse() (*Options, error) {
 	flag.StringVar(&o.Target, "target", "", "can be *.js file path or url")
 	flag.BoolVar(&o.Debug, "debug", false, "do you want to know what's inside the engine?")
+	flag.StringVar(&o.Output, "output", "", "output file name (currently support html)")
 	flag.Parse()
 
 	if o.Debug {

diff --git a/internal/output/output.go b/internal/output/output.go
@@ -1,5 +1,9 @@
 package output
 
+type Output interface {
+	RenderAndSave(data OutputData) error
+}
+
 type OutputData struct {
 	Target   string
 	Filename string

diff --git a/internal/output/output_html.go b/internal/output/output_html.go
@@ -1,12 +1,13 @@
 package output
 
 import (
+	"github.com/riza/linx/pkg/logger"
 	"html/template"
 	"os"
 )
 
-const templateFile = "./internal/output/output_html_template.html"
-const templateRaw = `<!doctype html>
+const htmlExtension = ".html"
+const htmlTemplate = `<!doctype html>
 <html lang="en">
 <head>
     <meta charset="utf-8">
@@ -17,13 +18,13 @@ const templateRaw = `<!doctype html>
 </head>
 <body>
 
-<div class="container">
+<div style="margin-top: 2%;margin-bottom: 5%;" class="container">
     <div class="mb-5 pb-3 fs-4 border-bottom">
         {{ .Target }}
     </div>
 
     <div class="table-responsive">
-        <table class="table table-striped table-hover">
+        <table style="table-layout: fixed" class="table table-striped table-hover">
             <thead>
             <tr>
                 <th scope="col">URL</th>
@@ -55,29 +56,27 @@ const templateRaw = `<!doctype html>
 </html>`
 
 type OutputHTML struct {
-	output OutputData
 }
 
-func NewOutputHTML(output OutputData) OutputHTML {
-	return OutputHTML{output: output}
-}
+func (oh OutputHTML) RenderAndSave(data OutputData) error {
+	fileName := data.Filename + htmlExtension
 
-func (oh OutputHTML) RenderAndSave() error {
-	f, err := os.Create(oh.output.Filename)
+	f, err := os.Create(fileName)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
 
-	t, err := template.New("output").Parse(templateRaw)
+	t, err := template.New("output").Parse(htmlTemplate)
 	if err != nil {
 		return err
 	}
 
-	err = t.Execute(f, oh.output)
+	err = t.Execute(f, data)
 	if err != nil {
 		return err
 	}
 
+	logger.Get().Infof("results saved: %s", fileName)
 	return nil
 }
diff --git a/internal/output/output_html_template.html b/internal/output/output_html_template.html
diff --git a/internal/output/output_noop.go b/internal/output/output_noop.go
@@ -0,0 +1,8 @@
+package output
+
+type OutputNoop struct {
+}
+
+func (o OutputNoop) RenderAndSave(data OutputData) error {
+	return nil
+}
diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go
@@ -2,39 +2,54 @@ package scanner
 
 import (
 	"fmt"
+	"github.com/riza/linx/internal/options"
 	"github.com/riza/linx/internal/output"
 	"github.com/riza/linx/internal/scanner/strategies"
 	"github.com/riza/linx/pkg/logger"
-	"os/exec"
+	"path/filepath"
 	"regexp"
-	"runtime"
 	"strings"
 	"unsafe"
 )
 
 // rule from LinkFinder https://github.com/GerbenJavado/LinkFinder/blob/master/linkfinder.py#L29 ty @GerbenJavado
-const rule = `(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')`
+const (
+	rule                = `(?:"|')(((?:[a-zA-Z]{1,10}://|//)[^"'/]{1,}\.[a-zA-Z]{2,}[^"']{0,})|((?:/|\.\./|\./)[^"'><,;| *()(%%$^/\\\[\]][^"'><,;|()]{1,})|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{1,}\.(?:[a-zA-Z]{1,4}|action)(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-/]{1,}/[a-zA-Z0-9_\-/]{3,}(?:[\?|#][^"|']{0,}|))|([a-zA-Z0-9_\-]{1,}\.(?:php|asp|aspx|jsp|json|action|html|js|txt|xml)(?:[\?|#][^"|']{0,}|)))(?:"|')`
+	excludeFileTypeRule = `.css|.jpg|.jpeg|.png|.svg|.img|.gif|.mp4|.flv|.ogv|.webm|.webp|.mov|.mp3|.m4a|.m4p|.scss|.tif|.tiff|.ttf|.otf|.woff|.woff2|.bmp|.ico|.eot|.htc|.rtf|.swf|.image|w3.org|doubleclick.net|youtube.com|.vue|jquery|bootstrap|font|jsdelivr.net|vimeo.com|pinterest.com|facebook|linkedin|twitter|instagram|google|mozilla.org|jibe.com|schema.org|schemas.microsoft.com|wordpress.org|w.org|wix.com|parastorage.com|whatwg.org|polyfill.io|typekit.net|schemas.openxmlformats.org|openweathermap.org|openoffice.org|reactjs.org|angularjs.org|java.com|purl.org|/image|/img|/css|/wp-json|/wp-content|/wp-includes|/theme|/audio|/captcha|/font|robots.txt|node_modules|.wav|.gltf|.js`
+	excludeMimeTypeRule = `text/css|image/jpeg|image/jpg|image/png|image/svg+xml|image/gif|image/tiff|image/webp|image/bmp|image/x-icon|image/vnd.microsoft.icon|font/ttf|font/woff|font/woff2|font/x-woff2|font/x-woff|font/otf|audio/mpeg|audio/wav|audio/webm|audio/aac|audio/ogg|audio/wav|audio/webm|video/mp4|video/mpeg|video/webm|video/ogg|video/mp2t|video/webm|video/x-msvideo|application/font-woff|application/font-woff2|application/vnd.android.package-archive|binary/octet-stream|application/octet-stream|application/pdf|application/x-font-ttf|application/x-font-otf|application/json|text/javascript|text/plain|text/x-yaml|text/html|text/babel|text/markdown|text/tsx|application/typescript|application/javascript|text/x-handlebars-template|application/x-typescript|text/x-gfm|text/jsx`
+)
+
+var (
+	outputEngines = map[string]output.Output{
+		"":      output.OutputNoop{},
+		".html": output.OutputHTML{},
+	}
+)
 
 type task struct {
 	target   string
+	output   string
 	strategy strategies.ScanStrategy
 }
 
 type scanner struct {
 	task task
 }
 
-func NewScanner(target string) scanner {
+func NewScanner(opts *options.Options) scanner {
 	return scanner{
 		task{
-			target:   target,
-			strategy: defineStrategyForTarget(target),
+			target:   opts.Target,
+			output:   opts.Output,
+			strategy: defineStrategyForTarget(opts.Target),
 		},
 	}
 }
 
 func (s scanner) Run() error {
 	r, _ := regexp.Compile(rule)
+	rFt, _ := regexp.Compile(excludeFileTypeRule)
+	rMt, _ := regexp.Compile(excludeMimeTypeRule)
 
 	strategy := s.task.strategy
 	content, err := strategy.GetContent()
@@ -44,14 +59,17 @@ func (s scanner) Run() error {
 
 	out := output.OutputData{
 		Target:   s.task.target,
-		Filename: strategy.GetFileName() + "_result.html",
+		Filename: strategy.GetFileName(),
 		Results:  []output.Result{},
 	}
 
 	for _, s := range r.FindAllStringSubmatchIndex(*(*string)(unsafe.Pointer(&content)), -1) {
 		url := content[s[0]:s[1]]
-		closeLines := content[s[0]-100 : s[1]+100]
+		if rFt.MatchString(string(url)) || rMt.MatchString(string(url)) {
+			continue
+		}
 
+		closeLines := content[s[0]-100 : s[1]+100]
 		out.Results = append(out.Results, output.Result{
 			URL:      string(url),
 			Location: string(closeLines),
@@ -61,42 +79,26 @@ func (s scanner) Run() error {
 	}
 
 	logger.Get().Infof("%d possible url found", len(out.Results))
-
-	err = output.NewOutputHTML(out).RenderAndSave()
-	if err != nil {
-		return fmt.Errorf(errOutputFailed, err)
+	oE, ok := outputEngines[s.getOutputEngineKey()]
+	if !ok {
+		return fmt.Errorf(errOutputEngineNotFound, s.getOutputEngineKey())
 	}
 
-	err = openResults(out.Filename)
+	err = oE.RenderAndSave(out)
 	if err != nil {
-		return fmt.Errorf(errOutputOpenFailed, err)
+		return fmt.Errorf(errOutputFailed, err)
 	}
 
 	return nil
 }
 
+func (s scanner) getOutputEngineKey() string {
+	return filepath.Ext(s.task.output)
+}
+
 func defineStrategyForTarget(target string) strategies.ScanStrategy {
 	if strings.Contains(target, "http://") || strings.Contains(target, "https://") {
 		return strategies.URLStrategy{target}
 	}
 	return strategies.FileStrategy{target}
 }
-
-func openResults(fileName string) error {
-	var err error
-	switch runtime.GOOS {
-	case "linux":
-		err = exec.Command("xdg-open", fileName).Start()
-	case "windows":
-		err = exec.Command("rundll32", "url.dll,FileProtocolHandler", fileName).Start()
-	case "darwin":
-		err = exec.Command("open", fileName).Start()
-	default:
-		err = fmt.Errorf("unsupported platform")
-	}
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
diff --git a/internal/scanner/scanner_errors.go b/internal/scanner/scanner_errors.go
@@ -1,7 +1,7 @@
 package scanner
 
 var (
-	errGetFileContent   = "error getting content err: %w"
-	errOutputFailed     = "output fail render and save err: %w"
-	errOutputOpenFailed = "output fail opening err: %w"
+	errGetFileContent       = "error getting content err: %w"
+	errOutputFailed         = "output fail render and save err: %w"
+	errOutputEngineNotFound = "output engine not fount ext: %s"
 )
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
@@ -21,47 +21,47 @@ func Get() *logger {
 }
 
 func (l *logger) Print(args ...interface{}) {
-	l.l.Print(args)
+	l.l.Print(args...)
 }
 
 func (l *logger) Printf(format string, args ...interface{}) {
-	l.l.Printf(format, args)
+	l.l.Printf(format, args...)
 }
 
 func (l *logger) Info(args ...interface{}) {
-	l.l.Info(args)
+	l.l.Info(args...)
 }
 
 func (l *logger) Infof(format string, args ...interface{}) {
-	l.l.Infof(format, args)
+	l.l.Infof(format, args...)
 }
 
 func (l *logger) Warn(args ...interface{}) {
-	l.l.Warn(args)
+	l.l.Warn(args...)
 }
 
 func (l *logger) Warnf(format string, args ...interface{}) {
-	l.l.Warnf(format, args)
+	l.l.Warnf(format, args...)
 }
 
 func (l *logger) Error(args ...interface{}) {
-	l.l.Error(args)
+	l.l.Error(args...)
 }
 
 func (l *logger) Errorf(format string, args ...interface{}) {
-	l.l.Errorf(format, args)
+	l.l.Errorf(format, args...)
 }
 
 func (l *logger) Fatal(args ...interface{}) {
-	l.l.Fatal(args)
+	l.l.Fatal(args...)
 }
 
 func (l *logger) Debug(args ...interface{}) {
-	l.l.Debug(args)
+	l.l.Debug(args...)
 }
 
 func (l *logger) Debugf(format string, args ...interface{}) {
-	l.l.Debugf(format, args)
+	l.l.Debugf(format, args...)
 }
 
 func (l *logger) SetLevelDebug() {