Skip to content

Commit

Permalink
Update: 增加调用ObserverWard获取应用系统的指纹信息
Browse files Browse the repository at this point in the history
  • Loading branch information
hanc00l committed Dec 7, 2021
1 parent e9a1f3b commit 7b53cae
Show file tree
Hide file tree
Showing 26 changed files with 46,797 additions and 81 deletions.
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ Nemo是用来进行自动化信息收集的一个简单平台,通过集成常
- [HTTPX](https://github.com/projectdiscovery/httpx)
- [ScreenShot](https://github.com/chromedp/chromedp) (调用chrome headless)
- [Wappalyzer](https://github.com/AliasIO/Wappalyzer) (基于[webanalyze](https://github.com/rverton/webanalyze) 代码,可[自定义指纹规则](thirdparty/wappalyzer/technologies_custom.json)
- [ObserverWard](https://github.com/0x727/ObserverWard_0x727) (指纹信息来源于https://github.com/0x727/FingerprintHub)


### 4、API接口 (需提供相应的Key)

Expand Down Expand Up @@ -67,7 +69,7 @@ docker-compose up -d

## Install

Tested on [ubuntu18.04 LTS](docs/install_linux.md)[macOS](docs/install_mac.md)
Tested on [ubuntu18.04/20.04 LTS](docs/install_linux.md)[macOS](docs/install_mac.md)



Expand Down Expand Up @@ -99,6 +101,7 @@ Tested on [ubuntu18.04 LTS](docs/install_linux.md)、[macOS](docs/install_mac.md

## 版本更新

- 2.4.5:2021-12-7,增加调用ObserverWard获取应用系统的指纹信息,指纹信息来源于 [FingerprintHub](https://github.com/0x727/FingerprintHub)
- 2.4.4:2021-10-18,对新建任务增加部份提示信息,便于掌握任务执行的参数;状态信息可手动刷新和查看正在执行的任务;
- 2.4.3:2021-10-13,增加IP扫描的masscan+nmap方法,masscan快速进行端口开放扫描,nmap用-sV进行详细扫描;
- 2.4.2:2021-10-9,增加IP扫描的“探测+扫描”模式任务,增加内网资产收集的便利性;去除whatweb的安装和使用(HTTPX已基本可替代其功能);
Expand Down
2 changes: 1 addition & 1 deletion cmd/worker/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func keepAlive() {

func main() {
var concurrency int
flag.IntVar(&concurrency, "c", 2, "concurrent number of tasks")
flag.IntVar(&concurrency, "c", 3, "concurrent number of tasks")
flag.Parse()

go keepAlive()
Expand Down
2 changes: 1 addition & 1 deletion docs/linux_install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ python3 -m pip install -U pip -i https://mirrors.aliyun.com/pypi/simple/ --user
sudo service mysql start \
&& mysql -u root -e 'CREATE DATABASE `nemo` DEFAULT CHARACTER SET utf8mb4;' \
&& mysql -u root -e 'CREATE USER "nemo"@"%" IDENTIFIED BY "nemo2020";GRANT ALL PRIVILEGES ON nemo.* TO "nemo"@"%";FLUSH PRIVILEGES;' \
&& mysql -u root nemo < docker/mysql/initdb.d/docs/nemo.sql \
&& mysql -u root nemo < ../docker/mysql/initdb.d/docs/nemo.sql \

2 changes: 1 addition & 1 deletion package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ tar -cvzf release/nemo_linux_amd64.tar \
tar -cvzf release/worker_linux_amd64.tar \
--exclude=thirdparty/xray/xray_darwin_amd64 --exclude=conf/app.conf --exclude=server.yml \
worker_linux_amd64 conf log thirdparty
tar -cvzf release/worker_darwin_amd64_amd64.tar \
tar -cvzf release/worker_darwin_amd64.tar \
--exclude=thirdparty/xray/xray_linux_amd64 --exclude=conf/app.conf --exclude=server.yml \
worker_darwin_amd64 conf log thirdparty

Expand Down
6 changes: 3 additions & 3 deletions pkg/task/custom/iplocation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ func TestIpLocation_FindPublicIP(t *testing.T) {
}

func TestLoadCustomIP(t *testing.T) {
t.Log(customMap)
t.Log(customBMap)
t.Log(customCMap)
//t.Log(customMap)
//t.Log(customBMap)
//t.Log(customCMap)
}

func TestIpLocation_FindCustomIP(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions pkg/task/domainscan/result.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type Config struct {
IsIPSubnetPortScan bool `json:"subnetPortscan"`
IsScreenshot bool `json:"screenshot"`
IsWappalyzer bool `json:"wappalyzer"`
IsFingerprintHub bool `json:"fingerprinthub"`
PortTaskMode int `json:"portTaskMode"`
}

Expand Down
126 changes: 126 additions & 0 deletions pkg/task/fingerprint/fingerprinthub.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
package fingerprint

import (
"encoding/json"
"fmt"
"github.com/hanc00l/nemo_go/pkg/conf"
"github.com/hanc00l/nemo_go/pkg/logging"
"github.com/hanc00l/nemo_go/pkg/task/domainscan"
"github.com/hanc00l/nemo_go/pkg/task/portscan"
"github.com/hanc00l/nemo_go/pkg/utils"
"github.com/remeh/sizedwaitgroup"
"os"
"os/exec"
"path/filepath"
"runtime"
)

type FingerprintHub struct {
Config Config
ResultPortScan portscan.Result
ResultDomainScan domainscan.Result
}

type FingerprintHubReult struct {
Url string `json:"url"`
WhatWebName []string `json:"what_web_name"`
Priority int `json:"priority"`
Length int `json:"length"`
Title string `json:"title"`
StatusCode int `json:"status_code"`
Plugins []string `json:"plugins"`
}

// NNewFingerprintHub 创建FingerprintHub对象
func NewFingerprintHub(config Config) *FingerprintHub {
return &FingerprintHub{Config: config}
}

// Do 调用ObserverWard,获取指纹
func (f *FingerprintHub) Do() {
swg := sizedwaitgroup.New(fpObserverWardThreadNumber)

if f.ResultPortScan.IPResult != nil {
bport := make(map[int]struct{})
for _, p := range IgnorePort {
bport[p] = struct{}{}
}
for ipName, ipResult := range f.ResultPortScan.IPResult {
for portNumber, _ := range ipResult.Ports {
if _, ok := bport[portNumber]; ok {
continue
}
url := fmt.Sprintf("%v:%v", ipName, portNumber)
swg.Add()
go func(ip string, port int, u string) {
fingerPrintResult := f.RunObserverWard(u)
if len(fingerPrintResult) > 0 {
for _, fpa := range fingerPrintResult {
for _,name := range fpa.WhatWebName {
par := portscan.PortAttrResult{
Source: "ObserverWard",
Tag: "fingerprint",
Content: name,
}
f.ResultPortScan.SetPortAttr(ip, port, par)
}
}
}
swg.Done()
}(ipName, portNumber, url)
}
}
}
if f.ResultDomainScan.DomainResult != nil {
for domain, _ := range f.ResultDomainScan.DomainResult {
swg.Add()
go func(d string) {
fingerPrintResult := f.RunObserverWard(d)
if len(fingerPrintResult) > 0 {
for _, fpa := range fingerPrintResult {
for _,name := range fpa.WhatWebName {
dar := domainscan.DomainAttrResult{
Source: "ObserverWard",
Tag: "fingerprint",
Content: name,
}
f.ResultDomainScan.SetDomainAttr(d, dar)
}
}
}
swg.Done()
}(domain)
}
}
swg.Wait()
}

// RunObserverWard 调用ObserverWard,获取一个目标的指纹
func (f *FingerprintHub) RunObserverWard(url string) []FingerprintHubReult {
resultTempFile := utils.GetTempPathFileName()
defer os.Remove(resultTempFile)

observerWardBinPath := filepath.Join(conf.GetRootPath(), "thirdparty/fingerprinthub", "observer_ward_darwin")
if runtime.GOOS == "linux" {
observerWardBinPath = filepath.Join(conf.GetRootPath(), "thirdparty/fingerprinthub", "observer_ward_amd64")
}
var cmdArgs []string
cmdArgs = append(cmdArgs, "-t", url, "-j", resultTempFile)
cmd := exec.Command(observerWardBinPath, cmdArgs...)
_, err := cmd.CombinedOutput()
if err != nil {
logging.RuntimeLog.Error(err.Error())
return nil
}
return parseObserverWardResult(resultTempFile)
}

// parseObserverWardResult 解析结果
func parseObserverWardResult(outputTempFile string) (result []FingerprintHubReult) {
content, err := os.ReadFile(outputTempFile)
if err != nil || len(content) == 0 {
return
}
json.Unmarshal(content, &result)
return
}
40 changes: 40 additions & 0 deletions pkg/task/fingerprint/fingerprinthub_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package fingerprint

import (
"github.com/hanc00l/nemo_go/pkg/task/portscan"
"testing"
)

func TestFingerprintHub_RunObserverWard(t *testing.T) {
f := NewFingerprintHub(Config{Target: "183.141.20.136:8082"})
rs := f.RunObserverWard("183.141.20.136:8082")
for _,fp := range rs{
t.Log(fp)
for _,n := range fp.WhatWebName {
t.Log(n)
}
}
}

func TestFingerprintHub_Do(t *testing.T) {
nmapConfig := portscan.Config{
Target: "183.60.156.84",
Port: "8088",
Rate: 1000,
IsPing: false,
Tech: "-sS",
CmdBin: "nmap",
}
nmap := portscan.NewNmap(nmapConfig)
nmap.Do()
t.Log(nmap.Result)

fp := NewFingerprintHub(Config{})
fp.ResultPortScan = nmap.Result
fp.Do()
for _,r := range fp.ResultPortScan.IPResult{
for port,p := range r.Ports{
t.Log(port,p)
}
}
}
1 change: 1 addition & 0 deletions pkg/task/fingerprint/result.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const (
fpWhatwebThreadNumber = 4
fpScreenshotThreadNum = 5
fpWappalyzerThreadNumber = 10
fpObserverWardThreadNumber = 10
)

type Config struct {
Expand Down
1 change: 0 additions & 1 deletion pkg/task/fingerprint/screenshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,4 @@ func TestScreenShot_Do(t *testing.T) {
t.Log(s)
}
}
ss.UploadResult()
}
33 changes: 17 additions & 16 deletions pkg/task/portscan/result.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,20 @@ const (

// Config 端口扫描的参数配置
type Config struct {
Target string `json:"target"`
ExcludeTarget string `json:"executeTarget"`
Port string `json:"port"`
OrgId *int `json:"orgId"`
Rate int `json:"rate"`
IsPing bool `json:"ping"`
Tech string `json:"tech"`
IsIpLocation bool `json:"ipLocation"`
IsHttpx bool `json:"httpx"`
IsWhatWeb bool `json:"whatweb"`
IsScreenshot bool `json:"screenshot"`
IsWappalyzer bool `json:"wappalyzer"`
CmdBin string `json:"cmdBin"`
Target string `json:"target"`
ExcludeTarget string `json:"executeTarget"`
Port string `json:"port"`
OrgId *int `json:"orgId"`
Rate int `json:"rate"`
IsPing bool `json:"ping"`
Tech string `json:"tech"`
IsIpLocation bool `json:"ipLocation"`
IsHttpx bool `json:"httpx"`
IsWhatWeb bool `json:"whatweb"`
IsScreenshot bool `json:"screenshot"`
IsWappalyzer bool `json:"wappalyzer"`
IsFingerprintHub bool `json:"fingerprinthub"`
CmdBin string `json:"cmdBin"`
}

// PortAttrResult 端口属性结果
Expand Down Expand Up @@ -139,11 +140,11 @@ func (r *Result) SaveResult(config Config) string {
}

// filterIPHasTooMuchPort 过滤有安全防护、显示太多端口开放的IP
func filterIPHasTooMuchPort(result Result){
func filterIPHasTooMuchPort(result Result) {
for ipName, ipResult := range result.IPResult {
if len(ipResult.Ports) > IpOpenedPortFilterNumber {
logging.RuntimeLog.Infof("ip:%s has too much open port:%d,discard to save!", ipName, len(ipResult.Ports))
delete(result.IPResult,ipName)
delete(result.IPResult, ipName)
}
}
}
}
5 changes: 5 additions & 0 deletions pkg/task/workerapi/batchscan.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,11 @@ func BatchScan(taskId, configJSON string) (result string, err error) {
httpx.ResultPortScan = resultPortScan
httpx.Do()
}
if config.IsFingerprintHub {
fp := fingerprint.NewFingerprintHub(fpConfig)
fp.ResultPortScan = resultPortScan
fp.Do()
}
}
// 保存结果
x := comm.NewXClient()
Expand Down
28 changes: 17 additions & 11 deletions pkg/task/workerapi/domainscan.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ func doFingerPrint(config domainscan.Config, resultDomainScan *domainscan.Result
wappalyzer.ResultDomainScan = *resultDomainScan
wappalyzer.Do()
}
if config.IsFingerprintHub {
fp := fingerprint.NewFingerprintHub(fpConfig)
fp.ResultDomainScan = *resultDomainScan
fp.Do()
}
}

// doPortScan 对IP进行端口扫描
Expand All @@ -138,17 +143,18 @@ func doPortScan(config domainscan.Config, resultDomainScan *domainscan.Result) {
for _, t := range targets {
for _, p := range ports {
configPortScan := portscan.Config{
OrgId: config.OrgId,
Target: t,
Port: p,
Rate: portsConfig.Rate,
CmdBin: portsConfig.Cmdbin,
IsPing: portsConfig.IsPing,
Tech: portsConfig.Tech,
IsIpLocation: true,
IsHttpx: config.IsHttpx,
IsWhatWeb: config.IsWhatWeb,
IsScreenshot: config.IsScreenshot,
OrgId: config.OrgId,
Target: t,
Port: p,
Rate: portsConfig.Rate,
CmdBin: portsConfig.Cmdbin,
IsPing: portsConfig.IsPing,
Tech: portsConfig.Tech,
IsIpLocation: true,
IsHttpx: config.IsHttpx,
IsWhatWeb: config.IsWhatWeb,
IsScreenshot: config.IsScreenshot,
IsFingerprintHub: config.IsFingerprintHub,
}
configPortScanJSON, _ := json.Marshal(configPortScan)
serverapi.NewTask("portscan", string(configPortScanJSON))
Expand Down
5 changes: 5 additions & 0 deletions pkg/task/workerapi/portscan.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,11 @@ func PortScan(taskId, configJSON string) (result string, err error) {
wappalyzer.ResultPortScan = resultPortScan
wappalyzer.Do()
}
if config.IsFingerprintHub {
fp := fingerprint.NewFingerprintHub(fpConfig)
fp.ResultPortScan = resultPortScan
fp.Do()
}
// 保存结果
resultArgs := comm.ScanResultArgs{
IPConfig: &config,
Expand Down
2 changes: 1 addition & 1 deletion pkg/web/controllers/domain.go
Original file line number Diff line number Diff line change
Expand Up @@ -564,7 +564,7 @@ func getDomainAttrFullInfo(id int, disableFofa bool) DomainAttrFullInfo {
if _, ok := r.TitleSet[da.Content]; !ok {
r.TitleSet[da.Content] = struct{}{}
}
} else if da.Tag == "server" {
} else if da.Tag == "server" || da.Tag == "fingerprint" {
if _, ok := r.BannerSet[da.Content]; !ok {
r.BannerSet[da.Content] = struct{}{}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/web/controllers/ip.go
Original file line number Diff line number Diff line change
Expand Up @@ -477,7 +477,7 @@ func getPortInfo(ip string, ipId int, disableFofa bool) (r PortInfo) {
if _, ok := r.TitleSet[pad.Content]; !ok {
r.TitleSet[pad.Content] = struct{}{}
}
} else if pad.Tag == "banner" || pad.Tag == "server" || pad.Tag == "tag" {
} else if pad.Tag == "banner" || pad.Tag == "server" || pad.Tag == "tag" || pad.Tag == "fingerprint"{
if pad.Content == "unknown" {
continue
}
Expand Down
Loading

0 comments on commit 7b53cae

Please sign in to comment.