Update: 增加IP扫描的“探测+扫描”模式任务，增加内网资产收集的便利性；去除whatweb的安装和使用（HTTPX已基本可替代其功能）

Dockerfile

@@ -10,7 +10,7 @@ RUN set -x \
     && apt-get update \
     && apt-get install -y  python3-pip python3-setuptools \
     wget curl vim net-tools  iputils-ping git unzip \
-    nmap whatweb masscan chromium-browser --fix-missing
+    nmap masscan chromium-browser --fix-missing
 
 # pip package
 RUN set -x \

README.md

@@ -23,7 +23,6 @@ Nemo是用来进行自动化信息收集的一个简单平台，通过集成常
 
 ### 3、指纹信息
 
-- [WhatWeb](https://github.com/urbanadventurer/WhatWeb)
 - [HTTPX](https://github.com/projectdiscovery/httpx) 
 - [ScreenShot](https://github.com/chromedp/chromedp) （调用chrome headless）
 - [Wappalyzer](https://github.com/AliasIO/Wappalyzer) （基于[webanalyze](https://github.com/rverton/webanalyze) 代码，可[自定义指纹规则](thirdparty/wappalyzer/technologies_custom.json)）
@@ -100,6 +99,7 @@ Tested on [ubuntu18.04 LTS](docs/install_linux.md)、[macOS](docs/install_mac.md
 
 ## 版本更新
 
+- 2.4.2：2021-10-9，增加IP扫描的“探测+扫描”模式任务，增加内网资产收集的便利性；去除whatweb的安装和使用（HTTPX已基本可替代其功能）；
 - 2.4.1：2021-9-15，支持扫描任务按IP和端口进行多维度切分，使任务在多个worker之间均衡分布执行；
 - 2.4.0：2021-9-10，使用RPC架构，优化server与worker之间的同步、server与worker的配置文件分离；增加在线的IP信息、登录验证码、按发现时间筛选资产功能。
 - 2.3：2021-8-25，使用docker-compose构建Docker，修改数据库连接重试功能，完善端口扫描及任务等信息；

docs/install_linux.md

@@ -106,7 +106,7 @@
   ```bash
   sudo apt-get update \
       && sudo apt-get install vim git python3-pip python3-setuptools \
-      nmap whatweb masscan chromium-browser --fix-missing
+      nmap masscan chromium-browser --fix-missing
   ```
 
 

docs/linux_install.sh

@@ -4,11 +4,11 @@
 
 #apt-get
 #只有server需要安装 mysql-server和rabbitmq-server
-#只有worker需要安装 nmap whatweb masscan chromium-browser python及pocsuite
+#只有worker需要安装 nmap masscan chromium-browser python及pocsuite
 sudo apt-get update \
     && sudo apt-get install wget curl vim net-tools git unzip \
     mysql-server rabbitmq-server \
-    python3-pip python3-setuptools nmap whatweb masscan chromium-browser --fix-missing
+    python3-pip python3-setuptools nmap masscan chromium-browser --fix-missing
 
 # pip package
 # 只有worker需要安装python

go.mod

@@ -3,23 +3,16 @@ module github.com/hanc00l/nemo_go
 go 1.16
 
 require (
-	cloud.google.com/go/pubsub v1.10.0 // indirect
 	github.com/PuerkitoBio/goquery v1.7.1 // indirect
 	github.com/RichardKnop/machinery/v2 v2.0.11
-	github.com/anacrolix/envpprof v1.0.0 // indirect
-	github.com/anacrolix/missinggo v1.1.0 // indirect
-	github.com/anacrolix/sync v0.0.0-20180808010631-44578de4e778 // indirect
-	github.com/anacrolix/utp v0.0.0-20180219060659-9e0e1d1d0572 // indirect
 	github.com/apache/thrift v0.14.2 // indirect
 	github.com/armon/go-metrics v0.3.9 // indirect
 	github.com/beego/beego/v2 v2.0.1
-	github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c // indirect
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chromedp/cdproto v0.0.0-20210713064928-7d28b402946a
 	github.com/chromedp/chromedp v0.7.4
 	github.com/disintegration/imaging v1.6.2
-	github.com/docker/libkv v0.2.1 // indirect
 	github.com/edwingeng/doublejump v0.0.0-20210724020454-c82f1bcb3280 // indirect
 	github.com/fatih/color v1.12.0 // indirect
 	github.com/fsnotify/fsnotify v1.5.1 // indirect
@@ -35,39 +28,28 @@ require (
 	github.com/hashicorp/go-hclog v0.16.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go.net v0.0.1 // indirect
 	github.com/joeguo/tldextract v0.0.0-20210326083850-1ec7be2de68a
-	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/klauspost/reedsolomon v1.9.13 // indirect
 	github.com/lib/pq v1.10.2 // indirect
-	github.com/marten-seemann/quic-conn v0.0.0-20190404134349-539f7de6a079 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/miekg/dns v1.1.43
-	github.com/mitchellh/gox v0.4.0 // indirect
-	github.com/mitchellh/iochan v1.0.0 // indirect
 	github.com/peterbourgon/g2s v0.0.0-20170223122336-d4e7ad98afea // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/projectdiscovery/httpx v1.0.9
-	github.com/projectdiscovery/mapcidr v0.0.7 // indirect
+	github.com/projectdiscovery/mapcidr v0.0.7
 	github.com/projectdiscovery/subfinder/v2 v2.4.8
-	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remeh/sizedwaitgroup v1.0.0
 	github.com/rs/cors v1.8.0 // indirect
 	github.com/rs/xid v1.3.0
-	github.com/rverton/webanalyze v0.3.3 // indirect
+	github.com/rverton/webanalyze v0.3.3
 	github.com/sirupsen/logrus v1.6.0
 	github.com/smallnest/quick v0.0.0-20210406061658-4bf95e372fbd // indirect
-	github.com/smallnest/rpcx v1.6.11 // indirect
+	github.com/smallnest/rpcx v1.6.11
 	github.com/soheilhy/cmux v0.1.5 // indirect
-	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e // indirect
 	github.com/tjfoc/gmsm v1.4.1 // indirect
-	github.com/u35s/rudp v0.0.0-20171228014240-b384c469e861 // indirect
 	github.com/valyala/fastrand v1.0.0 // indirect
-	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.opentelemetry.io/otel/internal/metric v0.23.0 // indirect
-	go.opentelemetry.io/otel/oteltest v0.20.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 // indirect
 	golang.org/x/mod v0.5.0 // indirect
 	golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f // indirect