diff --git a/Dockerfile b/Dockerfile index 2d99d62..588dd82 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,30 +1,29 @@ FROM ubuntu:20.04 -# RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \ -# && sed -i s/security.ubuntu.com/mirrors.aliyun.com/g /etc/apt/sources.list \ -# && apt-get clean \ -# && apt update - -#ADD sources.list /etc/apt/sources.list -RUN apt update ENV TZ=Asia/Shanghai ENV LANG C.UTF-8 ENV DEBIAN_FRONTEND=noninteractive -RUN apt install -y wget gnupg \ +RUN sed -i s@/archive.ubuntu.com/@/mirrors.aliyun.com/@g /etc/apt/sources.list \ +&& sed -i s/security.ubuntu.com/mirrors.aliyun.com/g /etc/apt/sources.list \ +&& apt-get clean \ +&& apt update \ +&& apt install -y wget gnupg zip\ && wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \ -&& echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list - -RUN apt update +&& echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \ +&& wget http://npm.taobao.org/mirrors/chromedriver/70.0.3538.16/chromedriver_linux64.zip -O /tmp/chrome.zip \ +&& unzip -d /opt /tmp/chrome.zip \ +&& ln -fs /opt/chromedriver /usr/local/bin/chromedriver \ +&& apt update ADD . /root WORKDIR /root/ +COPY config/SIMSUN.TTC /usr/share/fonts/ttf-dejavu/SIMSUN.TTC RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime \ && echo $TZ > /etc/timezone \ -&& apt install -y python3 python3-pip masscan wget whatweb nmap nikto zip tzdata google-chrome-stable \ -&& pip3 install IPy simplejson requests bs4 prettytable func_timeout xlrd\ +&& apt install -y wget python3 python3-pip masscan whatweb nmap nikto tzdata dnsutils google-chrome-stable \ && pip3 install -r requirements.txt ENTRYPOINT ["python3","main.py"] diff --git a/README.md b/README.md old mode 100755 new mode 100644 index edf79b9..19cceae --- a/README.md +++ b/README.md @@ -1,73 +1,43 @@ -# AutoScanner - -## AutoScanner是什么 -AutoScanner是一款自动化扫描器，其功能主要是遍历所有子域名、及遍历主机所有端口寻找出所有http服务，并使用集成的工具进行扫描，最后集成扫描报告； -工具目前有：oneforall、masscan、nmap、crawlergo、dirsearch、xray、awvs、whatweb等 - -是之前[hscan](https://www.freebuf.com/sectool/260394.html) 的重构版本； - - -## AutoScanner做了什么 -- 自动下载项目所需要的tools -- 使用oneforall遍历子域名 -- 使用masscan遍历主机所有开放端口 -- 使用nmap扫描开放端口；得出所有http服务端口 -- 使用crawlergo进行扫描 -- 动态添加crawlergo扫描到的域名至任务清单 -- 使用dirsearch进行目录文件扫描 -- 扫描到的目录、文件传递到xray -- 使用xray进行被动扫描 -- 扫描结束后生成两份报告，xray和 所有tools集成的一份报告 - -- 支持企查查导出资产一键扫描 -- 支持工具超时自动停止，防止程序卡死 -- 支持还原断点扫描 -- 支持所有扫描目标、扫描数据存入本地sqlite数据库，后续个人可调用 -- ... - -另外，在各个工具直接做了很多逻辑处理，如masscan扫描到过多开放端口，直接忽略；如nmap发现80和443同时开放http服务，忽略443；等等 -需要注意的是，项目中提供了awvs的扫描脚本，但是考虑到正版盗版的原因项目中未集成awvs的安装包； - -## 项目运行 -由于涉及过多pip包依赖及浏览器环境等，建议使用docker运行； -其中注意项目所需要的工具会自动下载，但是由于国内github网速问题可能会导致下载失败等问题，如果发生，可下载下方包解压到tools目录； -链接: https://pan.baidu.com/s/1FAP02yYK7CF9mxMD0yj08g 密码: a6p4 - -- 如工具是自动下载的话这步可以省略；如是百度云下载的话，将解压的tools目录放置项目主目录即main.py这一层； -- 执行`docker build -t auto .`构造镜像 -- 查看、修改、执行`./docker_run.sh`命令即可运行项目 - -### 脚本参数 -脚本支持-u -d --fu --fd --fq -r -f等参数，其中fq参数是直接使用企查查的备案域名导出文件扫描，-r参数待上线 - -#### 1 -u -d -f参数 --u扫描url，-d扫描域名，注意这儿扫描中扫描到的子域名都会动态添加到扫描任务中，如不需要的化添加-f参数即可 --f参数取自fastscan，使用-f时不会支持扫描到的域名动态添加到扫描列表中 -``` -docker run -ti --rm -v `pwd`/:/root/ auto:latest -u http://testphp.vulnweb.com -docker run -ti --rm -v `pwd`/:/root/ auto:latest -u http://testphp.vulnweb.com -f -``` - -#### 2 --fu --fd 参数 -这两个参数读取文件并扫描，区别就是url和域名的形式，写入时以换行符分隔即可 -``` -docker run -ti --rm -v `pwd`/:/root/ auto:latest --fu 1.txt -``` - -#### 3 --fq参数 -读取企查查导出的域名备案文件 -``` -docker run -ti --rm -v `pwd`/:/root/ auto:latest --fq 1.xls -``` -![image](lib/images/keda.png) - -#### 4 -r参数 -支持断点恢复扫描功能，待上线 - - -## 截图展示 -部分截图可以看之前的[hscan](https://www.freebuf.com/sectool/260394.html)； -这儿展示下单独的tools的报告 -![image](lib/images/1.png) -![image](lib/images/2.png) -![image](lib/images/3.png) +# AutoScanner + +## AutoScanner是什么 +AutoScanner是一款自动化扫描器，其功能功能分为两块： ++ 1 遍历所有子域名、子域名主机所有端口及所有http端口服务 ++ 2 对子域名主机信息进行相关检测，如cname解析判断是否是cdn、域名定位信息判断是否为云服务器、masscan扫端口、nmap等 ++ 3 对http端口服务截图、使用集成的工具如crawlergo、xray、dirsearch等进行扫描； ++ 4 集成扫描报告 + +AutoScanner对工具之间的调用衔接做了很多处理，及对渗透测试的相关信息收集做了记录；具体信息看报告文件即可， + + +## 项目运行 +由于涉及过多工具、python包依赖及浏览器环境等，建议使用docker运行； + +### 0x01 工具下载 +二选一即可 +- 工具在执行docker时自动下载, (国内从github下载，可能非常慢) +- 下载百度云,将解压的tools目录放置项目主目录即main.py这一层； + + 链接: https://pan.baidu.com/s/1FAP02yYK7CF9mxMD0yj08g 密码: a6p4 + +### 0x02 构建镜像 +- `docker build -t auto .` + +### 0x03 执行项目 +- docker运行命令参数已放入docker_run.sh文件中，直接修改执行`./docker_run.sh`即可 +- 其中支持参数为： + + -u url + + -d domain + + --fu 包含urls的文件 + + --fd 包含domains的文件 + + --fq 从企查查导出的企业备案域名xls文件 + +### 0x04 报告查看 +- 执行`python3 -m http.server 80 --directory report/`, 在浏览器中输入地址即可 + + + +## 截图展示 +![image](lib/img/1.png) +![image](lib/img/2.png) +![image](lib/img/3.png) + diff --git a/config/SIMSUN.TTC b/config/SIMSUN.TTC new file mode 100755 index 0000000..23d5c4a Binary files /dev/null and b/config/SIMSUN.TTC differ diff --git a/config/logging.ini b/config/logging.ini new file mode 100644 index 0000000..ce51931 --- /dev/null +++ b/config/logging.ini @@ -0,0 +1,37 @@ +[loggers] +keys=root,toolConsole + + +[handlers] +keys=consoleHandler,fileHandler + +[formatters] +keys=fmt + +[logger_root] +level=DEBUG +handlers=consoleHandler,fileHandler + +[logger_toolConsole] +level = DEBUG +handlers = consoleHandler,fileHandler +qualname=toolConsole +propagate=0 + + +[handler_consoleHandler] +class = StreamHandler +level = DEBUG +formatter = fmt +args = (sys.stdout,) + +[handler_fileHandler] +class = logging.handlers.RotatingFileHandler +level = DEBUG +formatter = fmt +args = ('log/test.log', 'a', 10000, 3,) +#args = ("test.log", mode="w", maxBytes=1000, backupCount=3, encoding="utf-8") + +[formatter_fmt] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= diff --git a/lib/Tools.py b/lib/Tools.py new file mode 100644 index 0000000..5b06156 --- /dev/null +++ b/lib/Tools.py @@ -0,0 +1,356 @@ +import os +import re +import csv +import logging.config +import simplejson +import subprocess +import requests +import tempfile +import threading +import sqlite3 +import time +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from lib.db import db_update + +now_time = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) +main_path = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] +logging.config.fileConfig(os.path.join(main_path, "config/logging.ini")) +tool_path = os.path.join(main_path,'tools') +XRAY_LISTEN_PORT = 7777 # 改动的话需要注意controller里面crawlergo推送也要改 + + +class Tools: + def __init__(self, command, logfile=None): + self.command = command + self.logfile = logfile + self.log = None # 存放工具原日志 + self.data = None # 存放自定义删选后的数据 + self.logger = logging.getLogger('toolConsole') + self.scan() + self.filter_log() + self.db_update() + + def scan(self): + self.logger.info(self.__class__.__name__ + ' - ' + ' start scanning ! ') + try: + _ = subprocess.run(self.command, shell=True, timeout=400, cwd=tool_path, stdout=subprocess.PIPE) + self.log = str(_.stdout, encoding='utf-8') + print(self.log) + if self.logfile: + self.read_report_file() + except Exception as e: + self.log = None + self.kill_process() + self.logger.error(self.__class__.__name__ + ' - ' + str(e)) + self.logger.info(self.__class__.__name__ + ' - ' + ' scanned over ! ') + + def read_report_file(self): + if self.logfile and os.path.exists(self.logfile): + with open(self.logfile) as f: + self.log = f.read() + + def filter_log(self): + pass + + def kill_process(self): + _ = "ps aux | grep '{name}'|grep -v 'color' | awk '{{print $2}}'".format(name=self.__class__.__name__.lower()) + process = os.popen(_).read() + if process: + os.popen('nohup kill -9 {} 2>&1 &'.format(process.replace('\n', ' '))) + + # 需要在main.py中创建列，在controller中调用 + # 默认记录url扫描的工具日志，其他如端口日志需要重构 + def db_update(self): + if self.log: + db_update('scanned_info', self.__class__.__name__.lower(), self.log) + + # with sqlite3.connect(os.path.join(main_path, 'scanned_info.db')) as conn: + # sql = 'update scanned_info set {}=? order by id desc limit 1;'.format(self.__class__.__name__.lower()) + # conn.execute(sql, (self.log,)) # 插入时必须是str + + +''' +oneforall +扫描所有子域名并筛选去重出所有子域名 +''' +class Oneforall(Tools): + def filter_log(self): + try: + if self.logfile and os.path.exists(self.logfile): + with open(self.logfile, 'r') as csvfile: + reader = csv.reader(csvfile) + column = [row[5] for row in reader] + del column[0] + self.data = list(set(column)) + self.log = '\n'.join(self.data) + except Exception as e: + print(e) + + def db_update(self): + if self.log: + db_update('target_info', self.__class__.__name__.lower(), self.log) + + +''' +nslookup , 查看是否有cdn +''' +class Nslookup(Tools): + def scan(self): + # IBM 阿里云 中国互联网络信息中心 + dns = ['9.9.9.9', '223.5.5.5', '1.2.4.8'] + self.log = '' + for _dns in dns: + r = os.popen('nslookup {t} {d}'.format(t=self.command, d=_dns)).read() + r = r.split('\n')[4:] + self.log += ('\n'.join(r)) + print(self.log) + + def db_update(self): + if self.log: + db_update('host_info', self.__class__.__name__.lower(), self.log) + + +''' +查询ip定位 主要看是不是云服务器 +''' +class IpLocation(Tools): + def scan(self): + # 此处IP和域名都行 + url = 'http://demo.ip-api.com/json/{ip}'.format(ip=self.command) + resp = Request().get(url) + if resp and resp.json(): + self.log = '' + r = resp.json() + l = ['status', 'country', 'city', 'isp', 'org', 'asname', 'mobile'] + for k, v in r.items(): + if k in l: + print('{}: {}'.format(k, r[k])) + self.log += '{}: {}'.format(k, r[k]) + '\n' + + def db_update(self): + if self.log: + db_update('host_info', self.__class__.__name__.lower(), self.log) + + +''' +masscan +调用self.data获取返回的ports list +masscan 只接收ip作为target +''' +class Masscan(Tools): + def filter_log(self): + if self.log: + ports = re.findall('\d{1,5}/tcp', self.log) + self.data = [x[:-4] for x in ports] + + def db_update(self): + if self.log: + db_update('host_info', self.__class__.__name__.lower(), self.log) + + +''' +nmap +遍历所有http https端口 +''' +class Nmap(Tools): + def filter_log(self): + if self.log: + http_ports = re.findall('\d{1,5}/tcp\s{1,}open\s{1,}[ssl/]*http', self.log) + http_ports = [int(x.split("/")[0]) for x in http_ports] + self.data = http_ports + + def db_update(self): + if self.log: + db_update('host_info', self.__class__.__name__.lower(), self.log) + + +''' +whatweb +''' +class Whatweb(Tools): + def filter_log(self): + if self.log: + log = [] + if '\n' in self.log.strip('\n'): + self.log = self.log.split('\n')[0] + + keys = ['IP', 'Title', 'PoweredBy', 'HTTPServer', 'X-Powered-By', 'Meta-Refresh-Redirect', 'Cookies'] + for _ in self.log.split(','): + for key in keys: + if _.strip().startswith(key): + log.append(_) + self.log = '\n'.join(log) + + +''' +crawlergo +发现的子域名将在controller模块中动态去重添加进入扫描 +''' +class Crawlergo(Tools): + def scan(self): + try: + rsp = subprocess.run(self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=300, shell=True, cwd=tool_path) + output = str(rsp.stdout, encoding='utf-8') + result = simplejson.loads(output.split("--[Mission Complete]--")[1]) + req_list = result["req_list"] + urls = [] + for req in req_list: + urls.append(req['url'] + ' ' + req['data']) + subdomains = result["sub_domain_list"] + domain = self.command.split()[-1] + self.data = self.filter_domain(domain, subdomains) + self.log = urls + ['\n'*2] + subdomains + self.log = '\n'.join(self.log) + print(self.log) + except Exception as e: + self.log = None + self.logger.error(self.__class__.__name__ + ' - ' + str(e)) + finally: + self.kill_process() # 可能还会余留chrome进程，为了避免杀掉用户的chrome，暂时保留 + + # crawlergo 在获取sub_domain_list时 在获取xx.com.cn这种3级域名时会默认com.cn为base域名 + @staticmethod + def filter_domain(domain, domains): + if domains: + if domain.count('.') > 2: + domain = domain.split('.', 1)[1] + for _ in domains: + if domain not in _: + domains.remove(_) + return domains + +class Xray: + def __init__(self): + self.logfile = os.path.join(main_path, 'report/{}-xray.html'.format(now_time)) + self.backup_file = tempfile.NamedTemporaryFile(delete=False).name + self.proxy = '127.0.0.1:{}'.format(XRAY_LISTEN_PORT) + self.kill_exists_process() + self.xray_wait_time = 0 + + def passive_scan(self): + def xray_passive(): + _ = "{path}/tools/xray_linux_amd64 webscan --listen {proxy} --html-output {logfile} | tee -a {backup_file}"\ + .format(path=main_path, proxy=self.proxy, logfile=self.logfile, backup_file=self.backup_file) + os.system(_) + + t = threading.Thread(target=xray_passive, daemon=True) + t.start() + + def initiative_scan(self, url): + def xray_initiative(u): + _ = "{path}/tools/xray_linux_amd64 webscan --basic-crawler {url} --html-output {logfile}.html" \ + .format(path=main_path, url=u, logfile=self.logfile) + os.system(_) + + t = threading.Thread(target=xray_initiative, args=(url,), daemon=True) + t.start() + + def wait_xray_ok(self): + __ = ''' + wc {0} | awk '{{print $1}}'; + sleep 5; + wc {0} | awk '{{print $1}}'; + '''.format(self.backup_file) + result = os.popen(__).read() + + if result.split('\n')[0] == result.split('\n')[1]: + _ = "tail -n 10 {}".format(self.backup_file) + s = os.popen(_).read() + + if "All pending requests have been scanned" in s: + os.system('echo "" > {}'.format(self.backup_file)) + return True + + if self.xray_wait_time == 2: + return True + else: + self.xray_wait_time += 1 + return False + + def kill_exists_process(self): + process = os.popen("ps aux | grep 'xray'|grep -v 'color' | awk '{print $2}'").read() + if process: + os.popen('nohup kill -9 {} 2>&1 &'.format(process.replace('\n', ' '))) + + +''' +dirsearch v0.4.1 +''' +class Dirsearch(Tools): + def read_report_file(self): + self.log, self.data = [], [] + with open(self.logfile, 'r') as f: + lines = [line.strip() for line in f.readlines()] + if lines: + lines.pop(0) + + for line in lines: + line = line.split(',') + try: + s = "{:<} - {:>5}B - {:<5}".format(line[2], line[3], line[1]) + self.log.append(s) + self.data.append(line[1]) + except Exception as e: + print(e) + continue + + self.log = '\n'.join(self.log) + + +''' +requests请求，将dirsearch扫描出的url推到xray +''' +class Request: + def __init__(self,): + self.proxy = {'http': 'http://127.0.0.1:{}'.format(XRAY_LISTEN_PORT), + 'https': 'http://127.0.0.1:{}'.format(XRAY_LISTEN_PORT)} + self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0)', + } + + def repeat(self, url): + try: + response = requests.get(url=url, headers=self.headers, proxies=self.proxy, verify=False, timeout=20) + # print(response) + return response + except Exception as e: + print(e) + + def get(self, url): + try: + response = requests.get(url=url, headers=self.headers, verify=False, timeout=20) + return response + except Exception as e: + print(e) + + +''' +截图， 考虑base64加到html里整个报告太大了，所以只能保存到本地，然后使用img src +暂时不写入db +''' +class Snapshot(Tools): + def scan(self): + option = Options() + option.add_argument('--headless') + option.add_argument('--no-sandbox') + option.add_argument('--start-maximized') + + try: + driver = webdriver.Chrome(chrome_options=option) + driver.set_window_size(1366, 768) + driver.implicitly_wait(4) + driver.get(self.command) + time.sleep(1) + driver.get_screenshot_as_file(os.path.join(main_path, 'report/img/{}.png'.format(self.format_img_name(self.command)))) + driver.quit() + except Exception as e: + pass + + @staticmethod + def format_img_name(url): + if url.startswith('http'): + url = url.split('/')[2] + if ':' in url: + url = url.replace(':', '_') + return url + diff --git a/lib/Tools.py.bak b/lib/Tools.py.bak new file mode 100644 index 0000000..392259b --- /dev/null +++ b/lib/Tools.py.bak @@ -0,0 +1,227 @@ +from func_timeout import func_set_timeout +import func_timeout +import os +import re +import csv +import logging.config +import simplejson +import subprocess +import requests +import tempfile +import threading +import sqlite3 +import time + +now_time = time.strftime("%Y%m%d-%H%M%S", time.localtime(time.time())) +main_path = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] +logging.config.fileConfig(os.path.join(main_path, "config/logging.ini")) +tool_path = os.path.join(main_path,'tools') +XRAY_LISTEN_PORT = 7777 # 改动的话需要注意controller里面crawlergo推送也要改 + + +class Tools: + def __init__(self, command, logfile): + self.command = command + self.logfile = logfile + self.log = None + self.data = None + self.logger = logging.getLogger('toolConsole') + + try: + self.scan() + except func_timeout.exceptions.FunctionTimedOut: + self.logger.error(self.__class__.__name__ + ' - ' + 'TimeoutError') + + self.filter_log() + + @func_set_timeout(300) + def scan(self): + try: + if not self.logfile: + self.log = os.popen(self.command).read() + else: + os.popen(self.command,'w') + self.read_report_file() + except Exception as e: + self.logger.error(self.__class__.__name__ + ' - ' + str(e)) + + self.logger.info(self.__class__.__name__ + ' - ' + ' scanned over ! ') + + def read_report_file(self): + if self.logfile and os.path.exists(self.logfile): + with open(self.logfile) as f: + self.log = f.read() + + # 赋予子类工具使用，用于处理筛选日志文件，并赋予self.data返回 + def filter_log(self): + pass + + # 需要在main.py中创建列，在controller中调用 + # def db_insert_log(self): + # if self.log: + # with sqlite3.connect('../scanned_info.db') as conn: + # sql = 'update scanned_info set {}=? where id=max(id)'.format(self.__class__.__name__) + # conn.execute(sql, tuple([self.log])) + + +''' +oneforall +扫描所有子域名并筛选去重出所有子域名 +''' +class Oneforall(Tools): + def filter_log(self): + try: + if self.logfile and os.path.exists(self.logfile): + with open(self.logfile, 'r') as csvfile: + reader = csv.reader(csvfile.__next__()) + column = [row[5] for row in reader] + self.data = list(set(column)) + except Exception as e: + print(e) + + # def db_insert_log(self): + # if self.log: + # with sqlite3.connect('../scanned_info.db') as conn: + # sql = 'update target_info set {}=? where id=max(id)'.format(self.__class__.__name__) + # conn.execute(sql, tuple([self.log])) + # 看看是不是可以把sql替换掉 + + +''' +masscan +调用self.data获取返回的ports list +masscan 只接收ip作为target +''' +class Masscan(Tools): + def filter_log(self): + ports = re.findall('\d{1,5}/tcp', self.log) + self.data = [x[:-4] for x in ports] + + +''' +nmap +遍历所有http https端口 +''' +class Nmap(Tools): + def filter_log(self): + http_ports = re.findall('\d{1,5}/tcp\s{1,}open\s{1,}[ssl/]*http', self.log) + http_ports = [int(x.split("/")[0]) for x in http_ports] + self.data = http_ports + + +''' +whatweb +''' +class Whatweb(Tools): + pass + + +''' +crawlergo +发现的子域名将在controller模块中动态去重添加进入扫描 +''' +class Crawlergo(Tools): + @func_set_timeout(300) + def scan(self): + try: + # cmd = [crawlergo, "-c", self.BROWERS, "-t", "5", "-f", "smart", "--push-to-proxy", self.XRAY_PROXY, "--push-pool-max", "10", "--fuzz-path", "-o", "json", self.target] + rsp = subprocess.Popen(self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = rsp.communicate() + result = simplejson.loads(output.decode().split("--[Mission Complete]--")[1]) + req_list = result["req_list"] + urls = [] + for req in req_list: + # print("crawlergo found :", req['url']) + urls.append(req['url']) + + subdomains = result["sub_domain_list"] + self.data = subdomains + self.log = urls + ['\n'*2] + subdomains + except Exception as e: + self.logger.error(self.__class__.__name__ + ' - ' + str(e)) + + self.logger.info(self.__class__.__name__ + ' - ' + ' scanned over ! ') + + +class Xray: + def __init__(self): + # 这儿要考虑tempfile怎么删除 + self.logfile = os.path.join(main_path,'/report/{}-xray.html'.format(now_time)) + self.backup_file = tempfile.NamedTemporaryFile(delete=False).name + self.proxy = '127.0.0.1:{}'.format(XRAY_LISTEN_PORT) + self.kill_exists_process() + + def scan(self): + t = threading.Thread(target=self.xray_run, daemon=True) + t.start() + + def xray_run(self): + _ = "{path}/tools/xray_linux_amd64 webscan --listen {proxy} --html-output {logfile} | tee -a {backup_file}"\ + .format(path=main_path, proxy=self.proxy, logfile=self.logfile, backup_file=self.backup_file) + os.system(_) + + def wait_xray_ok(self): + _ = "tail -n 10 {}".format(self.backup_file) + s = os.popen(_).read() + + # {{0}} 需要这样来转义大括号，结果为{0}字符串 + if "All pending requests have been scanned" in s: + __ = ''' + wc {0} | awk '{{print $1}}'; + sleep 5; + wc {0} | awk '{{print $1}}'; + '''.format(self.backup_file) + result = os.popen(__).read() + if result.split('\n')[0] == result.split('\n')[1]: + print('xray end scan') + os.system('echo "" > {}'.format(self.backup_file)) + return True + else: + return False + + def kill_exists_process(self): + process = os.popen("ps aux | grep 'xray' | awk '{print $2}'").read() + os.system('kill -9 {}'.format(process.replace('\n', ' '))) + + +''' +dirsearch v0.4.1 +''' +class Dirsearch(Tools): + def read_report_file(self): + self.log = [] + self.data = [] + with open(self.logfile, 'r') as f: + lines = f.readlines() + lines.pop(0) + + for line in lines: + line = line.strip().split(',') + try: + s = "{:<} - {:>5}B - {:<5}".format(line[2], line[3], line[1]) + self.log.append(s) + self.data.append(line[1]) + except Exception : + continue + finally: + self.kill_residual_process() + + def kill_residual_process(self): + process = os.popen("ps aux | grep 'dirsearch' | awk '{print $2}'").read() + os.system('kill -9 {}'.format(process.replace('\n', ' '))) + + +class Request: + def __init__(self,): + self.proxy = {'http': 'http://127.0.0.1:{}'.format(XRAY_LISTEN_PORT), + 'https': 'http://127.0.0.1:{}'.format(XRAY_LISTEN_PORT)} + self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0)', + } + + def repeat(self, url): + try: + response = requests.get(url=url, headers=self.headers, proxies=self.proxy, verify=False, timeout=20) + # print(response) + return response + except Exception as e: + print(e) diff --git a/lib/__pycache__/Tools.cpython-36.pyc b/lib/__pycache__/Tools.cpython-36.pyc new file mode 100644 index 0000000..dfaa29c Binary files /dev/null and b/lib/__pycache__/Tools.cpython-36.pyc differ diff --git a/lib/__pycache__/Tools.cpython-38.pyc b/lib/__pycache__/Tools.cpython-38.pyc new file mode 100644 index 0000000..ea39a72 Binary files /dev/null and b/lib/__pycache__/Tools.cpython-38.pyc differ diff --git a/lib/__pycache__/__init__.cpython-36.pyc b/lib/__pycache__/__init__.cpython-36.pyc new file mode 100644 index 0000000..e62cf64 Binary files /dev/null and b/lib/__pycache__/__init__.cpython-36.pyc differ diff --git a/lib/__pycache__/__init__.cpython-38.pyc b/lib/__pycache__/__init__.cpython-38.pyc index 6d4ecae..475893c 100644 Binary files a/lib/__pycache__/__init__.cpython-38.pyc and b/lib/__pycache__/__init__.cpython-38.pyc differ diff --git a/lib/__pycache__/arguments_parse.cpython-38.pyc b/lib/__pycache__/arguments_parse.cpython-38.pyc deleted file mode 100644 index a999660..0000000 Binary files a/lib/__pycache__/arguments_parse.cpython-38.pyc and /dev/null differ diff --git a/lib/__pycache__/controller.cpython-38.pyc b/lib/__pycache__/controller.cpython-38.pyc deleted file mode 100644 index 15dc3f5..0000000 Binary files a/lib/__pycache__/controller.cpython-38.pyc and /dev/null differ diff --git a/lib/__pycache__/download_tools.cpython-38.pyc b/lib/__pycache__/download_tools.cpython-38.pyc deleted file mode 100644 index 871c27a..0000000 Binary files a/lib/__pycache__/download_tools.cpython-38.pyc and /dev/null differ diff --git a/lib/__pycache__/general.cpython-38.pyc b/lib/__pycache__/general.cpython-38.pyc deleted file mode 100644 index 6f354b5..0000000 Binary files a/lib/__pycache__/general.cpython-38.pyc and /dev/null differ diff --git a/lib/__pycache__/logger.cpython-38.pyc b/lib/__pycache__/logger.cpython-38.pyc deleted file mode 100644 index 40c5026..0000000 Binary files a/lib/__pycache__/logger.cpython-38.pyc and /dev/null differ diff --git a/lib/__pycache__/report.cpython-38.pyc b/lib/__pycache__/report.cpython-38.pyc deleted file mode 100644 index d28cbce..0000000 Binary files a/lib/__pycache__/report.cpython-38.pyc and /dev/null differ diff --git a/lib/__pycache__/setting.cpython-38.pyc b/lib/__pycache__/setting.cpython-38.pyc deleted file mode 100644 index cecbeea..0000000 Binary files a/lib/__pycache__/setting.cpython-38.pyc and /dev/null differ diff --git a/lib/arguments_parse.py b/lib/arguments_parse.py index 780fb35..88740d2 100755 --- a/lib/arguments_parse.py +++ b/lib/arguments_parse.py @@ -1,77 +1,59 @@ from optparse import OptionParser, OptionGroup from IPy import IP -import os +from .general import get_file_content, read_xls import time -from .general import * -from lib.scanner.oneforall import OneForAll -class ArgumentParser(): - def __init__(self,): - self.args = self.parseArguments() - options = self.parseArguments() - - if options.restore: - print("restore模式，从上次中断处重新开始扫描中") - time.sleep(1) - - if options.url: - self.urlList = [options.url] - - elif options.domain: - self.urlList = [options.domain] - - elif options.urlsFile: - self.urlList = get_file_content(options.urlsFile) - - elif options.domainsFile: - self.urlList = get_file_content(options.domains_file) - - elif options.qccFile: #企查查的文件 - print("读取qichacha文件并扫描：") - time.sleep(1) - self.urlList = read_xls(options.qccFile).domains - elif options.domainsFileFromQichacha: - self.urlList = get_domains_from_qichacha_xlsx(options.domainsFileFromQichacha) - - elif options.cidr : +class ArgumentParser: + def __init__(self,): + self.args = self.parse_arguments() + urls, domains = [], [] + + if self.args.url: + urls += [self.args.url] + elif self.args.domain: + domains += [self.args.domain] + elif self.args.urlsFile: + urls += get_file_content(self.args.urlsFile) + elif self.args.domainsFile: + domains += get_file_content(self.args.domainsFile) + elif self.args.qccFile: + domains += read_xls(self.args.qccFile).domains + elif self.args.cidr: try: - self.urlList = [ip for ip in IP(options.cidr)] - except : - print("the input cidr is not correct") - - else : + urls += [ip for ip in IP(self.args.cidr)] + except Exception as e: + exit(e) + else: exit("need a target input") - print("urlList:",self.urlList) + self.args.urlList, self.args.domainList = urls, domains - self.toolList = [] - if options.tools: - for tool_name in options.tools.split(','): - self.toolList.append(tool_name) - - def parseArguments(self): + @staticmethod + def parse_arguments(): usage = "Usage: %prog [-u|--url] target [-e|--extensions] extensions [options]" - parser = OptionParser(usage, version="hscan v1 ",epilog="By hujiankai") + parser = OptionParser(usage, epilog="By hujiankai") - # Mandatory arguments mandatory = OptionGroup(parser, "Mandatory") - mandatory.add_option("-u", "--url", help="Target URL", action="store", type="string", dest="url", default=None) - mandatory.add_option("-d", "--domain", help="Target domain", action="store", type="string", dest="domain", default=None) - mandatory.add_option("--fu", help="Target URLS from file", action="store", type="string", dest="urlsFile", default=None) - mandatory.add_option("--fd", help="Target domains from file", action="store", type="string", dest="domainsFile", default=None) - mandatory.add_option("--fq", help="Target domains from file of qichacha", action="store",type='string',dest="qccFile", default=None) - mandatory.add_option("--cidr", help="Target cidr", action="store", type="string", dest="cidr", default=None) + mandatory.add_option("-u", "--url", help="Target URL", action="store", type="string", dest="url",) + mandatory.add_option("-d", "--domain", help="Target domain", action="store", type="string", dest="domain") + mandatory.add_option("--fu", help="Target URLS from file", action="store", type="string", dest="urlsFile", ) + mandatory.add_option("--fd", help="Target domains from file", action="store", type="string", dest="domainsFile") + mandatory.add_option("--fq", help="Target domains from qichacha file", action="store", type='string', dest="qccFile",) + mandatory.add_option("--cidr", help="Target cidr", action="store", type="string", dest="cidr",) arg = OptionGroup(parser, "arg") - arg.add_option("-r","--restore",action="store_true",dest="restore",default=False) + arg.add_option("-r", "--restore", action="store_true", dest="restore", default=False) arg.add_option("-f", "--fast", action="store_true", dest="fastscan", default=False,help="url scan only") tools = OptionGroup(parser, "tools") - tools.add_option("-t", "--tools", help="select tools run", action="store", dest="tools", default="dirsearch,") + tools.add_option("-t", "--tools", help="select tools run", action="store", dest="tools", default=None) parser.add_option_group(mandatory) + parser.add_option_group(arg) parser.add_option_group(tools) options, arguments = parser.parse_args() return options + + diff --git a/lib/scanner/awvs.py b/lib/awvs.py similarity index 100% rename from lib/scanner/awvs.py rename to lib/awvs.py diff --git a/lib/bannerscan.py b/lib/bannerscan.py new file mode 100644 index 0000000..65f8497 --- /dev/null +++ b/lib/bannerscan.py @@ -0,0 +1,331 @@ +# coding=utf-8 +__author__ = 'DM_' +# Modifed by le4f + +''' +该脚本来源于 https://github.com/x0day/bannerscan +原脚本为python2型， 此处手动改为python3型直接集成了 +''' + +import threading +import requests +import argparse +import time +import re + +PORTS = (80, + 81, + 82, + 443, + 4848, + 7001, + 8080, + 8090, + 8000, + 8082, + 8888, + 9043, + 8443, + 9200, + 9000, + 9060, + 9440, + 9090, + 8081, + 9043, + 41080, + 9080, + 18100, + 9956, + 8886, + 7778 +) + + +PATHS = ('/robots.txt', + '/admin/', + '/manager/html/', + '/jmx-console/', + '/web-console/', + '/jonasAdmin/', + '/manager/', + '/install/', + '/ibm/console/logon.jsp', + '/axis2/axis2-admin/', + '/CFIDE/administrator/index.cfm', + '/FCKeditor/', + '/fckeditor/', + '/fck/', + '/FCK/', + '/HFM/', + '/WEB-INF/', + '/ckeditor/', + '/console/', + '/phpMyAdmin/', + '/Struts2/index.action', + '/index.action', + '/phpinfo.php', + '/info.php', + '/1.php', + '/CHANGELOG.txt', + '/LICENSE.txt', + '/readme.html', + '/cgi-bin/', + '/invoker/', + '/.svn/', + '/test/', + '/CFIDE/', + '/.htaccess', + '/.git/' +) + +HTML_LOG_TEMPLATE=""" + + + + +Bannerscan Report + + + +

%s

+
+%s +
+ + +""" +css = """ +body{background-color:#FFF;color:#444;font-family:"Droid Serif",Georgia,"Times New Roman",STHeiti,serif;font-size:100%;} +a{color:#3354AA;text-decoration:none;} +a:hover,a:active{color:#444;} +pre,code{background:#F3F3F0;font-family:Menlo,Monaco,Consolas,"Lucida Console","Courier New",monospace;font-size:.92857em;padding:2px 4px;} +code{color:#B94A48;} +pre{overflow:auto;max-height:400px;padding:8px;} +pre code{color:#444;padding:0;} +h1,h2,h3{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;} +textarea{resize:vertical;}.report-meta a,.report-content a,.widget a,a{border-bottom:1px solid#EEE;}.report-meta a:hover,.report-content a:hover,.widget a:hover,a{border-bottom-color:transparent;}#header{padding-top:35px;border-bottom:1px solid#EEE;}#logo{color:#333;font-size:2.5em;}.description{color:#999;font-style:italic;margin:.5em 0 0;}.report{border-bottom:1px solid#EEE;padding:15px 0 20px;}.report-title{font-size:1.4em;margin:.83em 0;}.report-meta{margin-top:-.5em;color:#999;font-size:.92857em;padding:0;}.report-meta li{display:inline-block;padding-left:12px;border-left:1px solid#EEE;margin:0 8px 0 0;}.report-meta li:first-child{margin-left:0;padding-left:0;border:none;}.report-content{line-height:1.5;}.report-content hr,hr{margin:2em auto;width:100px;border:1px solid#E9E9E9;border-width:2px 0 0 0;} +""" + +ipPattern = "^([1]?\d\d?|2[0-4]\d|25[0-5])\." \ + "([1]?\d\d?|2[0-4]\d|25[0-5])\." \ + "([1]?\d\d?|2[0-4]\d|25[0-5])\." \ + "([1]?\d\d?|2[0-4]\d|25[0-5])$" + +iprangePattern = "^([1]?\d\d?|2[0-4]\d|25[0-5])\." \ + "([1]?\d\d?|2[0-4]\d|25[0-5])\." \ + "([1]?\d\d?|2[0-4]\d|25[0-5])\." \ + "([1]?\d\d?|2[0-4]\d|25[0-5])-([1]?\d\d?|2[0-4]\d|25[0-5])$" + +ua = "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6" + +headers = dict() +result = dict() + + +class bannerscan(threading.Thread): + def __init__(self, ip, timeout, headers): + self.ip = ip + self.req = requests + self.timeout = timeout + self.headers = headers + self.per = 0 + threading.Thread.__init__(self) + + def run(self): + result[self.ip] = dict() + for port in PORTS: + url_pre = "https://" if port == 443 else "http://" + site = url_pre + self.ip + ":" + str(port) + try: + print ("[*] %s\r" % (site[0:60].ljust(60, " "))), + resp = requests.head(site, + allow_redirects = False, + timeout=self.timeout, + headers=self.headers + ) + result[self.ip][port] = dict() + + except Exception as e: + pass + + else: + result[self.ip][port]["headers"] = resp.headers + result[self.ip][port]["available"] = list() + + for path in PATHS: + try: + url = site + path + print ("[*] %s\r" % (url[0:60].ljust(60, " "))), + resp = self.req.get(url, + allow_redirects = False, + timeout=self.timeout, + headers=self.headers + ) + + except Exception as e: + pass + else: + if resp.status_code in [200, 406, 401, 403, 500]: + r = re.findall("([\s\S]+?)", resp.content) + title = lambda r : r and r[0] or "" + result[self.ip][port]["available"].append((title(r), url, resp.status_code)) + +def getiplst(host, start=1, end=255): + iplst = [] + ip_pre = "" + for pre in host.split('.')[0:3]: + ip_pre = ip_pre + pre + '.' + for i in range(start, end): + iplst.append(ip_pre + str(i)) + return iplst + +def retiplst(ip): + iplst = [] + if ip: + if re.match(ipPattern, ip): + print("[*] job: {} \r".format(ip)) + iplst = getiplst(ip) + return iplst + else: + print("[!] not a valid ip given.") + exit() + +def retiprangelst(iprange): + iplst = [] + if re.match(iprangePattern, iprange): + ips = re.findall(iprangePattern, iprange)[0] + ip = ips[0] + "." + ips[1] + "." + ips[2] + "." + "1" + ipstart = int(ips[3]) + ipend = int(ips[4]) + 1 + # print("[*] job: %s.%s - %s" % (ips[0] + "." + ips[1] + "." + ips[2], ipstart, ipend)) + print("[*] job: {}.{} - {}{}{}".format(ips[0], ips[1], ips[2], ipstart, ipend)) + iplst = getiplst(ip, ipstart, ipend) + return iplst + else: + print("[!] not a valid ip range given.") + exit() + +def ip2int(s): + l = [int(i) for i in s.split('.')] + return (l[0] << 24) | (l[1] << 16) | (l[2] << 8) | l[3] + +def log(out, path): + logcnt = "" + centerhtml = lambda ips: len(ips)>1 and str(ips[0]) + " - " + str(ips[-1]) or str(ips[0]) + titlehtml = lambda x : x and "" + str(x) + "
" or "" + ips = out.keys() + ips.sort(lambda x, y: cmp(ip2int(x), ip2int(y))) + for ip in ips: + titled = False + if type(out[ip]) == type(dict()): + for port in out[ip].keys(): + if not titled: + if len(out[ip][port]['headers']): + logcnt += "

%s

" % ip + logcnt += "
" + titled = True + logcnt += "PORT: %s
" % port + logcnt += "Response Headers:
"
+                for key in out[ip][port]["headers"].keys():
+                    logcnt += key + ":" + out[ip][port]["headers"][key] + "\n"
+                logcnt += "
" + for title, url, status_code in out[ip][port]["available"]: + logcnt += titlehtml(title) + \ + "" + url + " "+ \ + "Status Code:" + str(status_code) + "
" + logcnt += "
" + center = centerhtml(ips) + logcnt = HTML_LOG_TEMPLATE % ( css, center, logcnt) + outfile = open(path, "a") + outfile.write(logcnt) + outfile.close() + +def scan(iplst, timeout, headers, savepath): + global result + start = time.time() + threads = [] + + for ip in iplst: + t = bannerscan(ip,timeout,headers) + threads.append(t) + + for t in threads: + t.start() + + for t in threads: + t.join() + + log(result, savepath) + result = dict() + print + +def main(): + parser = argparse.ArgumentParser(description='banner scanner. by DM_ http://x0day.me') + group = parser.add_mutually_exclusive_group() + + group.add_argument('-i', + action="store", + dest="ip", + ) + group.add_argument('-r', + action="store", + dest="iprange", + type=str, + ) + group.add_argument('-f', + action="store", + dest="ipfile", + type=argparse.FileType('r') + ) + parser.add_argument('-s', + action="store", + required=True, + dest="savepath", + type=str, + ) + parser.add_argument('-t', + action="store", + required=False, + type = int, + dest="timeout", + default=5 + ) + + args = parser.parse_args() + savepath = args.savepath + timeout = args.timeout + iprange = args.iprange + ipfile = args.ipfile + ip = args.ip + + headers['user-agent'] = ua + + print("[*] starting at %s" % time.ctime()) + + if ip: + iplst = retiplst(ip) + scan(iplst, timeout, headers, savepath) + + elif iprange: + iplst = retiprangelst(iprange) + scan(iplst, timeout, headers, savepath) + + elif ipfile: + lines = ipfile.readlines() + for line in lines: + if re.match(ipPattern, line): + iplst = retiplst(line) + scan(iplst, timeout, headers, savepath) + elif re.match(iprangePattern, line): + iplst = retiprangelst(line) + scan(iplst, timeout, headers, savepath) + + else: + parser.print_help() + exit() + +if __name__ == '__main__': + main() diff --git a/lib/controller.py b/lib/controller.py old mode 100755 new mode 100644 index ab9cd0b..107d5cf --- a/lib/controller.py +++ b/lib/controller.py @@ -1,147 +1,147 @@ import os -import time -import re -from lib.general import url_parse,get_ip_from_url -from lib.scanner.oneforall import OneForAll -from lib.scanner import xray,crawlergo,nmap,masscan,dirsearch,awvs,request_engine,whatweb +import tempfile +import sqlite3 +from lib.Tools import * +from lib.urlParser import Parse +from lib.report import Report +from lib.db import db_insert + +class Controller: + def __init__(self, arguments): + self.urls_target = arguments.urlList + self.domains_target = arguments.domainList + self.logfile = tempfile.NamedTemporaryFile(delete=False).name + self.log = {} + self.xray = Xray() + + if arguments.tools: + self.toolsList = [tool for tool in arguments.tools.split(',')] + + def assign_task(self): + def url_scan(urls_target): + if urls_target: + self.urls_target = sorted(set(self.urls_target), key=urls_target.index) + for _target in urls_target: + db_insert('insert into target_info (target, batch_num) values (?,?);', _target, now_time) + _target = Parse(_target) # return dict{ip,domain,http_url} + if _target.data: + db_insert('insert into scanned_info (domain, batch_num) values (?,?)', _target.data['http_url'], now_time) + self.urls_scan(_target) + Report().html_report_single() + + self.xray.passive_scan() + + if self.urls_target: + url_scan(self.urls_target) + + if self.domains_target: + self.domains_target = sorted(set(self.domains_target), key=self.domains_target.index) + for domain in self.domains_target: + if not Parse.isIP(domain): + if Parse(domain).data: # 域名存在解析不成功的情况 + subdomains = self.subdomains_scan(Parse(domain).data['domain']) + for subdomain in subdomains: + target = Parse(subdomain) + print(target) + if target.data: + db_insert('insert into host_info (domain, batch_num) values (?,?)', target.data['domain'], now_time) + http_urls = self.ports_scan(target) + url_scan(http_urls) + + else: + target = Parse(domain) + db_insert('insert into host_info (domain, batch_num) values (?,?)', target.data['domain'], now_time) + http_urls = self.ports_scan(target) + url_scan(http_urls) + + def subdomains_scan(self, target): + _ = "python3 oneforall/oneforall.py --target {target} run".format(path=tool_path, target=target) + logfile = '{path}/oneforall/results/{target}.csv'.format(path=tool_path, target=target) + oneforall = Oneforall(_, logfile) + return oneforall.data if oneforall.data else [target] + + def ports_scan(self, target): + nslookup = Nslookup(target.data['domain']) + + cdns = ['cdn', 'kunlun', 'bsclink.cn', 'ccgslb.com.cn', 'dwion.com', 'dnsv1.com', 'wsdvs.com', 'wsglb0.com', 'lxdns.com', 'chinacache.net.', 'ccgslb.com.cn', 'aliyun'] + for cdn in cdns: + if cdn in nslookup.log: + print('maybe the {} is cdn'.format(target.data['domain'])) + print(nslookup.log) + return [target.data['http_url']] + + _ = "masscan --open -sS -Pn -p 1-20000 {target} --rate 2000".format(target=target.data['ip']) + masscan = Masscan(_, None) + + ''' + 可能存在防火墙等设备,导致扫出的端口非常多 + ''' + if not masscan.data or len(masscan.data) > 20: + masscan.data = ['21', '22', '445', '80', '1433', '3306', '3389', '6379', '7001', '8080'] + + ''' + nmap 如果80和443同时开放，舍弃443端口 + ''' + _ = "nmap -sS -Pn -A -p {ports} {target_ip} -oN {logfile}".format(ports=','.join(masscan.data), target_ip=target.data['ip'], logfile=self.logfile) + nmap = Nmap(_, self.logfile) + if nmap.data: + if 80 in nmap.data and 443 in nmap.data: + nmap.data.remove("443") + return ['{}:{}'.format(target.data['http_url'], port) for port in nmap.data] + + return [target.data['http_url']] + + def urls_scan(self, target): + # 主要查看组织， 是否是云服务器 + iplocation = IpLocation(target.data['ip']) + + _ = "whatweb --color never {}".format(target.data['http_url']) + whatweb = Whatweb(_) + + # 截图 + snapshot = Snapshot(target.data['http_url']) + + ''' + crawlergo + 扫描出来的子域名动态添加到域名列表中 + 注意--push-to-proxy必须是http协议, 更换chrome为静态包执行不了 + ''' + _ = './crawlergo -c /usr/bin/google-chrome-stable -t 10 --push-to-proxy http://127.0.0.1:7777 -o json {}'.format(target.data['http_url']) + crawlergo = Crawlergo(_) + + if crawlergo.data: + if self.domains_target: + self.domains_target += [domain for domain in crawlergo.data if domain not in self.domains_target] + + ''' + 等待xray扫描结束，因为各类工具都是多线程高并发，不等待的话xray会一批红：timeout + ''' + if crawlergo.log: + print('xray start scanning') + while True: + if self.xray.wait_xray_ok(): + break + + ''' + 将dirsearch扫出的url添加到xray去 + ''' + _ = 'python3 dirsearch/dirsearch.py -x 301,302,403,404,405,500,501,502,503 --full-url -u {target} --csv-report {logfile}'.format( + target=target.data['http_url'], logfile=self.logfile) + dirsearch = Dirsearch(_, self.logfile) + + if dirsearch.data: + for url in dirsearch.data: + response = Request().repeat(url) + -class Controller(): - def __init__(self,arguments): - self.arguments = arguments - self.scanned_domains = [] - if self.arguments.args.restore: - exit("restore exit ") - def assign_task(self): - self.init_report() - self.xray = xray.Xray() - self.xray.scan() - for http_url in self.format_url(): - print("scanning : ",http_url) - if self.arguments.args.fastscan: # fastscan模式只web扫描，并且不重复添加扫描到的子域名 - self.url_scan(http_url) - continue - if http_url.count(":") < 2 and http_url.count("/") < 3 : # if like http://a.com:8080 or http://xx.com/1.php ,do self.url_scan() - ip = get_ip_from_url(http_url) - if not ip : - continue - open_ports = masscan.Masscan(ip).open_ports - if not open_ports or len(open_ports) > 20: - self.url_scan(http_url) - continue - http_open_ports = nmap.Nmap(url_parse(http_url).get_netloc(),open_ports).http_open_ports #use domain not ip in order to report - if http_open_ports: - for port in http_open_ports: - http_url_with_port = http_url + ":" + port - self.url_scan(http_url_with_port) - else: - print("nmap not found http server port at : ",http_url) - else: - self.url_scan(http_url) - - - def url_scan(self,target): - whatweb.Whatweb(target) - - c = crawlergo.Crawlergo(target) - if c.sub_domains and not self.arguments.args.fastscan : #将crawlergo扫描出的子域名添加到任务清单中 - print("crawlergo found domains : ",c.sub_domains) - for domains in c.sub_domains: - if domains not in self.arguments.urlList: - self.arguments.urlList.append(domains) - - time.sleep(5) - print("waiting xray scan to end") - while (True): #wait for xray end scan - if self.xray.check_xray_status(): - break - - urls = dirsearch.Dirsearch(target).urls - if urls: - request = request_engine.Request() #repeat urls found by dirsearch to xray - for url in urls: - request.repeat(url) - time.sleep(5) - - if "awvs" in self.arguments.toolList: - awvs.Awvs(target) - - self.scanned_domains.append(target) - - # 使用oneforall遍历子域名 - def format_url(self): - for url in self.arguments.urlList: - result = re.search(r'(([01]{0,1}\d{0,1}\d|2[0-4]\d|25[0-5])\.){3}([01]{0,1}\d{0,1}\d|2[0-4]\d|25[0-5])', url) - if result: - url = result.group() - http_url = url_parse(url).get_http_url() # - yield http_url - - elif url.startswith('http'): - yield url - - else: - # 判断域名是否已经扫描过，包括含有http这类的链接 - scanned_status = False - compile = '^[http://|https://]*' + url - for u in self.scanned_domains: - if re.findall(compile,u): - print("{} had in scanned_domains list .".format(url)) - scanned_status = True - break - - if scanned_status : - continue - - # 判断是否是二级域名， - if url.count('.') >= 2: - is_subdomain = True - for suffix in [".com.cn", ".edu.cn", ".net.cn", ".org.cn", ".co.jp",".gov.cn", ".co.uk", "ac.cn",]: - if suffix in url : - is_subdomain = False - break - - # 二级域名的话跳过，不再爆破三级域名 - if is_subdomain : - yield url_parse(url).get_http_url() - continue - - # 域名当作url先扫描 - yield url_parse(url).get_http_url() - - # 遍历子域名并扫描 - domains_list = OneForAll(url).scan() - domains_list = sorted(set(domains_list), key=domains_list.index) # 去重 保持顺序 - for domains in domains_list: - http_url = url_parse(domains).get_http_url() # - yield http_url - continue - - def init_report(self): - from .setting import TEMPLATE_FILE - from .setting import TOOLS_REPORT_FILE - - if not os.path.exists(TOOLS_REPORT_FILE): - with open(TOOLS_REPORT_FILE, 'w+') as w: - with open(TEMPLATE_FILE, 'r') as r: - w.write(r.read()) - - def restore(self): - main_path = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - restore_path = os.path.join(main_path,'.restore') - if not os.path.exists(restore_path): - exit('not found .restore file') - - with open(restore_path,'r') as f: # 判断域名情况 - url = f.read() - return url diff --git a/lib/db.py b/lib/db.py index 23f1c93..f6d714e 100644 --- a/lib/db.py +++ b/lib/db.py @@ -1,50 +1,54 @@ -from lib.setting import now_time -from lib.general import path_build import sqlite3 +import os -class db(): - def __init__(self,sql, value=None, dbfile='scanned_info.db'): - self.db_path = path_build(dbfile) - self.sql = sql - self.value = self.replace_date(value) - self.date = now_time.replace('-','') - self.db = sqlite3.connect(self.db_path) - self.c = self.db.cursor() - - - def __enter__(self): - def run(): - if self.value: - return self.c.execute(self.sql,self.value) - else: - return self.c.execute(self.sql) - - try: - run() - except sqlite3.OperationalError as e: - try : - if 'no such table' in str(e): - # target table - self.c.execute(''' - create table if not exists target_info ( - id INTEGER PRIMARY KEY,input_target text, found_domains text, date integer)''') - # scanned info - self.c.execute(''' - create table if not exists scanned_info ( - id INTEGER PRIMARY KEY,domain text, date integer, crawlergo text, dirsearch text - )''') - run() - except Exception as e: - print(e) - - def __exit__(self, exc_type, exc_val, exc_tb): - self.db.commit() - self.db.close() - - - def replace_date(self,value): - if 'date' in value: - v = list(value) - index = v.index('date') - v[index] = now_time - return tuple(v) \ No newline at end of file +main_path = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] + + +''' +初试化表结构 +''' +def db_init(): + with sqlite3.connect('scanned_info.db') as conn: + conn.execute(''' + create table if not exists target_info ( + id INTEGER PRIMARY KEY, + target text, + oneforall text, + crawlergo text, + batch_num integer, + date timestamp not null default (datetime('now','localtime'))) + ''') + + conn.execute(''' + create table if not exists host_info ( + id INTEGER PRIMARY KEY, + domain text, + nslookup text, + iplocation text, + masscan text, + nmap text, + batch_num integer, + date timestamp not null default (datetime('now','localtime'))) + ''') + + conn.execute(''' + create table if not exists scanned_info ( + id INTEGER PRIMARY KEY, + domain text, + whatweb text, + crawlergo text, + dirsearch text, + batch_num integer, + date timestamp not null default (datetime('now','localtime')) + )''') + + +def db_insert(sql, *value): + with sqlite3.connect(os.path.join(main_path, 'scanned_info.db')) as conn: + conn.execute(sql, value) # *value返回(1,) (1,2)这种元祖 + + +def db_update(table, name, text): + with sqlite3.connect(os.path.join(main_path, 'scanned_info.db')) as conn: + sql = 'update {table} set {column}=? order by id desc limit 1;'.format(table=table, column=name) + conn.execute(sql, (text,)) \ No newline at end of file diff --git a/lib/download_tools.py b/lib/download_tools.py index 55341bf..e7a3c0a 100644 --- a/lib/download_tools.py +++ b/lib/download_tools.py @@ -1,39 +1,31 @@ import os -from lib.setting import TOOLS_DIR -def download(): - tools = { - 'xray' : "https://download.xray.cool/xray/1.7.0/xray_linux_amd64.zip?download=true", - 'crawlergo' : 'https://github.com/0Kee-Team/crawlergo/releases/download/v0.4.0/crawlergo_linux_amd64.zip', - 'dirsearch' : 'https://github.com/maurosoria/dirsearch/archive/v0.4.1.zip', - 'oneforall' : 'https://github.com/shmilylty/OneForAll/archive/v0.4.3.zip', +TOOLS_DIR = os.path.join(os.path.split(os.path.dirname(os.path.realpath(__file__)))[0], 'tools') +TOOLS = { + 'xray': "https://download.xray.cool/xray/1.7.0/xray_linux_amd64.zip", + 'crawlergo': 'https://github.com/0Kee-Team/crawlergo/releases/download/v0.4.0/crawlergo_linux_amd64.zip', + 'dirsearch': 'https://github.com/maurosoria/dirsearch/archive/v0.4.1.zip', + 'oneforall': 'https://github.com/shmilylty/OneForAll/archive/v0.4.3.zip', } - os.chdir(TOOLS_DIR) +# github上下载的工具解压后会变成xxx-master , 需要变更为xxx +RENAME_DIRS = ['dirsearch', 'oneforall'] - #download - for key,value in tools.items(): - if key == 'xray' and not value.endswith('.zip'): - name = 'xray_linux_amd64.zip' - os.system('wget --no-check-certificate {url} -O {name}'.format(url=value,name=name)) - else: - #os.system('wget --no-check-certificate {url}'.format(url=value)) - os.system('wget --no-check-certificate {url}'.format(url=value)) +def download(): + os.chdir(TOOLS_DIR) - #extract + for key, value in TOOLS.items(): + os.system('wget --no-check-certificate {url}'.format(url=value)) os.system('unzip "*.zip"') - #rename dirsearch and oneforall dir - dirs = os.listdir(TOOLS_DIR) - for dir in dirs: - if dir.lower().startswith('dirsearch'): - os.system('mv {} dirsearch'.format(dir)) - - if dir.lower().startswith('oneforall'): - os.system('mv {} oneforall'.format(dir)) - + for name_src in os.listdir(TOOLS_DIR): + for name_dest in RENAME_DIRS: + if name_src.lower().startswith(name_dest): + os.system('mv {} {}'.format(name_src, name_dest)) - tool_name = ['crawlergo','dirsearch','oneforall','xray_linux_amd64'] - if set(tool_name) < set(os.listdir()): - os.system('touch install.lock') \ No newline at end of file + # 判断工具清单的文件是否都在目录了 + if set([name for name in TOOLS.keys()]).issubset(set(os.listdir())): + os.system('touch install.lock') + else: + exit('The tool has not been downloaded completely, check lib//download_tools.py for details') diff --git a/lib/general.py b/lib/general.py index 60cd26e..c41f63f 100755 --- a/lib/general.py +++ b/lib/general.py @@ -1,41 +1,38 @@ -import os from urllib.parse import urlparse +import os import socket import json import xlrd -class read_xls(): - def __init__(self,file): - self.base_str = list('abcdefghijklmnopqrstuvwxyz.-_') +class read_xls: + def __init__(self, file): + self.base_str = list('0123456789abcdefghijklmnopqrstuvwxyz.-_') self.domains = self.read_xls(file) - def read_xls(self,file): + def read_xls(self, file): try: workbook = xlrd.open_workbook(file) sheet1 = workbook.sheet_by_index(0) column = sheet1.col_values(3) + return self.filter(column) except Exception as e: exit(e) - return self.filter(column) - def filter(self,domains): + def filter(self, domains): domains_filterd = [] for domain in domains: if domain is None: break - if ';' in domain: domain = domain.split(';')[0] - # 判断域名内容是否标准，比如是否存在中文 if not set(list(domain)) < set(self.base_str): print('domain {} 不规范，忽略'.format(domain)) continue - if not len(domain) < 3: domains_filterd.append(domain) - return domains_filterd + return sorted(set(domains_filterd), key=domains_filterd.index) class Run(): def __init__(self,command,logfile='',delete_file=True): @@ -109,13 +106,10 @@ def get_ip_from_url(http_url): def get_file_content(file_path): if not os.path.exists(file_path): - print("the file path is not correct") + exit("not found file:{}".format(file_path)) - urlList = [] - with open(file_path,'r') as f: - for line in f.readlines(): - urlList.append(line.strip()) - return urlList + with open(file_path, 'r') as f: + return [line.strip() for line in f.readlines()] def dir_is_exists_or_create(*dir_path): @@ -137,7 +131,7 @@ def check_dict_key_vaild(dict,*keys): def path_build(*path): main_path = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] - path = os.path.join(main_path,*path) + path = os.path.join(main_path, *path) return path diff --git a/lib/img/1.png b/lib/img/1.png new file mode 100644 index 0000000..2220091 Binary files /dev/null and b/lib/img/1.png differ diff --git a/lib/img/2.png b/lib/img/2.png new file mode 100644 index 0000000..fd77708 Binary files /dev/null and b/lib/img/2.png differ diff --git a/lib/img/3.png b/lib/img/3.png new file mode 100644 index 0000000..79578fc Binary files /dev/null and b/lib/img/3.png differ diff --git a/lib/logger.py b/lib/logger.py deleted file mode 100755 index 1e3bf83..0000000 --- a/lib/logger.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging -import colorlog -log_colors_config = { - 'DEBUG': 'cyan', - 'INFO': 'green', - 'WARNING': 'yellow', - 'ERROR': 'red', - 'CRITICAL': 'red', -} - -logger = logging.getLogger('mylogger') -logger.setLevel(logging.DEBUG) - -fh = logging.FileHandler('test.log') -fh.setLevel(logging.DEBUG) - -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) - -file_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -color_formatter = colorlog.ColoredFormatter('%(log_color)s[%(levelname)s] - %(asctime)s - %(name)s - %(message)s',log_colors=log_colors_config) - -fh.setFormatter(file_formatter) -ch.setFormatter(color_formatter) - -logger.addHandler(fh) -logger.addHandler(ch) - -# logger.info("test info") -# logger.error("error") \ No newline at end of file diff --git a/lib/report.py b/lib/report.py index d60e717..0a388aa 100755 --- a/lib/report.py +++ b/lib/report.py @@ -1,38 +1,152 @@ -from .general import url_parse -from .setting import TOOLS_REPORT_FILE -from bs4 import BeautifulSoup import html +import os +import sqlite3 +from bs4 import BeautifulSoup +from lib.Tools import Snapshot + + +main_path = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0] +REPORT_PATH = os.path.join(main_path, 'report') +REPORT_TEMPLATE = os.path.join(main_path, "lib/template.html") + +REPORT_TAB = ''' +

{DOMAIN}

+
+ {IMG_TAB} +
+
+
    + {LI} +
+
+
+
+''' + +REPORT_LI = ''' +
  • + +
    +
    +

    {NAME}

    + 

    {REPORT}

    +
    +
    • +''' + + +class Report: + def __init__(self): + self.body = "" + self.report = None + self.batch_num = None + self.IMG_TAB = r'
    ' + + def html_report_single(self): + with sqlite3.connect(os.path.join(main_path, 'scanned_info.db')) as conn: + def parse(fetch): + li = '' + key = [i[0] for i in fetch.description] + for row in fetch.fetchall(): + value = [str(row[_]) for _ in range(len(row))] + domain = value[1] + self.batch_num = value[-2] + time = value[-1].split() + time1, time2 = time[0], time[1] + # 生成li模块 + for name, report in zip(key[2:-2], value[2:-2]): + li += REPORT_LI.format(TIME1=time1, TIME2=time2, NAME=name, REPORT=html.escape(report)) + + # 返回整个tab模块 + if domain.startswith('http'): + yield REPORT_TAB.format(DOMAIN=domain, IMG_TAB=self.IMG_TAB.format(Snapshot.format_img_name(domain)), LI=li) + else: + yield REPORT_TAB.format(DOMAIN=domain, IMG_TAB='', LI=li) + + tag = '' + + # 插入时从scanned_info确认是否有port扫描，如有需要先插入port信息 + sql = ''' + select * from host_info + where batch_num = (select batch_num from scanned_info order by id desc limit 1) + order by id desc limit 1 ; + ''' + fetch = conn.execute(sql) + for _ in parse(fetch): + tag += _ -class Report(): - def __init__(self,target): # all tools use http_url already - self.target = url_parse(target).get_netloc() - self.separate = "\n"*6 - - def report_insert(self,tool_log_file,title="",is_file=True): - soup = BeautifulSoup(self.read(TOOLS_REPORT_FILE),'html.parser') - if is_file == True: - report = self.read(tool_log_file) - else : - report = tool_log_file - report = title + '\n' + report - - if not soup.h3 or self.target not in soup.h3.string: - text = '

    {}

    {}
    \n'.format(self.target,html.escape(report)) - t = BeautifulSoup(text,'html.parser') + # 下面的limit 1，1 要删掉1 + # 如果port_info有信息， 那么要插入scanned_info中对应的domain数据在port下面 + if tag: + sql = ''' + SELECT * from scanned_info + where batch_num = (select batch_num from host_info order by id desc limit 1) + and domain LIKE '%'||(SELECT domain from host_info order by id desc limit 1)||'%' + order by id desc limit 1; + ''' + for _ in parse(conn.execute(sql)): + tag += _ + else: + sql = ''' + SELECT * from scanned_info order by id desc limit 1 ; + ''' + for _ in parse(conn.execute(sql)): + tag += _ + + if os.path.exists(os.path.join(REPORT_PATH, '{}-tools.html'.format(self.batch_num))): + soup = BeautifulSoup(self.read(os.path.join(REPORT_PATH, '{}-tools.html'.format(self.batch_num))), 'html.parser') + else: + soup = BeautifulSoup(self.read(REPORT_TEMPLATE), 'html.parser') + + if soup.h3: + t = BeautifulSoup(tag, 'html.parser') soup.h3.insert_before(t) + self.write(os.path.join(REPORT_PATH, '{}-tools.html'.format(self.batch_num)), str(soup)) else: - soup.div.pre.string += self.separate + report + print('Failed to write to report file ! ') - self.rewrite_template(str(soup)) + ''' + 获取单个batch_num， 并输出 + 此处未完成，瞎做 + ''' + def html_report_entire(self): + with sqlite3.connect(os.path.join(main_path, 'scanned_info.db')) as conn: + self.batch_num = conn.execute('select batch_num from scanned_info order by id desc limit 1;').fetchone()[0] + sql = 'select * from scanned_info where batch_num = {};'.format(self.batch_num) + fetch = conn.execute(sql).fetchall() + for row in fetch: + print(row) + title = row[1] + value = [str(row[_]) for _ in range(len(row)) if row[_] is not None] + value = '\n'.join(value[2:]) + self.body += '

    {}

    {}
    \n'.format(title, html.escape(value)) - def rewrite_template(self,text): - with open(TOOLS_REPORT_FILE,'w') as f: - f.write(text) + soup = BeautifulSoup(self.read(REPORT_TEMPLATE), 'html.parser') + if soup.h3: + t = BeautifulSoup(self.body, 'html.parser') + soup.h3.insert_before(t) + + self.write(os.path.join(REPORT_PATH, '{}-tools.html'.format(self.batch_num)), str(soup)) - def read(self,file): - with open(file,'r') as f: + @staticmethod + def read(file): + with open(file, 'r') as f: return f.read() -if __name__ == "__main__": - X = Report("http://a.testphp.vulnweb.com:9000/index.php") - X.insert("setting.py") + @staticmethod + def write(file, text): + with open(file, 'w+') as f: + f.write(text) + + def test(self): + with sqlite3.connect(os.path.join(main_path, 'scanned_info.db')) as conn: + sql = '''select * from scanned_info where batch_num = ( + select batch_num from scanned_info order by id desc limit 1 + );''' + + fetch = conn.execute(sql).fetchall() + for row in fetch: + v = [str(row[c]) for c in range(len(row)) if row[c] is not None] + print('\n'.join(v[2:])) + + diff --git a/lib/scanner/__init__.py b/lib/scanner/__init__.py deleted file mode 100755 index e69de29..0000000 diff --git a/lib/scanner/__pycache__/__init__.cpython-38.pyc b/lib/scanner/__pycache__/__init__.cpython-38.pyc deleted file mode 100644 index 0be7a35..0000000 Binary files a/lib/scanner/__pycache__/__init__.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/awvs.cpython-38.pyc b/lib/scanner/__pycache__/awvs.cpython-38.pyc deleted file mode 100644 index eba33c8..0000000 Binary files a/lib/scanner/__pycache__/awvs.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/crawlergo.cpython-38.pyc b/lib/scanner/__pycache__/crawlergo.cpython-38.pyc deleted file mode 100644 index c123ba9..0000000 Binary files a/lib/scanner/__pycache__/crawlergo.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/dirsearch.cpython-38.pyc b/lib/scanner/__pycache__/dirsearch.cpython-38.pyc deleted file mode 100644 index c6de570..0000000 Binary files a/lib/scanner/__pycache__/dirsearch.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/masscan.cpython-38.pyc b/lib/scanner/__pycache__/masscan.cpython-38.pyc deleted file mode 100644 index bfd9195..0000000 Binary files a/lib/scanner/__pycache__/masscan.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/nmap.cpython-38.pyc b/lib/scanner/__pycache__/nmap.cpython-38.pyc deleted file mode 100644 index 9010bbb..0000000 Binary files a/lib/scanner/__pycache__/nmap.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/oneforall.cpython-38.pyc b/lib/scanner/__pycache__/oneforall.cpython-38.pyc deleted file mode 100644 index d1b7310..0000000 Binary files a/lib/scanner/__pycache__/oneforall.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/request_engine.cpython-38.pyc b/lib/scanner/__pycache__/request_engine.cpython-38.pyc deleted file mode 100644 index c3c1e40..0000000 Binary files a/lib/scanner/__pycache__/request_engine.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/whatweb.cpython-38.pyc b/lib/scanner/__pycache__/whatweb.cpython-38.pyc deleted file mode 100644 index 0674bf4..0000000 Binary files a/lib/scanner/__pycache__/whatweb.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/__pycache__/xray.cpython-38.pyc b/lib/scanner/__pycache__/xray.cpython-38.pyc deleted file mode 100644 index 0335b54..0000000 Binary files a/lib/scanner/__pycache__/xray.cpython-38.pyc and /dev/null differ diff --git a/lib/scanner/crawlergo.py b/lib/scanner/crawlergo.py deleted file mode 100755 index fcc1952..0000000 --- a/lib/scanner/crawlergo.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/python3 -# coding: utf-8 - - -import simplejson -import subprocess -import os -from lib.report import Report -from lib.setting import TOOLS_DIR -from func_timeout import func_set_timeout -import func_timeout - -class Crawlergo(Report): - def __init__(self,target): - super().__init__(target) - self.BROWERS = '/usr/bin/google-chrome' - self.XRAY_PROXY = 'http://127.0.0.1:7777' - self.sub_domains = [] - self.target = target - - try: - self.scan() - except func_timeout.exceptions.FunctionTimedOut: - print('crawlergo timeout') - - - @func_set_timeout(300) - def scan(self): - print("crawlergo scanning : ",self.target) - - try: - crawlergo = os.path.join(TOOLS_DIR,"crawlergo") - print(1) - #cmd = [crawlergo, "-c", self.BROWERS, "-t", "5", "-f", "smart", "--push-to-proxy", self.XRAY_PROXY, "--push-pool-max", "10", "--fuzz-path", "-o", "json", self.target] - cmd = [crawlergo, "-c", self.BROWERS, "--push-to-proxy", self.XRAY_PROXY, "-o", "json", self.target] - rsp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - print(2) - output, error = rsp.communicate() - # "--[Mission Complete]--" 是任务结束的分隔字符s串 - result = simplejson.loads(output.decode().split("--[Mission Complete]--")[1]) - req_list = result["req_list"] - urls = [] - for req in req_list: - print("crawlergo found :" , req['url']) - urls.append(req['url']) - #self.put_to_file(req['url'],os.path.join(self.report_dir,url_parse(self.target).get_netloc()+"-crawlergo.url")) - #print(req_list[0]) - - #subdomain 在controller模块中已添加进入扫描 - print(type(result["sub_domain_list"])) - - self.sub_domains = result["sub_domain_list"] - # for domain in sub_domains: - # print("sub found :", domain) - #self.put_to_file(domain, os.path.join(self.report_dir, url_parse(self.target).get_netloc() + "-crawlergo.sub")) - - - rows = ["crawlergo urls found:"] + urls + ['\n'*1,"crawlergo subdomains found:"] + self.sub_domains - self.report_insert('\n'.join(rows),"crawlergo report:",is_file=False) - except Exception as f: - print(f) - pass - - # def put_to_file(self,row,filename): - # with open(filename,'a+') as f: - # f.write(row) - -if __name__ == '__main__': - A = Crawlergo("http://testphp.vulnweb.com") \ No newline at end of file diff --git a/lib/scanner/dirsearch.py b/lib/scanner/dirsearch.py deleted file mode 100755 index 0326d68..0000000 --- a/lib/scanner/dirsearch.py +++ /dev/null @@ -1,56 +0,0 @@ -from lib.setting import TOOLS_DIR -from lib.report import Report -from lib.general import Run -import os -from func_timeout import func_set_timeout -import func_timeout - -class Dirsearch(Report): - def __init__(self,target): - super().__init__(target) - self.LOG = '/tmp/dirsearch.csv' - self.target = target - self.urls = [] - - try: - self.scan() - except func_timeout.exceptions.FunctionTimedOut: - print('dirsearch timeout') - - @func_set_timeout(300) - def scan(self): - command = 'python3 {}/dirsearch/dirsearch.py -e * -x 301,403,404,405,500,501,502,503 -u {} --csv-report {}'.format(TOOLS_DIR,self.target,self.LOG) - print('dirsearch is running') - with Run(command,self.LOG,delete_file=False) as f: - if os.path.exists(self.LOG): - #print("dirsearch log file exists , and run report_insert") - print('\n'.join(self.report_parse(self.LOG))) - self.report_insert('\n'.join(self.report_parse(self.LOG)),"DIRSEARCH SCAN:",is_file=False) - os.system('rm -f ' + self.LOG) - else: - print("dirsearch log file is not exists , may be because of dirsearch cannot connect the {}".format(self.target)) - - # 将dirsearch的csv报告重新整理下 - def report_parse(self,log_file): - rows = [] - with open(log_file,'r') as f: - try: - next(f) #always raise a StopIteration error before - except StopIteration as s: - print(s) - finally: - for line in f.readlines(): - line = line.strip().split(',') - if not line[1]: - continue - - self.urls.append(line[1]) - try: - s = "{:<} - {:>5}B - {:<5}".format(line[2], line[3], line[1]) - except: - continue - rows.append(s) - return rows - -if __name__ == "__main__": - X = Dirsearch("http://testphp.vulnweb.com") diff --git a/lib/scanner/dirsearch1.py b/lib/scanner/dirsearch1.py deleted file mode 100755 index c351bc1..0000000 --- a/lib/scanner/dirsearch1.py +++ /dev/null @@ -1,38 +0,0 @@ -from lib.setting import TOOLS_DIR -from lib.report import Report -from lib.general import Run -import os -from func_timeout import func_set_timeout -import func_timeout - -class Dirsearch(Report): - def __init__(self,target): - super().__init__(target) - self.LOG = '/tmp/dirsearch.txt' - self.target = target - self.urls = [] - - try: - self.scan() - except func_timeout.exceptions.FunctionTimedOut: - print('dirsearch timeout') - - - def scan(self): - command = 'python3 {}/dirsearch/dirsearch.py -e * -u {} --plain-text-report {}'.format(TOOLS_DIR,self.target,self.LOG) - with Run(command,self.LOG,delete_file=False) as f: - if os.path.exists(self.LOG): - self.report_insert('\n'.join(self.report_parse(self.LOG)),"DIRSEARCH SCAN:",is_file=False) - self.parse() - os.system('rm -f ' + self.LOG) - else: - print("dirsearch log file is not exists") - - def parse(self): - with open(self.LOG,'r') as f: - for line in f.readlines(): - url = line.split(' ')[-1] - self.urls.append(url.strip()) - -if __name__ == "__main__": - X = Dirsearch("http://testphp.vulnweb.com") \ No newline at end of file diff --git a/lib/scanner/masscan.py b/lib/scanner/masscan.py deleted file mode 100755 index e74ae09..0000000 --- a/lib/scanner/masscan.py +++ /dev/null @@ -1,39 +0,0 @@ -import os -import re -from ..setting import TOOLS_DIR -from func_timeout import func_set_timeout -import func_timeout - - -class Masscan(): - def __init__(self,target): - self.target = target - - try: - self.open_ports = self.scan() - except func_timeout.exceptions.FunctionTimedOut: - self.open_ports = [] - - @func_set_timeout(300) - def scan(self): - try: - print("masscan scanning :", self.target) - #os.system("masscan --ports 0-65535 {0} --rate=10000 -oX {}/masscan.log".format(self.target,self.report_dir)) - result = os.popen("masscan --ports 0-65535 {0} -sS -Pn --rate=1000".format(self.target)).read() - open_ports = self.reg_port(result) - print("opening ports:",open_ports) - except Exception as e: - print(e) - - return open_ports - - def reg_port(self,text): - #masscan output : Discovered open port 8814/tcp on 192.168.1.225 - pattern = '\d{1,5}/tcp' - result = re.findall(pattern, text) - result = [x[:-4] for x in result] - return result - - -if __name__ == "__main__": - X = Masscan("192.168.1.225") \ No newline at end of file diff --git a/lib/scanner/nmap.py b/lib/scanner/nmap.py deleted file mode 100755 index 498f8a6..0000000 --- a/lib/scanner/nmap.py +++ /dev/null @@ -1,44 +0,0 @@ -import re -import os -from lib.report import Report -from lib.general import Run -from func_timeout import func_set_timeout -import func_timeout - - -class Nmap(Report): - def __init__(self,target,ports_scan=[]): - super().__init__(target) - self.LOG_FILE = "/tmp/nmap.log" - self.target = target - self.ports_scan = ports_scan - - try: - self.http_open_ports = self.scan() - except func_timeout.exceptions.FunctionTimedOut: - print('nmap timeout') - self.http_open_ports = [] - - - @func_set_timeout(500) - def scan(self): - if not self.ports_scan: - command = "nmap -sS -Pn -A -v {0} -oN {1}".format(self.target,self.LOG_FILE) - else: - command = "nmap -sS -Pn -A -p {0} {1} -v -oN {2}".format(",".join(self.ports_scan),self.target,self.LOG_FILE) - - with Run(command,self.LOG_FILE,) as f: - self.report_insert(self.LOG_FILE,"NMAP report:") - return self.get_http_ports(f) - - def get_http_ports(self,text): - http_ports = re.findall('\d{1,5}/tcp\s{1,}open\s{1,}[ssl/]*http', text) - http_ports = [x.split("/")[0] for x in http_ports] - - if '80' in http_ports and '443' in http_ports: - http_ports.remove("443") - - return http_ports - -if __name__ == "__main__": - X = Nmap("47.98.126.199",["80"]) diff --git a/lib/scanner/oneforall.py b/lib/scanner/oneforall.py deleted file mode 100755 index 7e48e83..0000000 --- a/lib/scanner/oneforall.py +++ /dev/null @@ -1,35 +0,0 @@ -from ..setting import TOOLS_REPORT_FILE -from ..setting import TOOLS_DIR -import os -import csv - -class OneForAll(): - def __init__(self,target): - self.target = target - - if os.path.exists(os.path.join(TOOLS_DIR,'OneForAll')): - os.system('cp -r {0}/OneForAll {0}/oneforall'.format(TOOLS_DIR)) - - def scan(self): - print("Brute domain: " + self.target) - - os.system('python3 {}/oneforall/oneforall.py --target {} run'.format(TOOLS_DIR, self.target)) - report_file = "{}/oneforall/results/{}.csv".format(TOOLS_DIR, self.target) - if not os.path.exists(report_file): - exit("Not found the oneforall's output file ") - - return self.get_subdomains(report_file) - - - def get_subdomains(self,report_file): - try: - with open(report_file, 'r') as csvfile: - csvfile.__next__() - reader = csv.reader(csvfile) - column = [row[5] for row in reader] - urlList = list(set(column)) - except Exception as e: - print(e) - urlList = [] - - return urlList diff --git a/lib/scanner/request_engine.py b/lib/scanner/request_engine.py deleted file mode 100644 index 43d57d7..0000000 --- a/lib/scanner/request_engine.py +++ /dev/null @@ -1,23 +0,0 @@ -import requests - -class Request(): - def __init__(self): - self.proxy = {'http': 'http://127.0.0.1:7777', - 'https': 'http://127.0.0.1:7777',} - self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:27.0) Gecko/20100101 Firefox/27.0)', - } - - def repeat(self,url): - try: - response = requests.get(url=url,headers=self.headers,proxies=self.proxy,verify=False,timeout=20) - #print(response) - return response - except Exception as e: - print(e) - - -if __name__ =="__main__": - X = Request() - X.repeat("http://testphp.vulnweb.com:80/.idea/") - - diff --git a/lib/scanner/web_path/dirb.py b/lib/scanner/web_path/dirb.py deleted file mode 100755 index 65d7eb2..0000000 --- a/lib/scanner/web_path/dirb.py +++ /dev/null @@ -1,3 +0,0 @@ - -class dirb(): - def __init__(self): diff --git a/lib/scanner/whatweb.py b/lib/scanner/whatweb.py deleted file mode 100644 index 2ba35f3..0000000 --- a/lib/scanner/whatweb.py +++ /dev/null @@ -1,18 +0,0 @@ -from lib.general import Run -from lib.report import Report - -class Whatweb(Report): - def __init__(self,target): - super().__init__(target) - self.target = target - self.scan() - - def scan(self): - command = "whatweb --color never {}".format(self.target) - with Run(command) as f: - for i in f.split('\n'): - if i.startswith('http'): - #print(i.replace(',','\n')) - self.report_insert(i.replace(',','\n'),'WHATWEB SCAN:',is_file=False) - - diff --git a/lib/scanner/xray.py b/lib/scanner/xray.py deleted file mode 100755 index 6f34bde..0000000 --- a/lib/scanner/xray.py +++ /dev/null @@ -1,62 +0,0 @@ -import os -import time -import threading -import re -from ..setting import XRAY_REPORT_FILE,TOOLS_DIR - -class Xray(): - def __init__(self): - self.LOG = "/tmp/xray.log" - self.PROXY = '127.0.0.1:7777' - self.vuln_scaned = 0 - - self.kill_previous_xray_process() - self.clear_previous_xray_log() - - def scan(self): - t = threading.Thread(target=self.xray_run, daemon=True) - t.start() - - ##阻塞模式，因为之前实测后面工具扫描时导致xray请求一片红， - # while(True): - # if self.check_xray_status(): - # os.system("rm /tmp/xray.log") - # self.xray_scan_over = 1 - # break - - def xray_run(self): - run_command = "{0}/xray_linux_amd64 webscan --listen {1} --html-output {2} | tee -a {3}".format(TOOLS_DIR,self.PROXY,XRAY_REPORT_FILE,self.LOG) - print("xray command : ",run_command) - os.system(run_command) - - - def check_xray_status(self): - cmd = "wc " + self.LOG +"| awk '{print $1}'" - rows0 = os.popen(cmd).read() - time.sleep(5) - rows1 = os.popen(cmd).read() - cmd = "tail -n 10 {}".format(self.LOG) - s = os.popen(cmd).read() - - if rows0 == rows1 and "All pending requests have been scanned" in s: - os.system('echo "" > {}'.format(self.LOG)) - return True - else: - return False - - def clear_previous_xray_log(self): - if os.path.exists(self.LOG): - os.system("rm -f {}".format(self.LOG)) - - def kill_previous_xray_process(self): - port = self.PROXY.rsplit(':')[-1] - process_status = os.popen("netstat -pantu | grep " + port).read() - if process_status: - process_num = re.findall("\d{1,}/xray", process_status) - if process_num: - process_num = ''.join(process_num)[:-5] - print(port," port exist previous xray process , killing") - os.system("kill " + str(process_num)) - -if __name__ == "__main__": - A = Xray() diff --git a/lib/setting.py b/lib/setting.py deleted file mode 100755 index 90c924f..0000000 --- a/lib/setting.py +++ /dev/null @@ -1,26 +0,0 @@ -from lib.general import path_build -from lib.general import dir_is_exists_or_create -import time - - -# TIME -now_time = time.strftime("%Y-%m-%d-%H-%M-%S-", time.localtime(time.time())) - - -# DIR PATH -REPORT_DIR = path_build("report") -TOOLS_DIR = path_build("tools") -dir_is_exists_or_create(REPORT_DIR,TOOLS_DIR) - -TOOLS_REPORT_NAME = now_time + "tools-scan.html" -TOOLS_REPORT_FILE = path_build('report',TOOLS_REPORT_NAME) - -XRAY_REPORT_NAME = now_time + "xray.html" -XRAY_REPORT_FILE = path_build('report',XRAY_REPORT_NAME) - -AWVS_REPORT_FILE = now_time + "{}.awvs.html" - -TEMPLATE_FILE = path_build('lib','template.html') - - - diff --git a/lib/template.html b/lib/template.html index 4927968..79ad1f0 100755 --- a/lib/template.html +++ b/lib/template.html @@ -1,104 +1,434 @@ + + + + + + + - - - -
    - -

    end

    sign
    - -
    - - - - - + #tab03 { position:relative; width:300px; height: auto; margin:20px; } + #tab03 h3 { position:relative; z-index:1; height:; padding-top:4px; margin-bottom:-1px; border:solid #ccc; border-width:1px 0 1px 1px; text-align:center; font-family: ;color : white;background:DeepSkyBlue; cursor:pointer; } + #tab03 h3.up { z-index:3; color:#c00; background:#fff; } + #tab03 div.tab { display:none; position: fixed; overflow-x: hidden; overflow-y: scroll; left: 450px; top:0; bottom: 0; z-index:2; width:auto; height:auto; padding:5px; border:solid 1px #ccc; color:#266; } + #tab03 div.tab.up { display:block;border:0px } + --> + + + + + + + +
    + +

    end

    sign
    +
    + + + + + diff --git a/lib/test.log b/lib/test.log deleted file mode 100644 index 0a923d7..0000000 --- a/lib/test.log +++ /dev/null @@ -1,2 +0,0 @@ -2021-04-23 10:45:19,556 - mylogger - INFO - test info -2021-04-23 10:45:19,556 - mylogger - ERROR - error diff --git a/lib/urlParser.py b/lib/urlParser.py new file mode 100644 index 0000000..05467e4 --- /dev/null +++ b/lib/urlParser.py @@ -0,0 +1,54 @@ +from socket import gethostbyname_ex +from IPy import IP +from urllib.parse import urlparse + + +class Parse: + def __init__(self, target): + if self.isIP(target): + self.data = { + 'ip': target, + 'domain': target, + 'http_url': 'http://' + target, + } + + else: + if not target.count('.') > 1: + target = 'www.' + target + + for suffix in [".com.cn", ".edu.cn", ".net.cn", ".org.cn", ".gov.cn"]: + if suffix in target: + if not target.count('.') > 2: + target = 'www.' + target + + if not target.startswith('http'): + target = 'http://' + target + + netloc = urlparse(target).netloc + if ':' in netloc: + netloc = netloc.split(':')[0] + + if self.isIP(netloc): + self.data = { + 'ip': netloc, + 'domain': netloc, + 'http_url': target, + } + else: + try: + data = list(gethostbyname_ex(netloc)) + self.data = {'ip': data[2][0], + 'domain': netloc, + 'http_url': target, + } + except Exception as e: + print(e) + self.data = None + + @staticmethod + def isIP(str): + try: + IP(str) + except ValueError: + return False + return True \ No newline at end of file diff --git a/log/test.log b/log/test.log new file mode 100644 index 0000000..3d9384b --- /dev/null +++ b/log/test.log @@ -0,0 +1,72 @@ +2021-06-17 22:05:56,151 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-17 22:06:00,413 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-19 23:20:15,020 - toolConsole - INFO - Oneforall - start scanning ! +2021-06-19 23:21:33,127 - toolConsole - INFO - Oneforall - scanned over ! +2021-06-19 23:21:33,199 - toolConsole - INFO - Masscan - start scanning ! +2021-06-19 23:26:33,085 - toolConsole - ERROR - Masscan - Command 'masscan --open -sS -Pn -p 1-15000 81.70.233.210' timed out after 300 seconds +2021-06-19 23:26:33,098 - toolConsole - INFO - Masscan - scanned over ! +2021-06-19 23:26:33,103 - toolConsole - INFO - Nmap - start scanning ! +2021-06-19 23:26:33,670 - toolConsole - INFO - Nmap - scanned over ! +2021-06-19 23:26:33,763 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-19 23:26:41,833 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-19 23:29:17,585 - toolConsole - INFO - Dirsearch - start scanning ! +2021-06-19 23:31:48,535 - toolConsole - INFO - Dirsearch - scanned over ! +2021-06-19 23:31:50,216 - toolConsole - INFO - Oneforall - start scanning ! +2021-06-19 23:32:06,546 - toolConsole - INFO - Oneforall - scanned over ! +2021-06-19 23:32:06,640 - toolConsole - INFO - Masscan - start scanning ! +2021-06-20 10:26:35,958 - toolConsole - INFO - Oneforall - start scanning ! +2021-06-20 10:27:48,809 - toolConsole - INFO - Oneforall - scanned over ! +2021-06-20 10:27:48,861 - toolConsole - INFO - Masscan - start scanning ! +2021-06-20 10:31:23,048 - toolConsole - INFO - Masscan - scanned over ! +2021-06-20 10:31:23,177 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-20 10:31:28,872 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-20 10:32:28,227 - toolConsole - INFO - Dirsearch - start scanning ! +2021-06-20 11:16:50,018 - toolConsole - INFO - Oneforall - start scanning ! +2021-06-20 11:19:13,931 - toolConsole - INFO - Oneforall - start scanning ! +2021-06-20 11:19:30,656 - toolConsole - INFO - Oneforall - scanned over ! +2021-06-20 11:19:30,711 - toolConsole - INFO - Masscan - start scanning ! +2021-06-20 11:23:12,989 - toolConsole - INFO - Masscan - scanned over ! +2021-06-20 11:23:13,118 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-20 11:23:16,974 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-20 11:24:06,881 - toolConsole - INFO - Dirsearch - start scanning ! +2021-06-20 11:26:59,964 - toolConsole - INFO - Dirsearch - scanned over ! +2021-06-20 11:49:03,076 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-20 11:49:07,369 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-20 12:03:40,396 - toolConsole - INFO - Dirsearch - start scanning ! +2021-06-20 12:03:41,953 - toolConsole - INFO - Dirsearch - scanned over ! +2021-06-24 16:42:48,370 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:42:49,224 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:43:22,107 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:43:22,815 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:44:55,745 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:44:56,404 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:47:01,371 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:47:02,548 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:47:38,497 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:47:39,669 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:48:13,326 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:48:14,530 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:50:19,294 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:50:20,622 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:52:14,318 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:52:15,863 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:53:33,898 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:53:35,225 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 16:55:25,922 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 16:55:27,040 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-24 17:00:25,847 - toolConsole - INFO - Whatweb - start scanning ! +2021-06-24 17:00:27,183 - toolConsole - INFO - Whatweb - scanned over ! +2021-06-26 19:06:04,594 - toolConsole - INFO - Nslookup - start scanning ! +2021-06-26 19:06:15,105 - toolConsole - INFO - Nslookup - scanned over ! +2021-06-27 11:53:21,784 - toolConsole - INFO - Masscan - start scanning ! +2021-06-27 11:53:21,795 - toolConsole - INFO - Masscan - scanned over ! +2021-06-27 11:53:45,953 - toolConsole - INFO - Masscan - start scanning ! +2021-06-27 11:53:45,970 - toolConsole - INFO - Masscan - scanned over ! +2021-06-27 11:53:54,407 - toolConsole - INFO - Masscan - start scanning ! +2021-06-27 11:54:16,727 - toolConsole - INFO - Masscan - scanned over ! +2021-06-27 11:54:33,837 - toolConsole - INFO - Masscan - start scanning ! +2021-06-27 11:54:56,719 - toolConsole - INFO - Masscan - scanned over ! +2021-06-27 11:59:28,662 - toolConsole - INFO - Masscan - start scanning ! +2021-06-27 11:59:50,843 - toolConsole - INFO - Masscan - scanned over ! +2021-06-27 12:03:17,550 - toolConsole - INFO - Masscan - start scanning ! +2021-06-27 12:03:39,847 - toolConsole - INFO - Masscan - scanned over ! diff --git a/main.py b/main.py index 5b21822..561fe86 100755 --- a/main.py +++ b/main.py @@ -1,25 +1,31 @@ -import os,stat +import os import time from lib.arguments_parse import ArgumentParser from lib.controller import Controller -from lib.setting import TOOLS_DIR +from lib.db import db_init def main(): + for dir in ['log', 'tools', 'report', 'report/img']: + if not os.path.exists(dir): + os.mkdir(dir) + ''' - 需要安装'crawlergo','dirsearch','oneforall','xray_linux_amd64'到tools目录 + 需要安装系列工具到tools目录 定义是自动下载，但是由于github问题可能经常会出错；出错的话手动下载解压最好； 具体链接在download_tools.py里有 ''' - if not os.path.exists(os.path.join(TOOLS_DIR,'install.lock')): + if not os.path.exists('tools/install.lock'): print('tools not exists; downloading ...') time.sleep(3) from lib.download_tools import download download() + db_init() arguments = ArgumentParser() - controller = Controller(arguments) + controller = Controller(arguments.args) controller.assign_task() + if __name__ == "__main__": main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 85ed146..7444e3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,3 +21,9 @@ tqdm==4.51.0 treelib==1.6.1 urllib3==1.25.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4' win32-setctime==1.0.3; sys_platform == 'win32' +xlrd==2.0.1 +simplejson==3.17.2 +fire==0.3.1 +beautifulsoup4==4.9.3 +IPy==1.01 +selenium diff --git a/script/find_port.py b/script/find_port.py new file mode 100644 index 0000000..1447ca1 --- /dev/null +++ b/script/find_port.py @@ -0,0 +1,94 @@ +''' + usage: python3 find_port.py 1.txt + 功能： 寻找开放某个端口的资产，导入资产文件中无论是url还是ip还是域名都可以 +''' +import socket +import queue +import threading +import sys +from socket import gethostbyname_ex +from IPy import IP +from urllib.parse import urlparse + + +PORT = 80 +class PortScan: + def __init__(self, file): + self.file = file + self.ips = queue.Queue() + self.readfile() + self.threads_run() + + def readfile(self): + with open(self.file, 'r') as f: + for url in f.readlines(): + target = Parse(url) + if target.data: + self.ips.put(target.data['ip']) + + def threads_run(self): + for i in range(20): + t = threading.Thread(target=self.check_port, ) + t.start() + + def check_port(self): + while True: + if self.ips.empty(): + exit('empty') + + ip = self.ips.get() + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.connect((ip, PORT)) + s.settimeout(3) + s.close() + print(ip) + return True + except socket.error as e: + return False + + +class Parse: + def __init__(self,target): + if self.isIP(target): + self.data = { + 'ip': target, + 'domain': None, + 'http_url': 'http://' + target, + } + + elif target.startswith('http'): + netloc = urlparse(target).netloc + if self.isIP(netloc): + self.data = { + 'ip': netloc, + 'domain': None, + 'http_url': target, + } + else: + try: + data = list(gethostbyname_ex(netloc)) + self.data = {'ip': data[2][0], + 'domain': netloc, + 'http_url': target, + } + except: + self.data = None + + def isIP(self, str): + try: + IP(str) + except ValueError: + return False + return True + + +if __name__ == '__main__': + if not sys.argv[1]: + print(''' + usage: python3 find_port.py 1.txt + + 功能： 寻找开放某个端口的资产，导入资产文件中无论是url还是ip还是域名都可以 + ''') + + a = PortScan(sys.argv[1]) diff --git a/sources.list b/sources.list deleted file mode 100644 index 13813bd..0000000 --- a/sources.list +++ /dev/null @@ -1,10 +0,0 @@ -deb http://mirrors.aliyun.com/ubuntu/ focal main restricted -deb http://mirrors.aliyun.com/ubuntu/ focal-updates main restricted -deb http://mirrors.aliyun.com/ubuntu/ focal universe -deb http://mirrors.aliyun.com/ubuntu/ focal-updates universe -deb http://mirrors.aliyun.com/ubuntu/ focal multiverse -deb http://mirrors.aliyun.com/ubuntu/ focal-updates multiverse -deb http://mirrors.aliyun.com/ubuntu/ focal-backports main restricted universe multiverse -deb http://mirrors.aliyun.com/ubuntu/ focal-security main restricted -deb http://mirrors.aliyun.com/ubuntu/ focal-security universe -deb http://mirrors.aliyun.com/ubuntu/ focal-security multiverse