Merge pull request #193 from GoVanguard/development

Merging in service bug fixes and travis update

.travis.yml

@@ -23,7 +23,7 @@ after_success:
   - (test $TRAVIS_PULL_REQUEST != "false") && exit 0
   - (test $TRAVIS_BRANCH != "master" && test $TRAVIS_BRANCH != "development") && exit 0
   - cd ./docker/
-  - docker login -u $DOCKER_USER -p $DOCKER_PASS
+  - echo "$DOCKER_PASS" | docker login --username $DOCKER_USER --password-stdin
   - export REPO=gvit/legion
   - docker build -f Dockerfile -t $REPO:$COMMIT . --no-cache
   - docker tag $REPO:$COMMIT $REPO:travis-$TRAVIS_BUILD_NUMBER

CHANGELOG.txt

@@ -1,6 +1,9 @@
 LEGION 0.3.7
 
-* Bug fixes
+* Bug fixes for several edge cases
+* Screenshot fixes
+* Service version data overwrite bug fixed
+* Stale service version data bug fixed
 * Refactor of docker base image
 
 LEGION 0.3.6

app/ApplicationInfo.py

@@ -19,12 +19,12 @@
 applicationInfo = {
     "name": "LEGION",
     "version": "0.3.7",
-    "build": '1593185290',
+    "build": '1596220187',
     "author": "GoVanguard",
     "copyright": "2020",
     "links": ["http://github.com/GoVanguard/legion/issues", "https://GoVanguard.com/legion"],
     "emails": [],
-    "update": '06/26/2020',
+    "update": '07/31/2020',
     "license": "GPL v3",
     "desc": "Legion is a fork of SECFORCE's Sparta, Legion is an open source, easy-to-use, \n" +
             "super-extensible and semi-automated network penetration testing tool that aids in " +

app/importers/NmapImporter.py

@@ -119,7 +119,6 @@ def run(self):
             session.commit()
 
             for h in allHosts:  # create all OS, service and port objects that need to be created
-                                ## does not take into account different service versions?
                 self.tsLog("Processing h {ip}".format(ip=h.ip))
 
                 db_host = self.hostRepository.getHostInformation(h.ip)
@@ -138,7 +137,7 @@ def run(self):
 
                     if not db_os:
                         t_osObj = osObj(os.name, os.family, os.generation, os.osType, os.vendor, os.accuracy,
-                                        db_host.id)
+                            db_host.id)
                         session.add(t_osObj)
 
                     createOsNodesProgress = createOsNodesProgress + ((100.0 / hostCount) / 5)
@@ -154,26 +153,22 @@ def run(self):
                     s = p.getService()
 
                     if not (s is None):  # check if service already exists to avoid adding duplicates
-                        # print("            Found service {service} for port {port}".format(service=str(s.name),
-                        # port=str(p.portId)))
-                        # db_service = session.query(serviceObj).filter_by(name=s.name).filter_by(product=s.product).
-                        # filter_by(version=s.version).filter_by(extrainfo=s.extrainfo).
-                        # filter_by(fingerprint=s.fingerprint).first()
-                        db_service = session.query(serviceObj).filter_by(name=s.name).filter_by(product=s.product) \
-                            .first()
+                        print("Processing service result *********** name={0} prod={1} ver={2} extra={3} fing={4}"
+                            .format(s.name, s.product, s.version, s.extrainfo, s.fingerprint))
+                        db_service = session.query(serviceObj).filter_by(hostId=db_host.id) \
+                            .filter_by(name=s.name).filter_by(product=s.product).filter_by(version=s.version) \
+                            .filter_by(extrainfo=s.extrainfo).filter_by(fingerprint=s.fingerprint).first()
                         if not db_service:
-                            # print("Did not find service *********** name={0} prod={1} ver={2} extra={3} fing={4}".
-                            # format(s.name, s.product, s.version, s.extrainfo, s.fingerprint))
-                            db_service = serviceObj(s.name, s.product, s.version, s.extrainfo, s.fingerprint)
+                            print("Did not find service *********** name={0} prod={1} ver={2} extra={3} fing={4}"
+                                .format(s.name, s.product, s.version, s.extrainfo, s.fingerprint))
+                            db_service = serviceObj(s.name, db_host.id, s.product, s.version, s.extrainfo,
+                                s.fingerprint)
                             session.add(db_service)
-                    # else:
-                    # print("FOUND service *************** name={0}".format(db_service.name))
-
                     else:  # else, there is no service info to parse
                         db_service = None
                         # fetch the port
-                    db_port = session.query(portObj).filter_by(hostId=db_host.id).filter_by(portId=p.portId).filter_by(
-                        protocol=p.protocol).first()
+                    db_port = session.query(portObj).filter_by(hostId=db_host.id).filter_by(portId=p.portId) \
+                        .filter_by(protocol=p.protocol).first()
 
                     if not db_port:
                         # print("Did not find port *********** portid={0} proto={1}".format(p.portId, p.protocol))
@@ -200,19 +195,21 @@ def run(self):
                     for scr in p.getScripts():
                         self.tsLog("        Processing script obj {scr}".format(scr=str(scr)))
                         print("        Processing script obj {scr}".format(scr=str(scr)))
-                        db_port = session.query(portObj).filter_by(hostId=db_host.id).filter_by(
-                            portId=p.portId).filter_by(protocol=p.protocol).first()
-                        db_script = session.query(l1ScriptObj).filter_by(scriptId=scr.scriptId).filter_by(
-                            portId=db_port.id).first()
+                        db_port = session.query(portObj).filter_by(hostId=db_host.id) \
+                            .filter_by(portId=p.portId).filter_by(protocol=p.protocol).first()
+                        #db_script = session.query(l1ScriptObj).filter_by(scriptId=scr.scriptId) \
+                        #   .filter_by(portId=db_port.id).first()
+                        db_script = session.query(l1ScriptObj).filter_by(hostId=db_host.id) \
+                           .filter_by(portId=db_port.id).first()
 
                         if not db_script:  # if this script object doesn't exist, create it
                             t_l1ScriptObj = l1ScriptObj(scr.scriptId, scr.output, db_port.id, db_host.id)
                             self.tsLog("        Adding l1ScriptObj obj {script}".format(script=scr.scriptId))
                             session.add(t_l1ScriptObj)
 
                 for hs in h.getHostScripts():
-                    db_script = session.query(l1ScriptObj).filter_by(scriptId=hs.scriptId).filter_by(
-                        hostId=db_host.id).first()
+                    db_script = session.query(l1ScriptObj).filter_by(scriptId=hs.scriptId) \
+                        .filter_by(hostId=db_host.id).first()
                     if not db_script:
                         t_l1ScriptObj = l1ScriptObj(hs.scriptId, hs.output, None, db_host.id)
                         session.add(t_l1ScriptObj)
@@ -253,9 +250,9 @@ def run(self):
 
                 os_nodes = h.getOs()
                 for os in os_nodes:
-                    db_os = session.query(osObj).filter_by(hostId=db_host.id).filter_by(name=os.name).filter_by(
-                        family=os.family).filter_by(generation=os.generation).filter_by(osType=os.osType).filter_by(
-                        vendor=os.vendor).first()
+                    db_os = session.query(osObj).filter_by(hostId=db_host.id).filter_by(name=os.name) \
+                        .filter_by(family=os.family).filter_by(generation=os.generation) \
+                        .filter_by(osType=os.osType).filter_by(vendor=os.vendor).first()
 
                     db_os.osAccuracy = os.accuracy  # update the accuracy
 
@@ -290,15 +287,17 @@ def run(self):
                 for p in h.all_ports():
                     s = p.getService()
                     if not (s is None):
-                        # db_service = session.query(serviceObj).filter_by(name=s.name).filter_by(product=s.product).
-                        # filter_by(version=s.version).filter_by(extrainfo=s.extrainfo).
-                        # filter_by(fingerprint=s.fingerprint).first()
-                        db_service = session.query(serviceObj).filter_by(name=s.name).first()
+                        db_service = session.query(serviceObj).filter_by(hostId=db_host.id) \
+                            .filter_by(name=s.name).filter_by(product=s.product) \
+                            .filter_by(version=s.version).filter_by(extrainfo=s.extrainfo) \
+                            .filter_by(fingerprint=s.fingerprint).first()
+                        #db_service = session.query(serviceObj).filter_by(hostId=db_host.id) \
+                        #    .filter_by(name=s.name).first()
                     else:
                         db_service = None
                         # fetch the port
-                    db_port = session.query(portObj).filter_by(hostId=db_host.id).filter_by(portId=p.portId).filter_by(
-                        protocol=p.protocol).first()
+                    db_port = session.query(portObj).filter_by(hostId=db_host.id).filter_by(portId=p.portId) \
+                        .filter_by(protocol=p.protocol).first()
                     if db_port:
                         # print("************************ Found {0}".format(db_port))
 
@@ -313,8 +312,8 @@ def run(self):
 
                     # store the script results (note that existing script outputs are also kept)
                     for scr in p.getScripts():
-                        db_script = session.query(l1ScriptObj).filter_by(scriptId=scr.scriptId).filter_by(
-                            portId=db_port.id).first()
+                        db_script = session.query(l1ScriptObj).filter_by(scriptId=scr.scriptId) \
+                            .filter_by(portId=db_port.id).first()
 
                         if not scr.output == '' and scr.output is not None:
                             db_script.output = scr.output

db/entities/service.py

@@ -15,7 +15,7 @@
 
 Author(s): Dmitriy Dubson (d.dubson@gmail.com)
 """
-from sqlalchemy import String, Column, Integer
+from sqlalchemy import String, Column, Integer, ForeignKey
 from sqlalchemy.orm import relationship
 
 from db.database import Base
@@ -32,13 +32,15 @@ class serviceObj(Base):
     version = Column(String)
     extrainfo = Column(String)
     fingerprint = Column(String)
+    hostId = Column(String, ForeignKey('hostObj.id'))
     port = relationship(portObj)
     cves = relationship(cve)
     application = relationship(appObj)
 
-    def __init__(self, name='', product='', version='', extrainfo='', fingerprint=''):
+    def __init__(self, name, host, product='', version='', extrainfo='', fingerprint=''):
         self.name = name
         self.product = product
         self.version = version
         self.extrainfo = extrainfo
         self.fingerprint = fingerprint
+        self.hostId = host

legion.conf

@@ -9,7 +9,7 @@ store-cleartext-passwords-on-exit=True
 username-wordlist-path=/usr/share/wordlists/
 
 [GUISettings]
-process-tab-column-widths="125,0,26,26,26,0,100,100,0,0,0,0,0,0,0,755,100"
+process-tab-column-widths="125,0,100,150,100,0,100,100,0,0,0,0,0,0,0,481,100"
 process-tab-detail=false
 
 [GeneralSettings]
@@ -281,8 +281,8 @@ telnet-default-router=Check for default telnet router credentials, hydra -s [POR
 tftp-enum.nse=tftp-enum.nse, "nmap -Pn [IP] -p [PORT] --script=tftp-enum.nse --script-args=unsafe=1", tftp
 theharvester=Run theharvester, theharvester -d [IP]:[PORT] -b all -n -c -t -h, dns
 vnc-brute.nse=vnc-brute.nse, "nmap -Pn [IP] -p [PORT] --script=vnc-brute.nse --script-args=unsafe=1", vnc
-vnc-info.nse=vnc-info.nse, "nmap -Pn [IP] -p [PORT] --script=vnc-info.nse --script-args=unsafe=1", vnc
 vnc-default=Check for default VNC credentials, hydra -s [PORT] -C ./wordlists/vnc-betterdefaultpasslist.txt -u -o \"[OUTPUT].txt\" -f [IP] vnc, vnc
+vnc-info.nse=vnc-info.nse, "nmap -Pn [IP] -p [PORT] --script=vnc-info.nse --script-args=unsafe=1", vnc
 wafw00f=Run wafw00f, wafw00f [IP]:[PORT], "https,ssl,https-alt"
 whatweb=Run whatweb, "whatweb [IP]:[PORT] --color=never --log-brief=[OUTPUT].txt", "http,https,ssl,https-alt"
 wpscan=Run wpscan, wpscan --url [IP], "http,https,ssl,https-alt"
@@ -302,7 +302,7 @@ ssh=Open with ssh client (as root), [term] ssh root@[IP] -p [PORT], ssh
 telnet=Open with telnet, [term] telnet [IP] [PORT], 
 vncviewer=Open with vncviewer, vncviewer [IP]:[PORT], vnc
 xephyr=Open with Xephyr, [term] Xephyr -query [IP] :1, xdmcp
-xterm=Open terminal, [term] bash,
+xterm=Open terminal, [term] bash, 
 
 [SchedulerSettings]
 ftp-default=ftp, tcp
@@ -318,7 +318,7 @@ snmpcheck=snmp, udp
 x11screen=X11, tcp
 
 [StagedNmapSettings]
-stage1-ports="T:80,443"
+stage1-ports="T:80,81,443,4443,8080,8081,8082"
 stage2-ports="T:25,135,137,139,445,1433,3306,5432,U:137,161,162,1434"
 stage3-ports="Vulners,CVE"
 stage4-ports="T:23,21,22,110,111,2049,3389,8080,U:500,5060"
@@ -329,4 +329,5 @@ stage6-ports=T:30000-65535
 cutycapt-path=/usr/bin/cutycapt
 hydra-path=/usr/bin/hydra
 nmap-path=/sbin/nmap
+pyshodan-api-key=SNYEkE0gdwNu9BRURVDjWPXePCquXqht
 texteditor-path=/usr/bin/leafpad