Merge pull request #24 from polarityio/develop

INT-1352: Ensure integration does not hang when only private IPs are searched

.github/workflows/run-int-dev-checklist.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   run-integration-development-checklist:
     runs-on: ubuntu-latest
-    container: 'centos:7'
+    container: 'rockylinux:8'
 
     steps:
     - uses: actions/checkout@v2

README.md

@@ -4,6 +4,8 @@
 
 Polarity's Shodan integration gives users access to automated IPv4 and IPv6 lookups against the Shodan Host REST API.  The Shodan REST API restricts searches to 1 per second.  The integration will automatically throttle lookups to stay below this limit and will queue up to 15 search requests per API key.  If the queue is full, you will receive back a response indicating that the queue is full and will have the option to rerun the search from the Overlay Window.  
 
+You must be using a paid Shodan Membership API key. 
+
 Please see [https://www.shodan.io/](https://www.shodan.io/) for more information.
 
 | ![image](assets/overlay.png)  |
@@ -14,7 +16,7 @@ Please see [https://www.shodan.io/](https://www.shodan.io/) for more information
 
 ### Shodan API Key
 
-Your Shodan API Key.
+Your Shodan API Key.  This must be a paid membership API key.
 
 ## Installation Instructions
 

config/config.js

@@ -84,8 +84,8 @@ module.exports = {
       description: 'Your Shodan API Key.',
       default: '',
       type: 'password',
-      userCanEdit: true,
-      adminOnly: false
+      userCanEdit: false,
+      adminOnly: true
     }
   ]
 };

config/config.json

@@ -32,8 +32,8 @@
       "description": "Your Shodan API Key.",
       "default": "",
       "type": "password",
-      "userCanEdit": true,
-      "adminOnly": false
+      "userCanEdit": false,
+      "adminOnly": true
     }
   ]
 }

integration.js

@@ -31,6 +31,7 @@ const IGNORED_IPS = new Set(['127.0.0.1', '255.255.255.255', '0.0.0.0']);
 const MAX_FACET_RESULTS = 1000;
 
 function doLookup(entities, options, cb) {
+  const ignoredEntityResults = [];
   let limiter = bottlneckApiKeyCache.get(options.apiKey);
 
   if (!limiter) {
@@ -47,9 +48,16 @@ function doLookup(entities, options, cb) {
   let requestResults = [];
   Logger.trace({ entities }, 'doLookup');
 
-  const validEntities = entities.filter(
-    (entity) => !entity.isPrivateIP && !IGNORED_IPS.has(entity.value)
-  );
+  const validEntities = entities.filter((entity) => {
+    if (entity.isPrivateIP || IGNORED_IPS.has(entity.value)) {
+      ignoredEntityResults.push({
+        entity,
+        data: null
+      });
+      return false;
+    } 
+    return true;
+  });
 
   let requestOptions;
   validEntities.forEach((entity) => {
@@ -129,10 +137,14 @@ function doLookup(entities, options, cb) {
         });
 
         Logger.trace({ lookupResults }, 'Lookup Results');
-        cb(null, lookupResults);
+        cb(null, lookupResults.concat(ignoredEntityResults));
       }
     });
   });
+
+  if (validEntities.length === 0) {
+    cb(null, ignoredEntityResults);
+  }
 }
 
 const parseErrorToReadableJSON = (error) =>
@@ -180,7 +192,7 @@ const requestEntity = (entity, requestOptions, callback) =>
       });
     } else {
       return callback({
-        detail: 'Unexpected HTTP Status Received',
+        detail: body && body.error ? body.error : 'Unexpected HTTP Status Received',
         httpStatus: res.statusCode,
         body
       });

package.json

@@ -1,12 +1,12 @@
 {
   "name": "shodan",
-  "version": "3.4.4",
+  "version": "3.4.6",
   "main": "./integration.js",
   "private": true,
   "dependencies": {
     "bottleneck": "^2.19.5",
     "lodash": "^4.17.21",
     "memory-cache": "^0.2.0",
-    "postman-request": "^2.88.1-postman.33"
+    "postman-request": "^2.88.1-postman.40"
   }
 }