Skip to content

Commit

Permalink
Finish 3.3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@sarus
sarus committed Feb 16, 2021
2 parents ab68af0 + b513c9c commit 56bb630
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 35 deletions.
4 changes: 4 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ This option allows you to specify a regex to used to ignore domains. Any domain

This option allows you to specify a regex used to ignore IPv4 Addresses. Any IPv4 matching the regex will not be searched.

### Proxy Screenshots through Polarity Server

If checked, the Overlay Window will receive the URL\'s screenshot from the Polarity server rather than directly from the urlscan website. If your screenshot images are not properly displaying try enabling this option as it can resolve issues in some environments with unsupported web proxy configurations on the client.

## Installation Instructions

Installation instructions for integrations are provided on the [PolarityIO GitHub Page](https://polarityio.github.io/).
Expand Down
16 changes: 13 additions & 3 deletions config/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ module.exports = {
* @type String
* @optional
*/
description: 'Searches the urlscan.io API and returns results from the most recent, relevant scan',
description:
'Searches the urlscan.io API and returns results from the most recent, relevant scan',
entityTypes: ['IPv4', 'IPv6', 'IPv4CIDR', 'domain', 'url', 'sha256'],
onDemandOnly: true,
/**
Expand Down Expand Up @@ -128,8 +129,7 @@ module.exports = {
{
key: 'domainBlocklistRegex',
name: 'Ignored Domain Regex',
description:
'Domains that match the given regex will not be looked up.',
description: 'Domains that match the given regex will not be looked up.',
default: '',
type: 'text',
userCanEdit: false,
Expand All @@ -143,6 +143,16 @@ module.exports = {
type: 'text',
userCanEdit: false,
adminOnly: false
},
{
key: 'downloadScreenshot',
name: 'Proxy Screenshots through Polarity Server',
description:
'If checked, the Overlay Window will receive the URL\'s screenshot from the Polarity server rather than directly from the urlscan website. If your screenshot images are not properly displaying try enabling this option as it can resolve issues in some environments with unsupported web proxy configurations on the client.',
default: false,
type: 'boolean',
userCanEdit: false,
adminOnly: true
}
]
};
155 changes: 125 additions & 30 deletions integration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ function startup(logger) {
defaults.key = fs.readFileSync(config.request.key);
}

if (typeof config.request.passphrase === 'string' && config.request.passphrase.length > 0) {
if (
typeof config.request.passphrase === 'string' &&
config.request.passphrase.length > 0
) {
defaults.passphrase = config.request.passphrase;
}

Expand All @@ -54,26 +57,38 @@ function startup(logger) {
}

function _setupRegexBlocklists(options) {
if (options.domainBlocklistRegex !== previousDomainRegexAsString && options.domainBlocklistRegex.length === 0) {
if (
options.domainBlocklistRegex !== previousDomainRegexAsString &&
options.domainBlocklistRegex.length === 0
) {
Logger.debug('Removing Domain Blocklist Regex Filtering');
previousDomainRegexAsString = '';
domainBlocklistRegex = null;
} else {
if (options.domainBlocklistRegex !== previousDomainRegexAsString) {
previousDomainRegexAsString = options.domainBlocklistRegex;
Logger.debug({ domainBlocklistRegex: previousDomainRegexAsString }, 'Modifying Domain Blocklist Regex');
Logger.debug(
{ domainBlocklistRegex: previousDomainRegexAsString },
'Modifying Domain Blocklist Regex'
);
domainBlocklistRegex = new RegExp(options.domainBlocklistRegex, 'i');
}
}

if (options.ipBlocklistRegex !== previousIpRegexAsString && options.ipBlocklistRegex.length === 0) {
if (
options.ipBlocklistRegex !== previousIpRegexAsString &&
options.ipBlocklistRegex.length === 0
) {
Logger.debug('Removing IP Blocklist Regex Filtering');
previousIpRegexAsString = '';
ipBlocklistRegex = null;
} else {
if (options.ipBlocklistRegex !== previousIpRegexAsString) {
previousIpRegexAsString = options.ipBlocklistRegex;
Logger.debug({ ipBlocklistRegex: previousIpRegexAsString }, 'Modifying IP Blocklist Regex');
Logger.debug(
{ ipBlocklistRegex: previousIpRegexAsString },
'Modifying IP Blocklist Regex'
);
ipBlocklistRegex = new RegExp(options.ipBlocklistRegex, 'i');
}
}
Expand All @@ -91,6 +106,40 @@ function getQuery(entity) {
}
}

async function getScreenshotAsBase64(imageUrl) {
const requestOptions = {
uri: imageUrl,
encoding: null,
method: 'get'
};

return new Promise((resolve, reject) => {
requestWithDefaults(requestOptions, (error, response, body) => {
if (error) {
return reject(error);
}

if (
![200, 404].includes(response.statusCode) &&
!(body && Buffer.from(body).toString('base64').length)
) {
return reject({
detail:
'Unexpected status code or Image Not Found when downloading screenshot from urlscan',
response
});
}
const data =
'data:' +
response.headers['content-type'] +
';base64,' +
Buffer.from(body).toString('base64');

resolve(data);
});
});
}

function doLookup(entities, options, cb) {
let lookupResults = [];
let tasks = [];
Expand All @@ -101,15 +150,18 @@ function doLookup(entities, options, cb) {

entities.forEach((entity) => {
if (!_isInvalidEntity(entity) && !_isEntityBlocklisted(entity, options)) {

tasks.push(function (done) {
async.waterfall(
[
function (next) {
searchIndicator(entity, options, next);
},
function (result, next) {
if (!_isMiss(result.body) && result.body.results[0] && result.body.results[0].result) {
if (
!_isMiss(result.body) &&
result.body.results[0] &&
result.body.results[0].result
) {
let uri = result.body.results[0].result;
getVerdicts(uri, entity, options, (err, { refererLinks, verdicts }) => {
if (err) return next(err);
Expand All @@ -122,9 +174,24 @@ function doLookup(entities, options, cb) {
} else {
next(null, result);
}
},
async function (result) {
if (
options.downloadScreenshot &&
fp.get('body.results.0.screenshot', result)
) {
const screenshot = await getScreenshotAsBase64(
result.body.results[0].screenshot
);
result.body.results[0].screenshotBase64 = screenshot;
}
return result;
}
],
(err, result) => {
if (err) {
Logger.error(err, 'doLookup Error');
}
done(err, result);
}
);
Expand Down Expand Up @@ -188,7 +255,7 @@ function doLookup(entities, options, cb) {
details: {
...result.body,
searchLimitTag:
dailySearchLimit.percentage > 50 &&
dailySearchLimit && dailySearchLimit.percent > 75 &&
`${dailySearchLimit.limit - dailySearchLimit.used}/${dailySearchLimit.limit}`
}
}
Expand All @@ -203,7 +270,6 @@ function doLookup(entities, options, cb) {
});
}


function getIsMalicious(result) {
if (
result.body &&
Expand Down Expand Up @@ -252,21 +318,21 @@ function getVerdicts(uri, entity, options, cb) {

requestWithDefaults(requestOptions, (error, response, body) => {
let parsedResult = _handleErrors(entity, error, response, body);

if (parsedResult.error) {
cb(parsedResult.error, {});
} else {
cb(null, {
refererLinks: _getRefererLinks(body),
verdicts: body.verdicts,
verdicts: body.verdicts
});
}
});
}

const _getRefererLinks = fp.flow(
fp.getOr([], "data.requests"),
fp.map(fp.getOr(false, "request.request.headers.Referer")),
fp.getOr([], 'data.requests'),
fp.map(fp.getOr(false, 'request.request.headers.Referer')),
fp.compact,
fp.uniq
);
Expand All @@ -282,7 +348,7 @@ function _handleErrors(entity, err, response, body) {
}

let result;
if(response) {
if (response) {
if (response.statusCode === 200) {
// we got data!
result = {
Expand Down Expand Up @@ -350,7 +416,7 @@ function _isInvalidEntity(entity) {
}
}

if(entity.isURL && entity.requestContext.requestType !== 'OnDemand') {
if (entity.isURL && entity.requestContext.requestType !== 'OnDemand') {
return true;
}

Expand Down Expand Up @@ -397,7 +463,6 @@ function _isEntityBlocklisted(entity, options) {

const _isMiss = (body) => !body || !body.results;


const submitUrl = ({ data: { entity, tags, submitAsPublic } }, options, cb) => {
const requestOptions = {
uri: `${API_URL}/v1/scan/`,
Expand All @@ -424,33 +489,63 @@ const submitUrl = ({ data: { entity, tags, submitAsPublic } }, options, cb) => {

requestWithDefaults(requestOptions, (error, response, body) => {
let parsedResult = _handleErrors(entity, error, response, body);

if (parsedResult.error) {
cb({ errors: [parsedResult.error] });
} else {
const { data: { body } } = parsedResult;
const {
data: { body }
} = parsedResult;
cb(null, {
...body,
results: [{
justSubmitted: true,
_id: body.uuid,
task: {
visibility: body.visibility
},
page: {
domain: entity.value,
url: body.url
results: [
{
justSubmitted: true,
_id: body.uuid,
task: {
visibility: body.visibility
},
page: {
domain: entity.value,
url: body.url
}
}
}]
]
});

}
});
};

function validateOptions(userOptions, cb){
let errors = [];
if (typeof userOptions.domainBlocklistRegex.value === 'string' && userOptions.domainBlocklistRegex.value.length > 0){
try{
new RegExp(userOptions.domainBlocklistRegex.value, 'i');
}catch(e){
errors.push({
key: 'domainBlocklistRegex',
message: 'You must provide a valid regular expression (do not surround your regex in forward slashes)'
});
}
}

if (typeof userOptions.ipBlocklistRegex.value === 'string' && userOptions.ipBlocklistRegex.value.length > 0){
try{
new RegExp(userOptions.ipBlocklistRegex.value, 'i');
}catch(e){
errors.push({
key: 'ipBlocklistRegex',
message: 'You must provide a valid regular expression (do not surround your regex in forward slashes)'
});
}
}

cb(null, errors);
}

module.exports = {
doLookup,
startup,
onMessage: submitUrl
onMessage: submitUrl,
validateOptions
};
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"main": "./integration.js",
"name": "urlscan",
"version": "3.2.6-beta",
"version": "3.3.0",
"private": true,
"dependencies": {
"request": "^2.88.2",
Expand Down
6 changes: 5 additions & 1 deletion templates/us-block.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,11 @@
<span class="p-title">{{fa-icon "browser" fixedWidth=true}} Screenshot</span>
{{#if result._id}}
<div class="image-container">
<img src="https://urlscan.io/screenshots/{{result._id}}.png">
{{#if result.getScreenshotAsBase64}}
<img src="{{result.getScreenshotAsBase64}}"/>
{{else}}
<img src="{{result.screenshot}}">
{{/if}}
</div>
<div>
<span class="p-key">Screenshot URL: </span>
Expand Down

0 comments on commit 56bb630

Please sign in to comment.