From c19e15b1b09cb9529c88c8e16aad5646e5874302 Mon Sep 17 00:00:00 2001 From: David Blackman Date: Fri, 21 Aug 2020 13:12:48 -0400 Subject: [PATCH 1/2] feat(auth): add Header mode for credentials We have a custom server in front of our pelias with some application level hacks. It uses http header auth. I would like to be able to use pelias fuzzy tester to see the differences, especially in autocomplete testing, between our raw backend pelias, and this hacked up proxied pelias. This change allows for that. --- README.md | 31 ++++++++++++++++++++++++++++- lib/apiKey.js | 19 ++++++++++++++++-- lib/gather_test_urls.js | 5 ----- lib/request_urls.js | 12 +++++++++++ test/apiKey.js | 44 +++++++++++++++++++++++++++++++++++++---- 5 files changed, 99 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 9753380..34c3f72 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,36 @@ override the default aliases and define your own in `pelias-config`: { "acceptance-tests": { "endpoints": { - "alias": "http://my.pelias.instance" + "prod": "http://pelias-prod.myhost.com/protectedpath/", + "staging": "http://pelias-staging.myhost.com", + "localhost": "http://localhost:3100" + } + } +} +``` + + +### Credentials +You can specify api keys for different hosts in the pelias-config. There is an optional dict of credentials in +`acceptance-tests.credentials`. The keys are bare hostnames (with no path or protocol). The values can either be a +string, which will be added to the urls as `&api_key=${value}` which is the default pelias api key schema, but it can +also be a dict that specifies `method=Header` (which will send credentials in the http authorization header) or +`method=GET` (which supports an optional key "keyName", which if specified means it will be added to urls like `&${keyName}=${value}`) + +```javascript +{ + "acceptance-tests": { + "endpoints": { + "prod": "http://pelias-prod.myhost.com/protectedpath/", + "staging": "http://pelias-staging.myhost.com", + "localhost": "http://localhost:3100" + }, + "credentials": { + "pelias-staging.myhost.com": 'secret_key_12342354', + "pelias-prod.myhost.com": { + "method": "Header", + "value": "prj_sk_XXXXXXXXX" + } } } } diff --git a/lib/apiKey.js b/lib/apiKey.js index ea3a559..e891db7 100644 --- a/lib/apiKey.js +++ b/lib/apiKey.js @@ -1,9 +1,24 @@ const url = require('url'); +const _ = require('lodash'); const logger = require('pelias-logger').get('fuzzy-tester'); const deprecation_message = 'Loading credentials from deprecated `mapzen.api_key` pelias.json property.' + 'Credentials should now live in `acceptance-tests.credentials`'; +function processCredentials(credentials) { + if (_.isObject(credentials)) { + return credentials; + } else if (_.isString(credentials)) { + return { + method: 'GET', + keyName: 'api_key', + value: credentials + }; + } else { + throw new Error('credentials entry must be a string or an object', credentials); + } +} + module.exports = function( uri ){ const config = require('pelias-config').generate(); const host = url.parse( uri ).host; @@ -13,12 +28,12 @@ module.exports = function( uri ){ const old_credentials = config.get('mapzen.api_key'); if (preferred_credentials && preferred_credentials[host] !== undefined) { - return preferred_credentials[host] || null; + return processCredentials(preferred_credentials[host]); } if (old_credentials && old_credentials[host]) { logger.warn(deprecation_message); - return old_credentials[ host ]; + return processCredentials(old_credentials[host]); } return null; diff --git a/lib/gather_test_urls.js b/lib/gather_test_urls.js index b34a7bc..9a3c218 100644 --- a/lib/gather_test_urls.js +++ b/lib/gather_test_urls.js @@ -1,4 +1,3 @@ -var apiKey = require( '../lib/apiKey' ); var url = require ('url'); var _ = require('lodash'); @@ -48,13 +47,9 @@ function gatherAutocompleteURLs(testCase, baseUrlObj) { function gatherTestSuiteURLs( config, testSuite ){ var apiUrl = config.endpoint.url; - var key = apiKey( apiUrl ); var baseUrlObj = url.parse(apiUrl); return _.flatten(testSuite.tests.map(function(testCase) { - if( key ){ - testCase.in.api_key = key; - } testCase.autocompleteURLs = []; var testCaseURLs = [gatherTestCaseURL(testCase, testSuite, baseUrlObj)]; diff --git a/lib/request_urls.js b/lib/request_urls.js index 717c107..c22e102 100644 --- a/lib/request_urls.js +++ b/lib/request_urls.js @@ -1,6 +1,7 @@ var http = require('http'); var https = require('https'); var request = require('request'); +var apiKey = require( '../lib/apiKey' ); var ExponentialBackoff = require( '../lib/ExponentialBackoff'); @@ -37,6 +38,9 @@ function request_urls(config, urls, callback) { var test_interval = new ExponentialBackoff(interval, 5, interval, 20000); var delay = test_interval.getBackoff(); + var apiUrl = config.endpoint.url; + var key = apiKey( apiUrl ); + var getOneUrl = function (){ // check if all responses have been recieved and call the next step in // processing via the `callback` function @@ -64,6 +68,14 @@ function request_urls(config, urls, callback) { gzip: true }; + if (key && key.method === 'Header') { + requestOpts.headers.authorization = `${key.value}`; + } + + if (key && key.method === 'GET') { + requestOpts.url = url + `&${key.keyName}=${key.value}`; + } + request( requestOpts, function ( err, res ){ if( err ){ console.error( err ); diff --git a/test/apiKey.js b/test/apiKey.js index b834e04..fc55ce9 100644 --- a/test/apiKey.js +++ b/test/apiKey.js @@ -24,7 +24,7 @@ tape( 'api_key not found in config', function ( test ){ test.end(); }); -tape( 'stage api_key imported from pelias config, old location', function ( test ){ +tape( 'stage string api_key imported from pelias config, old location', function ( test ){ var config = '{ "mapzen": { "api_key": { "pelias.stage.mapzen.com": "my_api_key" } } }'; @@ -35,7 +35,9 @@ tape( 'stage api_key imported from pelias config, old location', function ( test process.env.PELIAS_CONFIG = '/tmp/pelias_temp2.json'; // staging - test.equal( apiKey( 'http://pelias.stage.mapzen.com/foo' ), 'my_api_key', 'api key loaded' ); + test.deepEqual( apiKey( 'http://pelias.stage.mapzen.com/foo' ), + { method: 'GET', keyName: 'api_key', value: 'my_api_key' }, + 'api key loaded' ); // unset the PELIAS_CONFIG env var delete process.env.PELIAS_CONFIG; @@ -46,7 +48,7 @@ tape( 'stage api_key imported from pelias config, old location', function ( test test.end(); }); -tape( 'stage api_key imported from preferred config location first', function ( test ){ +tape( 'stage string api_key imported from preferred config location first', function ( test ){ const custom_config = { 'acceptance-tests': { credentials: { @@ -67,7 +69,9 @@ tape( 'stage api_key imported from preferred config location first', function ( process.env.PELIAS_CONFIG = '/tmp/pelias_temp3.json'; // staging - test.equal( apiKey( 'http://pelias.stage.mapzen.com/foo' ), 'my_new_api_key', 'api key loaded' ); + test.deepEqual( apiKey( 'http://pelias.stage.mapzen.com/foo' ), + { method: 'GET', keyName: 'api_key', value: 'my_new_api_key' }, + 'api key loaded' ); // unset the PELIAS_CONFIG env var delete process.env.PELIAS_CONFIG; @@ -99,3 +103,35 @@ tape( 'avoid matching partial urls', function ( test ){ test.end(); }); + + + +tape( 'stage object api_key imported correctly', function ( test ){ + const custom_config = { + 'acceptance-tests': { + credentials: { + 'pelias.stage.mapzen.com': { + method: 'Header', + value: '12345' + } + } + } + }; + + // write a temporary pelias config + fs.writeFileSync( '/tmp/pelias_temp4.json', JSON.stringify(custom_config), 'utf8' ); + + // set the PELIAS_CONFIG env var + process.env.PELIAS_CONFIG = '/tmp/pelias_temp4.json'; + + // staging + test.deepEqual( apiKey( 'http://pelias.stage.mapzen.com/foo' ), { method: 'Header',value: '12345' }, 'api key loaded' ); + + // unset the PELIAS_CONFIG env var + delete process.env.PELIAS_CONFIG; + + // delete temp file + fs.unlinkSync( '/tmp/pelias_temp4.json' ); + + test.end(); +}); \ No newline at end of file From e9ecaeecd141d420b12766ba1507eb0ae1c71306 Mon Sep 17 00:00:00 2001 From: Julian Simioni Date: Mon, 24 Aug 2020 07:04:30 -0400 Subject: [PATCH 2/2] Improve and extend credentials docs --- README.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 34c3f72..116d3fc 100644 --- a/README.md +++ b/README.md @@ -176,11 +176,21 @@ override the default aliases and define your own in `pelias-config`: ### Credentials -You can specify api keys for different hosts in the pelias-config. There is an optional dict of credentials in -`acceptance-tests.credentials`. The keys are bare hostnames (with no path or protocol). The values can either be a -string, which will be added to the urls as `&api_key=${value}` which is the default pelias api key schema, but it can -also be a dict that specifies `method=Header` (which will send credentials in the http authorization header) or -`method=GET` (which supports an optional key "keyName", which if specified means it will be added to urls like `&${keyName}=${value}`) + +You can specify api keys for different hosts via `pelias.json` in the `acceptance-tests.credentials` +section. + +The keys are bare hostnames (with no path or protocol). The values can either be a string, or an +object. + +If using a string, the string will be used as an API key and appended to the query URL with +`&api_key=${value}`. or an object. + +If using an object, the `method` property can be specified as either `GET` (the default`) or +`Header`. Selecting `Header` will send the API key in the `authorization:` HTTP header. + +The optional `keyName` property can be specified with `GET` if an authorization URL other than +`api_key` is required. ```javascript { @@ -195,6 +205,11 @@ also be a dict that specifies `method=Header` (which will send credentials in th "pelias-prod.myhost.com": { "method": "Header", "value": "prj_sk_XXXXXXXXX" + }, + "pelias-testing.myhost.com": { + "method": "GET", + "keyName": "my_auth_parameter", + "value": "prj_sk_XXXXXXXXX" } } }