From 5c151a477af6bc7d7b6259c21c46067789ed8347 Mon Sep 17 00:00:00 2001 From: Rhys Evans Date: Tue, 30 Aug 2016 14:48:20 +0100 Subject: [PATCH] Rhys/gte lte (#128) * added support for >=, <= * update build * update build * docs --- .gitignore | 3 +- Makefile | 8 +- README.md | 2 + lib/cli.js | 1 - lib/data/immutable-matrix.js | 2 - lib/keen-query/filters.js | 2 +- lib/keen-query/shorthands.js | 8 ++ lib/post-processing/reduce.js | 4 +- lib/printers/ascii.js | 6 -- n.Makefile | 149 ++++++++++++++++++++++++++++++++++ package.json | 4 +- test/url-generation.test.js | 18 ++-- 12 files changed, 182 insertions(+), 25 deletions(-) create mode 100644 n.Makefile diff --git a/.gitignore b/.gitignore index eca0c52..408e021 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /node_modules/ /bower_components/ /.idea/ -.env /test.js +.eslintrc.js +.editorconfig diff --git a/Makefile b/Makefile index b25f2e6..d4ab4e7 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,6 @@ -.PHONY: test - -install: - npm install +include n.Makefile test-unit: mocha test/helpers.js test -test: test-unit - nbt verify --skip-layout-checks +test: test-unit verify diff --git a/README.md b/README.md index 230b4ea..a8bda55 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,8 @@ Filter can be called as many times as you like | prop is not equal to value | `->filter(prop!=val)` | `kq.filter('prop!=val')` | | prop is greater than value | `->filter(prop>val)` | `kq.filter('prop>val')` | | prop is less than value | `->filter(propfilter(prop>=val)` | `kq.filter('prop>=val')` | +| prop is less than or equal to value | `->filter(prop<=val)` | `kq.filter('prop<=val')` | | prop contains value | `->filter(prop~val)` | `kq.filter('prop~val')` | | prop does not contain value | `->filter(prop!~val)` | `kq.filter('prop!~val')` | | prop is equal to val1, val2 ... | `->filter(prop?val1,val2,val3)` | `kq.filter('prop?val1,val2,val3')` | diff --git a/lib/cli.js b/lib/cli.js index 5339783..defe235 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -129,4 +129,3 @@ module.exports = { } } } - diff --git a/lib/data/immutable-matrix.js b/lib/data/immutable-matrix.js index ccb0139..48b48fd 100644 --- a/lib/data/immutable-matrix.js +++ b/lib/data/immutable-matrix.js @@ -359,5 +359,3 @@ function detectTableGaps (table1, table2) { module.exports.isGappy = function (table2, table1) { return detectTableGaps(table1, table2) || detectTableGaps(table2, table1); } - - diff --git a/lib/keen-query/filters.js b/lib/keen-query/filters.js index 53ba591..7ceba61 100644 --- a/lib/keen-query/filters.js +++ b/lib/keen-query/filters.js @@ -26,7 +26,7 @@ function transformValue (value, handleList) { } module.exports.parse = function (filter) { - let filterConf = /^([^!~\?]*)(=|!=|>|<|~|!~|\?|!\?)(.+)$/.exec(filter); + let filterConf = /^([^!~\?><]*)(>=?|<=?|=|!=|~|!~|\?|!\?)(.+)$/.exec(filter); if (filterConf) { if (!filterConf[1] && !filterConf[3]) { throw new Error(`Filter ${filter} must specify a property name to filter on and a value to match e.g. ->filter(user.uuid${filter}abcd)`); diff --git a/lib/keen-query/shorthands.js b/lib/keen-query/shorthands.js index dd3b0aa..1b70bb1 100644 --- a/lib/keen-query/shorthands.js +++ b/lib/keen-query/shorthands.js @@ -25,6 +25,12 @@ module.exports.filters = { '<': { operator: 'lt' }, + '>=': { + operator: 'gte' + }, + '<=': { + operator: 'lte' + }, '<<': { operator: 'in', handleList: true @@ -43,6 +49,8 @@ module.exports.reverseFilters = { 'not_contains': '!~', 'gt': '>', 'lt': '<', + 'gte': '>=', + 'lte': '<=', 'in': '?' }; diff --git a/lib/post-processing/reduce.js b/lib/post-processing/reduce.js index 4fa7826..fc17d5d 100644 --- a/lib/post-processing/reduce.js +++ b/lib/post-processing/reduce.js @@ -8,7 +8,7 @@ function linearRegression(arr){ var sum_y = 0; var sum_xy = 0; var sum_xx = 0; - var sum_yy = 0; + // var sum_yy = 0; for (var i = 0; i < arr.length; i++) { @@ -16,7 +16,7 @@ function linearRegression(arr){ sum_y += arr[i]; sum_xy += (i*arr[i]); sum_xx += (i*i); - sum_yy += (arr[i]*arr[i]); + // sum_yy += (arr[i]*arr[i]); } return (n * sum_xy - sum_x * sum_y) / (n*sum_xx - sum_x * sum_x); diff --git a/lib/printers/ascii.js b/lib/printers/ascii.js index 789a975..a943d24 100644 --- a/lib/printers/ascii.js +++ b/lib/printers/ascii.js @@ -26,9 +26,3 @@ function ascii () { module.exports = function () { return ascii.call(this); } - - - - - - diff --git a/n.Makefile b/n.Makefile new file mode 100644 index 0000000..89d9cfb --- /dev/null +++ b/n.Makefile @@ -0,0 +1,149 @@ +# Warning, don't edit this file, it's maintained on GitHub and updated by running `make update-tools` +# Submit PR's here: https://www.github.com/Financial-Times/n-makefile + + +# Setup environment variables +sinclude .env +export $(shell [ -f .env ] && sed 's/=.*//' .env) + +# ./node_modules/.bin on the PATH +export PATH := ./node_modules/.bin:$(PATH) + +# Use bash not sh +SHELL := /bin/bash + +# Some handy utilities +GLOB = git ls-files -z $1 | tr '\0' '\n' | xargs -I {} find {} ! -type l +NPM_INSTALL = npm prune --production=false && npm install +BOWER_INSTALL = bower prune && bower install --config.registry.search=http://registry.origami.ft.com --config.registry.search=https://bower.herokuapp.com +JSON_GET_VALUE = grep $1 | head -n 1 | sed 's/[," ]//g' | cut -d : -f 2 +IS_GIT_IGNORED = grep -q $(if $1, $1, $@) .gitignore +VERSION = v1.4.7 +APP_NAME = $(shell cat package.json 2>/dev/null | $(call JSON_GET_VALUE,name)) +DONE = echo ✓ $@ done +CONFIG_VARS = curl -fsL https://ft-next-config-vars.herokuapp.com/$1/$(call APP_NAME)$(if $2,.$2,) -H "Authorization: `heroku config:get APIKEY --app ft-next-config-vars`" + +# +# META TASKS +# + +.PHONY: test + +# +# COMMON TASKS +# + +# clean +clea%: +# HACK: Can't use -e option here because it's not supported by our Jenkins + @git clean -fxd + @$(DONE) + +# install +instal%: node_modules bower_components _install_scss_lint .editorconfig .eslintrc.js .scss-lint.yml .env webpack.config.js heroku-cli + @$(MAKE) $(foreach f, $(shell find functions/* -type d -maxdepth 0 2>/dev/null), $f/node_modules $f/bower_components) + @$(DONE) + +# deploy +deplo%: _deploy_apex + @$(DONE) + +# verify +verif%: _verify_lintspaces _verify_eslint _verify_scss_lint + @$(DONE) + +# assets (includes assets-production) +asset%: + @if [ -e webpack.config.js ]; then webpack $(if $(findstring assets-production,$@),--bail,--dev); fi + +# build (includes build-production) +buil%: public/__about.json + @if [ -e webpack.config.js ]; then $(MAKE) $(subst build,assets,$@); fi + @if [ -e Procfile ] && [ "$(findstring build-production,$@)" == "build-production" ]; then haikro build; fi + @$(DONE) + +# watch +watc%: + @if [ -e webpack.config.js ]; then webpack --watch --dev; fi + @$(DONE) + +# +# SUB-TASKS +# + +# INSTALL SUB-TASKS + +# Regular npm install +node_modules: package.json + @if [ -e package.json ]; then $(NPM_INSTALL) && $(DONE); fi + +# Regular bower install +bower_components: bower.json + @if [ -e bower.json ]; then $(BOWER_INSTALL) && $(DONE); fi + +# These tasks have been intentionally left blank +package.json: +bower.json: + +# node_modules for Lambda functions +functions/%/node_modules: + @cd $(dir $@) && if [ -e package.json ]; then $(NPM_INSTALL) && $(DONE); fi + +# bower_components for Lambda functions +functions/%/bower_components: + @cd $(dir $@) && if [ -e bower.json ]; then $(BOWER_INSTALL) && $(DONE); fi + +_install_scss_lint: + @if [ ! -x "$(shell which scss-lint)" ] && [ "$(shell $(call GLOB,'*.scss'))" != "" ]; then gem install scss-lint -v 0.35.0 && $(DONE); fi + +# Manage various dot/config files if they're in the .gitignore +.editorconfig .eslintrc.js .scss-lint.yml webpack.config.js: n.Makefile + @if $(call IS_GIT_IGNORED); then curl -sL https://raw.githubusercontent.com/Financial-Times/n-makefile/$(VERSION)/config/$@ > $@ && $(DONE); fi + +ENV_MSG_CANT_GET = "Cannot get config vars for this service. Check you are added to the ft-next-config-vars service on Heroku with operate permissions. Do that here - https://docs.google.com/spreadsheets/d/1mbJQYJOgXAH2KfgKUM1Vgxq8FUIrahumb39wzsgStu0 (or ask someone to do it for you). Check that your package.json's name property is correct. Check that your project has config-vars set up in models/development.js." +.env: + @if $(call IS_GIT_IGNORED,*.env*) && [ -e package.json ]; then ($(call CONFIG_VARS,development,env) > .env && perl -pi -e 's/="(.*)"/=\1/' .env && $(DONE)) || (echo $(ENV_MSG_CANT_GET) && rm .env && exit 1); fi + +MSG_HEROKU_CLI = "Please make sure the Heroku CLI toolbelt is installed - see https://toolbelt.heroku.com/. And make sure you are authenticated by running ‘heroku login’. If this is not an app, delete Procfile." +heroku-cli: + @if [ -e Procfile ]; then heroku auth:whoami &>/dev/null || (echo $(MSG_HEROKU_CLI) && exit 1); fi + + +# VERIFY SUB-TASKS + +_verify_eslint: + @if [ -e .eslintrc.js ]; then $(call GLOB,'*.js') | xargs eslint && $(DONE); fi + +_verify_lintspaces: + @if [ -e .editorconfig ] && [ -e package.json ]; then $(call GLOB) | xargs lintspaces -e .editorconfig -i js-comments -i html-comments && $(DONE); fi + +_verify_scss_lint: +# HACK: Use backticks rather than xargs because xargs swallow exit codes (everything becomes 1 and stoopidly scss-lint exits with 1 if warnings, 2 if errors) + @if [ -e .scss-lint.yml ]; then { scss-lint -c ./.scss-lint.yml `$(call GLOB,'*.scss')`; if [ $$? -ne 0 -a $$? -ne 1 ]; then exit 1; fi; $(DONE); } fi + +# DEPLOY SUB-TASKS + +APEX_PROD_ENV_FILE = .env.prod.json +_deploy_apex: + @if [ -e project.json ]; then $(call CONFIG_VARS,production) > $(APEX_PROD_ENV_FILE) && apex deploy --env-file $(APEX_PROD_ENV_FILE); fi + @if [ -e $(APEX_PROD_ENV_FILE) ]; then rm $(APEX_PROD_ENV_FILE) && $(DONE); fi + +npm-publis%: + npm-prepublish --verbose + npm publish --access public + +# BUILD SUB-TASKS + +# Only apply to Heroku apps for now +public/__about.json: + @if [ -e Procfile ]; then mkdir -p public && echo '{"description":"$(call APP_NAME)","support":"next.team@ft.com","supportStatus":"active","appVersion":"$(shell git rev-parse HEAD | xargs echo -n)","buildCompletionTime":"$(shell date -u +"%Y-%m-%dT%H:%M:%SZ")"}' > $@ && $(DONE); fi + +# UPDATE TASK +update-tools: + $(eval LATEST = $(shell curl -fs https://api.github.com/repos/Financial-Times/n-makefile/tags | $(call JSON_GET_VALUE,name))) + $(if $(filter $(LATEST), $(VERSION)), $(error Cannot update n-makefile, as it is already up to date!)) + @curl -sL https://raw.githubusercontent.com/Financial-Times/n-makefile/$(LATEST)/Makefile > n.Makefile + @sed -i "" "s/^VERSION = master/VERSION = $(LATEST)/" n.Makefile + @read -p "Updated tools from $(VERSION) to $(LATEST). Do you want to commit and push? [y/N] " Y;\ + if [ $$Y == "y" ]; then git add n.Makefile && git commit -m "Updated tools to $(LATEST)" && git push origin HEAD; fi + @$(DONE) diff --git a/package.json b/package.json index a1e6d67..581a34e 100644 --- a/package.json +++ b/package.json @@ -28,10 +28,12 @@ }, "homepage": "https://github.com/Financial-Times/keen-query#readme", "devDependencies": { + "bower": "^1.7.9", "chai": "^3.4.1", + "eslint": "^3.4.0", "fetch-mock": "^4.4.0", + "lintspaces-cli": "^0.4.0", "mocha": "^2.4.2", - "next-build-tools": "^5.16.1", "sinon": "^1.17.3" } } diff --git a/test/url-generation.test.js b/test/url-generation.test.js index a318971..f0e7891 100644 --- a/test/url-generation.test.js +++ b/test/url-generation.test.js @@ -7,6 +7,8 @@ KeenQuery.setConfig({ }); const expect = require('chai').expect; +//FYI - to run just a single onle of these prefix the entry with `only.` + const queryUrlMappings = { extractionTypes: { 'tomato:bread->count()': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days', @@ -24,6 +26,8 @@ const queryUrlMappings = { 'tomato:bread->count()->filter(prop!=false)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22ne%22%2C%22property_value%22%3Afalse%7D%5D', 'tomato:bread->count()->filter(prop>1)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22gt%22%2C%22property_value%22%3A1%7D%5D', 'tomato:bread->count()->filter(prop<2)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22lt%22%2C%22property_value%22%3A2%7D%5D', + 'tomato:bread->count()->filter(prop>=1)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22gte%22%2C%22property_value%22%3A1%7D%5D', + 'tomato:bread->count()->filter(prop<=2)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22lte%22%2C%22property_value%22%3A2%7D%5D', 'tomato:bread->count()->filter(prop~val)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22contains%22%2C%22property_value%22%3A%22val%22%7D%5D', 'tomato:bread->count()->filter(prop!~val)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22not_contains%22%2C%22property_value%22%3A%22val%22%7D%5D', 'tomato:bread->count()->filter(prop?val1,val2)': 'https://api.keen.io/3.0/projects/test_proj/queries/count?api_key=test_key&event_collection=tomato%3Abread&timeframe=this_14_days&filters=%5B%7B%22property_name%22%3A%22prop%22%2C%22operator%22%3A%22in%22%2C%22property_value%22%3A%5B%22val1%22%2C%22val2%22%5D%7D%5D', @@ -114,19 +118,23 @@ describe('Generation of keen urls from keen query strings', () => { Object.keys(queryUrlMappings).forEach(category => { const queries = queryUrlMappings[category]; describe(category, () => { - Object.keys(queries).forEach(kq => { + Object.keys(queries).forEach(key => { let func = it; - if (kq.indexOf('skip.') === 0) { + let kq = key; + if (key.indexOf('skip.') === 0) { func = it.skip; - kq = kq.replace('skip.', '') + kq = key.replace('skip.', '') + } else if (key.indexOf('only.') === 0) { + func = it.only; + kq = key.replace('only.', '') } func(`Should construct api url(s) correctly for ${kq}`, () => { return KeenQuery.build(kq).print('url') .then(url => { if (Array.isArray(url)) { - expect(url).to.deep.equal(queries[kq]); + expect(url).to.deep.equal(queries[key]); } else { - expect(url).to.equal(queries[kq]); + expect(url).to.equal(queries[key]); } }); });