From 95bccf0f047e4a1b986d477c882d395b7e35122c Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Mon, 5 Jun 2017 14:16:06 -0500 Subject: [PATCH 1/5] Fxied bug that prevented new URLs to be correctly parsed when putting nested parenthesis in lists. --- app/react/Pages/utils/pageItemLists.js | 16 +++--- .../Pages/utils/specs/pageItemLists.spec.js | 11 ++-- app/react/utils/markdownEscapedValues.js | 50 +++++++++++++++++++ .../utils/specs/markdownEscapedValues.spec.js | 24 +++++++++ 4 files changed, 90 insertions(+), 11 deletions(-) create mode 100644 app/react/utils/markdownEscapedValues.js create mode 100644 app/react/utils/specs/markdownEscapedValues.spec.js diff --git a/app/react/Pages/utils/pageItemLists.js b/app/react/Pages/utils/pageItemLists.js index 4c786c6e2d..4f8f27d5a2 100644 --- a/app/react/Pages/utils/pageItemLists.js +++ b/app/react/Pages/utils/pageItemLists.js @@ -1,15 +1,17 @@ +import markdownEscapedValues from 'app/utils/markdownEscapedValues'; + const listPlaceholder = '{---UWAZILIST---}'; +const listEscape = '{list}'; export default { generate: (originalText) => { - const listMatch = /{list}\((.*?)\)/g; - const params = []; - const originalContent = originalText || ''; + const values = markdownEscapedValues(originalText, '(...)', listEscape); - const content = originalContent.replace(listMatch, (_, list) => { - const listParams = /\?(.*)/g.exec(list); - params.push(listParams ? listParams[0] : ''); - return listPlaceholder; + let content = originalText || ''; + const params = values.map(match => { + content = content.replace(`${listEscape}(${match})`, listPlaceholder); + const urlParams = /\?(.*)/g.exec(match); + return urlParams ? urlParams[0] : ''; }); return {params, content}; diff --git a/app/react/Pages/utils/specs/pageItemLists.spec.js b/app/react/Pages/utils/specs/pageItemLists.spec.js index a7dc09f337..2931f27617 100644 --- a/app/react/Pages/utils/specs/pageItemLists.spec.js +++ b/app/react/Pages/utils/specs/pageItemLists.spec.js @@ -1,3 +1,5 @@ +/* eslint-disable max-len */ + import pageLists from '../pageItemLists'; describe('Pages: pageItemLists util', () => { @@ -6,10 +8,10 @@ describe('Pages: pageItemLists util', () => { beforeEach(() => { content = '## title\nSome text with a [URL](http://google.com) inside.' + '\n\n{list}(http://someurl:3000/es/?parameters=values)' + - '\n\nWhich should be in its own line, separated with TWO line breaks (to create a new

Element)' + + '\n\nWhich should be in its own line, "separated" with TWO line breaks (to create a new

Element)' + '\n\n{list}(http://someurl:3000/es/)' + '\n\nAnd should allow multiple lists with different values' + - '\n\n{list}(http://anotherurl:4000/es/?different=parameters)' + + '\n\n{list}(https://cejil.uwazi.io/es/library/?q=(filters:(mandatos_de_la_corte:(from:1496620800)),order:asc,sort:title,types:!(%2758b2f3a35d59f31e1345b4b6%27)))' + '\n\n{list}(http://anotherurl:5000/es/?a=b)' + '\n\n```javascript\nCode\n```'; }); @@ -19,14 +21,15 @@ describe('Pages: pageItemLists util', () => { expect(params.length).toBe(4); expect(params[0]).toBe('?parameters=values'); expect(params[1]).toBe(''); - expect(params[2]).toBe('?different=parameters'); + expect(params[2]).toBe('?q=(filters:(mandatos_de_la_corte:(from:1496620800)),order:asc,sort:title,types:!(%2758b2f3a35d59f31e1345b4b6%27))'); expect(params[3]).toBe('?a=b'); }); it('should return the content with list placeholders', () => { const newContent = pageLists.generate(content).content; expect(newContent).toContain('{---UWAZILIST---}'); - expect(newContent).not.toContain('?different=parameters'); + expect(newContent).not.toContain('?parameters=values'); + expect(newContent).not.toContain('order:asc,sort:title,types:!(%2758b2f3a35d59f31e1345b4b6%27)'); }); it('should return empty if no content', () => { diff --git a/app/react/utils/markdownEscapedValues.js b/app/react/utils/markdownEscapedValues.js new file mode 100644 index 0000000000..da4d791161 --- /dev/null +++ b/app/react/utils/markdownEscapedValues.js @@ -0,0 +1,50 @@ +/* eslint-disable max-depth */ + +// Adapted from http://blog.stevenlevithan.com/archives/javascript-match-nested +export default (function () { + const formatParts = /^([\S\s]+?)\.\.\.([\S\s]+)/; + const metaChar = /[-[\]{}()*+?.\\^$|,]/g; + const escape = function (str) { + return str.replace(metaChar, '\\$&'); + }; + + return function (str, format, escapeCode = '') { + const p = formatParts.exec(format); + if (!p) { + throw new Error('format must include start and end tokens separated by "..."'); + } + + if (p[1] === p[2]) { + throw new Error('start and end format tokens cannot be identical'); + } + + const opener = p[1]; + const closer = p[2]; + const iterator = new RegExp(format.length === 5 ? '[' + escape(opener + closer) + ']' : escape(opener) + '|' + escape(closer), 'g'); + const results = []; + let openTokens; + let matchStartIndex; + let match; + + do { + openTokens = 0; + while ((match = iterator.exec(str)) !== null) { + if (match[0] === opener) { + if (!openTokens) { + matchStartIndex = iterator.lastIndex; + } + openTokens += 1; + } else if (openTokens) { + openTokens -= 1; + if (!openTokens) { + if (str.slice(matchStartIndex - escapeCode.length - opener.length, matchStartIndex - opener.length) === escapeCode) { + results.push(str.slice(matchStartIndex, match.index)); + } + } + } + } + } while (openTokens && (iterator.lastIndex = matchStartIndex)); + + return results; + }; +}()); diff --git a/app/react/utils/specs/markdownEscapedValues.spec.js b/app/react/utils/specs/markdownEscapedValues.spec.js new file mode 100644 index 0000000000..58f9fbeaa8 --- /dev/null +++ b/app/react/utils/specs/markdownEscapedValues.spec.js @@ -0,0 +1,24 @@ +import markdownEscapedValues from '../markdownEscapedValues'; + +describe('markdownEscapedValues', () => { + it('should return an emtpy array when no match found', () => { + expect(markdownEscapedValues(null, '(...)')).toEqual([]); + expect(markdownEscapedValues('', '(...)')).toEqual([]); + expect(markdownEscapedValues('Unmatched text', '(...)')).toEqual([]); + }); + + it('should extract found matches, even in nested configurations', () => { + expect(markdownEscapedValues('This is a (match)', '(...)')).toEqual(['match']); + + expect(markdownEscapedValues('This should also (match(as(nested(parenthesis),with more data)))', '(...)')) + .toEqual(['match(as(nested(parenthesis),with more data))']); + }); + + it('should extract matches with custom escape code only, and avoid other type of escapes that may use similar patters', () => { + expect(markdownEscapedValues('This {should}(not match), this is a {a}(match), this other [a](should not)', '(...)', '{a}')) + .toEqual(['match']); + + expect(markdownEscapedValues('This {should}(not match), this is a {a}(should(match)), this other {a}(too)', '(...)', '{a}')) + .toEqual(['should(match)', 'too']); + }); +}); From 11b9865eddb58e4af1c4174a4eed3632b715d142 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Mon, 5 Jun 2017 15:12:15 -0500 Subject: [PATCH 2/5] Changed Page View to use new rison encoded URLs. --- app/react/Pages/PageView.js | 5 +++-- app/react/Pages/specs/PageView.spec.js | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app/react/Pages/PageView.js b/app/react/Pages/PageView.js index d23ab1ce58..18add6e2d4 100644 --- a/app/react/Pages/PageView.js +++ b/app/react/Pages/PageView.js @@ -1,4 +1,5 @@ import React from 'react'; +import rison from 'rison'; import RouteHandler from 'app/App/RouteHandler'; import api from 'app/Search/SearchAPI'; @@ -6,7 +7,6 @@ import PagesAPI from './PagesAPI'; import TemplatesAPI from 'app/Templates/TemplatesAPI'; import ThesaurisAPI from 'app/Thesauris/ThesaurisAPI'; import {actions} from 'app/BasicReducer'; -import queryString from 'query-string'; import PageViewer from './components/PageViewer'; import pageItemLists from './utils/pageItemLists'; @@ -15,7 +15,8 @@ function prepareLists(page) { const listsData = pageItemLists.generate(page.metadata.content); listsData.searchs = listsData.params.map(params => { - let query = params ? queryString.parse(params) : {filters: {}, types: []}; + const sanitizedParams = params ? params.replace(/%27/g, '\'') : ''; + let query = sanitizedParams ? rison.decode(sanitizedParams.replace('?q=', '') || '()') : {filters: {}, types: []}; query.limit = '6'; return api.search(query); }); diff --git a/app/react/Pages/specs/PageView.spec.js b/app/react/Pages/specs/PageView.spec.js index 1231e1af81..41aadfcacd 100644 --- a/app/react/Pages/specs/PageView.spec.js +++ b/app/react/Pages/specs/PageView.spec.js @@ -29,7 +29,7 @@ describe('PageView', () => { spyOn(ThesaurisAPI, 'get').and.returnValue(Promise.resolve('thesauris')); spyOn(pageItemLists, 'generate').and.returnValue({ content: 'parsedContent', - params: ['?a=1&b=2', '', '?x=1&y=2&limit=24'] + params: ['?q=(a:1,b:2)', '', '?q=(x:1,y:!(%27array%27),limit:24)'] }); RouteHandler.renderedFromServer = true; @@ -57,17 +57,17 @@ describe('PageView', () => { PageView.requestState({pageId: 'abc2'}) .then((response) => { expect(api.search.calls.count()).toBe(3); - expect(JSON.parse(JSON.stringify(api.search.calls.argsFor(0)[0]))).toEqual({a: '1', b: '2', limit: '6'}); + expect(JSON.parse(JSON.stringify(api.search.calls.argsFor(0)[0]))).toEqual({a: 1, b: 2, limit: '6'}); expect(api.search.calls.argsFor(1)[0]).toEqual({filters: {}, types: [], limit: '6'}); - expect(JSON.parse(JSON.stringify(api.search.calls.argsFor(2)[0]))).toEqual({x: '1', y: '2', limit: '6'}); + expect(JSON.parse(JSON.stringify(api.search.calls.argsFor(2)[0]))).toEqual({x: 1, y: ['array'], limit: '6'}); expect(response.page.itemLists.length).toBe(3); - expect(response.page.itemLists[0].params).toBe('?a=1&b=2'); + expect(response.page.itemLists[0].params).toBe('?q=(a:1,b:2)'); expect(response.page.itemLists[0].items).toEqual(['resultsFor:0']); expect(response.page.itemLists[1].params).toBe(''); expect(response.page.itemLists[1].items).toEqual(['resultsFor:1']); - expect(response.page.itemLists[2].params).toBe('?x=1&y=2&limit=24'); + expect(response.page.itemLists[2].params).toBe('?q=(x:1,y:!(%27array%27),limit:24)'); expect(response.page.itemLists[2].items).toEqual(['resultsFor:2']); done(); }) From ec9752393ebb1d5f3b3bd56e60e3a3ddf8f730da Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 6 Jun 2017 12:16:28 -0500 Subject: [PATCH 3/5] Fixed bug that through errors on previous lists --- app/react/Pages/PageView.js | 13 +++++++++++-- app/react/Pages/specs/PageView.spec.js | 9 ++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/app/react/Pages/PageView.js b/app/react/Pages/PageView.js index 18add6e2d4..3e99524e2c 100644 --- a/app/react/Pages/PageView.js +++ b/app/react/Pages/PageView.js @@ -15,8 +15,17 @@ function prepareLists(page) { const listsData = pageItemLists.generate(page.metadata.content); listsData.searchs = listsData.params.map(params => { - const sanitizedParams = params ? params.replace(/%27/g, '\'') : ''; - let query = sanitizedParams ? rison.decode(sanitizedParams.replace('?q=', '') || '()') : {filters: {}, types: []}; + const sanitizedParams = params ? decodeURI(params) : ''; + const queryDefault = {filters: {}, types: []}; + let query = queryDefault; + + if (sanitizedParams) { + query = rison.decode(sanitizedParams.replace('?q=', '') || '()'); + if (typeof query !== 'object') { + query = queryDefault; + } + } + query.limit = '6'; return api.search(query); }); diff --git a/app/react/Pages/specs/PageView.spec.js b/app/react/Pages/specs/PageView.spec.js index 41aadfcacd..cb95a679f8 100644 --- a/app/react/Pages/specs/PageView.spec.js +++ b/app/react/Pages/specs/PageView.spec.js @@ -29,7 +29,7 @@ describe('PageView', () => { spyOn(ThesaurisAPI, 'get').and.returnValue(Promise.resolve('thesauris')); spyOn(pageItemLists, 'generate').and.returnValue({ content: 'parsedContent', - params: ['?q=(a:1,b:2)', '', '?q=(x:1,y:!(%27array%27),limit:24)'] + params: ['?q=(a:1,b:2)', '', '?q=(x:1,y:!(%27array%27),limit:24)', '?order=metadata.form&treatAs=number'] }); RouteHandler.renderedFromServer = true; @@ -56,12 +56,13 @@ describe('PageView', () => { it('should request each list inside the content limited to 6 items and set the state', (done) => { PageView.requestState({pageId: 'abc2'}) .then((response) => { - expect(api.search.calls.count()).toBe(3); + expect(api.search.calls.count()).toBe(4); expect(JSON.parse(JSON.stringify(api.search.calls.argsFor(0)[0]))).toEqual({a: 1, b: 2, limit: '6'}); expect(api.search.calls.argsFor(1)[0]).toEqual({filters: {}, types: [], limit: '6'}); expect(JSON.parse(JSON.stringify(api.search.calls.argsFor(2)[0]))).toEqual({x: 1, y: ['array'], limit: '6'}); + expect(api.search.calls.argsFor(3)[0]).toEqual({filters: {}, types: [], limit: '6'}); - expect(response.page.itemLists.length).toBe(3); + expect(response.page.itemLists.length).toBe(4); expect(response.page.itemLists[0].params).toBe('?q=(a:1,b:2)'); expect(response.page.itemLists[0].items).toEqual(['resultsFor:0']); @@ -69,6 +70,8 @@ describe('PageView', () => { expect(response.page.itemLists[1].items).toEqual(['resultsFor:1']); expect(response.page.itemLists[2].params).toBe('?q=(x:1,y:!(%27array%27),limit:24)'); expect(response.page.itemLists[2].items).toEqual(['resultsFor:2']); + expect(response.page.itemLists[3].params).toBe('?order=metadata.form&treatAs=number'); + expect(response.page.itemLists[3].items).toEqual(['resultsFor:3']); done(); }) .catch(done.fail); From 0f6e09abea3c6341c454722778cdf7e336034d1e Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 6 Jun 2017 12:22:10 -0500 Subject: [PATCH 4/5] Fixed most ESLint warnings. --- app/react/Documents/components/SearchText.js | 4 ++-- app/react/Entities/components/EntityViewer.js | 2 +- app/react/I18N/t.js | 8 ++++---- app/react/Library/components/UploadEntityStatus.js | 2 +- .../actions/specs/relationTypesActions.spec.js | 1 + app/react/Viewer/actions/specs/documentActions.spec.js | 1 + 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/app/react/Documents/components/SearchText.js b/app/react/Documents/components/SearchText.js index 66f8cb6e7b..4404da5899 100644 --- a/app/react/Documents/components/SearchText.js +++ b/app/react/Documents/components/SearchText.js @@ -6,8 +6,8 @@ import {t} from 'app/I18N'; import {Field, LocalForm} from 'react-redux-form'; export class SearchText extends Component { - search(){} - resetSearch(){} + search() {} + resetSearch() {} render() { const {snippets} = this.props; diff --git a/app/react/Entities/components/EntityViewer.js b/app/react/Entities/components/EntityViewer.js index 15bb7bdbc7..620a1dd6f3 100644 --- a/app/react/Entities/components/EntityViewer.js +++ b/app/react/Entities/components/EntityViewer.js @@ -135,7 +135,7 @@ export class EntityViewer extends Component { entityBeingEdited={entityBeingEdited} /> - +

diff --git a/app/react/I18N/t.js b/app/react/I18N/t.js index 561824e383..172df539ff 100644 --- a/app/react/I18N/t.js +++ b/app/react/I18N/t.js @@ -8,10 +8,10 @@ let t = (contextId, key, _text) => { let context = translation.contexts.find((ctx) => ctx.id === contextId) || {values: {}}; if (!context.values) { - console.log(contextId); - console.log(key); - console.log(_text); - console.log(context); + console.log(contextId); // eslint-disable-line no-console + console.log(key); // eslint-disable-line no-console + console.log(_text); // eslint-disable-line no-console + console.log(context); // eslint-disable-line no-console } if (contextId === 'System' && !context.values[key]) { diff --git a/app/react/Library/components/UploadEntityStatus.js b/app/react/Library/components/UploadEntityStatus.js index 68211ca77d..f6ab1157fb 100644 --- a/app/react/Library/components/UploadEntityStatus.js +++ b/app/react/Library/components/UploadEntityStatus.js @@ -1,4 +1,4 @@ - import PropTypes from 'prop-types'; +import PropTypes from 'prop-types'; import React, {Component} from 'react'; import {ItemFooter} from 'app/Layout/Lists'; import {connect} from 'react-redux'; diff --git a/app/react/RelationTypes/actions/specs/relationTypesActions.spec.js b/app/react/RelationTypes/actions/specs/relationTypesActions.spec.js index b0b8ee9c78..fe9e62e3cd 100644 --- a/app/react/RelationTypes/actions/specs/relationTypesActions.spec.js +++ b/app/react/RelationTypes/actions/specs/relationTypesActions.spec.js @@ -1,3 +1,4 @@ +/* eslint-disable max-nested-callbacks */ import backend from 'fetch-mock'; import {APIURL} from 'app/config.js'; import {actions as formActions} from 'react-redux-form'; diff --git a/app/react/Viewer/actions/specs/documentActions.spec.js b/app/react/Viewer/actions/specs/documentActions.spec.js index 5381c2613c..6b1979937a 100644 --- a/app/react/Viewer/actions/specs/documentActions.spec.js +++ b/app/react/Viewer/actions/specs/documentActions.spec.js @@ -1,3 +1,4 @@ +/* eslint-disable max-nested-callbacks */ import configureMockStore from 'redux-mock-store'; import thunk from 'redux-thunk'; import backend from 'fetch-mock'; From 5b92524fbd7aea981fa4b46f44b80ca56111c7f9 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 7 Jun 2017 11:40:36 -0500 Subject: [PATCH 5/5] Added fast vectors mapping to elastic definition to improve search speed. --- database/elastic_mapping.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/elastic_mapping.js b/database/elastic_mapping.js index 6d780d60c3..eed86edba5 100644 --- a/database/elastic_mapping.js +++ b/database/elastic_mapping.js @@ -35,11 +35,11 @@ export default { path_match: 'fullText', match_mapping_type: 'string', mapping: { - type: 'string', + type: 'text', index: 'analyzed', omit_norms: true, analyzer: 'standard', - fielddata: {format: 'enabled'} + term_vector: 'with_positions_offsets' } } }, {