Merge pull request #1737 from huridocs/1712-thesauri-groups-aggregations

query builder now calculates aggregations for dictionary groups

app/api/search/documentQueryBuilder.js

@@ -447,6 +447,35 @@ export default function () {
       };
     },
 
+    aggregationWithGroupsOfOptions(key, should, filters, dictionary) {
+      const aggregation = {
+        filters: { filters: {} },
+        aggregations: {
+          filtered: {
+            filter: {
+              bool: {
+                should,
+                filter: filters
+              }
+            }
+          }
+        }
+      };
+      const addMatch = (value) => {
+        const match = { terms: {} };
+        match.terms[key] = value.values ? value.values.map(v => v.id) : [value.id];
+        aggregation.filters.filters[value.id.toString()] = match;
+        if (value.values) {
+          value.values.forEach(addMatch);
+        }
+      };
+      dictionary.values.forEach(addMatch);
+
+      const missingMatch = { bool: { must_not: { exists: { field: key } } } };
+      aggregation.filters.filters.missing = missingMatch;
+      return aggregation;
+    },
+
     nestedAggregation(property, should, readOnlyFilters) {
       const nestedAggregation = {
         nested: {
@@ -515,7 +544,7 @@ export default function () {
       return nestedAggregation;
     },
 
-    aggregations(properties) {
+    aggregations(properties, dictionaries) {
       properties.forEach((property) => {
         const path = `metadata.${property.name}.raw`;
         let filters = baseQuery.query.bool.filter.filter(match => match &&
@@ -528,7 +557,15 @@ export default function () {
           baseQuery.aggregations.all.aggregations[property.name] = this.nestedAggregation(property, should, filters);
           return;
         }
-
+        let dictionary;
+        if (property.content) {
+          dictionary = dictionaries.find(d => property.content.toString() === d._id.toString());
+        }
+        const isADictionaryWithGroups = dictionary && dictionary.values.find(v => v.values);
+        if (isADictionaryWithGroups) {
+          baseQuery.aggregations.all.aggregations[property.name] = this.aggregationWithGroupsOfOptions(path, should, filters, dictionary);
+          return;
+        }
         baseQuery.aggregations.all.aggregations[property.name] = this.aggregation(path, should, filters);
       });
       return this;

app/api/search/search.js

@@ -2,7 +2,7 @@ import { comonProperties, defaultFilters, allUniqueProperties, textFields } from
 import { detect as detectLanguage } from 'shared/languages';
 import { index as elasticIndex } from 'api/config/elasticIndexes';
 import languages from 'shared/languagesList';
-import dictionaries from 'api/thesauris/dictionariesModel';
+import dictionariesModel from 'api/thesauris/dictionariesModel';
 
 import documentQueryBuilder from './documentQueryBuilder';
 import elastic from './elastic';
@@ -52,7 +52,7 @@ function agregationProperties(properties) {
     if (property.type === 'nested') {
       return { name: property.name, nested: true, nestedProperties: property.nestedProperties };
     }
-    return { name: property.name, nested: false };
+    return { name: property.name, nested: false, content: property.content };
   });
 }
 
@@ -79,15 +79,15 @@ const search = {
     if (query.searchTerm) {
       searchEntitiesbyTitle = entities.get({ $text: { $search: query.searchTerm }, language });
       const regexp = `.*${query.searchTerm}.*`;
-      searchDictionariesByTitle = dictionaries.db.aggregate([
+      searchDictionariesByTitle = dictionariesModel.db.aggregate([
         { $match: { 'values.label': { $regex: regexp, $options: 'i' } } },
         { $unwind: '$values' },
         { $match: { 'values.label': { $regex: regexp, $options: 'i' } } }
       ]);
     }
 
-    return Promise.all([templatesModel.get(), searchEntitiesbyTitle, searchDictionariesByTitle])
-    .then(([templates, entitiesMatchedByTitle, dictionariesMatchByLabel]) => {
+    return Promise.all([templatesModel.get(), searchEntitiesbyTitle, searchDictionariesByTitle, dictionariesModel.get()])
+    .then(([templates, entitiesMatchedByTitle, dictionariesMatchByLabel, dictionaries]) => {
       const textFieldsToSearch = query.fields || textFields(templates).map(prop => `metadata.${prop.name}`).concat(['title', 'fullText']);
       const documentsQuery = documentQueryBuilder()
       .fullTextSearch(query.searchTerm, textFieldsToSearch, 2)
@@ -126,7 +126,7 @@ const search = {
 
       documentsQuery.filterMetadataByFullText(textSearchFilters);
       documentsQuery.filterMetadata(filters);
-      documentsQuery.aggregations(aggregations);
+      documentsQuery.aggregations(aggregations, dictionaries);
 
       if (query.geolocation) {
         searchGeolocation(documentsQuery, filteringTypes, templates);
@@ -154,6 +154,13 @@ const search = {
           result._id = hit._id;
           return result;
         });
+        Object.keys(response.aggregations.all).forEach((aggregationKey) => {
+          const aggregation = response.aggregations.all[aggregationKey];
+          if (aggregation.buckets && !Array.isArray(aggregation.buckets)) {
+            aggregation.buckets = Object.keys(aggregation.buckets).map(key => Object.assign({ key }, aggregation.buckets[key]));
+          }
+          response.aggregations.all[aggregationKey] = aggregation;
+        });
 
         return { rows, totalRows: response.hits.total, aggregations: response.aggregations };
       });

app/api/search/specs/elasticResult.js

@@ -1,19 +1,15 @@
 export default function () {
-
-
-  let hit = () => {
-    return Object.assign({}, {
+  const hit = () => Object.assign({}, {
       _index: 'uwazi',
       _type: 'logs',
       _id: 'id1',
       _score: 0.05050901,
       _source: {
       },
       highlight: {}
-    });
-  };
+  });
 
-  let result = {
+  const result = {
     took: 7,
     timed_out: false,
     _shards: {
@@ -25,6 +21,9 @@ export default function () {
       total: 10,
       max_score: 0.05050901,
       hits: []
+    },
+    aggregations: {
+      all: {}
     }
   };
 
@@ -36,13 +35,13 @@ export default function () {
     withDocs(docs) {
       result.hits.hits = [];
       docs.forEach((doc) => {
-        let newHit = hit();
+        const newHit = hit();
         newHit._id = doc._id;
         delete doc._id;
 
         newHit._source = doc;
         if (doc.snippets) {
-          newHit.inner_hits = {fullText: doc.snippets};
+          newHit.inner_hits = { fullText: doc.snippets };
           delete doc.snippets;
         }
         result.hits.hits.push(newHit);

app/api/search/specs/search.spec.js

@@ -147,6 +147,24 @@ describe('search', () => {
       .catch(catchErrors(done));
     });
 
+    it('should convert aggregation buckets that are objects to arrays', (done) => {
+      result.aggregations.all = {
+        dictionaryWithGroups: {
+          buckets: {
+            a: { doc_count: 2, filtered: { doc_count: 1 } },
+            b: { doc_count: 2, filtered: { doc_count: 1 } }
+          }
+        }
+      };
+      spyOn(elastic, 'search').and.returnValue(Promise.resolve(result));
+      search.search({ searchTerm: '', geolocation: true }, 'en')
+      .then((response) => {
+        const expectedBuckets = [{ key: 'a', doc_count: 2, filtered: { doc_count: 1 } }, { key: 'b', doc_count: 2, filtered: { doc_count: 1 } }];
+        expect(response.aggregations.all.dictionaryWithGroups.buckets).toEqual(expectedBuckets);
+        done();
+      });
+    });
+
     it('should match entities related somehow with other entities with a title that is the search term', (done) => {
       search.search({ searchTerm: 'egypt' }, 'en')
       .then(({ rows }) => {
@@ -239,7 +257,7 @@ describe('search', () => {
 
     describe('when the query is for geolocation', () => {
       it('should set size to 9999', (done) => {
-        spyOn(elastic, 'search').and.returnValue(Promise.resolve({ hits: { hits: [] } }));
+        spyOn(elastic, 'search').and.returnValue(Promise.resolve(result));
         search.search({ searchTerm: '', geolocation: true }, 'en')
         .then(() => {
           const elasticQuery = elastic.search.calls.argsFor(0)[0].body;

app/api/templates/specs/utils.spec.js

@@ -44,6 +44,20 @@ describe('templates utils', () => {
       const result = getUpdatedNames(oldProperties, newProperties);
       expect(result).toEqual({ my_prop_two: 'my_fancy_new_name' });
     });
+
+    it('should work for sub values too', () => {
+      const oldProperties = [
+        { id: 1, name: 'my_prop' },
+        { id: 2, name: 'my_prop_two', values: [{ id: 3, name: 'look_at_me' }] },
+      ];
+      const newProperties = [
+        { id: 1, name: 'my_prop' },
+        { id: 2, name: 'my_prop_two', values: [{ id: 3, name: 'I_changed' }] },
+      ];
+
+      const result = getUpdatedNames(oldProperties, newProperties);
+      expect(result).toEqual({ look_at_me: 'I_changed' });
+    });
   });
 
   describe('getDeletedProperties()', () => {
@@ -59,5 +73,18 @@ describe('templates utils', () => {
       const result = getDeletedProperties(oldProperties, newProperties);
       expect(result).toEqual(['boromir']);
     });
+
+    it('should check sub values too', () => {
+      const oldProperties = [
+        { id: 1, name: 'my_prop' },
+        { id: 2, name: 'my_prop_two', values: [{ id: 3, name: 'boromir' }] },
+      ];
+      const newProperties = [
+        { id: 2, name: 'my_prop_two', values: [] },
+        { id: 4, name: 'vip', values: [{ id: 1, name: 'my_prop' }] }
+      ];
+      const result = getDeletedProperties(oldProperties, newProperties);
+      expect(result).toEqual(['boromir']);
+    });
   });
 });

app/api/templates/utils.js

@@ -24,26 +24,53 @@ export function generateNamesAndIds(_properties = []) {
   return generateIds(properties);
 }
 
+const deepFind = (collection, matchFunction) => {
+  let match = collection.find(matchFunction);
+  if (match) {
+    return match;
+  }
+  collection.forEach((item) => {
+    if (item.values) {
+      match = item.values.find(matchFunction);
+    }
+  });
+
+  return match;
+};
+
 export function getUpdatedNames(oldProperties = [], newProperties, prop = 'name') {
   const propertiesWithNewName = {};
-  oldProperties.forEach((property) => {
-    const newProperty = newProperties.find(p => p.id === property.id);
+  const getUpdatedName = (property) => {
+    const newProperty = deepFind(newProperties, p => p.id === property.id);
     if (newProperty && newProperty[prop] !== property[prop]) {
       propertiesWithNewName[property[prop]] = newProperty[prop];
     }
+  };
+
+  oldProperties.forEach((property) => {
+    getUpdatedName(property);
+    if (property.values) {
+      property.values.forEach(getUpdatedName);
+    }
   });
 
   return propertiesWithNewName;
 }
 
 export function getDeletedProperties(oldProperties = [], newProperties, prop = 'name') {
   const deletedProperties = [];
-
-  oldProperties.forEach((property) => {
-    const newProperty = newProperties.find(p => p.id === property.id);
+  const checkDeleted = (property) => {
+    const newProperty = deepFind(newProperties, p => p.id === property.id);
     if (!newProperty) {
       deletedProperties.push(property[prop]);
     }
+  };
+
+  oldProperties.forEach((property) => {
+    checkDeleted(property);
+    if (property.values) {
+      property.values.forEach(checkDeleted);
+    }
   });
 
   return deletedProperties;