544 configure meilisearch (#561)

* configured the picture index in plugin.js, provided a hook to test meilisearch with one search term * minor correction in the config, added the necessary packages for frontend and backend * finished the search helper * plugin configuration is done, for now. The helper function should work just fine * forgot to add the config file in the last commit * Update projects/bp-gallery/src/components/views/search/SearchView.tsx Co-authored-by: Oliver Schulz <64468276+olschulz@users.noreply.github.com> * commit for fun * revert the quatsch from last commit * small change to the conscole output * small change to the conscole output * implemented the requested changes --------- Co-authored-by: Oliver Schulz <64468276+olschulz@users.noreply.github.com>
hpi-swa-lab · Jun 20, 2023 · a24e96e · a24e96e
1 parent a554b4d
commit a24e96e
Show file tree

Hide file tree

Showing 8 changed files with 269 additions and 27 deletions.
diff --git a/projects/bp-gallery/package.json b/projects/bp-gallery/package.json
@@ -29,11 +29,12 @@
     "jodit-react": "^1.3.31",
     "leaflet": "^1.9.3",
     "lodash": "^4.17.21",
+    "meilisearch": "^0.33.0",
     "nanoid": "^4.0.2",
     "node-sass": "^8.0.0",
     "puppeteer": "^19.4.0",
-    "react-currency-input-field": "^3.6.10",
     "react": "^18.2.0",
+    "react-currency-input-field": "^3.6.10",
     "react-date-range": "^1.4.0",
     "react-device-detect": "^2.2.2",
     "react-dom": "^18.2.0",

diff --git a/projects/bp-gallery/src/components/views/search/SearchView.tsx b/projects/bp-gallery/src/components/views/search/SearchView.tsx
@@ -12,11 +12,8 @@ import SearchBreadcrumbs from './SearchBreadcrumbs';
 import SearchHub from './SearchHub';
 import './SearchView.scss';
 import { isValidYear } from './helpers/addNewParamToSearchPath';
-import {
-  SearchType,
-  convertSearchParamsToPictureFilters,
-  paramToTime,
-} from './helpers/search-filters';
+import getSearchResultPictureIds from './helpers/getSearchResultPictureIds';
+import { SearchType, paramToTime } from './helpers/search-filters';
 import { toURLSearchParam } from './helpers/url-search-params';
 
 const isValidTimeSpecification = (searchRequest: string) => {
@@ -47,25 +44,26 @@ const SearchView = () => {
 
   // Builds query from search params in the path
   const queryParams = useMemo(() => {
-    if (isAllSearchActive) {
-      const allSearchTerms = searchParams
-        .getAll(toURLSearchParam(SearchType.ALL))
-        .map(decodeURIComponent);
-      const searchTimes: string[][] = [];
-      allSearchTerms.forEach(searchTerm => {
-        if (isValidTimeSpecification(searchTerm)) {
-          const { startTime, endTime } = paramToTime(searchTerm);
-          searchTimes.push([searchTerm, startTime, endTime]);
-        }
-      });
-      return {
-        searchTerms: allSearchTerms.filter(searchTerm => !isValidTimeSpecification(searchTerm)),
-        searchTimes,
-      };
-    }
-    return convertSearchParamsToPictureFilters(searchParams);
-  }, [isAllSearchActive, searchParams]);
-
+    // if (isAllSearchActive) {
+    const allSearchTerms = searchParams
+      .getAll(toURLSearchParam(SearchType.ALL))
+      .map(decodeURIComponent);
+    const searchTimes: string[][] = [];
+    allSearchTerms.forEach(searchTerm => {
+      if (isValidTimeSpecification(searchTerm)) {
+        const { startTime, endTime } = paramToTime(searchTerm);
+        searchTimes.push([searchTerm, startTime, endTime]);
+      }
+    });
+    return {
+      searchTerms: allSearchTerms.filter(searchTerm => !isValidTimeSpecification(searchTerm)),
+      searchTimes,
+    };
+    // }
+    // return convertSearchParamsToPictureFilters(searchParams);
+  }, [/*isAllSearchActive,*/ searchParams]);
+  if (import.meta.env.MODE === 'development')
+    getSearchResultPictureIds(queryParams, '').then(res => console.log('search results:', res));
   const { linkToCollection, bulkEdit } = useBulkOperations();
 
   return (

diff --git a/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts b/projects/bp-gallery/src/components/views/search/helpers/getSearchResultPictureIds.ts
@@ -0,0 +1,48 @@
+import { MeiliSearch } from 'meilisearch';
+
+const dateToTimeStamp = (date: string) => {
+  return Date.parse(date) / 1000;
+};
+
+const getSearchResultPictureIds = async (
+  { searchTerms, searchTimes }: { searchTerms: string[]; searchTimes: string[][] },
+  filter: string
+) => {
+  const client = new MeiliSearch({
+    host: 'localhost:7700',
+    apiKey: '',
+  });
+  const index = client.index('picture');
+
+  const TIME_RANGE_START = 'time_range_tag_start';
+  const TIME_RANGE_END = 'time_range_tag_end';
+  // when building a filter for meilisearch the filtered attribute always
+  //  has to come first i.e. time_range_start >= 0 works but 0 <= time_range_start does not work
+  if (searchTimes.length !== 0) {
+    const timeFilters = searchTimes.map(
+      searchTime =>
+        `(${TIME_RANGE_START} >= ${dateToTimeStamp(
+          searchTime[1]
+        )} AND ${TIME_RANGE_END} <= ${dateToTimeStamp(searchTime[2])})`
+    );
+
+    const timeFilter = timeFilters.join(' OR ');
+    filter = filter === '' ? timeFilter : filter.concat(' AND ', timeFilter);
+  }
+
+  const RESULT_LIMIT = 1000;
+  // this makes it so only documents that match all of the query terms are returned
+  const MATCHING_STRATEGY = 'all';
+
+  const settings = {
+    limit: RESULT_LIMIT,
+    showMatchesPosition: true,
+    matchingStrategy: MATCHING_STRATEGY,
+    filter: filter,
+  };
+  const query = searchTerms.length !== 0 ? searchTerms.join(' ') : '';
+  const searchResult = await index.search(query, settings);
+  return searchResult.hits;
+};
+
+export default getSearchResultPictureIds;
diff --git a/projects/bp-gallery/vite.config.ts b/projects/bp-gallery/vite.config.ts
@@ -1,7 +1,7 @@
 import react from '@vitejs/plugin-react-swc';
-import { defineConfig } from 'vite';
 import type { Plugin, PluginBuild } from 'esbuild';
 import path from 'path';
+import { defineConfig } from 'vite';
 import eslint from 'vite-plugin-eslint';
 
 const splitPackages = ['@mui/icons-material', '@mui/material'];

diff --git a/projects/bp-gallery/yarn.lock b/projects/bp-gallery/yarn.lock
@@ -3422,6 +3422,13 @@ cross-fetch@3.1.5, cross-fetch@^3.1.5:
   dependencies:
     node-fetch "2.6.7"
 
+cross-fetch@^3.1.6:
+  version "3.1.6"
+  resolved "https://registry.yarnpkg.com/cross-fetch/-/cross-fetch-3.1.6.tgz#bae05aa31a4da760969756318feeee6e70f15d6c"
+  integrity sha512-riRvo06crlE8HiqOwIpQhxwdOk4fOeR7FVM/wXoxchFEqMNUjvbs3bfo4OTgMEMHzppd4DxFBDbyySj8Cv781g==
+  dependencies:
+    node-fetch "^2.6.11"
+
 cross-spawn@^7.0.0, cross-spawn@^7.0.2, cross-spawn@^7.0.3:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -5978,6 +5985,13 @@ map-stream@~0.1.0:
   resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.1.0.tgz#e56aa94c4c8055a16404a0674b78f215f7c8e194"
   integrity sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==
 
+meilisearch@^0.33.0:
+  version "0.33.0"
+  resolved "https://registry.yarnpkg.com/meilisearch/-/meilisearch-0.33.0.tgz#25982b193cdd22e9ec534a022dbde89c42951dc4"
+  integrity sha512-bYPb9WyITnJfzf92e7QFK8Rc50DmshFWxypXCs3ILlpNh8pT15A7KSu9Xgnnk/K3G/4vb3wkxxtFS4sxNkWB8w==
+  dependencies:
+    cross-fetch "^3.1.6"
+
 meow@^9.0.0:
   version "9.0.0"
   resolved "https://registry.yarnpkg.com/meow/-/meow-9.0.0.tgz#cd9510bc5cac9dee7d03c73ee1f9ad959f4ea364"
@@ -6251,6 +6265,13 @@ node-fetch@2.6.7, node-fetch@^2.6.1, node-fetch@^2.6.7:
   dependencies:
     whatwg-url "^5.0.0"
 
+node-fetch@^2.6.11:
+  version "2.6.11"
+  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.11.tgz#cde7fc71deef3131ef80a738919f999e6edfff25"
+  integrity sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==
+  dependencies:
+    whatwg-url "^5.0.0"
+
 node-gyp@^8.4.1:
   version "8.4.1"
   resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-8.4.1.tgz#3d49308fc31f768180957d6b5746845fbd429937"

diff --git a/projects/bp-strapi/config/plugins.js b/projects/bp-strapi/config/plugins.js
@@ -1,4 +1,9 @@
 /* eslint-disable no-unused-vars */
+
+const dateToTimeStamp = (date) => {
+  return Date.parse(date) / 1000;
+};
+
 module.exports = ({ env }) => ({
   // disable i18n (all content is explicitly german as it's a german photo archive)
   i18n: false,
@@ -58,6 +63,120 @@ module.exports = ({ env }) => ({
       },
     },
   },
+  meilisearch:
+    env("MEILISEARCH_ENABLED", "false") === "true"
+      ? {
+          config: {
+            host: env("MEILISEARCH_HOST"),
+            apiKey: env("MEILISEARCH_API_KEY"),
+            picture: {
+              transformEntry({ entry }) {
+                const transformedEntry = {
+                  id: entry.id,
+                  likes: entry.likes,
+                  descriptions: entry.descriptions.map(
+                    (description) => description.text
+                  ),
+                  comments: entry.comments.map((comment) => comment.text),
+                  keyword_tags: entry.keyword_tags
+                    .map((tag) => tag.name)
+                    .concat(entry.verified_keyword_tags.map((tag) => tag.name)),
+                  person_tags: entry.person_tags
+                    .map((tag) => tag.name)
+                    .concat(entry.verified_person_tags.map((tag) => tag.name)),
+                  location_tags: entry.location_tags
+                    .map((tag) => tag.name)
+                    .concat(
+                      entry.verified_location_tags.map((tag) => tag.name)
+                    ),
+                  face_tags: entry.face_tags.map((tag) => tag.name),
+                  collections: entry.collections.map((tag) => tag.name),
+                  archive_tag: entry.archive_tag,
+                  time_range_tag_start: entry?.time_range_tag
+                    ? dateToTimeStamp(entry.time_range_tag.start)
+                    : entry.verified_time_range_tag
+                    ? dateToTimeStamp(entry.verified_time_range_tag.start)
+                    : null,
+                  time_range_tag_end: entry?.time_range_tag
+                    ? dateToTimeStamp(entry.time_range_tag.end)
+                    : entry.verified_time_range_tag
+                    ? dateToTimeStamp(entry.verified_time_range_tag.end)
+                    : null,
+                };
+
+                return transformedEntry;
+              },
+              settings: {
+                displayedAttributes: ["id"],
+                // the order of the attributes in searchableAttributes determines the priorization
+                // of search results i.e. a match in the first searchable attribute will always outrank a match in any other searchable attribute
+                searchableAttributes: [
+                  "descriptions",
+                  "keyword_tags",
+                  "location_tags",
+                  "time_range_tag_start",
+                  "time_range_tag_end",
+                  "face_tags",
+                  "person_tags",
+                  "collections",
+                  "archive_tag",
+                  "comments",
+                ],
+                filterableAttributes: [
+                  "keyword_tags",
+                  "location_tags",
+                  "time_range_tag_start",
+                  "time_range_tag_end",
+                  "face_tags",
+                  "person_tags",
+                  "descriptions",
+                  "comments",
+                  "collections",
+                  "archive_tag",
+                  "is_text",
+                ],
+                sortableAttributes: [
+                  "time_range_tag_start",
+                  "time_range_tag_end",
+                  "likes",
+                ],
+                rankingRules: [
+                  "words",
+                  "typo",
+                  "proximity",
+                  "attribute",
+                  "sort",
+                  "exactness",
+                ],
+                // words that are ignored during searches, useful for common words
+                //  that do not carry a meaning on their own like articles, pronomina etc.
+                // we do not use this setting, since our data on user searchers suggests, that
+                // users only search for proper names, people, locations and nouns in general
+                stopWords: [],
+                synonyms: {},
+                // returned documents will always be unigue in this attribute
+                distinctAttribute: null,
+                typoTolerance: {
+                  enabled: true,
+                  minWordSizeForTypos: { oneTypo: 3, twoTypos: 4 },
+                  disableOnWords: [],
+                  disableOnAttributes: [],
+                },
+                // faceting is currently not in use
+                faceting: {
+                  maxValuesPerFacet: 100,
+                },
+                // maxtotalHits determines the maximal possible amount
+                // of search results and overrides any other settings
+                // like the result_limit of the search settings in this regard
+                pagination: {
+                  maxTotalHits: 1000,
+                },
+              },
+            },
+          },
+        }
+      : null,
   upload:
     env("AWS_ENABLED", "false") === "true"
       ? {

diff --git a/projects/bp-strapi/package.json b/projects/bp-strapi/package.json
@@ -34,6 +34,8 @@
     "pg": "8.6.0",
     "pm2": "^5.2.0",
     "sendmail": "^1.6.1",
+    "sharp": "^0.32.1",
+    "strapi-plugin-meilisearch": "^0.9.2",
     "strapi-provider-upload-aws-s3-advanced": "^5.0.1",
     "xlsx": "^0.18.5"
   },