From 14d9f0d94837aea43390c054dcc825ad1d784ef4 Mon Sep 17 00:00:00 2001 From: Rackover Date: Sat, 6 Mar 2021 13:23:57 +0100 Subject: [PATCH] Added search engine --- app.js | 1 + app/express.js | 1 + app/layout/default.pug | 7 ++++ app/map.js | 7 +++- app/page.js | 9 ++-- app/routes/search.js | 42 +++++++++++++++++++ app/search.js | 90 ++++++++++++++++++++++++++++++++++++++++ app/views/read.pug | 1 - app/views/search.pug | 11 +++++ public/res/css/style.css | 26 +++++++++--- 10 files changed, 183 insertions(+), 12 deletions(-) create mode 100755 app/routes/search.js create mode 100755 app/search.js create mode 100755 app/views/search.pug diff --git a/app.js b/app.js index 72d021f..de59a49 100755 --- a/app.js +++ b/app.js @@ -92,6 +92,7 @@ global.wikiMap = require("./app/map.js") global.wikiPage = require("./app/page.js") global.wikiContents = require('./app/content.js') global.utility = require("./app/utility.js") +global.searchEngine = require("./app/search.js"); logger.debug(`Running in directory ${EXECUTION_ROOT} with application root ${APPLICATION_ROOT} and wiki path: ${WIKI_PATH}`) logger.debug("Parchment currently running as OS user: "+require('os').userInfo().username) diff --git a/app/express.js b/app/express.js index c4a2184..d3dbf89 100755 --- a/app/express.js +++ b/app/express.js @@ -39,6 +39,7 @@ module.exports = function(port){ const routes = [ {name:"read/*", isProtected: false}, {name:"write/*", isProtected: true}, + {name:"search", isProtected: false} ] // All routes diff --git a/app/layout/default.pug b/app/layout/default.pug index fa84b41..5abc143 100755 --- a/app/layout/default.pug +++ b/app/layout/default.pug @@ -22,6 +22,7 @@ html title #{website.name} link(rel="stylesheet", href="/res/css/font-awesome-all.css") link(rel="stylesheet", href="/res/css/highlight-js.min.css") + script(src="/res/js/jquery.min.js") block head link(rel="stylesheet", href=website.links.css) link(rel="icon", type="image/png", href=website.links.favicon) @@ -48,6 +49,12 @@ html label(for="pass") Pass input(type="password", name="pass") input(type="submit", value="🡆") + + + form.searchBar(method="GET", action="/search") + input(type="text", name="term", placeholder="Search...") + input(type="submit", value="🔍") + include elements/header.pug .mainContainer .navigation diff --git a/app/map.js b/app/map.js index b6f9ef4..819948e 100755 --- a/app/map.js +++ b/app/map.js @@ -22,7 +22,7 @@ async function updateTree(){ if (isOperating) await new Promise(resolve => bus.once('unlocked', resolve)); isOperating = true; - tree = await scanDirectory( WIKI_PATH) + tree = await scanDirectory(WIKI_PATH) isOperating = false; bus.emit('unlocked'); @@ -40,6 +40,7 @@ async function scanDirectory(dirPath){ const fullPath = fullPathNoExt+".md"; const contents = await readFile(fullPath) const virtualPath = fullPath.replace(WIKI_PATH, ""); + const meta = markdown.parseMeta(contents.toString()); entries[virtualPath] = { url: virtualPath, @@ -53,11 +54,13 @@ async function scanDirectory(dirPath){ .replace(".git", "/blob/"+process.env.GIT_REPO_BRANCH) + "/" + virtualPath, - name: markdown.parseMeta(contents.toString()).title || name, + name: meta.title || name, children: fs.existsSync(fullPathNoExt) ? await scanDirectory(fullPathNoExt) : false } pages[virtualPath] = entries[virtualPath] + searchEngine.updateIndexForPageIfNecessary(virtualPath, entries[virtualPath].name, contents.toString()); + logger.debug("Added "+cleanName+" to "+virtualPath) } return entries diff --git a/app/page.js b/app/page.js index d6c7108..569202d 100755 --- a/app/page.js +++ b/app/page.js @@ -26,8 +26,9 @@ async function addPage(virtualPath, contents){ const fileName = elems[elems.length-1] logger.info("Adding page "+fileName+"...") mkdirp.sync(diskPath.substring(0, diskPath.lastIndexOf('/'))) - - await writeFile(diskPath, contents) + + await writeFile(diskPath, contents); + searchEngine.updateIndexForPage(virtualPath, markdown.parseMeta(contents).title, contents); // This will update tree await git.checkAndUploadModifications("Updated "+fileName) @@ -54,7 +55,9 @@ async function destroyPage(virtualPath){ const fileName = elems[elems.length-1] logger.info("Destroying page "+fileName+"...") - const mdPath = path.join(EXECUTION_ROOT, diskPath) + searchEngine.removePageFromIndex(virtualPath); + + const mdPath = diskPath; const dirPath = mdPath.substring(0, mdPath.length-3); if (fs.existsSync(dirPath)){ diff --git a/app/routes/search.js b/app/routes/search.js new file mode 100755 index 0000000..9ddca46 --- /dev/null +++ b/app/routes/search.js @@ -0,0 +1,42 @@ +const maxSummaryLength = 40; +const backwardLength = 8; + +module.exports = async function (req, response){ + + const body = req.query; + const miss = utility.getMissingFields(body, ["term"]) + + if (miss.length > 0 || body.term.length <= 0){ + response.error = { + data: 'No search term was given' + }; + + return response; + } + + const results = await searchEngine.search(body.term); + + for (i in results){ + let wholePage = results[i].content; + let summary = wholePage + .slice(Math.max(0, results[i].position-backwardLength)); + + const originalLength = summary.length; + + summary = summary + .slice(0, Math.min(maxSummaryLength, summary.length)) + .join(" "); + + if (summary.length < originalLength){ + summary += "..."; + } + + results[i].summary = summary; + } + + response.searchTerm = body.term; + response.results = results; + + return response; +} + \ No newline at end of file diff --git a/app/search.js b/app/search.js new file mode 100755 index 0000000..6fd860b --- /dev/null +++ b/app/search.js @@ -0,0 +1,90 @@ + +const minWordLength = 4; +let index = {}; + +module.exports = { + updateIndexForPage: updateIndexForPage, + updateIndexForPageIfNecessary: updateIndexForPageIfNecessary, + removePageFromIndex: removePageFromIndex, + search: search +} + +async function updateIndexForPageIfNecessary(virtualPath, title, content){ + if (index[virtualPath] == undefined || index[virtualPath].length != content.length){ + logger.debug(`Updating search index for virtual path ${virtualPath}`); + await updateIndexForPage(virtualPath, title, content); + } +} + +async function updateIndexForPage(virtualPath, title, content){ + if (index[virtualPath] != undefined){ + removePageFromIndex(virtualPath); + } + + let entry = { + length: content.length, + title: title, + words: [] + }; + + let pageWords = content + .replace(/[^A-Za-zÀ-ÖØ-öø-ÿ ]/gi, '') + .split(" ") + .filter((o) => o.length >= minWordLength) + .map((o) => o.toLowerCase()); + + entry.words = pageWords; + + index[virtualPath] = entry; +} + +function removePageFromIndex(virtualPath){ + logger.debug(`Removing search index for virtual path ${virtualPath}`); + delete index[virtualPath]; +} + +function search(term){ + const microtime = (Date.now() % 1000) / 1000; + logger.info(`Searching for term [${term}] in the index...`); + + return new Promise((resolve, reject) => { + term = term.toLowerCase(); + let results = []; + for (path in index){ + if (index[path].title.includes(term)){ + + results.push({ + virtualPath: path, + title: index[path].title, + content: index[path].words, + position: 0 + }); + + continue; + } + + for(i in index[path].words){ + const word = index[path].words[i]; + + if (word.length < term.length){ + continue; + } + + if (word.includes(term)){ + results.push({ + virtualPath: path, + title: index[path].title, + content: index[path].words + .map((o)=>{ return o == word ? `${word}` : o; }), + position: i + }); + + break; + } + } + } + + logger.info(`Found ${results.length} for search [${term}] after ${((Date.now() % 1000) / 1000) - microtime}ms`); + resolve(results); + }); +} diff --git a/app/views/read.pug b/app/views/read.pug index 63fda8c..4229642 100755 --- a/app/views/read.pug +++ b/app/views/read.pug @@ -15,7 +15,6 @@ mixin create_toc(tree, depth=0) +create_toc(anchor.nodes, depth+1) block head - script(src="/res/js/jquery.min.js") script(src="/res/js/loginForm.js") block content diff --git a/app/views/search.pug b/app/views/search.pug new file mode 100755 index 0000000..4d49857 --- /dev/null +++ b/app/views/search.pug @@ -0,0 +1,11 @@ +extends ../layout/default.pug + +block head + script(src="/res/js/loginForm.js") + +block content + h1 Results for "#{searchTerm}" + each result in results + h2 + a(href="/read/"+result.virtualPath) #{result.title} + p !{result.summary} diff --git a/public/res/css/style.css b/public/res/css/style.css index 1d57360..8dc412e 100755 --- a/public/res/css/style.css +++ b/public/res/css/style.css @@ -57,31 +57,45 @@ body{ mix-blend-mode: difference; } -.header .userBar{ +.header .userBar, .header .searchBar{ font-style:italic; color:LIGHT; position:absolute; right:20PX; - BOTTOM:0; + TOP:0; margin:6px; text-align:right; } -.header .userBar input{ +.header .searchBar{ + top: auto; + bottom:0; +} + +.header input{ width:120px; margin-left:10px; margin-right:10px; background:rgba(0,0,0,0.4); - border:1px solid LIGHTEST; + border:1px solid DARK; color: LIGHT; font-family: inherit; padding-left:4px; } -.header .userBar input[type=submit]{ +.header .searchBar input{ + width:10vw; + border:1px solid LIGHTEST; +} + +.header input[type=submit]{ padding-left:10px; padding-right:10px; - width:min-content; + width:40px; +} + +.header input[type=submit][name="logout"]{ + width:6vw; } .header #loginError{