Skip to content

Commit

Permalink
Refactor docs to use ExDocs search_data model for global HexDocs inde…
Browse files Browse the repository at this point in the history 
…xing of Gleam packages
  • Loading branch information
@diemogebhardt
diemogebhardt committed Dec 10, 2024
1 parent 475debf commit 9725204
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 61 deletions.
115 changes: 78 additions & 37 deletions compiler-core/src/docs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ pub fn generate_html<IO: FileSystemReader>(

let mut files = vec![];

let mut search_indexes = vec![];
let mut search_items = vec![];

let modules_links: Vec<_> = modules
.clone()
Expand Down Expand Up @@ -148,11 +148,12 @@ pub fn generate_html<IO: FileSystemReader>(
content: Content::Text(temp.render().expect("Page template rendering")),
});

search_indexes.push(SearchIndex {
doc: config.name.to_string(),
search_items.push(SearchItem {
type_: SearchItemType::Page,
parent_title: config.name.to_string(),
title: config.name.to_string(),
content,
url: page.path.to_string(),
doc: content,
ref_: page.path.to_string(),
})
}

Expand Down Expand Up @@ -213,50 +214,54 @@ pub fn generate_html<IO: FileSystemReader>(
})
.join("\n");

search_indexes.push(SearchIndex {
doc: module.name.to_string(),
search_items.push(SearchItem {
type_: SearchItemType::Type,
parent_title: module.name.to_string(),
title: type_.name.to_string(),
content: format!(
doc: format!(
"{}\n{}\n{}\n{}",
type_.definition,
type_.text_documentation,
constructors,
import_synonyms(&module.name, type_.name)
),
url: format!("{}.html#{}", module.name, type_.name),
ref_: format!("{}.html#{}", module.name, type_.name),
})
});
constants.iter().for_each(|constant| {
search_indexes.push(SearchIndex {
doc: module.name.to_string(),
search_items.push(SearchItem {
type_: SearchItemType::Constant,
parent_title: module.name.to_string(),
title: constant.name.to_string(),
content: format!(
doc: format!(
"{}\n{}\n{}",
constant.definition,
constant.text_documentation,
import_synonyms(&module.name, constant.name)
),
url: format!("{}.html#{}", module.name, constant.name),
ref_: format!("{}.html#{}", module.name, constant.name),
})
});
functions.iter().for_each(|function| {
search_indexes.push(SearchIndex {
doc: module.name.to_string(),
search_items.push(SearchItem {
type_: SearchItemType::Function,
parent_title: module.name.to_string(),
title: function.name.to_string(),
content: format!(
doc: format!(
"{}\n{}\n{}",
function.signature,
function.text_documentation,
import_synonyms(&module.name, function.name)
),
url: format!("{}.html#{}", module.name, function.name),
ref_: format!("{}.html#{}", module.name, function.name),
})
});
search_indexes.push(SearchIndex {
doc: module.name.to_string(),
search_items.push(SearchItem {
type_: SearchItemType::Module,
parent_title: module.name.to_string(),
title: module.name.to_string(),
content: documentation_content,
url: format!("{}.html", module.name),
doc: documentation_content,
ref_: format!("{}.html", module.name),
});

let page_title = format!("{} · {} · v{}", name, config.name, config.version);
Expand Down Expand Up @@ -358,20 +363,27 @@ pub fn generate_html<IO: FileSystemReader>(
),
});

// lunr.min.js, search-data.js and index.js:
// lunr.min.js, search-data.js, search-data.json and index.js

files.push(OutputFile {
path: Utf8PathBuf::from("js/lunr.min.js"),
content: Content::Text(std::include_str!("../templates/docs-js/lunr.min.js").to_string()),
});

let search_data_json = serde_to_string(&SearchData {
items: escape_html_contents(search_items),
proglang: SearchProgLang::Gleam,
})
.expect("search index serialization");

files.push(OutputFile {
path: Utf8PathBuf::from("search-data.js"),
content: Content::Text(format!(
"window.Gleam.initSearch({});",
serde_to_string(&escape_html_contents(search_indexes))
.expect("search index serialization")
)),
content: Content::Text(format!("window.Gleam.initSearch({});", search_data_json)),
});

files.push(OutputFile {
path: Utf8PathBuf::from("search-data.json"),
content: Content::Text(search_data_json.to_string()),
});

files.push(OutputFile {
Expand Down Expand Up @@ -505,16 +517,17 @@ fn escape_html_content(it: String) -> String {
.replace('\'', "&#39;")
}

fn escape_html_contents(indexes: Vec<SearchIndex>) -> Vec<SearchIndex> {
fn escape_html_contents(indexes: Vec<SearchItem>) -> Vec<SearchItem> {
indexes
.into_iter()
.map(|idx| SearchIndex {
doc: idx.doc,
.map(|idx| SearchItem {
type_: idx.type_,
parent_title: idx.parent_title,
title: idx.title,
content: escape_html_content(idx.content),
url: idx.url,
doc: escape_html_content(idx.doc),
ref_: idx.ref_,
})
.collect::<Vec<SearchIndex>>()
.collect::<Vec<SearchItem>>()
}

fn import_synonyms(parent: &str, child: &str) -> String {
Expand Down Expand Up @@ -824,9 +837,37 @@ struct ModuleTemplate<'a> {
}

#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord, Clone)]
struct SearchIndex {
doc: String,
struct SearchData {
items: Vec<SearchItem>,
proglang: SearchProgLang,
}

#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord, Clone)]
struct SearchItem {
#[serde(rename = "type")]
type_: SearchItemType,
#[serde(rename = "parentTitle")]
parent_title: String,
title: String,
content: String,
url: String,
doc: String,
#[serde(rename = "ref")]
ref_: String,
}

#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord, Clone)]
#[serde(rename_all = "lowercase")]
enum SearchItemType {
Constant,
Function,
Module,
Page,
Type,
}

#[derive(Serialize, PartialEq, Eq, PartialOrd, Ord, Clone)]
#[serde(rename_all = "lowercase")]
enum SearchProgLang {
// Elixir,
// Erlang,
Gleam,
}
50 changes: 26 additions & 24 deletions compiler-core/templates/docs-js/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ window.Gleam = (function () {
else el.addEventListener(type, handler);
};

const searchLoaded = function (index, docs) {
const searchLoaded = function (index, searchItems) {
const preview_words_after = 10;
const preview_words_before = 5;
const previews = 3;
Expand Down Expand Up @@ -279,13 +279,13 @@ window.Gleam = (function () {
}

function addResult(resultsList, result) {
const doc = docs[result.ref];
const searchItem = searchItems[result.ref];
const resultsListItem = document.createElement("li");
resultsListItem.classList.add("search-results-list-item");
resultsList.appendChild(resultsListItem);
const resultLink = document.createElement("a");
resultLink.classList.add("search-result");
resultLink.setAttribute("href", `${window.unnest}/${doc.url}`);
resultLink.setAttribute("href", `${window.unnest}/${searchItem.ref}`);
resultsListItem.appendChild(resultLink);
const resultTitle = document.createElement("div");
resultTitle.classList.add("search-result-title");
Expand All @@ -297,14 +297,14 @@ window.Gleam = (function () {
resultTitle.appendChild(resultDoc);
const resultDocTitle = document.createElement("div");
resultDocTitle.classList.add("search-result-doc-title");
resultDocTitle.innerHTML = doc.doc;
resultDocTitle.innerHTML = searchItem.parentTitle;
resultDoc.appendChild(resultDocTitle);
let resultDocOrSection = resultDocTitle;
if (doc.doc != doc.title) {
if (searchItem.parentTitle != searchItem.title) {
resultDoc.classList.add("search-result-doc-parent");
const resultSection = document.createElement("div");
resultSection.classList.add("search-result-section");
resultSection.innerHTML = doc.title;
resultSection.innerHTML = searchItem.title;
resultTitle.appendChild(resultSection);
resultDocOrSection = resultSection;
}
Expand All @@ -328,11 +328,11 @@ window.Gleam = (function () {
let ellipsesBefore = true;
let ellipsesAfter = true;
for (let k = 0; k < preview_words_before; k++) {
const nextSpace = doc.content.lastIndexOf(
const nextSpace = searchItem.doc.lastIndexOf(
" ",
previewStart - 2
);
const nextDot = doc.content.lastIndexOf(". ", previewStart - 2);
const nextDot = searchItem.doc.lastIndexOf(". ", previewStart - 2);
if (nextDot >= 0 && nextDot > nextSpace) {
previewStart = nextDot + 1;
ellipsesBefore = false;
Expand All @@ -346,15 +346,15 @@ window.Gleam = (function () {
previewStart = nextSpace + 1;
}
for (let k = 0; k < preview_words_after; k++) {
const nextSpace = doc.content.indexOf(" ", previewEnd + 1);
const nextDot = doc.content.indexOf(". ", previewEnd + 1);
const nextSpace = searchItem.doc.indexOf(" ", previewEnd + 1);
const nextDot = searchItem.doc.indexOf(". ", previewEnd + 1);
if (nextDot >= 0 && nextDot < nextSpace) {
previewEnd = nextDot;
ellipsesAfter = false;
break;
}
if (nextSpace < 0) {
previewEnd = doc.content.length;
previewEnd = searchItem.doc.length;
ellipsesAfter = false;
break;
}
Expand All @@ -377,9 +377,9 @@ window.Gleam = (function () {
resultDocOrSection.innerHTML = "";
addHighlightedText(
resultDocOrSection,
doc.title,
searchItem.title,
0,
doc.title.length,
searchItem.title.length,
titlePositions
);
}
Expand Down Expand Up @@ -416,7 +416,7 @@ window.Gleam = (function () {
const resultPreviews = document.createElement("div");
resultPreviews.classList.add("search-result-previews");
resultLink.appendChild(resultPreviews);
const content = doc.content;
const content = searchItem.doc;
for (
let j = 0;
j < Math.min(previewPositions.length, previews);
Expand All @@ -443,7 +443,7 @@ window.Gleam = (function () {
}
const resultRelUrl = document.createElement("span");
resultRelUrl.classList.add("search-result-rel-url");
resultRelUrl.innerText = doc.url;
resultRelUrl.innerText = searchItem.ref;
resultTitle.appendChild(resultRelUrl);
}

Expand Down Expand Up @@ -542,28 +542,30 @@ window.Gleam = (function () {
});
};

self.initSearch = function initSeach(docs) {
self.initSearch = function initSeach(searchData) {
// enable support for hyphenated search words
lunr.tokenizer.separator = /[\s/]+/;

const index = lunr(function () {
this.ref("id");
// this.field("type");
// this.field("parentTitle");
this.field("title", { boost: 200 });
this.field("content", { boost: 2 });
this.field("url");
this.field("doc", { boost: 2 });
this.field("ref");
this.metadataWhitelist = ["position"];

for (let [i, entry] of docs.entries()) {
for (let [i, entry] of searchData.items.entries()) {
this.add({
id: i,
// type: entry.type,
parentTitle: entry.parentTitle,
title: entry.title,
content: entry.content,
url: `${window.unnest}/${entry.url}`,
doc: entry.doc,
ref: `${window.unnest}/${entry.ref}`,
});
}
});

searchLoaded(index, docs);
searchLoaded(index, searchData.items);
};

const init = function () {
Expand Down

0 comments on commit 9725204

Please sign in to comment.