Skip to content

Commit

Permalink
refactor: simplified the parsing of the tree without using a transformer
Browse files Browse the repository at this point in the history
  • Loading branch information
@ovflowd
ovflowd committed Jul 9, 2024
1 parent 19c4a01 commit 410770e
Show file tree
Hide file tree
Showing 3 changed files with 137 additions and 162 deletions.
29 changes: 21 additions & 8 deletions src/metadata.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
* to many files to create a full Navigation for a given version of the API
*
* @param {InstanceType<typeof import('github-slugger').default>} slugger A GitHub Slugger
* @param {ReturnType<typeof remark>} remarkProcessor A Remark processor
*/
const createMetadata = slugger => {
const createMetadata = (slugger, remarkProcessor) => {
// This holds a temporary buffer of raw metadata before being
// transformed into NavigationEntries and MetadataEntries
const internalMetadata = {
Expand Down Expand Up @@ -53,7 +54,7 @@ const createMetadata = slugger => {
* as it can be manipulated outside of the scope of the generation of the content
*
* @param {string} apiDoc The name of the API doc
* @param {import('vfile').VFile} section The content of the current Metadata entry
* @param {import('unist').Parent} section An AST tree containing the Nodes of the API doc entry section
* @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries
*/
create: (apiDoc, section) => {
Expand All @@ -76,27 +77,39 @@ const createMetadata = slugger => {
internalMetadata.heading.type =
yaml_type || internalMetadata.heading.type;

// A metadata entry is all the metadata we have about a certain API section
// with the content being a VFile (Virtual File) containing the Markdown content
section.data = {
const apiEntryMetadata = {
// The API file basename (without the extension)
api: yaml_name || apiDoc,
// The path/slug of the API section
slug: `${apiDoc}.html${slugHash}`,
// The source link of said API section
sourceLink: source_link,
// The latest update to an API section
// The latest updates to an API section
updates,
// The full-changeset to an API section
changes,
// The Heading metadata
heading: internalMetadata.heading,
// The Stability Index of the API section
stability: stability_index,
// The AST tree of the API section
content: section,
};

// Returns the updated VFile with the extra metadata
return section;
// Returns the Metadata entry for the API doc
return {
// Appends the base Metadata entry
...apiEntryMetadata,

// Overrides the toJSON method to allow for custom serialization
toJSON: () => ({
...apiEntryMetadata,

// We stringify the AST tree to a string
// since this is what we wanbt to render within a JSON object
content: remarkProcessor.stringify(section),
}),
};
},
};
};
Expand Down
243 changes: 102 additions & 141 deletions src/parser.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
'use strict';

import { VFile } from 'vfile';

import { remark } from 'remark';
import remarkGfm from 'remark-gfm';

Expand All @@ -16,16 +14,13 @@ import createQueries from './queries.mjs';

import { createNodeSlugger } from './utils/slugger.mjs';

// Retrieves an unfrozen instance of the Remark processor with GFM support
const getRemarkProcessor = () => remark().use(remarkGfm);

/**
* Creates an API doc parser for a given Markdown API doc file
*/
const createParser = () => {
// Creates an instance of the Remark processor with GFM support
// which is used for stringifying the AST tree back to Markdown
const defaultRemarkGfmProcessor = getRemarkProcessor();
const remarkGfmProcessor = remark().use(remarkGfm);

const {
updateLinkReference,
Expand All @@ -36,132 +31,6 @@ const createParser = () => {
addStabilityIndexMetadata,
} = createQueries();

/**
* This creates a Unified Plugin (Transformer) for parsing the source API doc file
* with numerous transformations into API doc Metadata entries.
*
* This transformer iterates on several sort of AST Nodes and applies transformations
* and then grabs a subtree of Nodes (grouped by a Heading Node) and then stringifies them into a VFile
* and finally creates a Metadata entry for the given section.
*
* @type {import('unified').Plugin<[{
* apiDoc: import('vfile').VFile,
* metadatas: Array<import('./types.d.ts').ApiDocMetadataEntry>,
* slugger: ReturnType<import('./utils/slugger.mjs').createNodeSlugger>
* }]>}
*/
const apiDocTransformer = ({ apiDoc, metadatas, slugger }) => {
/**
* Iterates through the AST tree and creates Metadata entries for each API doc section
*
* @param {import('unist').Parent} tree The root AST tree for the API doc file
*/
return tree => {
// Get all Markdown Footnote definitions from the tree
const markdownDefinitions = selectAll('definition', tree);

// Handles Link References
visit(tree, createQueries.UNIST.isLinkReference, node => {
updateLinkReference(node, markdownDefinitions);

return SKIP;
});

// Removes all the original definitions from the tree as they are not needed
// anymore, since all link references got updated to be plain links
remove(tree, markdownDefinitions);

// Handles API type references transformation into links
visit(tree, createQueries.UNIST.isTextWithType, node => {
updateTypeToReferenceLink(node);

return SKIP;
});

// Handles normalisation of Markdown URLs
visit(tree, createQueries.UNIST.isMarkdownUrl, node => {
updateMarkdownLink(node);

return SKIP;
});

visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => {
// Creates a new Metadata entry for the current API doc file
const apiEntryMetadata = createMetadata(slugger);

// Adds the Metadata of the current Heading Node to the Metadata entry
addHeadingMetadata(headingNode, apiEntryMetadata);

// We retrieve the immediate next Heading if it exists
// This is used for ensuring that we don't include items that would
// belong only to the next heading to the current Heading metadata
// Note that if there is no next heading, we use the current node as the next one
const nextHeadingNode =
findAfter(tree, index, createQueries.UNIST.isHeadingNode) ??
headingNode;

// This is the cutover index of the subtree that we should get
// of all the Nodes within the AST tree that belong to this section
// If `next` is equals the current heading, it means ther's no next heading
// and we are reaching the end of the document, hence the cutover should be the end of
// the document itself,
// Note.: This index needs to be retrieved after any modification to the tree occurs,
// otherwise the index will be off (out of sync with the tree)
const stop =
headingNode === nextHeadingNode
? tree.children.length - 1
: tree.children.indexOf(nextHeadingNode);

// Retrieves all the Nodes that should belong to the current API doc section
// `index + 1` is used to skip the current Heading Node
const subtree = u('root', tree.children.slice(index + 1, stop));

// Visits all Stability Index Nodes from the current subtree if there's any
// and then apply the Stability Index Metadata to the current Metadata entry
visit(subtree, createQueries.UNIST.isStabilityIndex, node => {
// Adds the Stability Index Metadata to the current Metadata entry
addStabilityIndexMetadata(node, apiEntryMetadata);

return SKIP;
});

// Visits all YAML Nodes from the current subtree if there's any
// and then apply the YAML Metadata to the current Metadata entry
visit(subtree, createQueries.UNIST.isYamlNode, node => {
// Adds the YAML Metadata to the current Metadata entry
addYAMLMetadata(node, apiEntryMetadata);

return SKIP;
});

// Removes already parsed items from the subtree so that they aren't included in the final content
remove(subtree, [
createQueries.UNIST.isStabilityIndex,
createQueries.UNIST.isYamlNode,
]);

// The stringified (back to Markdown) content of the section
const parsedSection = defaultRemarkGfmProcessor.stringify(
subtree,
apiDoc
);

// We seal and create the API doc entry Metadata and push them to the collection
// Creates the API doc entry Metadata and pushes it to the collection
const parsedApiEntryMetadata = apiEntryMetadata.create(
apiDoc.stem,
// Creates a VFile for the current section content
new VFile({ ...apiDoc, value: parsedSection })
);

// We push the parsed API doc entry Metadata to the collection
metadatas.push(parsedApiEntryMetadata);

return SKIP;
});
};
};

/**
* Parses a given API doc metadata file into a list of Metadata entries
*
Expand All @@ -184,18 +53,110 @@ const createParser = () => {
// hence we want to ensure that it first resolves before we pass it to the parser
const resolvedApiDoc = await Promise.resolve(apiDoc);

// Creates a new Remark processor with GFM (GitHub Flavoured Markdown) support
const apiDocProcessor = getRemarkProcessor().use(apiDocTransformer, {
apiDoc: resolvedApiDoc,
metadatas: metadataCollection,
slugger: createNodeSlugger(),
});
// Creates a new Slugger instance for the current API doc file
const nodeSlugger = createNodeSlugger();

// Parses the API doc into an AST tree using `unified` and `remark`
const apiDocTree = apiDocProcessor.parse(resolvedApiDoc);
const tree = remarkGfmProcessor.parse(resolvedApiDoc);

// Get all Markdown Footnote definitions from the tree
const markdownDefinitions = selectAll('definition', tree);

// Handles Link References
visit(tree, createQueries.UNIST.isLinkReference, node => {
updateLinkReference(node, markdownDefinitions);

return SKIP;
});

// Removes all the original definitions from the tree as they are not needed
// anymore, since all link references got updated to be plain links
remove(tree, markdownDefinitions);

// Handles API type references transformation into links
visit(tree, createQueries.UNIST.isTextWithType, node => {
updateTypeToReferenceLink(node);

return SKIP;
});

// Applies the AST transformers defined before to the API doc tree
await apiDocProcessor.run(apiDocTree);
// Handles normalisation of Markdown URLs
visit(tree, createQueries.UNIST.isMarkdownUrl, node => {
updateMarkdownLink(node);

return SKIP;
});

visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => {
// Creates a new Metadata entry for the current API doc file
const apiEntryMetadata = createMetadata(nodeSlugger, remarkGfmProcessor);

// Adds the Metadata of the current Heading Node to the Metadata entry
addHeadingMetadata(headingNode, apiEntryMetadata);

// We retrieve the immediate next Heading if it exists
// This is used for ensuring that we don't include items that would
// belong only to the next heading to the current Heading metadata
// Note that if there is no next heading, we use the current node as the next one
const nextHeadingNode =
findAfter(tree, index, createQueries.UNIST.isHeadingNode) ??
headingNode;

// This is the cutover index of the subtree that we should get
// of all the Nodes within the AST tree that belong to this section
// If `next` is equals the current heading, it means ther's no next heading
// and we are reaching the end of the document, hence the cutover should be the end of
// the document itself,
// Note.: This index needs to be retrieved after any modification to the tree occurs,
// otherwise the index will be off (out of sync with the tree)
const stop =
headingNode === nextHeadingNode
? tree.children.length - 1
: tree.children.indexOf(nextHeadingNode);

// Retrieves all the Nodes that should belong to the current API doc section
// `index + 1` is used to skip the current Heading Node
const subtree = u('root', tree.children.slice(index + 1, stop));

// Visits all Stability Index Nodes from the current subtree if there's any
// and then apply the Stability Index Metadata to the current Metadata entry
visit(subtree, createQueries.UNIST.isStabilityIndex, node => {
// Adds the Stability Index Metadata to the current Metadata entry
addStabilityIndexMetadata(node, apiEntryMetadata);

return SKIP;
});

// Visits all YAML Nodes from the current subtree if there's any
// and then apply the YAML Metadata to the current Metadata entry
visit(subtree, createQueries.UNIST.isYamlNode, node => {
// Adds the YAML Metadata to the current Metadata entry
addYAMLMetadata(node, apiEntryMetadata);

return SKIP;
});

// Removes already parsed items from the subtree so that they aren't included in the final content
remove(subtree, [
createQueries.UNIST.isStabilityIndex,
createQueries.UNIST.isYamlNode,
]);

// We seal and create the API doc entry Metadata and push them to the collection
// Creates the API doc entry Metadata and pushes it to the collection
const parsedApiEntryMetadata = apiEntryMetadata.create(
resolvedApiDoc.stem,
// Applies the AST transformations to the subtree based on the API doc entry Metadata
// Note that running the transformation on the subtree isn't costly as it is a reduced tree
// and the GFM transformations aren't that heavy
remarkGfmProcessor.runSync(subtree)
);

// We push the parsed API doc entry Metadata to the collection
metadataCollection.push(parsedApiEntryMetadata);

return SKIP;
});

// Returns the Metadata entries for the given API doc file
return metadataCollection;
Expand Down
27 changes: 14 additions & 13 deletions src/types.d.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { VFile } from 'vfile';
import { Parent } from 'unist';

export interface StabilityIndexMetadataEntry {
index: number;
Expand Down Expand Up @@ -29,7 +29,7 @@ export interface ApiDocMetadataChange {
}

export interface ApiDocMetadataUpdate {
// The type of the API Doc Metadata update
// The type of the API doc Metadata update
type: 'added' | 'removed' | 'deprecated' | 'introduced_in' | 'napiVersion';
// The Node.js version or versions where said metadata stability index changed
version: string[];
Expand All @@ -44,23 +44,24 @@ export interface ApiDocRawMetadataEntry {
stability_index?: StabilityIndexMetadataEntry;
}

export interface ApiDocNavigationEntry {
// The name of the API Doc file without the file extension (basename)
export interface ApiDocMetadataEntry {
// The name of the API doc file without the file extension (basename)
api: string;
// The unique slug of a Heading/Navigation Entry which is linkable through an anchor
// The unique slug of a Heading/Navigation entry which is linkable through an anchor
slug: string;
// The GitHub URL to the source of the API Entry
// The GitHub URL to the source of the API entry
sourceLink: string | undefined;
// Any updates to the API Doc Metadata
// Any updates to the API doc Metadata
updates: ApiDocMetadataUpdate[];
// Any changes to the API Doc Metadata
// Any changes to the API doc Metadata
changes: ApiDocMetadataChange[];
// The parsed Markdown content of a Navigation Entry
heading: HeadingMetadataEntry;
// The API Doc Metadata Entry Stability Index if exists
// The API doc metadata Entry Stability Index if exists
stability: StabilityIndexMetadataEntry | undefined;
}

export interface ApiDocMetadataEntry extends VFile {
data: ApiDocNavigationEntry;
// The subtree containing all Nodes of the API doc entry
content: Parent;
// String serialization of the AST tree
// @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior
toJSON: () => string;
}

0 comments on commit 410770e

Please sign in to comment.