From 410770e5e9c07278345c4960e5c2050ec83d7cca Mon Sep 17 00:00:00 2001 From: Claudio Wunder Date: Wed, 10 Jul 2024 00:15:44 +0200 Subject: [PATCH] refactor: simplified the parsing of the tree without using a transformer --- src/metadata.mjs | 29 ++++-- src/parser.mjs | 243 ++++++++++++++++++++--------------------------- src/types.d.ts | 27 +++--- 3 files changed, 137 insertions(+), 162 deletions(-) diff --git a/src/metadata.mjs b/src/metadata.mjs index 8a8f8ff..558921a 100644 --- a/src/metadata.mjs +++ b/src/metadata.mjs @@ -8,8 +8,9 @@ * to many files to create a full Navigation for a given version of the API * * @param {InstanceType} slugger A GitHub Slugger + * @param {ReturnType} remarkProcessor A Remark processor */ -const createMetadata = slugger => { +const createMetadata = (slugger, remarkProcessor) => { // This holds a temporary buffer of raw metadata before being // transformed into NavigationEntries and MetadataEntries const internalMetadata = { @@ -53,7 +54,7 @@ const createMetadata = slugger => { * as it can be manipulated outside of the scope of the generation of the content * * @param {string} apiDoc The name of the API doc - * @param {import('vfile').VFile} section The content of the current Metadata entry + * @param {import('unist').Parent} section An AST tree containing the Nodes of the API doc entry section * @returns {import('./types').ApiDocMetadataEntry} The locally created Metadata entries */ create: (apiDoc, section) => { @@ -76,16 +77,14 @@ const createMetadata = slugger => { internalMetadata.heading.type = yaml_type || internalMetadata.heading.type; - // A metadata entry is all the metadata we have about a certain API section - // with the content being a VFile (Virtual File) containing the Markdown content - section.data = { + const apiEntryMetadata = { // The API file basename (without the extension) api: yaml_name || apiDoc, // The path/slug of the API section slug: `${apiDoc}.html${slugHash}`, // The source link of said API section sourceLink: source_link, - // The latest update to an API section + // The latest updates to an API section updates, // The full-changeset to an API section changes, @@ -93,10 +92,24 @@ const createMetadata = slugger => { heading: internalMetadata.heading, // The Stability Index of the API section stability: stability_index, + // The AST tree of the API section + content: section, }; - // Returns the updated VFile with the extra metadata - return section; + // Returns the Metadata entry for the API doc + return { + // Appends the base Metadata entry + ...apiEntryMetadata, + + // Overrides the toJSON method to allow for custom serialization + toJSON: () => ({ + ...apiEntryMetadata, + + // We stringify the AST tree to a string + // since this is what we wanbt to render within a JSON object + content: remarkProcessor.stringify(section), + }), + }; }, }; }; diff --git a/src/parser.mjs b/src/parser.mjs index ffd6c9f..40b062b 100644 --- a/src/parser.mjs +++ b/src/parser.mjs @@ -1,7 +1,5 @@ 'use strict'; -import { VFile } from 'vfile'; - import { remark } from 'remark'; import remarkGfm from 'remark-gfm'; @@ -16,16 +14,13 @@ import createQueries from './queries.mjs'; import { createNodeSlugger } from './utils/slugger.mjs'; -// Retrieves an unfrozen instance of the Remark processor with GFM support -const getRemarkProcessor = () => remark().use(remarkGfm); - /** * Creates an API doc parser for a given Markdown API doc file */ const createParser = () => { // Creates an instance of the Remark processor with GFM support // which is used for stringifying the AST tree back to Markdown - const defaultRemarkGfmProcessor = getRemarkProcessor(); + const remarkGfmProcessor = remark().use(remarkGfm); const { updateLinkReference, @@ -36,132 +31,6 @@ const createParser = () => { addStabilityIndexMetadata, } = createQueries(); - /** - * This creates a Unified Plugin (Transformer) for parsing the source API doc file - * with numerous transformations into API doc Metadata entries. - * - * This transformer iterates on several sort of AST Nodes and applies transformations - * and then grabs a subtree of Nodes (grouped by a Heading Node) and then stringifies them into a VFile - * and finally creates a Metadata entry for the given section. - * - * @type {import('unified').Plugin<[{ - * apiDoc: import('vfile').VFile, - * metadatas: Array, - * slugger: ReturnType - * }]>} - */ - const apiDocTransformer = ({ apiDoc, metadatas, slugger }) => { - /** - * Iterates through the AST tree and creates Metadata entries for each API doc section - * - * @param {import('unist').Parent} tree The root AST tree for the API doc file - */ - return tree => { - // Get all Markdown Footnote definitions from the tree - const markdownDefinitions = selectAll('definition', tree); - - // Handles Link References - visit(tree, createQueries.UNIST.isLinkReference, node => { - updateLinkReference(node, markdownDefinitions); - - return SKIP; - }); - - // Removes all the original definitions from the tree as they are not needed - // anymore, since all link references got updated to be plain links - remove(tree, markdownDefinitions); - - // Handles API type references transformation into links - visit(tree, createQueries.UNIST.isTextWithType, node => { - updateTypeToReferenceLink(node); - - return SKIP; - }); - - // Handles normalisation of Markdown URLs - visit(tree, createQueries.UNIST.isMarkdownUrl, node => { - updateMarkdownLink(node); - - return SKIP; - }); - - visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { - // Creates a new Metadata entry for the current API doc file - const apiEntryMetadata = createMetadata(slugger); - - // Adds the Metadata of the current Heading Node to the Metadata entry - addHeadingMetadata(headingNode, apiEntryMetadata); - - // We retrieve the immediate next Heading if it exists - // This is used for ensuring that we don't include items that would - // belong only to the next heading to the current Heading metadata - // Note that if there is no next heading, we use the current node as the next one - const nextHeadingNode = - findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? - headingNode; - - // This is the cutover index of the subtree that we should get - // of all the Nodes within the AST tree that belong to this section - // If `next` is equals the current heading, it means ther's no next heading - // and we are reaching the end of the document, hence the cutover should be the end of - // the document itself, - // Note.: This index needs to be retrieved after any modification to the tree occurs, - // otherwise the index will be off (out of sync with the tree) - const stop = - headingNode === nextHeadingNode - ? tree.children.length - 1 - : tree.children.indexOf(nextHeadingNode); - - // Retrieves all the Nodes that should belong to the current API doc section - // `index + 1` is used to skip the current Heading Node - const subtree = u('root', tree.children.slice(index + 1, stop)); - - // Visits all Stability Index Nodes from the current subtree if there's any - // and then apply the Stability Index Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isStabilityIndex, node => { - // Adds the Stability Index Metadata to the current Metadata entry - addStabilityIndexMetadata(node, apiEntryMetadata); - - return SKIP; - }); - - // Visits all YAML Nodes from the current subtree if there's any - // and then apply the YAML Metadata to the current Metadata entry - visit(subtree, createQueries.UNIST.isYamlNode, node => { - // Adds the YAML Metadata to the current Metadata entry - addYAMLMetadata(node, apiEntryMetadata); - - return SKIP; - }); - - // Removes already parsed items from the subtree so that they aren't included in the final content - remove(subtree, [ - createQueries.UNIST.isStabilityIndex, - createQueries.UNIST.isYamlNode, - ]); - - // The stringified (back to Markdown) content of the section - const parsedSection = defaultRemarkGfmProcessor.stringify( - subtree, - apiDoc - ); - - // We seal and create the API doc entry Metadata and push them to the collection - // Creates the API doc entry Metadata and pushes it to the collection - const parsedApiEntryMetadata = apiEntryMetadata.create( - apiDoc.stem, - // Creates a VFile for the current section content - new VFile({ ...apiDoc, value: parsedSection }) - ); - - // We push the parsed API doc entry Metadata to the collection - metadatas.push(parsedApiEntryMetadata); - - return SKIP; - }); - }; - }; - /** * Parses a given API doc metadata file into a list of Metadata entries * @@ -184,18 +53,110 @@ const createParser = () => { // hence we want to ensure that it first resolves before we pass it to the parser const resolvedApiDoc = await Promise.resolve(apiDoc); - // Creates a new Remark processor with GFM (GitHub Flavoured Markdown) support - const apiDocProcessor = getRemarkProcessor().use(apiDocTransformer, { - apiDoc: resolvedApiDoc, - metadatas: metadataCollection, - slugger: createNodeSlugger(), - }); + // Creates a new Slugger instance for the current API doc file + const nodeSlugger = createNodeSlugger(); // Parses the API doc into an AST tree using `unified` and `remark` - const apiDocTree = apiDocProcessor.parse(resolvedApiDoc); + const tree = remarkGfmProcessor.parse(resolvedApiDoc); + + // Get all Markdown Footnote definitions from the tree + const markdownDefinitions = selectAll('definition', tree); + + // Handles Link References + visit(tree, createQueries.UNIST.isLinkReference, node => { + updateLinkReference(node, markdownDefinitions); + + return SKIP; + }); + + // Removes all the original definitions from the tree as they are not needed + // anymore, since all link references got updated to be plain links + remove(tree, markdownDefinitions); + + // Handles API type references transformation into links + visit(tree, createQueries.UNIST.isTextWithType, node => { + updateTypeToReferenceLink(node); + + return SKIP; + }); - // Applies the AST transformers defined before to the API doc tree - await apiDocProcessor.run(apiDocTree); + // Handles normalisation of Markdown URLs + visit(tree, createQueries.UNIST.isMarkdownUrl, node => { + updateMarkdownLink(node); + + return SKIP; + }); + + visit(tree, createQueries.UNIST.isHeadingNode, (headingNode, index) => { + // Creates a new Metadata entry for the current API doc file + const apiEntryMetadata = createMetadata(nodeSlugger, remarkGfmProcessor); + + // Adds the Metadata of the current Heading Node to the Metadata entry + addHeadingMetadata(headingNode, apiEntryMetadata); + + // We retrieve the immediate next Heading if it exists + // This is used for ensuring that we don't include items that would + // belong only to the next heading to the current Heading metadata + // Note that if there is no next heading, we use the current node as the next one + const nextHeadingNode = + findAfter(tree, index, createQueries.UNIST.isHeadingNode) ?? + headingNode; + + // This is the cutover index of the subtree that we should get + // of all the Nodes within the AST tree that belong to this section + // If `next` is equals the current heading, it means ther's no next heading + // and we are reaching the end of the document, hence the cutover should be the end of + // the document itself, + // Note.: This index needs to be retrieved after any modification to the tree occurs, + // otherwise the index will be off (out of sync with the tree) + const stop = + headingNode === nextHeadingNode + ? tree.children.length - 1 + : tree.children.indexOf(nextHeadingNode); + + // Retrieves all the Nodes that should belong to the current API doc section + // `index + 1` is used to skip the current Heading Node + const subtree = u('root', tree.children.slice(index + 1, stop)); + + // Visits all Stability Index Nodes from the current subtree if there's any + // and then apply the Stability Index Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isStabilityIndex, node => { + // Adds the Stability Index Metadata to the current Metadata entry + addStabilityIndexMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Visits all YAML Nodes from the current subtree if there's any + // and then apply the YAML Metadata to the current Metadata entry + visit(subtree, createQueries.UNIST.isYamlNode, node => { + // Adds the YAML Metadata to the current Metadata entry + addYAMLMetadata(node, apiEntryMetadata); + + return SKIP; + }); + + // Removes already parsed items from the subtree so that they aren't included in the final content + remove(subtree, [ + createQueries.UNIST.isStabilityIndex, + createQueries.UNIST.isYamlNode, + ]); + + // We seal and create the API doc entry Metadata and push them to the collection + // Creates the API doc entry Metadata and pushes it to the collection + const parsedApiEntryMetadata = apiEntryMetadata.create( + resolvedApiDoc.stem, + // Applies the AST transformations to the subtree based on the API doc entry Metadata + // Note that running the transformation on the subtree isn't costly as it is a reduced tree + // and the GFM transformations aren't that heavy + remarkGfmProcessor.runSync(subtree) + ); + + // We push the parsed API doc entry Metadata to the collection + metadataCollection.push(parsedApiEntryMetadata); + + return SKIP; + }); // Returns the Metadata entries for the given API doc file return metadataCollection; diff --git a/src/types.d.ts b/src/types.d.ts index c43b0e9..c5ff16c 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,4 +1,4 @@ -import { VFile } from 'vfile'; +import { Parent } from 'unist'; export interface StabilityIndexMetadataEntry { index: number; @@ -29,7 +29,7 @@ export interface ApiDocMetadataChange { } export interface ApiDocMetadataUpdate { - // The type of the API Doc Metadata update + // The type of the API doc Metadata update type: 'added' | 'removed' | 'deprecated' | 'introduced_in' | 'napiVersion'; // The Node.js version or versions where said metadata stability index changed version: string[]; @@ -44,23 +44,24 @@ export interface ApiDocRawMetadataEntry { stability_index?: StabilityIndexMetadataEntry; } -export interface ApiDocNavigationEntry { - // The name of the API Doc file without the file extension (basename) +export interface ApiDocMetadataEntry { + // The name of the API doc file without the file extension (basename) api: string; - // The unique slug of a Heading/Navigation Entry which is linkable through an anchor + // The unique slug of a Heading/Navigation entry which is linkable through an anchor slug: string; - // The GitHub URL to the source of the API Entry + // The GitHub URL to the source of the API entry sourceLink: string | undefined; - // Any updates to the API Doc Metadata + // Any updates to the API doc Metadata updates: ApiDocMetadataUpdate[]; - // Any changes to the API Doc Metadata + // Any changes to the API doc Metadata changes: ApiDocMetadataChange[]; // The parsed Markdown content of a Navigation Entry heading: HeadingMetadataEntry; - // The API Doc Metadata Entry Stability Index if exists + // The API doc metadata Entry Stability Index if exists stability: StabilityIndexMetadataEntry | undefined; -} - -export interface ApiDocMetadataEntry extends VFile { - data: ApiDocNavigationEntry; + // The subtree containing all Nodes of the API doc entry + content: Parent; + // String serialization of the AST tree + // @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify#tojson_behavior + toJSON: () => string; }