Allow tags to start with a number and be escaped with angle brackets

Includes tests and Markdown/Hashtags page for website. Closes #1072

common/markdown_parser/constants.ts

@@ -1,5 +1,5 @@
 export const wikiLinkRegex = /(!?\[\[)([^\]\|]+)(?:\|([^\]]+))?(\]\])/g; // [fullMatch, firstMark, url, alias, lastMark]
 export const mdLinkRegex = /!?\[(?<title>[^\]]*)\]\((?<url>.+)\)/g; // [fullMatch, alias, url]
 export const tagRegex =
-  /#[^\d\s!@#$%^&*(),.?":{}|<>\\][^\s!@#$%^&*(),.?":{}|<>\\]*/;
+  /#(?:(?:\d*[^\d\s!@#$%^&*(),.?":{}|<>\\][^\s!@#$%^&*(),.?":{}|<>\\]*)|(?:<[^>\n]+>))/;
 export const pWikiLinkRegex = new RegExp("^" + wikiLinkRegex.source); // Modified regex used only in parser

common/markdown_parser/parser.test.ts

@@ -4,7 +4,7 @@ import {
   renderToText,
 } from "@silverbulletmd/silverbullet/lib/tree";
 import { assert, assertEquals, assertNotEquals } from "@std/assert";
-import { parseMarkdown } from "./parser.ts";
+import { hashtagName, hashtagText, parseMarkdown } from "./parser.ts";
 
 const sample1 = `---
 type: page
@@ -162,3 +162,53 @@ Deno.test("Test lua directive parser", () => {
   const simpleExample = `Simple \${{a=}}`;
   console.log(JSON.stringify(parseMarkdown(simpleExample), null, 2));
 });
+
+const hashtagSample = `
+Hashtags, e.g. #mytag but ignore in code \`#mytag\`.
+They can contain slashes like #level/beginner, single quotes, and dashes: #Mike's-idea.
+Can be just #a single letter.
+But no other #interpunction: #exclamation! #question?
+There is a way to write #<tag with spaces>
+These cannot span #<multiple
+lines>
+#no#spacing also works.
+Hashtags can start with number if there's something after it: #3dprint #15-52_Trip-to-NYC.
+But magazine issue #1 or #123 are not hashtags.
+Should support other languages, like #żółć or #井号
+`;
+
+Deno.test("Test hashtag parser", () => {
+  const tree = parseMarkdown(hashtagSample);
+  const hashtags = collectNodesOfType(tree, "Hashtag");
+  assertEquals(hashtags.length, 14);
+
+  assertEquals(hashtags[0].children![0].text, "#mytag");
+  assertEquals(hashtags[1].children![0].text, "#level/beginner");
+  assertEquals(hashtags[2].children![0].text, "#Mike's-idea");
+  assertEquals(hashtags[3].children![0].text, "#a");
+  assertEquals(hashtags[4].children![0].text, "#interpunction");
+  assertEquals(hashtags[5].children![0].text, "#exclamation");
+  assertEquals(hashtags[6].children![0].text, "#question");
+  assertEquals(hashtags[7].children![0].text, "#<tag with spaces>");
+  // multiple lines not allowed
+  assertEquals(hashtags[8].children![0].text, "#no");
+  assertEquals(hashtags[9].children![0].text, "#spacing");
+  assertEquals(hashtags[10].children![0].text, "#3dprint");
+  assertEquals(hashtags[11].children![0].text, "#15-52_Trip-to-NYC");
+  assertEquals(hashtags[12].children![0].text, "#żółć");
+  assertEquals(hashtags[13].children![0].text, "#井号");
+});
+
+Deno.test("Test hashtag helper functions", () => {
+  assertEquals(hashtagName("#name"), "name");
+  assertEquals(hashtagName("#123-content"), "123-content");
+  assertEquals(hashtagName("#<escaped tag>"), "escaped tag");
+  assertEquals(hashtagName("#<allow < and # inside>"), "allow < and # inside");
+
+  assertEquals(hashtagText("simple"), "#simple");
+  assertEquals(hashtagText("123-content"), "#123-content");
+  assertEquals(hashtagText("with spaces"), "#<with spaces>");
+  assertEquals(hashtagText("single'quote"), "#single'quote");
+  // should behave like this for all characters in tagRegex
+  assertEquals(hashtagText("exclamation!"), "#<exclamation!>");
+});

common/markdown_parser/parser.ts

@@ -594,6 +594,32 @@ const Hashtag = regexParser(
   },
 );
 
+/** Extract the name from hashtag text, removing # prefix and <angle brackets> if necessary */
+export function hashtagName(text: string): string {
+  if (text[0] !== "#") { // you shouldn't call this function at all
+    console.error("hashtagName called on already clean string", text);
+    return text;
+  } else if (text[1] === "<") {
+    if (text.slice(-1) !== ">") { // this is malformed: #<name but maybe we're trying to autocomplete
+      return text.slice(2);
+    } else { // this is correct #<name>
+      return text.slice(2, -1);
+    }
+  } else { // this is just #name
+    return text.slice(1);
+  }
+}
+
+/** Get markup for a hashtag name with # prefix and angle brackets if necessary */
+export function hashtagText(name: string): string {
+  // detect with the same regex as the parser
+  const simple: string = "#".concat(name);
+  const match = simple.match(tagRegex);
+  if (!match || match[0].length !== simple.length) {
+    return `#<${name}>`;
+  } else return simple;
+}
+
 const TaskDeadline = regexParser({
   firstCharCode: 55357, // 📅
   regex: /^📅\s*\d{4}\-\d{2}\-\d{2}/,

plug-api/lib/frontmatter.ts

@@ -7,6 +7,7 @@ import {
 } from "./tree.ts";
 import { cleanupJSON } from "./json.ts";
 import { YAML } from "../syscalls.ts";
+import { hashtagName } from "$common/markdown_parser/parser.ts";
 
 export type FrontMatter = { tags?: string[] } & Record<string, any>;
 
@@ -48,7 +49,7 @@ export async function extractFrontmatter(
             break;
           }
         } else if (child.type === "Hashtag") {
-          const tagname = child.children![0].text!.substring(1);
+          const tagname = hashtagName(child.children![0].text!);
           collectedTags.add(tagname);
 
           if (

plug-api/lib/tags.ts

@@ -4,6 +4,7 @@ import {
   type ParseTree,
   traverseTree,
 } from "@silverbulletmd/silverbullet/lib/tree";
+import { hashtagName } from "$common/markdown_parser/parser.ts";
 
 export function updateITags<T>(obj: ObjectValue<T>, frontmatter: FrontMatter) {
   const itags = [obj.tag, ...frontmatter.tags || []];
@@ -26,7 +27,7 @@ export function extractHashTags(n: ParseTree): string[] {
   const tags = new Set<string>();
   traverseTree(n, (n) => {
     if (n.type === "Hashtag") {
-      tags.add(n.children![0].text!.substring(1));
+      tags.add(hashtagName(n.children![0].text!));
       return true;
     } else if (n.type === "OrderedList" || n.type === "BulletList") {
       // Don't traverse into sub-lists

plugs/editor/navigate.ts

@@ -20,6 +20,7 @@ import {
   parsePageRef,
 } from "@silverbulletmd/silverbullet/lib/page_ref";
 import { tagPrefix } from "../index/constants.ts";
+import { hashtagName } from "$common/markdown_parser/parser.ts";
 
 async function actionClickOrActionEnter(
   mdTree: ParseTree | null,
@@ -123,7 +124,7 @@ async function actionClickOrActionEnter(
       break;
     }
     case "Hashtag": {
-      const hashtag = mdTree.children![0].text!.slice(1);
+      const hashtag = hashtagName(mdTree.children![0].text!);
       await editor.navigate(
         { page: `${tagPrefix}${hashtag}`, pos: 0 },
         false,

plugs/index/header.ts

@@ -11,6 +11,7 @@ import type {
 import { indexObjects, queryObjects } from "./api.ts";
 import { parsePageRef } from "@silverbulletmd/silverbullet/lib/page_ref";
 import { extractAttributes } from "@silverbulletmd/silverbullet/lib/attribute";
+import { hashtagName } from "$common/markdown_parser/parser.ts";
 
 type HeaderObject = ObjectValue<
   {
@@ -35,7 +36,7 @@ export async function indexHeaders({ name: pageName, tree }: IndexTreeEvent) {
 
     collectNodesOfType(n, "Hashtag").forEach((h) => {
       // Push tag to the list, removing the initial #
-      tags.add(h.children![0].text!.substring(1));
+      tags.add(hashtagName(h.children![0].text!));
       h.children = [];
     });
 

plugs/index/paragraph.ts

@@ -10,6 +10,7 @@ import { extractAttributes } from "@silverbulletmd/silverbullet/lib/attribute";
 import type { ObjectValue } from "../../plug-api/types.ts";
 import { updateITags } from "@silverbulletmd/silverbullet/lib/tags";
 import { extractFrontmatter } from "@silverbulletmd/silverbullet/lib/frontmatter";
+import { hashtagName } from "$common/markdown_parser/parser.ts";
 
 /** ParagraphObject  An index object for the top level text nodes */
 export type ParagraphObject = ObjectValue<
@@ -40,7 +41,7 @@ export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) {
     // Collect tags and remove from the tree
     const tags = new Set<string>();
     collectNodesOfType(p, "Hashtag").forEach((tagNode) => {
-      tags.add(tagNode.children![0].text!.substring(1));
+      tags.add(hashtagName(tagNode.children![0].text!));
       // Hacky way to remove the hashtag
       tagNode.children = [];
     });

plugs/index/table.ts

@@ -5,6 +5,7 @@ import {
   type ParseTree,
 } from "@silverbulletmd/silverbullet/lib/tree";
 import { indexObjects } from "./api.ts";
+import { hashtagName } from "$common/markdown_parser/parser.ts";
 
 type TableRowObject =
   & ObjectValue<{
@@ -51,7 +52,7 @@ export async function indexTables({ name: pageName, tree }: IndexTreeEvent) {
         const tags = new Set<string>();
         collectNodesOfType(row, "Hashtag").forEach((h) => {
           // Push tag to the list, removing the initial #
-          tags.add(h.children![0].text!.substring(1));
+          tags.add(hashtagName(h.children![0].text!));
         });
 
         const cells = collectNodesOfType(row, "TableCell");

plugs/index/tag_page.ts

@@ -2,6 +2,7 @@ import type { FileMeta } from "../../plug-api/types.ts";
 import { markdown, system } from "@silverbulletmd/silverbullet/syscalls";
 import { renderToText } from "@silverbulletmd/silverbullet/lib/tree";
 import { tagPrefix } from "./constants.ts";
+import { hashtagText } from "$common/markdown_parser/parser.ts";
 
 export async function readFileTag(
   name: string,
@@ -10,7 +11,7 @@ export async function readFileTag(
     tagPrefix.length,
     name.length - ".md".length,
   );
-  const text = `All objects in your space tagged with #${tagName}:
+  const text = `All objects in your space tagged with ${hashtagText(tagName)}:
 \`\`\`template
 template: |
     {{#if .}}

plugs/index/tags.ts

@@ -7,6 +7,7 @@ import {
   findParentMatching,
 } from "@silverbulletmd/silverbullet/lib/tree";
 import type { ObjectValue } from "../../plug-api/types.ts";
+import { hashtagName, hashtagText } from "$common/markdown_parser/parser.ts";
 
 export type TagObject = ObjectValue<{
   name: string;
@@ -22,7 +23,7 @@ export async function indexTags({ name, tree }: IndexTreeEvent) {
     tags.add(`${pageTag}:page`);
   }
   collectNodesOfType(tree, "Hashtag").forEach((h) => {
-    const tagName = h.children![0].text!.substring(1);
+    const tagName = hashtagName(h.children![0].text!);
     // Check if this occurs in the context of a task
     if (findParentMatching(h, (n) => n.type === "Task")) {
       tags.add(`${tagName}:task`);
@@ -58,11 +59,10 @@ export async function tagComplete(completeEvent: CompleteEvent) {
     return null;
   }
 
-  const match = /#[^#\d\s\[\]]+\w*$/.exec(completeEvent.linePrefix);
+  const match = /#[^#\s\[\]]+\w*$/.exec(completeEvent.linePrefix);
   if (!match) {
     return null;
   }
-  const tagPrefix = match[0].substring(1);
 
   // Query all tags with a matching parent
   const allTags: any[] = await queryObjects<TagObject>("tag", {
@@ -71,9 +71,9 @@ export async function tagComplete(completeEvent: CompleteEvent) {
   }, 5);
 
   return {
-    from: completeEvent.pos - tagPrefix.length,
+    from: completeEvent.pos - match[0].length,
     options: allTags.map((tag) => ({
-      label: tag.name,
+      label: hashtagText(tag.name),
       type: "tag",
     })),
   };

website/Markdown/Extensions.md

@@ -10,7 +10,7 @@ In addition to supporting [[Markdown/Basics|markdown basics]] as standardized by
 * [[Transclusions]] syntax
 * [[Markdown/Anchors]]
 * [[Markdown/Admonitions]]
-* Hashtags, e.g. `#mytag`.
+* [[Markdown/Hashtags]]
 * [[Markdown/Command links]] syntax
 * [Tables](https://www.markdownguide.org/extended-syntax/#tables)
 * [Task lists](https://www.markdownguide.org/extended-syntax/#task-lists)

website/Markdown/Hashtags.md

@@ -0,0 +1,14 @@
+#level/beginner
+
+These can be used in text to assign an [[Objects#tag]]. If hashtags are the only content of first paragraph, they are applied to the entire page.
+
+Hashtags can contain letters, dashes, underscores and other characters, but not:
+- Whitespace (space, newline etc.)
+- Characters from this list: `!@#$%^&*(),.?":{}|<>\`
+- Consist of digits only like #123
+
+If you need your tags to contain these characters, you have to surround the tag content with angle brackets like this: #<my tag>
+
+```query
+tag where page = @page.name
+```