Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(#250): handle case sensitive tags #251

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions .changeset/poor-pears-marry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
---
"@bbob/parser": minor
"@bbob/types": minor
"@bbob/cli": minor
"@bbob/core": minor
"@bbob/html": minor
"@bbob/plugin-helper": minor
"@bbob/preset": minor
"@bbob/preset-html5": minor
"@bbob/preset-react": minor
"@bbob/preset-vue": minor
"@bbob/react": minor
"@bbob/vue2": minor
"@bbob/vue3": minor
---

New option flag `caseFreeTags` has been added

This flag allows to parse case insensitive tags like `[h1]some[/H1]` -> `<h1>some</h1>`

```js
import html from '@bbob/html'
import presetHTML5 from '@bbob/preset-html5'

const processed = html(`[h1]some[/H1]`, presetHTML5(), { caseFreeTags: true })

console.log(processed); // <h1>some</h1>
```

Also now you can pass `caseFreeTags` to `parse` function

```js
import { parse } from '@bbob/parser'

const ast = parse('[h1]some[/H1]', {
caseFreeTags: true
});
```

BREAKING CHANGE: `isTokenNested` function now accepts string `tokenValue` instead of `token`

Changed codecov.io to coveralls.io for test coverage
9 changes: 8 additions & 1 deletion .github/workflows/pr.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
name: Pull Request
on:
# workflow_run:
# workflows:
# - Tests
# - Benchmark
# types:
# - completed
pull_request:
paths-ignore:
- '.changeset/**'
- '.husky/**'
workflow_dispatch:

concurrency:
group: ci-pull-request=${{github.ref}}-1
Expand All @@ -30,7 +37,7 @@ jobs:
- name: Set SHA
id: sha
run: |
SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
SHORT_SHA=$(git rev-parse --short "${{ github.event.pull_request.head.sha }}")
echo "short_sha=${SHORT_SHA}" >> $GITHUB_OUTPUT

- name: Install pnpm
Expand Down
7 changes: 2 additions & 5 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,8 @@ jobs:
- name: Run the lint
run: pnpm run lint

- name: Install coverage
run: pnpm install --global codecov

- name: Run the coverage
run: pnpm run cover

- name: Run the coverage
run: codecov
- name: Coveralls
uses: coverallsapp/github-action@v2
28 changes: 25 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ written in pure javascript, no dependencies

[![Tests](https://github.com/JiLiZART/BBob/actions/workflows/test.yml/badge.svg)](https://github.com/JiLiZART/BBob/actions/workflows/test.yml)
[![Benchmark](https://github.com/JiLiZART/BBob/actions/workflows/benchmark.yml/badge.svg)](https://github.com/JiLiZART/BBob/actions/workflows/benchmark.yml)
<a href="https://codecov.io/gh/JiLiZART/bbob">
<img src="https://codecov.io/gh/JiLiZART/bbob/branch/master/graph/badge.svg" alt="codecov">
</a>
<a href='https://coveralls.io/github/JiLiZART/BBob?branch=master'>
<img src='https://coveralls.io/repos/github/JiLiZART/BBob/badge.svg?branch=master' alt='Coverage Status' />
</a>
<a href="https://www.codefactor.io/repository/github/jilizart/bbob">
<img src="https://www.codefactor.io/repository/github/jilizart/bbob/badge" alt="CodeFactor">
</a>
Expand Down Expand Up @@ -230,6 +230,28 @@ const processed = bbobHTML(`[b]Text[/b]'\\[b\\]Text\\[/b\\]'`, presetHTML5(), {
console.log(processed); // <span style="font-weight: bold;">Text</span>[b]Text[/b]
```

#### caseFreeTags

Allows to parse case insensitive tags like `[h1]some[/H1]` -> `<h1>some</h1>`

```js
import bbobHTML from '@bbob/html'
import presetHTML5 from '@bbob/preset-html5'

const processed = bbobHTML(`[h1]some[/H1]`, presetHTML5(), { caseFreeTags: true })

console.log(processed); // <h1>some</h1>
```

```js
import bbobHTML from '@bbob/html'
import presetHTML5 from '@bbob/preset-html5'

const processed = bbobHTML(`[b]Text[/b]'\\[b\\]Text\\[/b\\]'`, presetHTML5(), { enableEscapeTags: true })

console.log(processed); // <span style="font-weight: bold;">Text</span>[b]Text[/b]
```


### Presets <a name="basic"></a>

Expand Down
3 changes: 1 addition & 2 deletions benchmark/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,7 @@
"cpupro": "node --require cpupro benchmark.js"
},
"author": {
"name": "Nikolay Kostyurin <jilizart@gmail.com>",
"url": "https://artkost.ru/"
"name": "Nikolay Kostyurin <jilizart@gmail.com>"
},
"dependencies": {
"@bbob/parser": "*",
Expand Down
3 changes: 1 addition & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,7 @@
"cleanup": "node scripts/cleanup"
},
"author": {
"name": "Nikolay Kostyurin <jilizart@gmail.com>",
"url": "https://artkost.ru/"
"name": "Nikolay Kostyurin <jilizart@gmail.com>"
},
"license": "MIT",
"devDependencies": {
Expand Down
10 changes: 5 additions & 5 deletions packages/bbob-parser/src/Token.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,11 @@ const getTagName = (token: Token) => {
return isTagEnd(token) ? value.slice(1) : value;
};

const tokenToText = (token: Token) => {
let text = OPEN_BRAKET;
const tokenToText = (token: Token, openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET) => {
let text = openTag;

text += getTokenValue(token);
text += CLOSE_BRAKET;
text += closeTag;

return text;
};
Expand Down Expand Up @@ -167,8 +167,8 @@ class Token<TokenValue = string> implements TokenInterface {
return getEndPosition(this);
}

toString() {
return tokenToText(this);
toString({ openTag = OPEN_BRAKET, closeTag = CLOSE_BRAKET } = {}) {
return tokenToText(this, openTag, closeTag);
}
}

Expand Down
11 changes: 5 additions & 6 deletions packages/bbob-parser/src/lexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,13 +51,14 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
let stateMode = STATE_WORD;
let tagMode = TAG_STATE_NAME;
let contextFreeTag = '';
const tokens = new Array<Token<string>>(Math.floor(buffer.length));
const tokens = new Array<Token>(Math.floor(buffer.length));
const openTag = options.openTag || OPEN_BRAKET;
const closeTag = options.closeTag || CLOSE_BRAKET;
const escapeTags = !!options.enableEscapeTags;
const contextFreeTags = (options.contextFreeTags || [])
.filter(Boolean)
.map((tag) => tag.toLowerCase());
const caseFreeTags = options.caseFreeTags || false;
const nestedMap = new Map<string, boolean>();
const onToken = options.onToken || (() => {
});
Expand Down Expand Up @@ -88,8 +89,6 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo

/**
* Emits newly created token to subscriber
* @param {Number} type
* @param {String} value
*/
function emitToken(type: number, value: string, startPos?: number, endPos?: number) {
const token = createTokenOfType(type, value, row, prevCol, startPos, endPos);
Expand Down Expand Up @@ -352,13 +351,13 @@ export function createLexer(buffer: string, options: LexerOptions = {}): LexerTo
return tokens;
}

function isTokenNested(token: Token) {
const value = openTag + SLASH + token.getValue();
function isTokenNested(tokenValue: string) {
const value = openTag + SLASH + tokenValue;

if (nestedMap.has(value)) {
return !!nestedMap.get(value);
} else {
const status = (buffer.indexOf(value) > -1);
const status = caseFreeTags ? (buffer.toLowerCase().indexOf(value.toLowerCase()) > -1) : (buffer.indexOf(value) > -1);

nestedMap.set(value, status);

Expand Down
33 changes: 21 additions & 12 deletions packages/bbob-parser/src/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ function parse(input: string, opts: ParseOptions = {}) {
const onlyAllowTags = (options.onlyAllowTags || [])
.filter(Boolean)
.map((tag) => tag.toLowerCase());
const caseFreeTags = options.caseFreeTags || false;

let tokenizer: LexerTokenizer | null = null;
let tokenizer: ReturnType<typeof createLexer> | null = null;

/**
* Result AST of nodes
Expand Down Expand Up @@ -85,10 +86,11 @@ function parse(input: string, opts: ParseOptions = {}) {
const nestedTagsMap = new Set<string>();

function isTokenNested(token: Token) {
const value = token.getValue();
const tokenValue = token.getValue();
const value = caseFreeTags ? tokenValue.toLowerCase() : tokenValue;
const { isTokenNested } = tokenizer || {};

if (!nestedTagsMap.has(value) && isTokenNested && isTokenNested(token)) {
if (!nestedTagsMap.has(value) && isTokenNested && isTokenNested(value)) {
nestedTagsMap.add(value);

return true;
Expand All @@ -101,7 +103,7 @@ function parse(input: string, opts: ParseOptions = {}) {
* @private
*/
function isTagNested(tagName: string) {
return Boolean(nestedTagsMap.has(tagName));
return Boolean(nestedTagsMap.has(caseFreeTags ? tagName.toLowerCase() : tagName));
}

/**
Expand Down Expand Up @@ -203,17 +205,23 @@ function parse(input: string, opts: ParseOptions = {}) {
* @param {Token} token
*/
function handleTagEnd(token: Token) {
const lastTagNode = nestedNodes.last();
if (isTagNode(lastTagNode)) {
lastTagNode.setEnd({ from: token.getStart(), to: token.getEnd() });
}
flushTagNodes();

const tagName = token.getValue().slice(1);
const lastNestedNode = nestedNodes.flush();

flushTagNodes();

if (lastNestedNode) {
const nodes = getNodes();

if (isTagNode(lastNestedNode)) {
lastNestedNode.setEnd({ from: token.getStart(), to: token.getEnd() });
}

appendNodes(nodes, lastNestedNode);
} else if (!isTagNested(tagName)) { // when we have only close tag [/some] without any open tag
const nodes = getNodes();

appendNodes(nodes, token.toString({ openTag, closeTag }));
} else if (typeof options.onError === "function") {
const tag = token.getValue();
const line = token.getLine();
Expand Down Expand Up @@ -281,13 +289,13 @@ function parse(input: string, opts: ParseOptions = {}) {
}
} else if (token.isTag()) {
// if tag is not allowed, just pass it as is
appendNodes(nodes, token.toString());
appendNodes(nodes, token.toString({ openTag, closeTag }));
}
} else if (token.isText()) {
appendNodes(nodes, tokenValue);
} else if (token.isTag()) {
// if tag is not allowed, just pass it as is
appendNodes(nodes, token.toString());
appendNodes(nodes, token.toString({ openTag, closeTag }));
}
}

Expand All @@ -311,6 +319,7 @@ function parse(input: string, opts: ParseOptions = {}) {
closeTag,
onlyAllowTags: options.onlyAllowTags,
contextFreeTags: options.contextFreeTags,
caseFreeTags: options.caseFreeTags,
enableEscapeTags: options.enableEscapeTags,
});

Expand Down
Loading