Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(deps): update @textlint/ast-node-types #47

Merged
merged 4 commits into from
Nov 25, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [ 16, 18 ]
node-version: [ 18, 20 ]
steps:
- name: checkout
uses: actions/checkout@v3
Expand Down
18 changes: 9 additions & 9 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -83,20 +83,20 @@
"trailingComma": "none"
},
"dependencies": {
"@textlint/ast-node-types": "^13.2.0",
"@textlint/ast-node-types": "^13.4.1",
"structured-source": "^4.0.0"
},
"devDependencies": {
"@textlint/markdown-to-ast": "^13.2.0",
"@types/mocha": "^10.0.1",
"@types/node": "^18.13.0",
"lint-staged": "^13.1.1",
"@textlint/markdown-to-ast": "^13.4.1",
"@types/mocha": "^10.0.6",
"@types/node": "^20.10.0",
"lint-staged": "^15.1.0",
"mocha": "^10.2.0",
"prettier": "^2.8.4",
"prettier": "^3.1.0",
"ts-node": "^10.9.1",
"tsconfig-to-dual-package": "^1.1.1",
"typescript": "^4.9.5",
"vite": "^4.1.1"
"tsconfig-to-dual-package": "^1.2.0",
"typescript": "^5.3.2",
"vite": "^5.0.2"
},
"packageManager": "yarn@1.22.19",
"email": "azuciao@gmail.com"
Expand Down
21 changes: 15 additions & 6 deletions src/sentence-splitter.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { TxtNode, TxtParagraphNode, TxtParentNode, TxtStrNode, TxtTextNode } from "@textlint/ast-node-types";
import type { TxtParagraphNode, TxtParentNode, TxtStrNode, TxtTextNode } from "@textlint/ast-node-types";
import { ASTNodeTypes } from "@textlint/ast-node-types";

import { SourceCode } from "./parser/SourceCode.js";
Expand All @@ -11,7 +11,7 @@ import {
SeparatorParserOptions
} from "./parser/SeparatorParser.js";
import { AnyValueParser } from "./parser/AnyValueParser.js";
import { DefaultOptions as DefaultAbbrMarkerOptions, AbbrMarker, AbbrMarkerOptions } from "./parser/AbbrMarker.js";
import { AbbrMarker, AbbrMarkerOptions, DefaultOptions as DefaultAbbrMarkerOptions } from "./parser/AbbrMarker.js";
import { PairMaker } from "./parser/PairMaker.js";
import { nodeLog } from "./logger.js";

Expand All @@ -37,13 +37,22 @@ export type SentencePairMarkContext = {
};
};
};
export type TxtSentenceNode = Omit<TxtParentNode, "type"> & {
// SentenceNode does not have sentence
// Nested SentenceNode is not allowed
export type TxtSentenceNodeChildren =
| TxtParentNode["children"][number]
| TxtWhiteSpaceNode
| TxtPunctuationNode
| TxtStrNode;
export type TxtSentenceNode = Omit<TxtParentNode, "type" | "children"> & {
readonly type: "Sentence";
/**
* SentenceNode includes some context information
* - "PairMark": pair mark information
*/
readonly contexts: SentencePairMarkContext[];

children: TxtSentenceNodeChildren[];
};
export type TxtWhiteSpaceNode = Omit<TxtTextNode, "type"> & {
readonly type: "WhiteSpace";
Expand Down Expand Up @@ -71,7 +80,7 @@ class SplitParser {
return this.sentenceNodeList[this.sentenceNodeList.length - 1];
}

pushNodeToCurrent(node: TxtParentNodeWithSentenceNodeContent) {
pushNodeToCurrent(node: TxtSentenceNodeChildren) {
const current = this.current;
if (current) {
current.children.push(node);
Expand Down Expand Up @@ -122,7 +131,7 @@ class SplitParser {
if (currentNode.children.length === 0) {
return;
}
const firstChildNode: TxtNode = currentNode.children[0];
const firstChildNode = currentNode.children[0];
const endNow = this.source.now();
// update Sentence node's location and range
const rawValue = this.source.sliceRange(firstChildNode.range[0], endNow.offset);
Expand Down Expand Up @@ -303,7 +312,7 @@ function createWhiteSpaceNode(
column: endPosition.column
}
},
range: [startPosition.offset, endPosition.offset]
range: [startPosition.offset, endPosition.offset] as const
};
}

Expand Down
58 changes: 37 additions & 21 deletions test/sentence-splitter-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,33 @@ import {
DefaultSentenceSplitterOptions,
SentenceSplitterSyntax,
split as splitSentences,
splitAST
splitAST,
TxtParentNodeWithSentenceNode,
TxtSentenceNode,
TxtWhiteSpaceNode
} from "../src/sentence-splitter.js";
import { English } from "sentence-splitter/lang";

const getSentence = (children: TxtParentNodeWithSentenceNode["children"], index: number): TxtSentenceNode => {
const sentence = children[index];
assert.strictEqual(sentence.type, SentenceSplitterSyntax.Sentence);
return sentence;
};
const getWhitespace = (children: TxtParentNodeWithSentenceNode["children"], index: number): TxtWhiteSpaceNode => {
const sentence = children[index];
assert.strictEqual(sentence.type, SentenceSplitterSyntax.WhiteSpace);
return sentence;
};
describe("sentence-splitter", function () {
it("should return array", function () {
const sentences = splitSentences("text");
assert.equal(sentences.length, 1);
const sentence = sentences[0];
const sentence = getSentence(sentences, 0);
assert.strictEqual(sentence.raw, "text");
// @ts-expect-error
assert.strictEqual(typeof sentence.value, "undefined");
assert.deepStrictEqual(sentences[0].loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(sentences[0].loc.end, { line: 1, column: 4 });
assert.deepStrictEqual(getSentence(sentences, 0).loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(getSentence(sentences, 0).loc.end, { line: 1, column: 4 });
});

it("should not split number", function () {
Expand Down Expand Up @@ -58,13 +72,13 @@ describe("sentence-splitter", function () {
it("should return sentences split by first line break", function () {
const sentences = splitSentences("\ntext");
assert.equal(sentences.length, 2);
const whiteSpace0 = sentences[0];
const whiteSpace0 = getWhitespace(sentences, 0);
assert.strictEqual(whiteSpace0.type, SentenceSplitterSyntax.WhiteSpace);
assert.strictEqual(whiteSpace0.raw, "\n");
assert.deepStrictEqual(whiteSpace0.loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(whiteSpace0.loc.end, { line: 2, column: 0 });
assert.equal(sentences[1].children.length, 1);
const text = sentences[1].children[0];
assert.equal(getSentence(sentences, 1).children.length, 1);
const text = getSentence(sentences, 1).children[0];
assert.strictEqual(text.type, ASTNodeTypes.Str);
assert.strictEqual(text.raw, "text");
assert.deepStrictEqual(text.loc.start, { line: 2, column: 0 });
Expand All @@ -73,7 +87,7 @@ describe("sentence-splitter", function () {
it("should return sentences split by last line break", function () {
const sentences = splitSentences("text\n");
assert.equal(sentences.length, 1);
const [sentence0, whiteSpace1] = sentences[0].children;
const [sentence0, whiteSpace1] = getSentence(sentences, 0).children;
assert.strictEqual(sentence0.type, ASTNodeTypes.Str);
assert.strictEqual(sentence0.raw, "text");
assert.deepStrictEqual(sentence0.loc.start, { line: 1, column: 0 });
Expand All @@ -86,8 +100,8 @@ describe("sentence-splitter", function () {
it("should return sentences split by line break*2", function () {
const sentences = splitSentences("text\n\ntext");
assert.equal(sentences.length, 1);
assert.equal(sentences[0].children.length, 3);
const [sentence0, whiteSpace1, sentence3] = sentences[0].children;
assert.equal(getSentence(sentences, 0).children.length, 3);
const [sentence0, whiteSpace1, sentence3] = getSentence(sentences, 0).children;
assert.strictEqual(sentence0.type, ASTNodeTypes.Str);
assert.strictEqual(sentence0.raw, "text");
assert.deepStrictEqual(sentence0.loc.start, { line: 1, column: 0 });
Expand All @@ -104,26 +118,26 @@ describe("sentence-splitter", function () {
it("should return sentences split by 。", function () {
const sentences = splitSentences("text。。text");
assert.equal(sentences.length, 2);
const [sentence0, punctuation] = sentences[0].children;
const [sentence0, punctuation] = getSentence(sentences, 0).children;
assert.strictEqual(sentence0.raw, "text");
assert.deepStrictEqual(sentence0.loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(sentence0.loc.end, { line: 1, column: 4 });
assert.strictEqual(punctuation.raw, "。。");
assert.deepStrictEqual(punctuation.loc.start, { line: 1, column: 4 });
assert.deepStrictEqual(punctuation.loc.end, { line: 1, column: 6 });
const [sentence1] = sentences[1].children;
const [sentence1] = getSentence(sentences, 1).children;
assert.strictEqual(sentence1.raw, "text");
assert.deepStrictEqual(sentence1.loc.start, { line: 1, column: 6 });
assert.deepStrictEqual(sentence1.loc.end, { line: 1, column: 10 });
});
it("should return sentences split by 。 and linebreak", function () {
const sentences = splitSentences("text。\ntext");
assert.equal(sentences.length, 3);
const sentence0 = sentences[0];
const sentence0 = getSentence(sentences, 0);
assert.strictEqual(sentence0.raw, "text。");
assert.deepStrictEqual(sentence0.loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(sentence0.loc.end, { line: 1, column: 5 });
const whiteSpace1 = sentences[1];
const whiteSpace1 = getWhitespace(sentences, 1);
assert.strictEqual(whiteSpace1.raw, "\n");
assert.deepStrictEqual(whiteSpace1.loc.start, { line: 1, column: 5 });
assert.deepStrictEqual(whiteSpace1.loc.end, { line: 2, column: 0 });
Expand All @@ -135,11 +149,11 @@ describe("sentence-splitter", function () {
it("should return sentences split by . and whitespace", function () {
const sentences = splitSentences("1st text. 2nd text");
assert.equal(sentences.length, 3);
const sentence0 = sentences[0];
const sentence0 = getSentence(sentences, 0);
assert.strictEqual(sentence0.raw, "1st text.");
assert.deepStrictEqual(sentence0.loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(sentence0.loc.end, { line: 1, column: 9 });
const whiteSpace1 = sentences[1];
const whiteSpace1 = getWhitespace(sentences, 1);
assert.strictEqual(whiteSpace1.raw, " ");
assert.deepStrictEqual(whiteSpace1.loc.start, { line: 1, column: 9 });
assert.deepStrictEqual(whiteSpace1.loc.end, { line: 1, column: 10 });
Expand Down Expand Up @@ -182,32 +196,34 @@ describe("sentence-splitter", function () {
assert.strictEqual(sentence0.raw, "1st text.");
assert.deepStrictEqual(sentence0.range, [0, 9]);
assert.strictEqual(lineBreak.type, SentenceSplitterSyntax.WhiteSpace);
assert.strictEqual(whitespace1.type, SentenceSplitterSyntax.WhiteSpace);
assert.strictEqual(whitespace1.value, " ");
assert.strictEqual(lineBreak.type, SentenceSplitterSyntax.WhiteSpace);
assert.strictEqual(lineBreak.value, "\n");
assert.strictEqual(whitespace2.type, SentenceSplitterSyntax.WhiteSpace);
assert.strictEqual(whitespace2.value, " ");
assert.strictEqual(sentence1.raw, "2nd text");
assert.deepStrictEqual(sentence1.range, [12, 20]);
});
it("should return sentences split by !?", function () {
const sentences = splitSentences("text!?text");
assert.equal(sentences.length, 2);
const sentence0 = sentences[0];
const sentence0 = getSentence(sentences, 0);
assert.strictEqual(sentence0.raw, "text!?");
assert.deepStrictEqual(sentence0.loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(sentence0.loc.end, { line: 1, column: 6 });
const sentence1 = sentences[1];
const sentence1 = getSentence(sentences, 1);
assert.strictEqual(sentence1.raw, "text");
assert.deepStrictEqual(sentence1.loc.start, { line: 1, column: 6 });
assert.deepStrictEqual(sentence1.loc.end, { line: 1, column: 10 });
});
it("should sentences split by last 。", function () {
const sentences = splitSentences("text。");
assert.equal(sentences.length, 1);
const sentence = sentences[0];
const sentence = getSentence(sentences, 0);
assert.strictEqual(sentence.raw, "text。");
assert.deepStrictEqual(sentences[0].loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(sentences[0].loc.end, { line: 1, column: 5 });
assert.deepStrictEqual(getSentence(sentences, 0).loc.start, { line: 1, column: 0 });
assert.deepStrictEqual(getSentence(sentences, 0).loc.end, { line: 1, column: 5 });
});
it("should not split line indexes", function () {
const sentences = splitSentences("1. 1st text.\n2. 2nd text.");
Expand Down
2 changes: 1 addition & 1 deletion tsconfig.cjs.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"extends": "./tsconfig.json",
"compilerOptions": {
"module": "CommonJS",
"moduleResolution": "NodeNext",
"moduleResolution": "node",
"outDir": "./lib/",
}
}
2 changes: 1 addition & 1 deletion tsconfig.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"compilerOptions": {
/* Basic Options */
"module": "ESNext",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"esModuleInterop": true,
"newLine": "LF",
Expand Down
Loading