diff --git a/docs/adding-new-languages.md b/docs/adding-new-languages.md index 6b331ad654..9d6bc5e0f1 100644 --- a/docs/adding-new-languages.md +++ b/docs/adding-new-languages.md @@ -7,7 +7,7 @@ Please note that this list of steps reflects the state of Semantic as is, not wh ## The procedure 1. **Find or write a [tree-sitter](https://tree-sitter.github.io) parser for your language.** The tree-sitter [organization page](https://github.com/tree-sitter) has a number of parsers beyond those we currently support in Semantic; look there first to make sure you're not duplicating work. The tree-sitter [documentation on creating parsers](http://tree-sitter.github.io/tree-sitter/creating-parsers) provides an exhaustive look at the process of developing and debugging tree-sitter parsers. Though we do not support grammars written with other toolkits such as [ANTLR](https://www.antlr.org), translating an ANTLR or other BNF-style grammar into a tree-sitter grammar is usually straightforward. -2. **Create a Haskell library providing an interface to that C source.** The [`haskell-tree-sitter`](https://github.com/tree-sitter/haskell-tree-sitter/tree/master/languages) repository provides a Cabal package for each supported language. You can find an example of a pull request to add such a package here. Each package needs to provide two API surfaces: +2. **Create a Haskell library providing an interface to that C source.** The [`haskell-tree-sitter`](https://github.com/tree-sitter/haskell-tree-sitter) repository provides a Cabal package for each supported language. You can find an example of a pull request to add such a package here. Each package needs to provide two API surfaces: * a bridged (via the FFI) reference to the toplevel parser in the generated file ([example](https://github.com/tree-sitter/haskell-tree-sitter/blob/master/tree-sitter-json/internal/TreeSitter/JSON/Internal.hs)) * symbol datatypes for each syntax node in the parser, generated with the `mkSymbolDatatype` Template Haskell splice ([example](https://github.com/tree-sitter/haskell-tree-sitter/blob/master/tree-sitter-json/TreeSitter/JSON.hs)) 3. **Identify the new syntax nodes required to represent your language.** While we provide an extensive library of reusable AST nodes for [literals](https://github.com/github/semantic/blob/master/src/Data/Syntax/Literal.hs), [expressions](https://github.com/github/semantic/blob/master/src/Data/Syntax/Expression.hs), [statements](https://github.com/github/semantic/blob/master/src/Data/Syntax/Statement.hs), and [types](https://github.com/github/semantic/blob/master/src/Data/Syntax/Type.hs), most languages will require some syntax nodes not found in other languages. You'll need to create a new module providing those data types, and those data types must be written as an open union: [here](https://github.com/github/semantic/commits/master/src/Language/Ruby/Syntax.hs?author=charliesome) is an example for Ruby's syntactic details. diff --git a/src/Language/TSX/Assignment.hs b/src/Language/TSX/Assignment.hs index 839f7aeca1..2f79a0bae7 100644 --- a/src/Language/TSX/Assignment.hs +++ b/src/Language/TSX/Assignment.hs @@ -429,7 +429,7 @@ jsxAttribute' :: Assignment Term jsxAttribute' = jsxAttribute <|> jsxExpression' jsxOpeningElement' :: Assignment Term -jsxOpeningElement' = makeTerm <$> symbol Grammar.JsxOpeningElement <*> children (TSX.Syntax.JsxOpeningElement <$> term jsxElementName <*> manyTerm jsxAttribute') +jsxOpeningElement' = makeTerm <$> symbol Grammar.JsxOpeningElement <*> children (TSX.Syntax.JsxOpeningElement <$> term jsxElementName <*> term (typeArguments' <|> emptyTerm) <*> manyTerm jsxAttribute') jsxElementName :: Assignment Term jsxElementName = choice [ identifier, nestedIdentifier, jsxNamespaceName ] diff --git a/src/Language/TSX/Syntax/JSX.hs b/src/Language/TSX/Syntax/JSX.hs index 9430e67435..113b0375a9 100644 --- a/src/Language/TSX/Syntax/JSX.hs +++ b/src/Language/TSX/Syntax/JSX.hs @@ -28,7 +28,7 @@ newtype JsxExpression a = JsxExpression { jsxExpression :: a } instance Evaluatable JsxExpression -data JsxOpeningElement a = JsxOpeningElement { jsxOpeningElementIdentifier :: !a, jsxAttributes :: ![a] } +data JsxOpeningElement a = JsxOpeningElement { jsxOpeningElementIdentifier :: !a, jsxOpeningElementTypeArguments :: a, jsxAttributes :: ![a] } deriving (Declarations1, Diffable, Eq, Foldable, FreeVariables1, Functor, Generic1, Hashable1, NFData1, Ord, Show, ToJSONFields1, Traversable) deriving (Eq1, Show1, Ord1) via Generically JsxOpeningElement diff --git a/src/Parsing/Parser.hs b/src/Parsing/Parser.hs index c4bca8f5a5..09c37e63e3 100644 --- a/src/Parsing/Parser.hs +++ b/src/Parsing/Parser.hs @@ -191,9 +191,13 @@ someASTParser :: Language -> Maybe SomeASTParser someASTParser Go = Just (SomeASTParser (ASTParser tree_sitter_go :: Parser (AST [] Go.Grammar))) someASTParser Haskell = Just (SomeASTParser (ASTParser tree_sitter_haskell :: Parser (AST [] Haskell.Grammar))) someASTParser Java = Just (SomeASTParser (ASTParser tree_sitter_java :: Parser (AST [] Java.Grammar))) -someASTParser JavaScript = Just (SomeASTParser (ASTParser tree_sitter_typescript :: Parser (AST [] TypeScript.Grammar))) someASTParser JSON = Just (SomeASTParser (ASTParser tree_sitter_json :: Parser (AST [] JSON.Grammar))) -someASTParser JSX = Just (SomeASTParser (ASTParser tree_sitter_typescript :: Parser (AST [] TypeScript.Grammar))) + +-- Use the TSX parser for `.js` and `.jsx` files in case they use Flow type-annotation syntax. +-- The TSX and Flow syntaxes are the same, whereas the normal TypeScript syntax is different. +someASTParser JavaScript = Just (SomeASTParser (ASTParser tree_sitter_tsx :: Parser (AST [] TSX.Grammar))) +someASTParser JSX = Just (SomeASTParser (ASTParser tree_sitter_tsx :: Parser (AST [] TSX.Grammar))) + someASTParser Python = Just (SomeASTParser (ASTParser tree_sitter_python :: Parser (AST [] Python.Grammar))) someASTParser Ruby = Just (SomeASTParser (ASTParser tree_sitter_ruby :: Parser (AST [] Ruby.Grammar))) someASTParser TypeScript = Just (SomeASTParser (ASTParser tree_sitter_typescript :: Parser (AST [] TypeScript.Grammar))) diff --git a/test/fixtures/javascript/corpus/jsx.A.js b/test/fixtures/javascript/corpus/jsx.A.js new file mode 100644 index 0000000000..5b4c905660 --- /dev/null +++ b/test/fixtures/javascript/corpus/jsx.A.js @@ -0,0 +1,5 @@ +function Something() { + return
+ hello +
; +} diff --git a/test/fixtures/javascript/corpus/jsx.B.js b/test/fixtures/javascript/corpus/jsx.B.js new file mode 100644 index 0000000000..237faaca55 --- /dev/null +++ b/test/fixtures/javascript/corpus/jsx.B.js @@ -0,0 +1,5 @@ +function Something() { + return
+ >goodbye +
; +} diff --git a/test/fixtures/javascript/corpus/jsx.diffA-B.txt b/test/fixtures/javascript/corpus/jsx.diffA-B.txt new file mode 100644 index 0000000000..9286a8dc28 --- /dev/null +++ b/test/fixtures/javascript/corpus/jsx.diffA-B.txt @@ -0,0 +1,25 @@ +(Statements + (Function + (Empty) + (Empty) + (Identifier) + (StatementBlock + (Return + (JsxElement + (JsxOpeningElement + (Identifier) + (Empty)) + (JsxText) + (JsxElement + (JsxOpeningElement + (Identifier) + { (Empty) + ->(TypeArguments + {+(TypeIdentifier)+}) }) + { (JsxText) + ->(JsxText) } + (JsxClosingElement + (Identifier))) + (JsxText) + (JsxClosingElement + (Identifier))))))) diff --git a/test/fixtures/javascript/corpus/jsx.diffB-A.txt b/test/fixtures/javascript/corpus/jsx.diffB-A.txt new file mode 100644 index 0000000000..708286acdb --- /dev/null +++ b/test/fixtures/javascript/corpus/jsx.diffB-A.txt @@ -0,0 +1,25 @@ +(Statements + (Function + (Empty) + (Empty) + (Identifier) + (StatementBlock + (Return + (JsxElement + (JsxOpeningElement + (Identifier) + (Empty)) + (JsxText) + (JsxElement + (JsxOpeningElement + (Identifier) + { (TypeArguments + {-(TypeIdentifier)-}) + ->(Empty) }) + { (JsxText) + ->(JsxText) } + (JsxClosingElement + (Identifier))) + (JsxText) + (JsxClosingElement + (Identifier))))))) diff --git a/test/fixtures/javascript/corpus/jsx.parseA.txt b/test/fixtures/javascript/corpus/jsx.parseA.txt new file mode 100644 index 0000000000..c42723ce3d --- /dev/null +++ b/test/fixtures/javascript/corpus/jsx.parseA.txt @@ -0,0 +1,22 @@ +(Statements + (Function + (Empty) + (Empty) + (Identifier) + (StatementBlock + (Return + (JsxElement + (JsxOpeningElement + (Identifier) + (Empty)) + (JsxText) + (JsxElement + (JsxOpeningElement + (Identifier) + (Empty)) + (JsxText) + (JsxClosingElement + (Identifier))) + (JsxText) + (JsxClosingElement + (Identifier))))))) diff --git a/test/fixtures/javascript/corpus/jsx.parseB.txt b/test/fixtures/javascript/corpus/jsx.parseB.txt new file mode 100644 index 0000000000..4916b0be6b --- /dev/null +++ b/test/fixtures/javascript/corpus/jsx.parseB.txt @@ -0,0 +1,23 @@ +(Statements + (Function + (Empty) + (Empty) + (Identifier) + (StatementBlock + (Return + (JsxElement + (JsxOpeningElement + (Identifier) + (Empty)) + (JsxText) + (JsxElement + (JsxOpeningElement + (Identifier) + (TypeArguments + (TypeIdentifier))) + (JsxText) + (JsxClosingElement + (Identifier))) + (JsxText) + (JsxClosingElement + (Identifier))))))) diff --git a/test/fixtures/tsx/corpus/jsx-elements.diffA-B.txt b/test/fixtures/tsx/corpus/jsx-elements.diffA-B.txt index 9e330c8d5c..04d7fa6870 100644 --- a/test/fixtures/tsx/corpus/jsx-elements.diffA-B.txt +++ b/test/fixtures/tsx/corpus/jsx-elements.diffA-B.txt @@ -6,6 +6,7 @@ (JsxElement (JsxOpeningElement (Identifier) + (Empty) {+(JsxAttribute {+(Identifier)+} {+(JsxExpression @@ -31,7 +32,8 @@ {-(JsxOpeningElement {-(NestedIdentifier {-(Identifier)-} - {-(Identifier)-})-})-} + {-(Identifier)-})-} + {-(Empty)-})-} {-(JsxClosingElement {-(NestedIdentifier {-(Identifier)-} diff --git a/test/fixtures/tsx/corpus/jsx-elements.diffB-A.txt b/test/fixtures/tsx/corpus/jsx-elements.diffB-A.txt index 6f4ba9f23e..bb6402bef0 100644 --- a/test/fixtures/tsx/corpus/jsx-elements.diffB-A.txt +++ b/test/fixtures/tsx/corpus/jsx-elements.diffB-A.txt @@ -6,6 +6,7 @@ (JsxElement (JsxOpeningElement (Identifier) + (Empty) {+(JsxExpression {+(Call {+(Identifier)+} @@ -31,7 +32,8 @@ {+(JsxOpeningElement {+(NestedIdentifier {+(Identifier)+} - {+(Identifier)+})+})+} + {+(Identifier)+})+} + {+(Empty)+})+} {+(JsxClosingElement {+(NestedIdentifier {+(Identifier)+} diff --git a/test/fixtures/tsx/corpus/jsx-elements.parseA.txt b/test/fixtures/tsx/corpus/jsx-elements.parseA.txt index 85d98a0356..579b272fe3 100644 --- a/test/fixtures/tsx/corpus/jsx-elements.parseA.txt +++ b/test/fixtures/tsx/corpus/jsx-elements.parseA.txt @@ -6,6 +6,7 @@ (JsxElement (JsxOpeningElement (Identifier) + (Empty) (JsxExpression (Call (Identifier) @@ -25,7 +26,8 @@ (JsxOpeningElement (NestedIdentifier (Identifier) - (Identifier))) + (Identifier)) + (Empty)) (JsxClosingElement (NestedIdentifier (Identifier) diff --git a/test/fixtures/tsx/corpus/jsx-elements.parseB.txt b/test/fixtures/tsx/corpus/jsx-elements.parseB.txt index 32aee1e269..6e1a3a3ac9 100644 --- a/test/fixtures/tsx/corpus/jsx-elements.parseB.txt +++ b/test/fixtures/tsx/corpus/jsx-elements.parseB.txt @@ -6,6 +6,7 @@ (JsxElement (JsxOpeningElement (Identifier) + (Empty) (JsxAttribute (Identifier) (JsxExpression