From 87cd5a2de8fe3366633762255b2a35b513fa4cd2 Mon Sep 17 00:00:00 2001 From: David Paz Date: Fri, 5 May 2017 10:34:22 +0200 Subject: [PATCH 1/6] splitByDots created --- native/src/driver_app.erl | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/native/src/driver_app.erl b/native/src/driver_app.erl index 53c3ccb..474edb0 100644 --- a/native/src/driver_app.erl +++ b/native/src/driver_app.erl @@ -3,7 +3,7 @@ -behaviour(application). %% Application callbacks --export([start/2, stop/1,loop/0,parseExpr/1,tokenize/1,parse/1,format/1]). +-export([start/2, stop/1,loop/0,parseExpr/1,splitByDots/1,parse/1,format/1]). -include_lib("eunit/include/eunit.hrl"). @@ -60,8 +60,8 @@ process() -> {error,BadJSON} -> throw(BadJSON) end, - ExprList = tokenize(Content), - {ok,ParseList} = parse(ExprList), + TokExprList = tokenize(Content), + {ok,ParseList} = parse(TokExprList), FormatParse = format(ParseList), JSON = jsx:encode([{<<"status">>,<<"ok">>},{<<"ast">>,FormatParse}]), io:format("~s~n",[binary_to_list(JSON)]) @@ -79,10 +79,16 @@ decode(InputSrt) -> end; false-> {error,{json,<<"Input is not a valid JSON">>}} - end. + end. + tokenize(Content) -> - Formated= string:join(lists:map(fun erlang:binary_to_list/1,[Content]),""), - string:tokens(Formated,"."). + TokExprList = case erl_scan:string(Content) of + {ok,Tokens,_} -> + splitByDots(Tokens); + {error,BadScan,_}-> throw({scan,BadScan}) + end, + TokExprList. + parse(ExprList) -> List = lists:foldl(fun (Expr,ParseList)-> @@ -113,6 +119,23 @@ parseExpr(Tokens) -> end end. + +splitByDots(TokenList)-> + splitByDots(TokenList,[],[]). + +splitByDots([H|T],Acc,Final)-> + if + element(1,H) == dot -> + SubList = [lists:append(Acc,H)], + splitByDots(T,[],lists:append(Final,SubList)); + true -> + splitByDots(T,lists:append(Acc,[H]),Final) + end; +splitByDots([],_,Final)-> + Final. + + + %% Format do a conversion of erlang tuples to list, also change strings to binaries format(T) when is_list(T)-> format(list_to_tuple(T)); From 4ddd47ef2e14003c874402404bde2e641f797076 Mon Sep 17 00:00:00 2001 From: David Paz Date: Fri, 5 May 2017 11:31:50 +0200 Subject: [PATCH 2/6] Refactored parse --- native/src/driver_app.erl | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/native/src/driver_app.erl b/native/src/driver_app.erl index 474edb0..87fa5c8 100644 --- a/native/src/driver_app.erl +++ b/native/src/driver_app.erl @@ -3,7 +3,7 @@ -behaviour(application). %% Application callbacks --export([start/2, stop/1,loop/0,parseExpr/1,splitByDots/1,parse/1,format/1]). +-export([start/2, stop/1,loop/0,parseExpr/1,splitByDots/1,parse/1]). -include_lib("eunit/include/eunit.hrl"). @@ -60,7 +60,7 @@ process() -> {error,BadJSON} -> throw(BadJSON) end, - TokExprList = tokenize(Content), + TokExprList = tokenize(binary_to_list(Content)), {ok,ParseList} = parse(TokExprList), FormatParse = format(ParseList), JSON = jsx:encode([{<<"status">>,<<"ok">>},{<<"ast">>,FormatParse}]), @@ -68,7 +68,6 @@ process() -> end. decode(InputSrt) -> - SubS = string:substr(InputSrt,1,string:len(InputSrt)-1), case jsx:is_json(list_to_binary(SubS)) of true->Data= jsx:decode(list_to_binary(SubS)), @@ -92,20 +91,10 @@ tokenize(Content) -> parse(ExprList) -> List = lists:foldl(fun (Expr,ParseList)-> - case string:equal("\n",Expr) of - true -> - lists:append(ParseList,""); - false -> - ExprDot = string:concat(Expr,"."), - case erl_scan:string(ExprDot) of - {ok,Tokens,_} -> - case parseExpr(Tokens) of - {ok,AST} -> lists:append(ParseList,AST); - {error,BadParse} -> throw({parse,BadParse}) - end; - {error,BadScan,_}-> throw({scan,BadScan}) - end - end + case parseExpr(Expr) of + {ok,AST} -> lists:append(ParseList,AST); + {error,BadParse} -> throw({parse,BadParse}) + end end,[],ExprList), {ok,List}. @@ -119,19 +108,19 @@ parseExpr(Tokens) -> end end. - splitByDots(TokenList)-> splitByDots(TokenList,[],[]). splitByDots([H|T],Acc,Final)-> if element(1,H) == dot -> - SubList = [lists:append(Acc,H)], + SubList = [lists:append(Acc,[H])], splitByDots(T,[],lists:append(Final,SubList)); true -> splitByDots(T,lists:append(Acc,[H]),Final) end; splitByDots([],_,Final)-> + io:format("~p~n",[Final]), Final. From 54309196912efdf245546732dcb153b11ec18555 Mon Sep 17 00:00:00 2001 From: David Paz Date: Fri, 5 May 2017 12:14:06 +0200 Subject: [PATCH 3/6] Reformat --- native/src/driver_app.erl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/native/src/driver_app.erl b/native/src/driver_app.erl index 87fa5c8..3c789cd 100644 --- a/native/src/driver_app.erl +++ b/native/src/driver_app.erl @@ -11,7 +11,7 @@ start(_StartType, _StartArgs) -> try %If you are modifying the code comment the next line to show the exceptions - %error_logger:tty(false), + error_logger:tty(false), loop() catch throw:{eofErr,_}-> @@ -88,7 +88,6 @@ tokenize(Content) -> end, TokExprList. - parse(ExprList) -> List = lists:foldl(fun (Expr,ParseList)-> case parseExpr(Expr) of @@ -120,11 +119,8 @@ splitByDots([H|T],Acc,Final)-> splitByDots(T,lists:append(Acc,[H]),Final) end; splitByDots([],_,Final)-> - io:format("~p~n",[Final]), Final. - - %% Format do a conversion of erlang tuples to list, also change strings to binaries format(T) when is_list(T)-> format(list_to_tuple(T)); From 9f32c1449fffafe36ddc5852b3fefacf44dafbc8 Mon Sep 17 00:00:00 2001 From: David Paz Date: Mon, 8 May 2017 12:19:22 +0200 Subject: [PATCH 4/6] Added tests for splitByDots, solved some bugs --- native/src/driver_app.erl | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/native/src/driver_app.erl b/native/src/driver_app.erl index 3c789cd..05e8ad4 100644 --- a/native/src/driver_app.erl +++ b/native/src/driver_app.erl @@ -3,7 +3,7 @@ -behaviour(application). %% Application callbacks --export([start/2, stop/1,loop/0,parseExpr/1,splitByDots/1,parse/1]). +-export([start/2, stop/1,loop/0,splitByDots/1]). -include_lib("eunit/include/eunit.hrl"). @@ -118,8 +118,14 @@ splitByDots([H|T],Acc,Final)-> true -> splitByDots(T,lists:append(Acc,[H]),Final) end; -splitByDots([],_,Final)-> - Final. +splitByDots([],Acc,Final)-> + if + length(Acc) == 0 -> + Res = Final; + true -> + Res =lists:append(Final,[Acc]) + end, + Res. %% Format do a conversion of erlang tuples to list, also change strings to binaries format(T) when is_list(T)-> @@ -182,3 +188,21 @@ format_test_()-> ?_assert(format({"hello","world"})=:= [<<"hello">>,<<"world">>]), ?_assert(format({a,b,[c,"hello",{"world",["this","is","a","test"]}]})=:= [a,b,[c,<<"hello">>,[<<"world">>,[<<"this">>,<<"is">>,<<"a">>,<<"test">>]]]]), ?_assert(format({"hello",<<"world">>})=:= [<<"hello">>,<<"world">>])]. + + splitByDots_test_()-> + String1 = "ok . ok . ok . ok", + String2 = ". . . . ok", + String3 = "ok", + String4 = ".", + {_,Tokens1,_} = erl_scan:string(String1), + {_,Tokens2,_} = erl_scan:string(String2), + {_,Tokens3,_} = erl_scan:string(String3), + {_,Tokens4,_} = erl_scan:string(String4), + Res1 = length(splitByDots(Tokens1)), + Res2 = length(splitByDots(Tokens2)), + Res3 = length(splitByDots(Tokens3)), + Res4 = length(splitByDots(Tokens4)), + [?_assertEqual(4,Res1), + ?_assertEqual(5,Res2), + ?_assertEqual(1,Res3), + ?_assertEqual(1,Res4)]. From ae9499e89df901fd73d40e8a377ddc9ecc16e344 Mon Sep 17 00:00:00 2001 From: David Paz Date: Tue, 9 May 2017 11:21:20 +0200 Subject: [PATCH 5/6] addded a method to delete '?' in macros to avoid crashes --- native/src/driver_app.erl | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/native/src/driver_app.erl b/native/src/driver_app.erl index 05e8ad4..241f62d 100644 --- a/native/src/driver_app.erl +++ b/native/src/driver_app.erl @@ -83,7 +83,8 @@ decode(InputSrt) -> tokenize(Content) -> TokExprList = case erl_scan:string(Content) of {ok,Tokens,_} -> - splitByDots(Tokens); + PTokens = removeMacros(Tokens), + splitByDots(PTokens); {error,BadScan,_}-> throw({scan,BadScan}) end, TokExprList. @@ -147,6 +148,18 @@ format(T,N,Acc) when is_list(element(N,T)) -> format(T,N,Acc)-> format(T,N-1,[element(N,T)|Acc]). +removeMacros(Tokens) -> + removeMacros(Tokens,[]). + +removeMacros([H|T],Acc) -> + if + element(1,H) == '?' -> + removeMacros(T,Acc); + true -> + removeMacros(T,lists:append(Acc,[H])) + end; +removeMacros([],Acc)-> + Acc. %%Tests From 3a15a92156a1f1e7913c2f74fc802d4475022b99 Mon Sep 17 00:00:00 2001 From: David Paz Date: Tue, 9 May 2017 11:26:28 +0200 Subject: [PATCH 6/6] Updated SDK --- driver/main.go | 12 ++++++------ driver/normalizer/annotation.go | 9 +++++++++ driver/normalizer/parser.go | 17 +++++++++++++++++ 3 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 driver/normalizer/annotation.go create mode 100644 driver/normalizer/parser.go diff --git a/driver/main.go b/driver/main.go index 43ec7f4..df68a1f 100644 --- a/driver/main.go +++ b/driver/main.go @@ -1,7 +1,7 @@ package main import ( - "github.com/bblfsh/sdk/protocol" + "github.com/bblfsh/sdk/protocol/driver" "github.com/bblfsh/erlang-driver/driver/normalizer" ) @@ -10,11 +10,11 @@ var version string var build string func main() { - d := protocol.Driver{ - Version: version, - Build: build, - ToNoder: normalizer.NativeToNoder, - Annotate: normalizer.AnnotationRules, + d := driver.Driver{ + Version: version, + Build: build, + ASTParserBuilder: normalizer.ASTParserBuilder, + Annotate: normalizer.AnnotationRules, } d.Exec() } diff --git a/driver/normalizer/annotation.go b/driver/normalizer/annotation.go new file mode 100644 index 0000000..22bf2af --- /dev/null +++ b/driver/normalizer/annotation.go @@ -0,0 +1,9 @@ +package normalizer + +import ( + . "github.com/bblfsh/sdk/uast" + . "github.com/bblfsh/sdk/uast/ann" +) + +// AnnotationRules annotate a UAST with roles. +var AnnotationRules = On(Any).Roles(File) diff --git a/driver/normalizer/parser.go b/driver/normalizer/parser.go new file mode 100644 index 0000000..532307c --- /dev/null +++ b/driver/normalizer/parser.go @@ -0,0 +1,17 @@ +package normalizer + +import ( + "github.com/bblfsh/sdk/protocol/driver" + "github.com/bblfsh/sdk/protocol/native" +) + +// ASTParserBuilder creates a parser that transform source code files into *uast.Node. +func ASTParserBuilder(opts driver.ASTParserOptions) (driver.ASTParser, error) { + toNoder := &native.ObjectToNoder{} + parser, err := native.ExecParser(toNoder, opts.NativeBin) + if err != nil { + return nil, err + } + + return parser, nil +}