Preprocesor walkaround

driver/main.go

@@ -1,7 +1,7 @@
 package main
 
 import (
-	"github.com/bblfsh/sdk/protocol"
+	"github.com/bblfsh/sdk/protocol/driver"
 
 	"github.com/bblfsh/erlang-driver/driver/normalizer"
 )
@@ -10,11 +10,11 @@ var version string
 var build string
 
 func main() {
-	d := protocol.Driver{
-		Version:  version,
-		Build:    build,
-		ToNoder:  normalizer.NativeToNoder,
-		Annotate: normalizer.AnnotationRules,
+	d := driver.Driver{
+		Version:          version,
+		Build:            build,
+		ASTParserBuilder: normalizer.ASTParserBuilder,
+		Annotate:         normalizer.AnnotationRules,
 	}
 	d.Exec()
 }

driver/normalizer/annotation.go

@@ -0,0 +1,9 @@
+package normalizer
+
+import (
+	. "github.com/bblfsh/sdk/uast"
+	. "github.com/bblfsh/sdk/uast/ann"
+)
+
+// AnnotationRules annotate a UAST with roles.
+var AnnotationRules = On(Any).Roles(File)

driver/normalizer/parser.go

@@ -0,0 +1,17 @@
+package normalizer
+
+import (
+	"github.com/bblfsh/sdk/protocol/driver"
+	"github.com/bblfsh/sdk/protocol/native"
+)
+
+// ASTParserBuilder creates a parser that transform source code files into *uast.Node.
+func ASTParserBuilder(opts driver.ASTParserOptions) (driver.ASTParser, error) {
+	toNoder := &native.ObjectToNoder{}
+	parser, err := native.ExecParser(toNoder, opts.NativeBin)
+	if err != nil {
+		return nil, err
+	}
+
+	return parser, nil
+}

native/src/driver_app.erl

@@ -3,15 +3,15 @@
 -behaviour(application).
 
 %% Application callbacks
--export([start/2, stop/1,loop/0,parseExpr/1,tokenize/1,parse/1,format/1]).
+-export([start/2, stop/1,loop/0,splitByDots/1]).
 -include_lib("eunit/include/eunit.hrl").
 
 
 
 start(_StartType, _StartArgs) ->
     try
       %If you are modifying the code comment the next line to show the exceptions
-      %error_logger:tty(false),
+      error_logger:tty(false),
       loop()
     catch
       throw:{eofErr,_}->
@@ -60,15 +60,14 @@ process() ->
               {error,BadJSON} ->
                   throw(BadJSON)
             end,
-            ExprList = tokenize(Content),
-            {ok,ParseList} = parse(ExprList),
+            TokExprList = tokenize(binary_to_list(Content)),
+            {ok,ParseList} = parse(TokExprList),
             FormatParse = format(ParseList),
             JSON = jsx:encode([{<<"status">>,<<"ok">>},{<<"ast">>,FormatParse}]),
             io:format("~s~n",[binary_to_list(JSON)])
     end.
 
 decode(InputSrt) ->
-
     SubS = string:substr(InputSrt,1,string:len(InputSrt)-1),
     case jsx:is_json(list_to_binary(SubS)) of
       true->Data= jsx:decode(list_to_binary(SubS)),
@@ -79,27 +78,23 @@ decode(InputSrt) ->
           end;
       false->
           {error,{json,<<"Input is not a valid JSON">>}}
-      end.
+    end.
+
 tokenize(Content) ->
-    Formated= string:join(lists:map(fun erlang:binary_to_list/1,[Content]),""),
-    string:tokens(Formated,".").
+    TokExprList = case erl_scan:string(Content) of
+      {ok,Tokens,_} ->
+            PTokens = removeMacros(Tokens),
+            splitByDots(PTokens);
+      {error,BadScan,_}-> throw({scan,BadScan})
+    end,
+    TokExprList.
 
 parse(ExprList) ->
     List = lists:foldl(fun (Expr,ParseList)->
-        case string:equal("\n",Expr) of
-          true ->
-            lists:append(ParseList,"");
-          false ->
-            ExprDot = string:concat(Expr,"."),
-            case erl_scan:string(ExprDot) of
-              {ok,Tokens,_} ->
-                  case parseExpr(Tokens) of
-                    {ok,AST} -> lists:append(ParseList,AST);
-                    {error,BadParse} -> throw({parse,BadParse})
-                  end;
-              {error,BadScan,_}-> throw({scan,BadScan})
-            end
-        end
+      case parseExpr(Expr) of
+        {ok,AST} -> lists:append(ParseList,AST);
+        {error,BadParse} -> throw({parse,BadParse})
+      end
     end,[],ExprList),
     {ok,List}.
 
@@ -113,6 +108,26 @@ parseExpr(Tokens) ->
             end
     end.
 
+splitByDots(TokenList)->
+    splitByDots(TokenList,[],[]).
+
+splitByDots([H|T],Acc,Final)->
+    if
+      element(1,H) == dot ->
+        SubList = [lists:append(Acc,[H])],
+        splitByDots(T,[],lists:append(Final,SubList));
+      true ->
+        splitByDots(T,lists:append(Acc,[H]),Final)
+    end;
+splitByDots([],Acc,Final)->
+    if
+      length(Acc) == 0 ->
+        Res = Final;
+      true ->
+        Res =lists:append(Final,[Acc])
+    end,
+    Res.
+
 %% Format do a conversion of erlang tuples to list, also change strings to binaries
 format(T) when is_list(T)->
     format(list_to_tuple(T));
@@ -133,6 +148,18 @@ format(T,N,Acc) when is_list(element(N,T)) ->
 format(T,N,Acc)->
     format(T,N-1,[element(N,T)|Acc]).
 
+removeMacros(Tokens) ->
+  removeMacros(Tokens,[]).
+
+removeMacros([H|T],Acc) ->
+  if
+    element(1,H) == '?' ->
+      removeMacros(T,Acc);
+    true ->
+      removeMacros(T,lists:append(Acc,[H]))
+  end;
+removeMacros([],Acc)->
+  Acc.
 
 
 %%Tests
@@ -174,3 +201,21 @@ format_test_()->
    ?_assert(format({"hello","world"})=:= [<<"hello">>,<<"world">>]),
    ?_assert(format({a,b,[c,"hello",{"world",["this","is","a","test"]}]})=:= [a,b,[c,<<"hello">>,[<<"world">>,[<<"this">>,<<"is">>,<<"a">>,<<"test">>]]]]),
    ?_assert(format({"hello",<<"world">>})=:= [<<"hello">>,<<"world">>])].
+
+ splitByDots_test_()->
+   String1 = "ok . ok . ok . ok",
+   String2 = ". . . . ok",
+   String3 = "ok",
+   String4 = ".",
+   {_,Tokens1,_} = erl_scan:string(String1),
+   {_,Tokens2,_} = erl_scan:string(String2),
+   {_,Tokens3,_} = erl_scan:string(String3),
+   {_,Tokens4,_} = erl_scan:string(String4),
+   Res1 = length(splitByDots(Tokens1)),
+   Res2 = length(splitByDots(Tokens2)),
+   Res3 = length(splitByDots(Tokens3)),
+   Res4 = length(splitByDots(Tokens4)),
+   [?_assertEqual(4,Res1),
+   ?_assertEqual(5,Res2),
+   ?_assertEqual(1,Res3),
+   ?_assertEqual(1,Res4)].