Skip to content

Commit

Permalink
kernel: Fix group to return translation errors correctly
Browse files Browse the repository at this point in the history 
When file is used to read from a group, we should return a translation
error if the input stream contains unicode characters (as file expects
latin1).

Closes erlang#7591
  • Loading branch information
@garazdawi
garazdawi committed Oct 10, 2023
1 parent d52f2f6 commit 0880242
Show file tree
Hide file tree
Showing 2 changed files with 110 additions and 18 deletions.
65 changes: 48 additions & 17 deletions lib/kernel/src/group.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -496,13 +496,14 @@ get_chars_loop(Pbs, M, F, Xa, Drv, Shell, Buf0, State, LineCont0, Encoding) ->
false ->
%% get_line_echo_off only deals with lists,
%% so convert to list before calling it.
get_line_echo_off(cast(Buf0, list, Encoding), Pbs, Drv, Shell)
get_line_echo_off(cast(Buf0, list), Encoding, Pbs, Drv, Shell)
end,
case Result of
{done,LineCont1,Buf} ->
get_chars_apply(Pbs, M, F, Xa, Drv, Shell, append(Buf, [], Encoding),
State, LineCont1, Encoding);

{no_translation, unicode, latin1} ->
{error,{error,{no_translation, unicode, latin1}}, []};
interrupted ->
{error,{error,interrupted},[]};
terminated ->
Expand Down Expand Up @@ -543,20 +544,25 @@ get_chars_apply(Pbs, M, F, Xa, Drv, Shell, Buf, State0, LineCont, Encoding) ->
end.

get_chars_n_loop(Pbs, M, F, Xa, Drv, Shell, Buf0, State, Encoding) ->
try M:F(State, cast(Buf0, get(read_mode), Encoding), Encoding, Xa) of
{stop,Result,Rest} ->
{ok, Result, append(Rest,[],Encoding)};
State1 ->
case get_chars_echo_off(Pbs, Drv, Shell) of
interrupted ->
{error,{error,interrupted},[]};
terminated ->
{exit,terminated};
Buf ->
get_chars_n_loop(Pbs, M, F, Xa, Drv, Shell, Buf, State1, Encoding)
case check_encoding(Buf0, Encoding) of
false ->
{error,{error,{no_translation,unicode,Encoding}},[]};
true ->
try M:F(State, cast(Buf0, get(read_mode), Encoding), Encoding, Xa) of
{stop,Result,Rest} ->
{ok, Result, append(Rest,[],Encoding)};
State1 ->
case get_chars_echo_off(Pbs, Drv, Shell) of
interrupted ->
{error,{error,interrupted},[]};
terminated ->
{exit,terminated};
Buf ->
get_chars_n_loop(Pbs, M, F, Xa, Drv, Shell, Buf, State1, Encoding)
end
catch _:_ ->
{error,{error,err_func(M, F, Xa)},[]}
end
catch _:_ ->
{error,{error,err_func(M, F, Xa)},[]}
end.

%% Convert error code to make it look as before
Expand Down Expand Up @@ -816,9 +822,19 @@ more_data(What, Cont0, Drv, Shell, Ls, Encoding) ->
get_line1(edlin:edit_line([], Cont0), Drv, Shell, Ls, Encoding)
end.

get_line_echo_off(Chars, Pbs, Drv, Shell) ->
get_line_echo_off(Chars, ToEnc, Pbs, Drv, Shell) ->
send_drv_reqs(Drv, [{put_chars, unicode,Pbs}]),
get_line_echo_off1(edit_line(Chars,[]), Drv, Shell).
case get_line_echo_off1(edit_line(Chars,[]), Drv, Shell) of
{done, Line, _Rest} = Res when ToEnc =:= latin1 ->
case check_encoding(Line, ToEnc) of
false ->
{no_translation, unicode, ToEnc};
true ->
Res
end;
Res ->
Res
end.

get_line_echo_off1({Chars,[]}, Drv, Shell) ->
receive
Expand Down Expand Up @@ -1055,3 +1071,18 @@ append(eof, L, _) ->
L;
append(B, L, FromEnc) ->
unicode:characters_to_list(B, FromEnc) ++ L.

check_encoding(eof, _) ->
true;
check_encoding(ListOrBinary, unicode) when is_list(ListOrBinary); is_binary(ListOrBinary) ->
true;
check_encoding(List, latin1) when is_list(List) ->
is_latin1(List).

is_latin1([H|T]) when 0 =< H, H =< 255 ->
is_latin1(T);
is_latin1([]) ->
true;
is_latin1(_) ->
false.

63 changes: 62 additions & 1 deletion lib/stdlib/test/io_proto_SUITE.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@
read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1,
unicode_prompt/1, shell_slogan/1, raw_stdout/1, raw_stdout_isatty/1,
file_read_stdin_binary_mode/1, file_read_stdin_list_mode/1,
file_read_stdin_unicode_translation_error_binary_mode/1,
file_read_stdin_unicode_translation_error_list_mode/1,
file_read_line_stdin_unicode_translation_error_binary_mode/1,
file_read_line_stdin_unicode_translation_error_list_mode/1,
io_get_chars_stdin_binary_mode/1, io_get_chars_stdin_list_mode/1,
io_get_chars_file_read_stdin_binary_mode/1,
file_read_stdin_latin1_binary_mode/1,
Expand Down Expand Up @@ -63,6 +67,10 @@ all() ->
shell_slogan, raw_stdout, raw_stdout_isatty,
file_read_stdin_binary_mode,
file_read_stdin_list_mode,
file_read_stdin_unicode_translation_error_binary_mode,
file_read_stdin_unicode_translation_error_list_mode,
file_read_line_stdin_unicode_translation_error_binary_mode,
file_read_line_stdin_unicode_translation_error_list_mode,
io_get_chars_stdin_binary_mode,
io_get_chars_stdin_list_mode,
io_get_chars_file_read_stdin_binary_mode,
Expand Down Expand Up @@ -311,6 +319,54 @@ file_read_stdin_list_mode(_Config) ->

ok.

%% Test that reading from stdin using file:read returns
%% correct error when in binary mode
file_read_stdin_unicode_translation_error_binary_mode(_Config) ->
{ok, P, ErlPort} = start_stdin_node(fun() -> file:read(standard_io, 3) end, [binary]),

erlang:port_command(ErlPort, <<"ęö€"/utf8>>),
{ok, "error: {no_translation,unicode,latin1}\n"} = gen_tcp:recv(P, 0),
ErlPort ! {self(), close},
{ok, "got: eof"} = gen_tcp:recv(P, 0),

ok.

%% Test that reading from stdin using file:read returns
%% correct error when in list mode
file_read_stdin_unicode_translation_error_list_mode(_Config) ->
{ok, P, ErlPort} = start_stdin_node(fun() -> file:read(standard_io, 3) end, [list]),

erlang:port_command(ErlPort, <<"ęö€"/utf8>>),
{ok, "error: {no_translation,unicode,latin1}\n"} = gen_tcp:recv(P, 0),
ErlPort ! {self(), close},
{ok, "got: eof"} = gen_tcp:recv(P, 0),

ok.

%% Test that reading from stdin using file:read_line returns
%% correct error when in binary mode
file_read_line_stdin_unicode_translation_error_binary_mode(_Config) ->
{ok, P, ErlPort} = start_stdin_node(fun() -> file:read_line(standard_io) end, [binary]),

erlang:port_command(ErlPort, <<"ę\nö\n€"/utf8>>),
{ok, "error: {no_translation,unicode,latin1}\n"} = gen_tcp:recv(P, 0),
ErlPort ! {self(), close},
{ok, "got: eof"} = gen_tcp:recv(P, 0),

ok.

%% Test that reading from stdin using file:read_line returns
%% correct error when in list mode
file_read_line_stdin_unicode_translation_error_list_mode(_Config) ->
{ok, P, ErlPort} = start_stdin_node(fun() -> file:read_line(standard_io) end, [list]),

erlang:port_command(ErlPort, <<"ę\nö\n€"/utf8>>),
{ok, "error: {no_translation,unicode,latin1}\n"} = gen_tcp:recv(P, 0),
ErlPort ! {self(), close},
{ok, "got: eof"} = gen_tcp:recv(P, 0),

ok.

%% Test that reading from stdin using file:read works when io is in binary mode
io_get_chars_stdin_binary_mode(_Config) ->
{ok, P, ErlPort} = start_stdin_node(
Expand Down Expand Up @@ -493,7 +549,8 @@ start_stdin_node(ReadFun, IoOptions, ExtraArgs) ->
" -pa ", filename:dirname(code:which(?MODULE)),
" -s ", atom_to_list(?MODULE), " read_raw_from_stdin ", integer_to_list(Port)]),
ct:log("~p~n", [Cmd]),
ErlPort = open_port({spawn, Cmd}, [stream, eof, stderr_to_stdout]),
ErlPort = open_port({spawn, Cmd}, [stream, eof, stderr_to_stdout,
{env, [{"LC_CTYPE","en_US.UTF-8"}]}]),
{ok, P} = gen_tcp:accept(L),
gen_tcp:send(P, term_to_binary(IoOptions)),
gen_tcp:send(P, term_to_binary(ReadFun)),
Expand Down Expand Up @@ -526,6 +583,10 @@ read_raw_from_stdin(ReadFun, P) ->
{ok, Fmt, Char} ->
gen_tcp:send(P, unicode:characters_to_binary(
io_lib:format("got: "++Fmt++"\n",[Char]))),
read_raw_from_stdin(ReadFun, P);
{error, Reason} ->
gen_tcp:send(P, unicode:characters_to_binary(
io_lib:format("error: ~p\n",[Reason]))),
read_raw_from_stdin(ReadFun, P)
end.

Expand Down

0 comments on commit 0880242

Please sign in to comment.