From 338958e904bd3a7803a277b5e7af67985999b1f2 Mon Sep 17 00:00:00 2001 From: frazze-jobb Date: Fri, 5 May 2023 11:07:53 +0200 Subject: kernel: support sending raw byte data over stdout After rewriting the shell, group.erl converts everything to Unicode. However, this is problematic if you want to send raw byte data over stdout which may contain Erlang terms converted to binary that you want to convert back to Erlang term after data transfer. If io:setopts(_, {encoding, latin1}) is set, group.erl will just send the data directly to user_drv.erl which will send the data to prim_tty.erl without converting it to Unicode. --- lib/kernel/src/group.erl | 35 ++++++++++++++++++++++++----------- lib/kernel/src/prim_tty.erl | 5 +++++ lib/kernel/src/user_drv.erl | 4 ++++ 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/lib/kernel/src/group.erl b/lib/kernel/src/group.erl index 665b326ea0..311c2f3c5e 100644 --- a/lib/kernel/src/group.erl +++ b/lib/kernel/src/group.erl @@ -40,6 +40,7 @@ server(Ancestors, Drv, Shell, Options) -> put(line_buffer, proplists:get_value(line_buffer, Options, group_history:load())), put(read_mode, list), put(user_drv, Drv), + put(unicode_state, true), ExpandFun = normalize_expand_fun(Options, fun edlin_expand:expand/2), put(expand_fun, ExpandFun), put(echo, proplists:get_value(echo, Options, true)), @@ -239,17 +240,27 @@ io_request({put_chars,unicode,M,F,As}, Drv, _Shell, From, Buf) -> end end; io_request({put_chars,latin1,Binary}, Drv, _Shell, From, Buf) when is_binary(Binary) -> - send_drv(Drv, {put_chars_sync, unicode, - unicode:characters_to_binary(Binary,latin1), - From}), + IsUnicode = get(unicode_state), + if IsUnicode -> + send_drv(Drv, + {put_chars_sync, unicode, + unicode:characters_to_binary(Binary,latin1), + From}); + true -> + send_drv(Drv, {put_chars_sync, latin1, Binary, From}) + end, {noreply,Buf}; io_request({put_chars,latin1,Chars}, Drv, _Shell, From, Buf) -> - case catch unicode:characters_to_binary(Chars,latin1) of - Binary when is_binary(Binary) -> - send_drv(Drv, {put_chars_sync, unicode, Binary, From}), - {noreply,Buf}; - _ -> - {error,{error,{put_chars,latin1,Chars}},Buf} + IsUnicode = get(unicode_state), + if IsUnicode -> + case catch unicode:characters_to_binary(Chars,latin1) of + Binary when is_binary(Binary) -> + send_drv(Drv, {put_chars_sync, unicode, Binary, From}), + {noreply,Buf}; + _ -> + {error,{error,{put_chars,latin1,Chars}},Buf} + end; + true -> send_drv(Drv, {put_chars_sync, latin1, Chars, From}) end; io_request({put_chars,latin1,M,F,As}, Drv, _Shell, From, Buf) -> case catch apply(M, F, As) of @@ -396,9 +407,11 @@ do_setopts(Opts, Drv, Buf) -> put(echo, proplists:get_value(echo, Opts, get(echo))), case proplists:get_value(encoding,Opts) of Valid when Valid =:= unicode; Valid =:= utf8 -> - set_unicode_state(Drv,true); + set_unicode_state(Drv,true), + put(unicode_state, true); latin1 -> - set_unicode_state(Drv,false); + set_unicode_state(Drv,false), + put(unicode_state, false); _ -> ok end, diff --git a/lib/kernel/src/prim_tty.erl b/lib/kernel/src/prim_tty.erl index 7ed418de5e..4ef10c752d 100644 --- a/lib/kernel/src/prim_tty.erl +++ b/lib/kernel/src/prim_tty.erl @@ -164,6 +164,7 @@ sig => boolean() }. -type request() :: + {putc_raw, binary()} | {putc, unicode:unicode_binary()} | {expand, unicode:unicode_binary()} | {insert, unicode:unicode_binary()} | @@ -525,6 +526,8 @@ writer_loop(TTY, WriterRef) -> -spec handle_request(state(), request()) -> {erlang:iovec(), state()}. handle_request(State = #state{ options = #{ tty := false } }, Request) -> case Request of + {putc_raw, Binary} -> + {Binary, State}; {putc, Binary} -> {encode(Binary, State#state.unicode), State}; beep -> @@ -565,6 +568,8 @@ handle_request(State = #state{ unicode = U }, {putc, Binary}) -> {_, _, _, NewBA} = split(NewLength - OldLength, NewState#state.buffer_after, U), {encode(PutBuffer, U), NewState#state{ buffer_after = NewBA }} end; +handle_request(State, {putc_raw, Binary}) -> + handle_request(State, {putc, unicode:characters_to_binary(Binary, latin1)}); handle_request(State = #state{ unicode = U }, {delete, N}) when N > 0 -> {_DelNum, DelCols, _, NewBA} = split(N, State#state.buffer_after, U), BBCols = cols(State#state.buffer_before, U), diff --git a/lib/kernel/src/user_drv.erl b/lib/kernel/src/user_drv.erl index bcc0d2b78b..fc1a4434fb 100644 --- a/lib/kernel/src/user_drv.erl +++ b/lib/kernel/src/user_drv.erl @@ -765,6 +765,10 @@ io_request({put_chars_sync, unicode, Chars, Reply}, TTY) -> {Output, NewTTY} = prim_tty:handle_request(TTY, {putc, unicode:characters_to_binary(Chars)}), {ok, MonitorRef} = prim_tty:write(NewTTY, Output, self()), {Reply, MonitorRef, NewTTY}; +io_request({put_chars_sync, latin1, Chars, Reply}, TTY) -> + {Output, NewTTY} = prim_tty:handle_request(TTY, {putc_raw, Chars}), + {ok, MonitorRef} = prim_tty:write(NewTTY, Output, self()), + {Reply, MonitorRef, NewTTY}; io_request({put_expand, unicode, Chars}, TTY) -> write(prim_tty:handle_request(TTY, {expand, unicode:characters_to_binary(Chars)})); io_request({move_rel, N}, TTY) -> -- cgit v1.2.1 From 0fa5e8b1f7dcd96fc1fcce271d475b36dc8984b6 Mon Sep 17 00:00:00 2001 From: Rickard Green Date: Fri, 5 May 2023 18:45:19 +0200 Subject: Testcase io_proto_SUITE:raw_stdout --- lib/stdlib/test/io_proto_SUITE.erl | 39 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index bc96992ce2..81579c05c2 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -25,7 +25,7 @@ -export([setopts_getopts/1,unicode_options/1,unicode_options_gen/1, binary_options/1, read_modes_gl/1, read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1, - unicode_prompt/1, shell_slogan/1]). + unicode_prompt/1, shell_slogan/1, raw_stdout/1]). -export([io_server_proxy/1,start_io_server_proxy/0, proxy_getall/1, @@ -35,6 +35,8 @@ -export([uprompt/1, slogan/0, session_slogan/0]). +-export([write_raw_to_stdout/0]). + %%-define(debug, true). -ifdef(debug). @@ -51,7 +53,7 @@ all() -> [setopts_getopts, unicode_options, unicode_options_gen, binary_options, read_modes_gl, read_modes_ogl, broken_unicode, eof_on_pipe, unicode_prompt, - shell_slogan]. + shell_slogan, raw_stdout]. groups() -> []. @@ -1091,6 +1093,39 @@ eof_on_pipe(Config) when is_list(Config) -> {skipped,"Only on linux"} end. +raw_stdout(Config) when is_list(Config) -> + Cmd = lists:append( + [ct:get_progname(), + " -noshell -noinput", + " -pa ", filename:dirname(code:which(?MODULE)), + " -s ", atom_to_list(?MODULE), " write_raw_to_stdout"]), + ct:log("~p~n", [Cmd]), + Port = open_port({spawn, Cmd}, [stream, eof]), + Expected = lists:seq(0,255), + Expected = get_all_port_data(Port, []), + Port ! {self(), close}, + ok. + +get_all_port_data(Port, Acc) -> + receive + {Port, {data, Data}} -> + get_all_port_data(Port, [Acc|Data]); + {Port, eof} -> + lists:flatten(Acc) + end. + +write_raw_to_stdout() -> + try + ok = io:setopts(standard_io, [{encoding, latin1}]), + ok = file:write(standard_io, lists:seq(0,255)), + halt(0) + catch + Class:Reason:StackTrace -> + io:format(standard_error, "~p~p~p", [Class, Reason, StackTrace]), + halt(17) + end. + + %% %% Test I/O-server %% -- cgit v1.2.1 From 37556967e4ddc6b32c748f8bb0bd4245b1612671 Mon Sep 17 00:00:00 2001 From: frazze-jobb Date: Wed, 10 May 2023 14:50:30 +0200 Subject: Testcase io_proto_SUITE:raw_stdout_isatty --- lib/stdlib/test/io_proto_SUITE.erl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/stdlib/test/io_proto_SUITE.erl b/lib/stdlib/test/io_proto_SUITE.erl index 81579c05c2..0260b3251c 100644 --- a/lib/stdlib/test/io_proto_SUITE.erl +++ b/lib/stdlib/test/io_proto_SUITE.erl @@ -25,7 +25,7 @@ -export([setopts_getopts/1,unicode_options/1,unicode_options_gen/1, binary_options/1, read_modes_gl/1, read_modes_ogl/1, broken_unicode/1,eof_on_pipe/1, - unicode_prompt/1, shell_slogan/1, raw_stdout/1]). + unicode_prompt/1, shell_slogan/1, raw_stdout/1, raw_stdout_isatty/1]). -export([io_server_proxy/1,start_io_server_proxy/0, proxy_getall/1, @@ -53,7 +53,7 @@ all() -> [setopts_getopts, unicode_options, unicode_options_gen, binary_options, read_modes_gl, read_modes_ogl, broken_unicode, eof_on_pipe, unicode_prompt, - shell_slogan, raw_stdout]. + shell_slogan, raw_stdout, raw_stdout_isatty]. groups() -> []. @@ -1124,8 +1124,14 @@ write_raw_to_stdout() -> io:format(standard_error, "~p~p~p", [Class, Reason, StackTrace]), halt(17) end. - +raw_stdout_isatty(Config) when is_list(Config) -> + rtnode:run( + [{putline,"io:setopts(group_leader(), [{encoding, latin1}])."}, + {putline,"file:write(group_leader(),[90, 127, 128, 255, 131, 90, 10])."},% + {expect, "\\QZ^?\\200\\377\\203Z\\E"} + ],[]), + ok. %% %% Test I/O-server %% -- cgit v1.2.1