pty: Add support for PTY packet mode on sysv-like systems

Fixes: https://gitlab.gnome.org/GNOME/vte/-/issues/318

3 files changed, 79 insertions, 3 deletions

diff --git a/meson.build b/meson.build
index ea1d7bf3..ce52cb8d 100644
--- a/meson.build
+++ b/meson.build
@@ -152,6 +152,7 @@ check_headers = [
   'stropts.h',
   'sys/resource.h',
   'sys/select.h',
+  'sys/stream.h',
   'sys/syslimits.h',
   'sys/termios.h',
   'sys/types.h',
diff --git a/src/pty.cc b/src/pty.cc
index ce77106b..c6fcdc9d 100644
--- a/src/pty.cc
+++ b/src/pty.cc
@@ -357,10 +357,18 @@ Pty::get_size(int* rows,
 static int
 fd_set_cpkt(vte::libc::FD& fd)
 {
+        auto ret = 0;
+#if defined(TIOCPKT)
         /* tty_ioctl(4) -> every read() gives an extra byte at the beginning
          * notifying us of stop/start (^S/^Q) events. */
         int one = 1;
-        return ioctl(fd.get(), TIOCPKT, &one);
+        ret = ioctl(fd.get(), TIOCPKT, &one);
+#elif defined(__sun) && defined(HAVE_STROPTS_H)
+        if (isastream(fd.get()) == 1 &&
+            ioctl(fd.get(), I_FIND, "pckt") == 0)
+                ret = ioctl(fd.get(), I_PUSH, "pckt");
+#endif
+        return ret;
 }
 
 static int
@@ -400,7 +408,7 @@ fd_setup(vte::libc::FD& fd)
                 auto errsv = vte::libc::ErrnoSaver{};
                 _vte_debug_print(VTE_DEBUG_PTY,
                                  "%s failed: %s",
-                                 "ioctl(TIOCPKT)", g_strerror(errsv));
+                                 "Setting packet mode", g_strerror(errsv));
                 return -1;
         }
 
@@ -466,7 +474,7 @@ _vte_pty_open_posix(void)
                 auto errsv = vte::libc::ErrnoSaver{};
                 _vte_debug_print(VTE_DEBUG_PTY,
                                  "%s failed: %s",
-                                 "ioctl(TIOCPKT)", g_strerror(errsv));
+                                 "Setting packet mode", g_strerror(errsv));
                 return {};
         }
 
diff --git a/src/vte.cc b/src/vte.cc
index b5df3d24..4674241d 100644
--- a/src/vte.cc
+++ b/src/vte.cc
@@ -29,6 +29,12 @@
 #ifdef HAVE_SYS_TERMIOS_H
 #include <sys/termios.h>
 #endif
+#ifdef HAVE_STROPTS_H
+#include <stropts.h>
+#endif
+#ifdef HAVE_SYS_STREAM_H
+#include <sys/stream.h>
+#endif
 
 #include <glib.h>
 #include <glib-unix.h>
@@ -3966,6 +3972,7 @@ Terminal::pty_io_read(int const fd,
 			bp = chunk->begin_writing();
 			len = 0;
 			do {
+#if defined(TIOCPKT)
                                 /* We'd like to read (fd, bp, rem); but due to TIOCPKT mode
                                  * there's an extra input byte returned at the beginning.
                                  * We need to see what that byte is, but otherwise drop it
@@ -4014,6 +4021,66 @@ Terminal::pty_io_read(int const fd,
                                                 }
 						break;
 				}
+#elif defined(__sun) && defined(HAVE_STROPTS_H)
+				static unsigned char ctl_s[128];
+				struct strbuf ctlbuf, databuf;
+				int ret, flags = 0;
+				bool have_data = false;
+
+				ctlbuf.buf = (caddr_t)ctl_s;
+				ctlbuf.maxlen = sizeof(ctl_s);
+				databuf.buf = (caddr_t)bp;
+				databuf.maxlen = rem;
+
+				ret = getmsg(fd, &ctlbuf, &databuf, &flags);
+				if (ret == -1) {
+					err = errno;
+					goto out;
+				} else if (ctlbuf.len == 1) {
+					switch (ctl_s[0]) {
+					case M_IOCTL:
+						pty_termios_changed();
+						break;
+					case M_STOP:
+						pty_scroll_lock_changed(true);
+						break;
+					case M_START:
+						pty_scroll_lock_changed(false);
+						break;
+					case M_DATA:
+						have_data = true;
+						break;
+					}
+				} else if (ctlbuf.len == -1 && databuf.len != -1) {
+					// MOREDATA
+					have_data = true;
+				}
+
+				if (have_data) {
+					if (databuf.len == 0) {
+						eos = true;
+						goto out;
+					}
+					bp += databuf.len;
+					rem -= databuf.len;
+					len += databuf.len;
+				}
+#else /* neither TIOCPKT nor STREAMS pty */
+				int ret = read(fd, bp, rem);
+				switch (ret) {
+					case -1:
+						err = errno;
+						goto out;
+					case 0:
+						eos = true;
+						goto out;
+					default:
+						bp += ret;
+						rem -= ret;
+						len += ret;
+						break;
+				}
+#endif /* */
 			} while (rem);
 out:
 			chunk->add_size(len);