runtime, syscall: work around FreeBSD/amd64 kernel bug

The kernel implementation of the fast system call path,
the one invoked by the SYSCALL instruction, is broken for
restarting system calls. A C program demonstrating this is below.

Change the system calls to use INT $0x80 instead, because
that (perhaps slightly slower) system call path actually works.

I filed http://www.freebsd.org/cgi/query-pr.cgi?pr=182161.

The C program demonstrating that it is FreeBSD's fault is below.
It reports the same "Bad address" failures from wait.

#include <sys/time.h>
#include <sys/signal.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

static void handler(int);
static void* looper(void*);

int
main(void)
{
        int i;
        struct sigaction sa;
        pthread_cond_t cond;
        pthread_mutex_t mu;

        memset(&sa, 0, sizeof sa);
        sa.sa_handler = handler;
        sa.sa_flags = SA_RESTART;
        memset(&sa.sa_mask, 0xff, sizeof sa.sa_mask);
        sigaction(SIGCHLD, &sa, 0);

        for(i=0; i<2; i++)
                pthread_create(0, 0, looper, 0);

        pthread_mutex_init(&mu, 0);
        pthread_mutex_lock(&mu);
        pthread_cond_init(&cond, 0);
        for(;;)
                pthread_cond_wait(&cond, &mu);

        return 0;
}

static void
handler(int sig)
{
}

int
mywait4(int pid, int *stat, int options, struct rusage *rusage)
{
        int result;

        asm("movq %%rcx, %%r10; syscall"
                : "=a" (result)
                : "a" (7),
                  "D" (pid),
                  "S" (stat),
                  "d" (options),
                  "c" (rusage));
}

static void*
looper(void *v)
{
        int pid, stat, out;
        struct rusage rusage;

        for(;;) {
                if((pid = fork()) == 0)
                        _exit(0);
                out = mywait4(pid, &stat, 0, &rusage);
                if(out != pid) {
                        printf("wait4 returned %d\n", out);
                }
        }
}

Fixes issue 6372.

R=golang-dev, bradfitz
CC=golang-dev
https://codereview.appspot.com/13582047

2 files changed, 30 insertions, 0 deletions

diff --git a/src/pkg/runtime/sys_freebsd_amd64.s b/src/pkg/runtime/sys_freebsd_amd64.s
index e4aa7aabd..ef3beaedc 100644
--- a/src/pkg/runtime/sys_freebsd_amd64.s
+++ b/src/pkg/runtime/sys_freebsd_amd64.s
@@ -8,6 +8,31 @@
 
 #include "zasm_GOOS_GOARCH.h"
 #include "../../cmd/ld/textflag.h"
+
+// FreeBSD 8, FreeBSD 9, and older versions that I have checked
+// do not restore R10 on exit from a "restarted" system call
+// if you use the SYSCALL instruction. This means that, for example,
+// if a signal arrives while the wait4 system call is executing,
+// the wait4 internally returns ERESTART, which makes the kernel
+// back up the PC to execute the SYSCALL instruction a second time.
+// However, since the kernel does not restore R10, the fourth
+// argument to the system call has been lost. (FreeBSD 9 also fails
+// to restore the fifth and sixth arguments, R8 and R9, although
+// some earlier versions did restore those correctly.)
+// The broken code is in fast_syscall in FreeBSD's amd64/amd64/exception.S.
+// It restores only DI, SI, DX, AX, and RFLAGS on system call return.
+// http://fxr.watson.org/fxr/source/amd64/amd64/exception.S?v=FREEBSD91#L399
+//
+// The INT $0x80 system call path (int0x80_syscall in FreeBSD's 
+// amd64/ia32/ia32_exception.S) does not have this problem,
+// but it expects the third argument in R10. Instead of rewriting
+// all the assembly in this file, #define SYSCALL to a safe simulation
+// using INT $0x80.
+/
+// INT $0x80 is a little slower than SYSCALL, but correctness wins.
+//
+// See golang.org/issue/6372.
+#define SYSCALL MOVQ R10, CX; INT $0x80
 	
 TEXT runtime·sys_umtx_op(SB),NOSPLIT,$0
 	MOVQ 8(SP), DI
diff --git a/src/pkg/syscall/asm_freebsd_amd64.s b/src/pkg/syscall/asm_freebsd_amd64.s
index 72279d0af..fca7f371e 100644
--- a/src/pkg/syscall/asm_freebsd_amd64.s
+++ b/src/pkg/syscall/asm_freebsd_amd64.s
@@ -8,6 +8,11 @@
 // System call support for AMD64, FreeBSD
 //
 
+// The SYSCALL variant for invoking system calls is broken in FreeBSD.
+// See comment at top of ../runtime/sys_freebsd_amd64.c and
+// golang.org/issue/6372.
+#define SYSCALL MOVQ R10, CX; INT $0x80
+
 // func Syscall(trap int64, a1, a2, a3 int64) (r1, r2, err int64);
 // func Syscall6(trap int64, a1, a2, a3, a4, a5, a6 int64) (r1, r2, err int64);
 // func Syscall9(trap int64, a1, a2, a3, a4, a5, a6, a7, a8, a9 int64) (r1, r2, err int64)