diff options
Diffstat (limited to 'src/net')
240 files changed, 54136 insertions, 0 deletions
diff --git a/src/net/cgo_android.go b/src/net/cgo_android.go new file mode 100644 index 000000000..3819ce56a --- /dev/null +++ b/src/net/cgo_android.go @@ -0,0 +1,14 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo,!netgo + +package net + +//#include <netdb.h> +import "C" + +func cgoAddrInfoFlags() C.int { + return C.AI_CANONNAME +} diff --git a/src/net/cgo_bsd.go b/src/net/cgo_bsd.go new file mode 100644 index 000000000..3090d3019 --- /dev/null +++ b/src/net/cgo_bsd.go @@ -0,0 +1,17 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !netgo +// +build darwin dragonfly freebsd solaris + +package net + +/* +#include <netdb.h> +*/ +import "C" + +func cgoAddrInfoFlags() C.int { + return (C.AI_CANONNAME | C.AI_V4MAPPED | C.AI_ALL) & C.AI_MASK +} diff --git a/src/net/cgo_linux.go b/src/net/cgo_linux.go new file mode 100644 index 000000000..4ef2d0cd1 --- /dev/null +++ b/src/net/cgo_linux.go @@ -0,0 +1,22 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !android,cgo,!netgo + +package net + +/* +#include <netdb.h> +*/ +import "C" + +func cgoAddrInfoFlags() C.int { + // NOTE(rsc): In theory there are approximately balanced + // arguments for and against including AI_ADDRCONFIG + // in the flags (it includes IPv4 results only on IPv4 systems, + // and similarly for IPv6), but in practice setting it causes + // getaddrinfo to return the wrong canonical name on Linux. + // So definitely leave it out. + return C.AI_CANONNAME | C.AI_V4MAPPED | C.AI_ALL +} diff --git a/src/net/cgo_netbsd.go b/src/net/cgo_netbsd.go new file mode 100644 index 000000000..09c5ad2d9 --- /dev/null +++ b/src/net/cgo_netbsd.go @@ -0,0 +1,16 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo,!netgo + +package net + +/* +#include <netdb.h> +*/ +import "C" + +func cgoAddrInfoFlags() C.int { + return C.AI_CANONNAME +} diff --git a/src/net/cgo_openbsd.go b/src/net/cgo_openbsd.go new file mode 100644 index 000000000..09c5ad2d9 --- /dev/null +++ b/src/net/cgo_openbsd.go @@ -0,0 +1,16 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo,!netgo + +package net + +/* +#include <netdb.h> +*/ +import "C" + +func cgoAddrInfoFlags() C.int { + return C.AI_CANONNAME +} diff --git a/src/net/cgo_stub.go b/src/net/cgo_stub.go new file mode 100644 index 000000000..f533c1421 --- /dev/null +++ b/src/net/cgo_stub.go @@ -0,0 +1,25 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !cgo netgo + +// Stub cgo routines for systems that do not use cgo to do network lookups. + +package net + +func cgoLookupHost(name string) (addrs []string, err error, completed bool) { + return nil, nil, false +} + +func cgoLookupPort(network, service string) (port int, err error, completed bool) { + return 0, nil, false +} + +func cgoLookupIP(name string) (addrs []IP, err error, completed bool) { + return nil, nil, false +} + +func cgoLookupCNAME(name string) (cname string, err error, completed bool) { + return "", nil, false +} diff --git a/src/net/cgo_unix.go b/src/net/cgo_unix.go new file mode 100644 index 000000000..1f366ee5c --- /dev/null +++ b/src/net/cgo_unix.go @@ -0,0 +1,164 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !netgo +// +build darwin dragonfly freebsd linux netbsd openbsd + +package net + +/* +#include <sys/types.h> +#include <sys/socket.h> +#include <netinet/in.h> +#include <netdb.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +*/ +import "C" + +import ( + "syscall" + "unsafe" +) + +func cgoLookupHost(name string) (addrs []string, err error, completed bool) { + ip, err, completed := cgoLookupIP(name) + for _, p := range ip { + addrs = append(addrs, p.String()) + } + return +} + +func cgoLookupPort(net, service string) (port int, err error, completed bool) { + acquireThread() + defer releaseThread() + + var res *C.struct_addrinfo + var hints C.struct_addrinfo + + switch net { + case "": + // no hints + case "tcp", "tcp4", "tcp6": + hints.ai_socktype = C.SOCK_STREAM + hints.ai_protocol = C.IPPROTO_TCP + case "udp", "udp4", "udp6": + hints.ai_socktype = C.SOCK_DGRAM + hints.ai_protocol = C.IPPROTO_UDP + default: + return 0, UnknownNetworkError(net), true + } + if len(net) >= 4 { + switch net[3] { + case '4': + hints.ai_family = C.AF_INET + case '6': + hints.ai_family = C.AF_INET6 + } + } + + s := C.CString(service) + defer C.free(unsafe.Pointer(s)) + if C.getaddrinfo(nil, s, &hints, &res) == 0 { + defer C.freeaddrinfo(res) + for r := res; r != nil; r = r.ai_next { + switch r.ai_family { + default: + continue + case C.AF_INET: + sa := (*syscall.RawSockaddrInet4)(unsafe.Pointer(r.ai_addr)) + p := (*[2]byte)(unsafe.Pointer(&sa.Port)) + return int(p[0])<<8 | int(p[1]), nil, true + case C.AF_INET6: + sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(r.ai_addr)) + p := (*[2]byte)(unsafe.Pointer(&sa.Port)) + return int(p[0])<<8 | int(p[1]), nil, true + } + } + } + return 0, &AddrError{"unknown port", net + "/" + service}, true +} + +func cgoLookupIPCNAME(name string) (addrs []IP, cname string, err error, completed bool) { + acquireThread() + defer releaseThread() + + var res *C.struct_addrinfo + var hints C.struct_addrinfo + + hints.ai_flags = cgoAddrInfoFlags() + hints.ai_socktype = C.SOCK_STREAM + + h := C.CString(name) + defer C.free(unsafe.Pointer(h)) + gerrno, err := C.getaddrinfo(h, nil, &hints, &res) + if gerrno != 0 { + var str string + if gerrno == C.EAI_NONAME { + str = noSuchHost + } else if gerrno == C.EAI_SYSTEM { + if err == nil { + // err should not be nil, but sometimes getaddrinfo returns + // gerrno == C.EAI_SYSTEM with err == nil on Linux. + // The report claims that it happens when we have too many + // open files, so use syscall.EMFILE (too many open files in system). + // Most system calls would return ENFILE (too many open files), + // so at the least EMFILE should be easy to recognize if this + // comes up again. golang.org/issue/6232. + err = syscall.EMFILE + } + str = err.Error() + } else { + str = C.GoString(C.gai_strerror(gerrno)) + } + return nil, "", &DNSError{Err: str, Name: name}, true + } + defer C.freeaddrinfo(res) + if res != nil { + cname = C.GoString(res.ai_canonname) + if cname == "" { + cname = name + } + if len(cname) > 0 && cname[len(cname)-1] != '.' { + cname += "." + } + } + for r := res; r != nil; r = r.ai_next { + // We only asked for SOCK_STREAM, but check anyhow. + if r.ai_socktype != C.SOCK_STREAM { + continue + } + switch r.ai_family { + default: + continue + case C.AF_INET: + sa := (*syscall.RawSockaddrInet4)(unsafe.Pointer(r.ai_addr)) + addrs = append(addrs, copyIP(sa.Addr[:])) + case C.AF_INET6: + sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(r.ai_addr)) + addrs = append(addrs, copyIP(sa.Addr[:])) + } + } + return addrs, cname, nil, true +} + +func cgoLookupIP(name string) (addrs []IP, err error, completed bool) { + addrs, _, err, completed = cgoLookupIPCNAME(name) + return +} + +func cgoLookupCNAME(name string) (cname string, err error, completed bool) { + _, cname, err, completed = cgoLookupIPCNAME(name) + return +} + +func copyIP(x IP) IP { + if len(x) < 16 { + return x.To16() + } + y := make(IP, len(x)) + copy(y, x) + return y +} diff --git a/src/net/cgo_unix_test.go b/src/net/cgo_unix_test.go new file mode 100644 index 000000000..33566ce9c --- /dev/null +++ b/src/net/cgo_unix_test.go @@ -0,0 +1,24 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build cgo,!netgo +// +build darwin dragonfly freebsd linux netbsd openbsd + +package net + +import "testing" + +func TestCgoLookupIP(t *testing.T) { + host := "localhost" + _, err, ok := cgoLookupIP(host) + if !ok { + t.Errorf("cgoLookupIP must not be a placeholder") + } + if err != nil { + t.Errorf("cgoLookupIP failed: %v", err) + } + if _, err := goLookupIP(host); err != nil { + t.Errorf("goLookupIP failed: %v", err) + } +} diff --git a/src/net/conn_test.go b/src/net/conn_test.go new file mode 100644 index 000000000..9c9d1a805 --- /dev/null +++ b/src/net/conn_test.go @@ -0,0 +1,124 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements API tests across platforms and will never have a build +// tag. + +package net + +import ( + "os" + "runtime" + "testing" + "time" +) + +var connTests = []struct { + net string + addr string +}{ + {"tcp", "127.0.0.1:0"}, + {"unix", testUnixAddr()}, + {"unixpacket", testUnixAddr()}, +} + +// someTimeout is used just to test that net.Conn implementations +// don't explode when their SetFooDeadline methods are called. +// It isn't actually used for testing timeouts. +const someTimeout = 10 * time.Second + +func TestConnAndListener(t *testing.T) { + for _, tt := range connTests { + switch tt.net { + case "unix": + switch runtime.GOOS { + case "nacl", "plan9", "windows": + continue + } + case "unixpacket": + switch runtime.GOOS { + case "android", "darwin", "nacl", "openbsd", "plan9", "windows": + continue + case "freebsd": // FreeBSD 8 doesn't support unixpacket + continue + } + } + + ln, err := Listen(tt.net, tt.addr) + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer func(ln Listener, net, addr string) { + ln.Close() + switch net { + case "unix", "unixpacket": + os.Remove(addr) + } + }(ln, tt.net, tt.addr) + if ln.Addr().Network() != tt.net { + t.Fatalf("got %v; expected %v", ln.Addr().Network(), tt.net) + } + + done := make(chan int) + go transponder(t, ln, done) + + c, err := Dial(tt.net, ln.Addr().String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c.Close() + if c.LocalAddr().Network() != tt.net || c.LocalAddr().Network() != tt.net { + t.Fatalf("got %v->%v; expected %v->%v", c.LocalAddr().Network(), c.RemoteAddr().Network(), tt.net, tt.net) + } + c.SetDeadline(time.Now().Add(someTimeout)) + c.SetReadDeadline(time.Now().Add(someTimeout)) + c.SetWriteDeadline(time.Now().Add(someTimeout)) + + if _, err := c.Write([]byte("CONN TEST")); err != nil { + t.Fatalf("Conn.Write failed: %v", err) + } + rb := make([]byte, 128) + if _, err := c.Read(rb); err != nil { + t.Fatalf("Conn.Read failed: %v", err) + } + + <-done + } +} + +func transponder(t *testing.T, ln Listener, done chan<- int) { + defer func() { done <- 1 }() + + switch ln := ln.(type) { + case *TCPListener: + ln.SetDeadline(time.Now().Add(someTimeout)) + case *UnixListener: + ln.SetDeadline(time.Now().Add(someTimeout)) + } + c, err := ln.Accept() + if err != nil { + t.Errorf("Listener.Accept failed: %v", err) + return + } + defer c.Close() + network := ln.Addr().Network() + if c.LocalAddr().Network() != network || c.LocalAddr().Network() != network { + t.Errorf("got %v->%v; expected %v->%v", c.LocalAddr().Network(), c.RemoteAddr().Network(), network, network) + return + } + c.SetDeadline(time.Now().Add(someTimeout)) + c.SetReadDeadline(time.Now().Add(someTimeout)) + c.SetWriteDeadline(time.Now().Add(someTimeout)) + + b := make([]byte, 128) + n, err := c.Read(b) + if err != nil { + t.Errorf("Conn.Read failed: %v", err) + return + } + if _, err := c.Write(b[:n]); err != nil { + t.Errorf("Conn.Write failed: %v", err) + return + } +} diff --git a/src/net/dial.go b/src/net/dial.go new file mode 100644 index 000000000..e6f0436cd --- /dev/null +++ b/src/net/dial.go @@ -0,0 +1,302 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "time" +) + +// A Dialer contains options for connecting to an address. +// +// The zero value for each field is equivalent to dialing +// without that option. Dialing with the zero value of Dialer +// is therefore equivalent to just calling the Dial function. +type Dialer struct { + // Timeout is the maximum amount of time a dial will wait for + // a connect to complete. If Deadline is also set, it may fail + // earlier. + // + // The default is no timeout. + // + // With or without a timeout, the operating system may impose + // its own earlier timeout. For instance, TCP timeouts are + // often around 3 minutes. + Timeout time.Duration + + // Deadline is the absolute point in time after which dials + // will fail. If Timeout is set, it may fail earlier. + // Zero means no deadline, or dependent on the operating system + // as with the Timeout option. + Deadline time.Time + + // LocalAddr is the local address to use when dialing an + // address. The address must be of a compatible type for the + // network being dialed. + // If nil, a local address is automatically chosen. + LocalAddr Addr + + // DualStack allows a single dial to attempt to establish + // multiple IPv4 and IPv6 connections and to return the first + // established connection when the network is "tcp" and the + // destination is a host name that has multiple address family + // DNS records. + DualStack bool + + // KeepAlive specifies the keep-alive period for an active + // network connection. + // If zero, keep-alives are not enabled. Network protocols + // that do not support keep-alives ignore this field. + KeepAlive time.Duration +} + +// Return either now+Timeout or Deadline, whichever comes first. +// Or zero, if neither is set. +func (d *Dialer) deadline() time.Time { + if d.Timeout == 0 { + return d.Deadline + } + timeoutDeadline := time.Now().Add(d.Timeout) + if d.Deadline.IsZero() || timeoutDeadline.Before(d.Deadline) { + return timeoutDeadline + } else { + return d.Deadline + } +} + +func parseNetwork(net string) (afnet string, proto int, err error) { + i := last(net, ':') + if i < 0 { // no colon + switch net { + case "tcp", "tcp4", "tcp6": + case "udp", "udp4", "udp6": + case "ip", "ip4", "ip6": + case "unix", "unixgram", "unixpacket": + default: + return "", 0, UnknownNetworkError(net) + } + return net, 0, nil + } + afnet = net[:i] + switch afnet { + case "ip", "ip4", "ip6": + protostr := net[i+1:] + proto, i, ok := dtoi(protostr, 0) + if !ok || i != len(protostr) { + proto, err = lookupProtocol(protostr) + if err != nil { + return "", 0, err + } + } + return afnet, proto, nil + } + return "", 0, UnknownNetworkError(net) +} + +func resolveAddr(op, net, addr string, deadline time.Time) (netaddr, error) { + afnet, _, err := parseNetwork(net) + if err != nil { + return nil, err + } + if op == "dial" && addr == "" { + return nil, errMissingAddress + } + switch afnet { + case "unix", "unixgram", "unixpacket": + return ResolveUnixAddr(afnet, addr) + } + return resolveInternetAddr(afnet, addr, deadline) +} + +// Dial connects to the address on the named network. +// +// Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only), +// "udp", "udp4" (IPv4-only), "udp6" (IPv6-only), "ip", "ip4" +// (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and +// "unixpacket". +// +// For TCP and UDP networks, addresses have the form host:port. +// If host is a literal IPv6 address it must be enclosed +// in square brackets as in "[::1]:80" or "[ipv6-host%zone]:80". +// The functions JoinHostPort and SplitHostPort manipulate addresses +// in this form. +// +// Examples: +// Dial("tcp", "12.34.56.78:80") +// Dial("tcp", "google.com:http") +// Dial("tcp", "[2001:db8::1]:http") +// Dial("tcp", "[fe80::1%lo0]:80") +// +// For IP networks, the network must be "ip", "ip4" or "ip6" followed +// by a colon and a protocol number or name and the addr must be a +// literal IP address. +// +// Examples: +// Dial("ip4:1", "127.0.0.1") +// Dial("ip6:ospf", "::1") +// +// For Unix networks, the address must be a file system path. +func Dial(network, address string) (Conn, error) { + var d Dialer + return d.Dial(network, address) +} + +// DialTimeout acts like Dial but takes a timeout. +// The timeout includes name resolution, if required. +func DialTimeout(network, address string, timeout time.Duration) (Conn, error) { + d := Dialer{Timeout: timeout} + return d.Dial(network, address) +} + +// Dial connects to the address on the named network. +// +// See func Dial for a description of the network and address +// parameters. +func (d *Dialer) Dial(network, address string) (Conn, error) { + ra, err := resolveAddr("dial", network, address, d.deadline()) + if err != nil { + return nil, &OpError{Op: "dial", Net: network, Addr: nil, Err: err} + } + dialer := func(deadline time.Time) (Conn, error) { + return dialSingle(network, address, d.LocalAddr, ra.toAddr(), deadline) + } + if ras, ok := ra.(addrList); ok && d.DualStack && network == "tcp" { + dialer = func(deadline time.Time) (Conn, error) { + return dialMulti(network, address, d.LocalAddr, ras, deadline) + } + } + c, err := dial(network, ra.toAddr(), dialer, d.deadline()) + if d.KeepAlive > 0 && err == nil { + if tc, ok := c.(*TCPConn); ok { + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(d.KeepAlive) + testHookSetKeepAlive() + } + } + return c, err +} + +var testHookSetKeepAlive = func() {} // changed by dial_test.go + +// dialMulti attempts to establish connections to each destination of +// the list of addresses. It will return the first established +// connection and close the other connections. Otherwise it returns +// error on the last attempt. +func dialMulti(net, addr string, la Addr, ras addrList, deadline time.Time) (Conn, error) { + type racer struct { + Conn + error + } + // Sig controls the flow of dial results on lane. It passes a + // token to the next racer and also indicates the end of flow + // by using closed channel. + sig := make(chan bool, 1) + lane := make(chan racer, 1) + for _, ra := range ras { + go func(ra Addr) { + c, err := dialSingle(net, addr, la, ra, deadline) + if _, ok := <-sig; ok { + lane <- racer{c, err} + } else if err == nil { + // We have to return the resources + // that belong to the other + // connections here for avoiding + // unnecessary resource starvation. + c.Close() + } + }(ra.toAddr()) + } + defer close(sig) + lastErr := errTimeout + nracers := len(ras) + for nracers > 0 { + sig <- true + racer := <-lane + if racer.error == nil { + return racer.Conn, nil + } + lastErr = racer.error + nracers-- + } + return nil, lastErr +} + +// dialSingle attempts to establish and returns a single connection to +// the destination address. +func dialSingle(net, addr string, la, ra Addr, deadline time.Time) (c Conn, err error) { + if la != nil && la.Network() != ra.Network() { + return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: errors.New("mismatched local address type " + la.Network())} + } + switch ra := ra.(type) { + case *TCPAddr: + la, _ := la.(*TCPAddr) + c, err = dialTCP(net, la, ra, deadline) + case *UDPAddr: + la, _ := la.(*UDPAddr) + c, err = dialUDP(net, la, ra, deadline) + case *IPAddr: + la, _ := la.(*IPAddr) + c, err = dialIP(net, la, ra, deadline) + case *UnixAddr: + la, _ := la.(*UnixAddr) + c, err = dialUnix(net, la, ra, deadline) + default: + return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: addr}} + } + if err != nil { + return nil, err // c is non-nil interface containing nil pointer + } + return c, nil +} + +// Listen announces on the local network address laddr. +// The network net must be a stream-oriented network: "tcp", "tcp4", +// "tcp6", "unix" or "unixpacket". +// See Dial for the syntax of laddr. +func Listen(net, laddr string) (Listener, error) { + la, err := resolveAddr("listen", net, laddr, noDeadline) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: err} + } + var l Listener + switch la := la.toAddr().(type) { + case *TCPAddr: + l, err = ListenTCP(net, la) + case *UnixAddr: + l, err = ListenUnix(net, la) + default: + return nil, &OpError{Op: "listen", Net: net, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}} + } + if err != nil { + return nil, err // l is non-nil interface containing nil pointer + } + return l, nil +} + +// ListenPacket announces on the local network address laddr. +// The network net must be a packet-oriented network: "udp", "udp4", +// "udp6", "ip", "ip4", "ip6" or "unixgram". +// See Dial for the syntax of laddr. +func ListenPacket(net, laddr string) (PacketConn, error) { + la, err := resolveAddr("listen", net, laddr, noDeadline) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: err} + } + var l PacketConn + switch la := la.toAddr().(type) { + case *UDPAddr: + l, err = ListenUDP(net, la) + case *IPAddr: + l, err = ListenIP(net, la) + case *UnixAddr: + l, err = ListenUnixgram(net, la) + default: + return nil, &OpError{Op: "listen", Net: net, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}} + } + if err != nil { + return nil, err // l is non-nil interface containing nil pointer + } + return l, nil +} diff --git a/src/net/dial_gen.go b/src/net/dial_gen.go new file mode 100644 index 000000000..ada623300 --- /dev/null +++ b/src/net/dial_gen.go @@ -0,0 +1,46 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows plan9 + +package net + +import ( + "time" +) + +var testingIssue5349 bool // used during tests + +// dialChannel is the simple pure-Go implementation of dial, still +// used on operating systems where the deadline hasn't been pushed +// down into the pollserver. (Plan 9 and some old versions of Windows) +func dialChannel(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { + var timeout time.Duration + if !deadline.IsZero() { + timeout = deadline.Sub(time.Now()) + } + if timeout <= 0 { + return dialer(noDeadline) + } + t := time.NewTimer(timeout) + defer t.Stop() + type racer struct { + Conn + error + } + ch := make(chan racer, 1) + go func() { + if testingIssue5349 { + time.Sleep(time.Millisecond) + } + c, err := dialer(noDeadline) + ch <- racer{c, err} + }() + select { + case <-t.C: + return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: errTimeout} + case racer := <-ch: + return racer.Conn, racer.error + } +} diff --git a/src/net/dial_gen_test.go b/src/net/dial_gen_test.go new file mode 100644 index 000000000..c857acd06 --- /dev/null +++ b/src/net/dial_gen_test.go @@ -0,0 +1,11 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows plan9 + +package net + +func init() { + testingIssue5349 = true +} diff --git a/src/net/dial_test.go b/src/net/dial_test.go new file mode 100644 index 000000000..c5c3236cc --- /dev/null +++ b/src/net/dial_test.go @@ -0,0 +1,536 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "flag" + "fmt" + "io" + "os" + "os/exec" + "reflect" + "regexp" + "runtime" + "strconv" + "sync" + "testing" + "time" +) + +func newLocalListener(t *testing.T) Listener { + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + ln, err = Listen("tcp6", "[::1]:0") + } + if err != nil { + t.Fatal(err) + } + return ln +} + +func TestDialTimeout(t *testing.T) { + origBacklog := listenerBacklog + defer func() { + listenerBacklog = origBacklog + }() + listenerBacklog = 1 + + ln := newLocalListener(t) + defer ln.Close() + + errc := make(chan error) + + numConns := listenerBacklog + 100 + + // TODO(bradfitz): It's hard to test this in a portable + // way. This is unfortunate, but works for now. + switch runtime.GOOS { + case "linux": + // The kernel will start accepting TCP connections before userspace + // gets a chance to not accept them, so fire off a bunch to fill up + // the kernel's backlog. Then we test we get a failure after that. + for i := 0; i < numConns; i++ { + go func() { + _, err := DialTimeout("tcp", ln.Addr().String(), 200*time.Millisecond) + errc <- err + }() + } + case "darwin", "plan9", "windows": + // At least OS X 10.7 seems to accept any number of + // connections, ignoring listen's backlog, so resort + // to connecting to a hopefully-dead 127/8 address. + // Same for windows. + // + // Use an IANA reserved port (49151) instead of 80, because + // on our 386 builder, this Dial succeeds, connecting + // to an IIS web server somewhere. The data center + // or VM or firewall must be stealing the TCP connection. + // + // IANA Service Name and Transport Protocol Port Number Registry + // <http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xml> + go func() { + c, err := DialTimeout("tcp", "127.0.71.111:49151", 200*time.Millisecond) + if err == nil { + err = fmt.Errorf("unexpected: connected to %s!", c.RemoteAddr()) + c.Close() + } + errc <- err + }() + default: + // TODO(bradfitz): + // OpenBSD may have a reject route to 127/8 except 127.0.0.1/32 + // by default. FreeBSD likely works, but is untested. + // TODO(rsc): + // The timeout never happens on Windows. Why? Issue 3016. + t.Skipf("skipping test on %q; untested.", runtime.GOOS) + } + + connected := 0 + for { + select { + case <-time.After(15 * time.Second): + t.Fatal("too slow") + case err := <-errc: + if err == nil { + connected++ + if connected == numConns { + t.Fatal("all connections connected; expected some to time out") + } + } else { + terr, ok := err.(timeout) + if !ok { + t.Fatalf("got error %q; want error with timeout interface", err) + } + if !terr.Timeout() { + t.Fatalf("got error %q; not a timeout", err) + } + // Pass. We saw a timeout error. + return + } + } + } +} + +func TestSelfConnect(t *testing.T) { + if runtime.GOOS == "windows" { + // TODO(brainman): do not know why it hangs. + t.Skip("skipping known-broken test on windows") + } + // Test that Dial does not honor self-connects. + // See the comment in DialTCP. + + // Find a port that would be used as a local address. + l, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + c, err := Dial("tcp", l.Addr().String()) + if err != nil { + t.Fatal(err) + } + addr := c.LocalAddr().String() + c.Close() + l.Close() + + // Try to connect to that address repeatedly. + n := 100000 + if testing.Short() { + n = 1000 + } + switch runtime.GOOS { + case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "solaris", "windows": + // Non-Linux systems take a long time to figure + // out that there is nothing listening on localhost. + n = 100 + } + for i := 0; i < n; i++ { + c, err := DialTimeout("tcp", addr, time.Millisecond) + if err == nil { + c.Close() + t.Errorf("#%d: Dial %q succeeded", i, addr) + } + } +} + +var runErrorTest = flag.Bool("run_error_test", false, "let TestDialError check for dns errors") + +type DialErrorTest struct { + Net string + Raddr string + Pattern string +} + +var dialErrorTests = []DialErrorTest{ + { + "datakit", "mh/astro/r70", + "dial datakit mh/astro/r70: unknown network datakit", + }, + { + "tcp", "127.0.0.1:☺", + "dial tcp 127.0.0.1:☺: unknown port tcp/☺", + }, + { + "tcp", "no-such-name.google.com.:80", + "dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)", + }, + { + "tcp", "no-such-name.no-such-top-level-domain.:80", + "dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)", + }, + { + "tcp", "no-such-name:80", + `dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`, + }, + { + "tcp", "mh/astro/r70:http", + "dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name", + }, + { + "unix", "/etc/file-not-found", + "dial unix /etc/file-not-found: no such file or directory", + }, + { + "unix", "/etc/", + "dial unix /etc/: (permission denied|socket operation on non-socket|connection refused)", + }, + { + "unixpacket", "/etc/file-not-found", + "dial unixpacket /etc/file-not-found: no such file or directory", + }, + { + "unixpacket", "/etc/", + "dial unixpacket /etc/: (permission denied|socket operation on non-socket|connection refused)", + }, +} + +var duplicateErrorPattern = `dial (.*) dial (.*)` + +func TestDialError(t *testing.T) { + if !*runErrorTest { + t.Logf("test disabled; use -run_error_test to enable") + return + } + for i, tt := range dialErrorTests { + c, err := Dial(tt.Net, tt.Raddr) + if c != nil { + c.Close() + } + if err == nil { + t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern) + continue + } + s := err.Error() + match, _ := regexp.MatchString(tt.Pattern, s) + if !match { + t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern) + } + match, _ = regexp.MatchString(duplicateErrorPattern, s) + if match { + t.Errorf("#%d: %q, duplicate error return from Dial", i, s) + } + } +} + +var invalidDialAndListenArgTests = []struct { + net string + addr string + err error +}{ + {"foo", "bar", &OpError{Op: "dial", Net: "foo", Addr: nil, Err: UnknownNetworkError("foo")}}, + {"baz", "", &OpError{Op: "listen", Net: "baz", Addr: nil, Err: UnknownNetworkError("baz")}}, + {"tcp", "", &OpError{Op: "dial", Net: "tcp", Addr: nil, Err: errMissingAddress}}, +} + +func TestInvalidDialAndListenArgs(t *testing.T) { + for _, tt := range invalidDialAndListenArgTests { + var err error + switch tt.err.(*OpError).Op { + case "dial": + _, err = Dial(tt.net, tt.addr) + case "listen": + _, err = Listen(tt.net, tt.addr) + } + if !reflect.DeepEqual(tt.err, err) { + t.Fatalf("got %#v; expected %#v", err, tt.err) + } + } +} + +func TestDialTimeoutFDLeak(t *testing.T) { + if runtime.GOOS != "linux" { + // TODO(bradfitz): test on other platforms + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t) + defer ln.Close() + + type connErr struct { + conn Conn + err error + } + dials := listenerBacklog + 100 + // used to be listenerBacklog + 5, but was found to be unreliable, issue 4384. + maxGoodConnect := listenerBacklog + runtime.NumCPU()*10 + resc := make(chan connErr) + for i := 0; i < dials; i++ { + go func() { + conn, err := DialTimeout("tcp", ln.Addr().String(), 500*time.Millisecond) + resc <- connErr{conn, err} + }() + } + + var firstErr string + var ngood int + var toClose []io.Closer + for i := 0; i < dials; i++ { + ce := <-resc + if ce.err == nil { + ngood++ + if ngood > maxGoodConnect { + t.Errorf("%d good connects; expected at most %d", ngood, maxGoodConnect) + } + toClose = append(toClose, ce.conn) + continue + } + err := ce.err + if firstErr == "" { + firstErr = err.Error() + } else if err.Error() != firstErr { + t.Fatalf("inconsistent error messages: first was %q, then later %q", firstErr, err) + } + } + for _, c := range toClose { + c.Close() + } + for i := 0; i < 100; i++ { + if got := numFD(); got < dials { + // Test passes. + return + } + time.Sleep(10 * time.Millisecond) + } + if got := numFD(); got >= dials { + t.Errorf("num fds after %d timeouts = %d; want <%d", dials, got, dials) + } +} + +func numTCP() (ntcp, nopen, nclose int, err error) { + lsof, err := exec.Command("lsof", "-n", "-p", strconv.Itoa(os.Getpid())).Output() + if err != nil { + return 0, 0, 0, err + } + ntcp += bytes.Count(lsof, []byte("TCP")) + for _, state := range []string{"LISTEN", "SYN_SENT", "SYN_RECEIVED", "ESTABLISHED"} { + nopen += bytes.Count(lsof, []byte(state)) + } + for _, state := range []string{"CLOSED", "CLOSE_WAIT", "LAST_ACK", "FIN_WAIT_1", "FIN_WAIT_2", "CLOSING", "TIME_WAIT"} { + nclose += bytes.Count(lsof, []byte(state)) + } + return ntcp, nopen, nclose, nil +} + +func TestDialMultiFDLeak(t *testing.T) { + if !supportsIPv4 || !supportsIPv6 { + t.Skip("neither ipv4 nor ipv6 is supported") + } + + halfDeadServer := func(dss *dualStackServer, ln Listener) { + for { + if c, err := ln.Accept(); err != nil { + return + } else { + // It just keeps established + // connections like a half-dead server + // does. + dss.putConn(c) + } + } + } + dss, err := newDualStackServer([]streamListener{ + {net: "tcp4", addr: "127.0.0.1"}, + {net: "tcp6", addr: "[::1]"}, + }) + if err != nil { + t.Fatalf("newDualStackServer failed: %v", err) + } + defer dss.teardown() + if err := dss.buildup(halfDeadServer); err != nil { + t.Fatalf("dualStackServer.buildup failed: %v", err) + } + + _, before, _, err := numTCP() + if err != nil { + t.Skipf("skipping test; error finding or running lsof: %v", err) + } + + var wg sync.WaitGroup + portnum, _, _ := dtoi(dss.port, 0) + ras := addrList{ + // Losers that will fail to connect, see RFC 6890. + &TCPAddr{IP: IPv4(198, 18, 0, 254), Port: portnum}, + &TCPAddr{IP: ParseIP("2001:2::254"), Port: portnum}, + + // Winner candidates of this race. + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: portnum}, + &TCPAddr{IP: IPv6loopback, Port: portnum}, + + // Losers that will have established connections. + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: portnum}, + &TCPAddr{IP: IPv6loopback, Port: portnum}, + } + const T1 = 10 * time.Millisecond + const T2 = 2 * T1 + const N = 10 + for i := 0; i < N; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if c, err := dialMulti("tcp", "fast failover test", nil, ras, time.Now().Add(T1)); err == nil { + c.Close() + } + }() + } + wg.Wait() + time.Sleep(T2) + + ntcp, after, nclose, err := numTCP() + if err != nil { + t.Skipf("skipping test; error finding or running lsof: %v", err) + } + t.Logf("tcp sessions: %v, open sessions: %v, closing sessions: %v", ntcp, after, nclose) + + if after != before { + t.Fatalf("got %v open sessions; expected %v", after, before) + } +} + +func numFD() int { + if runtime.GOOS == "linux" { + f, err := os.Open("/proc/self/fd") + if err != nil { + panic(err) + } + defer f.Close() + names, err := f.Readdirnames(0) + if err != nil { + panic(err) + } + return len(names) + } + // All tests using this should be skipped anyway, but: + panic("numFDs not implemented on " + runtime.GOOS) +} + +func TestDialer(t *testing.T) { + ln, err := Listen("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer ln.Close() + ch := make(chan error, 1) + go func() { + c, err := ln.Accept() + if err != nil { + ch <- fmt.Errorf("Accept failed: %v", err) + return + } + defer c.Close() + ch <- nil + }() + + laddr, err := ResolveTCPAddr("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("ResolveTCPAddr failed: %v", err) + } + d := &Dialer{LocalAddr: laddr} + c, err := d.Dial("tcp4", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c.Close() + c.Read(make([]byte, 1)) + err = <-ch + if err != nil { + t.Error(err) + } +} + +func TestDialDualStackLocalhost(t *testing.T) { + if ips, err := LookupIP("localhost"); err != nil { + t.Fatalf("LookupIP failed: %v", err) + } else if len(ips) < 2 || !supportsIPv4 || !supportsIPv6 { + t.Skip("localhost doesn't have a pair of different address family IP addresses") + } + + touchAndByeServer := func(dss *dualStackServer, ln Listener) { + for { + if c, err := ln.Accept(); err != nil { + return + } else { + c.Close() + } + } + } + dss, err := newDualStackServer([]streamListener{ + {net: "tcp4", addr: "127.0.0.1"}, + {net: "tcp6", addr: "[::1]"}, + }) + if err != nil { + t.Fatalf("newDualStackServer failed: %v", err) + } + defer dss.teardown() + if err := dss.buildup(touchAndByeServer); err != nil { + t.Fatalf("dualStackServer.buildup failed: %v", err) + } + + d := &Dialer{DualStack: true} + for range dss.lns { + if c, err := d.Dial("tcp", "localhost:"+dss.port); err != nil { + t.Errorf("Dial failed: %v", err) + } else { + if addr := c.LocalAddr().(*TCPAddr); addr.IP.To4() != nil { + dss.teardownNetwork("tcp4") + } else if addr.IP.To16() != nil && addr.IP.To4() == nil { + dss.teardownNetwork("tcp6") + } + c.Close() + } + } +} + +func TestDialerKeepAlive(t *testing.T) { + ln := newLocalListener(t) + defer ln.Close() + defer func() { + testHookSetKeepAlive = func() {} + }() + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + } + }() + for _, keepAlive := range []bool{false, true} { + got := false + testHookSetKeepAlive = func() { got = true } + var d Dialer + if keepAlive { + d.KeepAlive = 30 * time.Second + } + c, err := d.Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + c.Close() + if got != keepAlive { + t.Errorf("Dialer.KeepAlive = %v: SetKeepAlive called = %v, want %v", d.KeepAlive, got, !got) + } + } +} diff --git a/src/net/dialgoogle_test.go b/src/net/dialgoogle_test.go new file mode 100644 index 000000000..df5895afa --- /dev/null +++ b/src/net/dialgoogle_test.go @@ -0,0 +1,209 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "flag" + "fmt" + "io" + "strings" + "syscall" + "testing" +) + +// If an IPv6 tunnel is running, we can try dialing a real IPv6 address. +var testIPv6 = flag.Bool("ipv6", false, "assume ipv6 tunnel is present") + +func TestResolveGoogle(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + for _, network := range []string{"tcp", "tcp4", "tcp6"} { + addr, err := ResolveTCPAddr(network, "www.google.com:http") + if err != nil { + if (network == "tcp" || network == "tcp4") && !supportsIPv4 { + t.Logf("ipv4 is not supported: %v", err) + } else if network == "tcp6" && !supportsIPv6 { + t.Logf("ipv6 is not supported: %v", err) + } else { + t.Errorf("ResolveTCPAddr failed: %v", err) + } + continue + } + if (network == "tcp" || network == "tcp4") && addr.IP.To4() == nil { + t.Errorf("got %v; expected an IPv4 address on %v", addr, network) + } else if network == "tcp6" && (addr.IP.To16() == nil || addr.IP.To4() != nil) { + t.Errorf("got %v; expected an IPv6 address on %v", addr, network) + } + } +} + +func TestDialGoogle(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + d := &Dialer{DualStack: true} + for _, network := range []string{"tcp", "tcp4", "tcp6"} { + if network == "tcp" && !supportsIPv4 && !supportsIPv6 { + t.Logf("skipping test; both ipv4 and ipv6 are not supported") + continue + } else if network == "tcp4" && !supportsIPv4 { + t.Logf("skipping test; ipv4 is not supported") + continue + } else if network == "tcp6" && !supportsIPv6 { + t.Logf("skipping test; ipv6 is not supported") + continue + } else if network == "tcp6" && !*testIPv6 { + t.Logf("test disabled; use -ipv6 to enable") + continue + } + if c, err := d.Dial(network, "www.google.com:http"); err != nil { + t.Errorf("Dial failed: %v", err) + } else { + c.Close() + } + } +} + +// fd is already connected to the destination, port 80. +// Run an HTTP request to fetch the appropriate page. +func fetchGoogle(t *testing.T, fd Conn, network, addr string) { + req := []byte("GET /robots.txt HTTP/1.0\r\nHost: www.google.com\r\n\r\n") + n, err := fd.Write(req) + + buf := make([]byte, 1000) + n, err = io.ReadFull(fd, buf) + + if n < 1000 { + t.Errorf("fetchGoogle: short HTTP read from %s %s - %v", network, addr, err) + return + } +} + +func doDial(t *testing.T, network, addr string) { + fd, err := Dial(network, addr) + if err != nil { + t.Errorf("Dial(%q, %q, %q) = _, %v", network, "", addr, err) + return + } + fetchGoogle(t, fd, network, addr) + fd.Close() +} + +var googleaddrsipv4 = []string{ + "%d.%d.%d.%d:80", + "www.google.com:80", + "%d.%d.%d.%d:http", + "www.google.com:http", + "%03d.%03d.%03d.%03d:0080", + "[::ffff:%d.%d.%d.%d]:80", + "[::ffff:%02x%02x:%02x%02x]:80", + "[0:0:0:0:0000:ffff:%d.%d.%d.%d]:80", + "[0:0:0:0:000000:ffff:%d.%d.%d.%d]:80", + "[0:0:0:0::ffff:%d.%d.%d.%d]:80", +} + +func TestDialGoogleIPv4(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + // Insert an actual IPv4 address for google.com + // into the table. + addrs, err := LookupIP("www.google.com") + if err != nil { + t.Fatalf("lookup www.google.com: %v", err) + } + var ip IP + for _, addr := range addrs { + if x := addr.To4(); x != nil { + ip = x + break + } + } + if ip == nil { + t.Fatalf("no IPv4 addresses for www.google.com") + } + + for i, s := range googleaddrsipv4 { + if strings.Contains(s, "%") { + googleaddrsipv4[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3]) + } + } + + for i := 0; i < len(googleaddrsipv4); i++ { + addr := googleaddrsipv4[i] + if addr == "" { + continue + } + t.Logf("-- %s --", addr) + doDial(t, "tcp", addr) + if addr[0] != '[' { + doDial(t, "tcp4", addr) + if supportsIPv6 { + // make sure syscall.SocketDisableIPv6 flag works. + syscall.SocketDisableIPv6 = true + doDial(t, "tcp", addr) + doDial(t, "tcp4", addr) + syscall.SocketDisableIPv6 = false + } + } + } +} + +var googleaddrsipv6 = []string{ + "[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:80", + "ipv6.google.com:80", + "[%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x]:http", + "ipv6.google.com:http", +} + +func TestDialGoogleIPv6(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + // Only run tcp6 if the kernel will take it. + if !supportsIPv6 { + t.Skip("skipping test; ipv6 is not supported") + } + if !*testIPv6 { + t.Skip("test disabled; use -ipv6 to enable") + } + + // Insert an actual IPv6 address for ipv6.google.com + // into the table. + addrs, err := LookupIP("ipv6.google.com") + if err != nil { + t.Fatalf("lookup ipv6.google.com: %v", err) + } + var ip IP + for _, addr := range addrs { + if x := addr.To16(); x != nil { + ip = x + break + } + } + if ip == nil { + t.Fatalf("no IPv6 addresses for ipv6.google.com") + } + + for i, s := range googleaddrsipv6 { + if strings.Contains(s, "%") { + googleaddrsipv6[i] = fmt.Sprintf(s, ip[0], ip[1], ip[2], ip[3], ip[4], ip[5], ip[6], ip[7], ip[8], ip[9], ip[10], ip[11], ip[12], ip[13], ip[14], ip[15]) + } + } + + for i := 0; i < len(googleaddrsipv6); i++ { + addr := googleaddrsipv6[i] + if addr == "" { + continue + } + t.Logf("-- %s --", addr) + doDial(t, "tcp", addr) + doDial(t, "tcp6", addr) + } +} diff --git a/src/net/dnsclient.go b/src/net/dnsclient.go new file mode 100644 index 000000000..e8014e4ff --- /dev/null +++ b/src/net/dnsclient.go @@ -0,0 +1,249 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "math/rand" + "sort" +) + +// DNSError represents a DNS lookup error. +type DNSError struct { + Err string // description of the error + Name string // name looked for + Server string // server used + IsTimeout bool +} + +func (e *DNSError) Error() string { + if e == nil { + return "<nil>" + } + s := "lookup " + e.Name + if e.Server != "" { + s += " on " + e.Server + } + s += ": " + e.Err + return s +} + +func (e *DNSError) Timeout() bool { return e.IsTimeout } +func (e *DNSError) Temporary() bool { return e.IsTimeout } + +const noSuchHost = "no such host" + +// reverseaddr returns the in-addr.arpa. or ip6.arpa. hostname of the IP +// address addr suitable for rDNS (PTR) record lookup or an error if it fails +// to parse the IP address. +func reverseaddr(addr string) (arpa string, err error) { + ip := ParseIP(addr) + if ip == nil { + return "", &DNSError{Err: "unrecognized address", Name: addr} + } + if ip.To4() != nil { + return itoa(int(ip[15])) + "." + itoa(int(ip[14])) + "." + itoa(int(ip[13])) + "." + + itoa(int(ip[12])) + ".in-addr.arpa.", nil + } + // Must be IPv6 + buf := make([]byte, 0, len(ip)*4+len("ip6.arpa.")) + // Add it, in reverse, to the buffer + for i := len(ip) - 1; i >= 0; i-- { + v := ip[i] + buf = append(buf, hexDigit[v&0xF]) + buf = append(buf, '.') + buf = append(buf, hexDigit[v>>4]) + buf = append(buf, '.') + } + // Append "ip6.arpa." and return (buf already has the final .) + buf = append(buf, "ip6.arpa."...) + return string(buf), nil +} + +// Find answer for name in dns message. +// On return, if err == nil, addrs != nil. +func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs []dnsRR, err error) { + addrs = make([]dnsRR, 0, len(dns.answer)) + + if dns.rcode == dnsRcodeNameError && dns.recursion_available { + return "", nil, &DNSError{Err: noSuchHost, Name: name} + } + if dns.rcode != dnsRcodeSuccess { + // None of the error codes make sense + // for the query we sent. If we didn't get + // a name error and we didn't get success, + // the server is behaving incorrectly. + return "", nil, &DNSError{Err: "server misbehaving", Name: name, Server: server} + } + + // Look for the name. + // Presotto says it's okay to assume that servers listed in + // /etc/resolv.conf are recursive resolvers. + // We asked for recursion, so it should have included + // all the answers we need in this one packet. +Cname: + for cnameloop := 0; cnameloop < 10; cnameloop++ { + addrs = addrs[0:0] + for _, rr := range dns.answer { + if _, justHeader := rr.(*dnsRR_Header); justHeader { + // Corrupt record: we only have a + // header. That header might say it's + // of type qtype, but we don't + // actually have it. Skip. + continue + } + h := rr.Header() + if h.Class == dnsClassINET && h.Name == name { + switch h.Rrtype { + case qtype: + addrs = append(addrs, rr) + case dnsTypeCNAME: + // redirect to cname + name = rr.(*dnsRR_CNAME).Cname + continue Cname + } + } + } + if len(addrs) == 0 { + return "", nil, &DNSError{Err: noSuchHost, Name: name, Server: server} + } + return name, addrs, nil + } + + return "", nil, &DNSError{Err: "too many redirects", Name: name, Server: server} +} + +func isDomainName(s string) bool { + // See RFC 1035, RFC 3696. + if len(s) == 0 { + return false + } + if len(s) > 255 { + return false + } + + last := byte('.') + ok := false // Ok once we've seen a letter. + partlen := 0 + for i := 0; i < len(s); i++ { + c := s[i] + switch { + default: + return false + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_': + ok = true + partlen++ + case '0' <= c && c <= '9': + // fine + partlen++ + case c == '-': + // Byte before dash cannot be dot. + if last == '.' { + return false + } + partlen++ + case c == '.': + // Byte before dot cannot be dot, dash. + if last == '.' || last == '-' { + return false + } + if partlen > 63 || partlen == 0 { + return false + } + partlen = 0 + } + last = c + } + if last == '-' || partlen > 63 { + return false + } + + return ok +} + +// An SRV represents a single DNS SRV record. +type SRV struct { + Target string + Port uint16 + Priority uint16 + Weight uint16 +} + +// byPriorityWeight sorts SRV records by ascending priority and weight. +type byPriorityWeight []*SRV + +func (s byPriorityWeight) Len() int { return len(s) } + +func (s byPriorityWeight) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (s byPriorityWeight) Less(i, j int) bool { + return s[i].Priority < s[j].Priority || + (s[i].Priority == s[j].Priority && s[i].Weight < s[j].Weight) +} + +// shuffleByWeight shuffles SRV records by weight using the algorithm +// described in RFC 2782. +func (addrs byPriorityWeight) shuffleByWeight() { + sum := 0 + for _, addr := range addrs { + sum += int(addr.Weight) + } + for sum > 0 && len(addrs) > 1 { + s := 0 + n := rand.Intn(sum) + for i := range addrs { + s += int(addrs[i].Weight) + if s > n { + if i > 0 { + addrs[0], addrs[i] = addrs[i], addrs[0] + } + break + } + } + sum -= int(addrs[0].Weight) + addrs = addrs[1:] + } +} + +// sort reorders SRV records as specified in RFC 2782. +func (addrs byPriorityWeight) sort() { + sort.Sort(addrs) + i := 0 + for j := 1; j < len(addrs); j++ { + if addrs[i].Priority != addrs[j].Priority { + addrs[i:j].shuffleByWeight() + i = j + } + } + addrs[i:].shuffleByWeight() +} + +// An MX represents a single DNS MX record. +type MX struct { + Host string + Pref uint16 +} + +// byPref implements sort.Interface to sort MX records by preference +type byPref []*MX + +func (s byPref) Len() int { return len(s) } + +func (s byPref) Less(i, j int) bool { return s[i].Pref < s[j].Pref } + +func (s byPref) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// sort reorders MX records as specified in RFC 5321. +func (s byPref) sort() { + for i := range s { + j := rand.Intn(i + 1) + s[i], s[j] = s[j], s[i] + } + sort.Sort(s) +} + +// An NS represents a single DNS NS record. +type NS struct { + Host string +} diff --git a/src/net/dnsclient_test.go b/src/net/dnsclient_test.go new file mode 100644 index 000000000..435eb3550 --- /dev/null +++ b/src/net/dnsclient_test.go @@ -0,0 +1,69 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "math/rand" + "testing" +) + +func checkDistribution(t *testing.T, data []*SRV, margin float64) { + sum := 0 + for _, srv := range data { + sum += int(srv.Weight) + } + + results := make(map[string]int) + + count := 1000 + for j := 0; j < count; j++ { + d := make([]*SRV, len(data)) + copy(d, data) + byPriorityWeight(d).shuffleByWeight() + key := d[0].Target + results[key] = results[key] + 1 + } + + actual := results[data[0].Target] + expected := float64(count) * float64(data[0].Weight) / float64(sum) + diff := float64(actual) - expected + t.Logf("actual: %v diff: %v e: %v m: %v", actual, diff, expected, margin) + if diff < 0 { + diff = -diff + } + if diff > (expected * margin) { + t.Errorf("missed target weight: expected %v, %v", expected, actual) + } +} + +func testUniformity(t *testing.T, size int, margin float64) { + rand.Seed(1) + data := make([]*SRV, size) + for i := 0; i < size; i++ { + data[i] = &SRV{Target: string('a' + i), Weight: 1} + } + checkDistribution(t, data, margin) +} + +func TestUniformity(t *testing.T) { + testUniformity(t, 2, 0.05) + testUniformity(t, 3, 0.10) + testUniformity(t, 10, 0.20) + testWeighting(t, 0.05) +} + +func testWeighting(t *testing.T, margin float64) { + rand.Seed(1) + data := []*SRV{ + {Target: "a", Weight: 60}, + {Target: "b", Weight: 30}, + {Target: "c", Weight: 10}, + } + checkDistribution(t, data, margin) +} + +func TestWeighting(t *testing.T) { + testWeighting(t, 0.05) +} diff --git a/src/net/dnsclient_unix.go b/src/net/dnsclient_unix.go new file mode 100644 index 000000000..abe7da05c --- /dev/null +++ b/src/net/dnsclient_unix.go @@ -0,0 +1,423 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +// DNS client: see RFC 1035. +// Has to be linked into package net for Dial. + +// TODO(rsc): +// Could potentially handle many outstanding lookups faster. +// Could have a small cache. +// Random UDP source port (net.Dial should do that for us). +// Random request IDs. + +package net + +import ( + "errors" + "io" + "math/rand" + "os" + "sync" + "time" +) + +// A dnsConn represents a DNS transport endpoint. +type dnsConn interface { + Conn + + // readDNSResponse reads a DNS response message from the DNS + // transport endpoint and returns the received DNS response + // message. + readDNSResponse() (*dnsMsg, error) + + // writeDNSQuery writes a DNS query message to the DNS + // connection endpoint. + writeDNSQuery(*dnsMsg) error +} + +func (c *UDPConn) readDNSResponse() (*dnsMsg, error) { + b := make([]byte, 512) // see RFC 1035 + n, err := c.Read(b) + if err != nil { + return nil, err + } + msg := &dnsMsg{} + if !msg.Unpack(b[:n]) { + return nil, errors.New("cannot unmarshal DNS message") + } + return msg, nil +} + +func (c *UDPConn) writeDNSQuery(msg *dnsMsg) error { + b, ok := msg.Pack() + if !ok { + return errors.New("cannot marshal DNS message") + } + if _, err := c.Write(b); err != nil { + return err + } + return nil +} + +func (c *TCPConn) readDNSResponse() (*dnsMsg, error) { + b := make([]byte, 1280) // 1280 is a reasonable initial size for IP over Ethernet, see RFC 4035 + if _, err := io.ReadFull(c, b[:2]); err != nil { + return nil, err + } + l := int(b[0])<<8 | int(b[1]) + if l > len(b) { + b = make([]byte, l) + } + n, err := io.ReadFull(c, b[:l]) + if err != nil { + return nil, err + } + msg := &dnsMsg{} + if !msg.Unpack(b[:n]) { + return nil, errors.New("cannot unmarshal DNS message") + } + return msg, nil +} + +func (c *TCPConn) writeDNSQuery(msg *dnsMsg) error { + b, ok := msg.Pack() + if !ok { + return errors.New("cannot marshal DNS message") + } + l := uint16(len(b)) + b = append([]byte{byte(l >> 8), byte(l)}, b...) + if _, err := c.Write(b); err != nil { + return err + } + return nil +} + +func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { + switch network { + case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": + default: + return nil, UnknownNetworkError(network) + } + // Calling Dial here is scary -- we have to be sure not to + // dial a name that will require a DNS lookup, or Dial will + // call back here to translate it. The DNS config parser has + // already checked that all the cfg.servers[i] are IP + // addresses, which Dial will use without a DNS lookup. + c, err := d.Dial(network, server) + if err != nil { + return nil, err + } + switch network { + case "tcp", "tcp4", "tcp6": + return c.(*TCPConn), nil + case "udp", "udp4", "udp6": + return c.(*UDPConn), nil + } + panic("unreachable") +} + +// exchange sends a query on the connection and hopes for a response. +func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg, error) { + d := Dialer{Timeout: timeout} + out := dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + recursion_desired: true, + }, + question: []dnsQuestion{ + {name, qtype, dnsClassINET}, + }, + } + for _, network := range []string{"udp", "tcp"} { + c, err := d.dialDNS(network, server) + if err != nil { + return nil, err + } + defer c.Close() + if timeout > 0 { + c.SetDeadline(time.Now().Add(timeout)) + } + out.id = uint16(rand.Int()) ^ uint16(time.Now().UnixNano()) + if err := c.writeDNSQuery(&out); err != nil { + return nil, err + } + in, err := c.readDNSResponse() + if err != nil { + return nil, err + } + if in.id != out.id { + return nil, errors.New("DNS message ID mismatch") + } + if in.truncated { // see RFC 5966 + continue + } + return in, nil + } + return nil, errors.New("no answer from DNS server") +} + +// Do a lookup for a single name, which must be rooted +// (otherwise answer will not find the answers). +func tryOneName(cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { + if len(cfg.servers) == 0 { + return "", nil, &DNSError{Err: "no DNS servers", Name: name} + } + if len(name) >= 256 { + return "", nil, &DNSError{Err: "DNS name too long", Name: name} + } + timeout := time.Duration(cfg.timeout) * time.Second + var lastErr error + for i := 0; i < cfg.attempts; i++ { + for _, server := range cfg.servers { + server = JoinHostPort(server, "53") + msg, err := exchange(server, name, qtype, timeout) + if err != nil { + lastErr = &DNSError{ + Err: err.Error(), + Name: name, + Server: server, + } + if nerr, ok := err.(Error); ok && nerr.Timeout() { + lastErr.(*DNSError).IsTimeout = true + } + continue + } + cname, addrs, err := answer(name, server, msg, qtype) + if err == nil || err.(*DNSError).Err == noSuchHost { + return cname, addrs, err + } + lastErr = err + } + } + return "", nil, lastErr +} + +func convertRR_A(records []dnsRR) []IP { + addrs := make([]IP, len(records)) + for i, rr := range records { + a := rr.(*dnsRR_A).A + addrs[i] = IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a)) + } + return addrs +} + +func convertRR_AAAA(records []dnsRR) []IP { + addrs := make([]IP, len(records)) + for i, rr := range records { + a := make(IP, IPv6len) + copy(a, rr.(*dnsRR_AAAA).AAAA[:]) + addrs[i] = a + } + return addrs +} + +var cfg struct { + ch chan struct{} + mu sync.RWMutex // protects dnsConfig and dnserr + dnsConfig *dnsConfig + dnserr error +} +var onceLoadConfig sync.Once + +// Assume dns config file is /etc/resolv.conf here +func loadDefaultConfig() { + loadConfig("/etc/resolv.conf", 5*time.Second, nil) +} + +func loadConfig(resolvConfPath string, reloadTime time.Duration, quit <-chan chan struct{}) { + var mtime time.Time + cfg.ch = make(chan struct{}, 1) + if fi, err := os.Stat(resolvConfPath); err != nil { + cfg.dnserr = err + } else { + mtime = fi.ModTime() + cfg.dnsConfig, cfg.dnserr = dnsReadConfig(resolvConfPath) + } + go func() { + for { + time.Sleep(reloadTime) + select { + case qresp := <-quit: + qresp <- struct{}{} + return + case <-cfg.ch: + } + + // In case of error, we keep the previous config + fi, err := os.Stat(resolvConfPath) + if err != nil { + continue + } + // If the resolv.conf mtime didn't change, do not reload + m := fi.ModTime() + if m.Equal(mtime) { + continue + } + mtime = m + // In case of error, we keep the previous config + ncfg, err := dnsReadConfig(resolvConfPath) + if err != nil || len(ncfg.servers) == 0 { + continue + } + cfg.mu.Lock() + cfg.dnsConfig = ncfg + cfg.dnserr = nil + cfg.mu.Unlock() + } + }() +} + +func lookup(name string, qtype uint16) (cname string, addrs []dnsRR, err error) { + if !isDomainName(name) { + return name, nil, &DNSError{Err: "invalid domain name", Name: name} + } + onceLoadConfig.Do(loadDefaultConfig) + + select { + case cfg.ch <- struct{}{}: + default: + } + + cfg.mu.RLock() + defer cfg.mu.RUnlock() + + if cfg.dnserr != nil || cfg.dnsConfig == nil { + err = cfg.dnserr + return + } + // If name is rooted (trailing dot) or has enough dots, + // try it by itself first. + rooted := len(name) > 0 && name[len(name)-1] == '.' + if rooted || count(name, '.') >= cfg.dnsConfig.ndots { + rname := name + if !rooted { + rname += "." + } + // Can try as ordinary name. + cname, addrs, err = tryOneName(cfg.dnsConfig, rname, qtype) + if rooted || err == nil { + return + } + } + + // Otherwise, try suffixes. + for i := 0; i < len(cfg.dnsConfig.search); i++ { + rname := name + "." + cfg.dnsConfig.search[i] + if rname[len(rname)-1] != '.' { + rname += "." + } + cname, addrs, err = tryOneName(cfg.dnsConfig, rname, qtype) + if err == nil { + return + } + } + + // Last ditch effort: try unsuffixed only if we haven't already, + // that is, name is not rooted and has less than ndots dots. + if count(name, '.') < cfg.dnsConfig.ndots { + cname, addrs, err = tryOneName(cfg.dnsConfig, name+".", qtype) + if err == nil { + return + } + } + + if e, ok := err.(*DNSError); ok { + // Show original name passed to lookup, not suffixed one. + // In general we might have tried many suffixes; showing + // just one is misleading. See also golang.org/issue/6324. + e.Name = name + } + return +} + +// goLookupHost is the native Go implementation of LookupHost. +// Used only if cgoLookupHost refuses to handle the request +// (that is, only if cgoLookupHost is the stub in cgo_stub.go). +// Normally we let cgo use the C library resolver instead of +// depending on our lookup code, so that Go and C get the same +// answers. +func goLookupHost(name string) (addrs []string, err error) { + // Use entries from /etc/hosts if they match. + addrs = lookupStaticHost(name) + if len(addrs) > 0 { + return + } + ips, err := goLookupIP(name) + if err != nil { + return + } + addrs = make([]string, 0, len(ips)) + for _, ip := range ips { + addrs = append(addrs, ip.String()) + } + return +} + +// goLookupIP is the native Go implementation of LookupIP. +// Used only if cgoLookupIP refuses to handle the request +// (that is, only if cgoLookupIP is the stub in cgo_stub.go). +// Normally we let cgo use the C library resolver instead of +// depending on our lookup code, so that Go and C get the same +// answers. +func goLookupIP(name string) (addrs []IP, err error) { + // Use entries from /etc/hosts if possible. + haddrs := lookupStaticHost(name) + if len(haddrs) > 0 { + for _, haddr := range haddrs { + if ip := ParseIP(haddr); ip != nil { + addrs = append(addrs, ip) + } + } + if len(addrs) > 0 { + return + } + } + type racer struct { + qtype uint16 + rrs []dnsRR + error + } + lane := make(chan racer, 1) + qtypes := [...]uint16{dnsTypeA, dnsTypeAAAA} + for _, qtype := range qtypes { + go func(qtype uint16) { + _, rrs, err := lookup(name, qtype) + lane <- racer{qtype, rrs, err} + }(qtype) + } + var lastErr error + for range qtypes { + racer := <-lane + if racer.error != nil { + lastErr = racer.error + continue + } + switch racer.qtype { + case dnsTypeA: + addrs = append(addrs, convertRR_A(racer.rrs)...) + case dnsTypeAAAA: + addrs = append(addrs, convertRR_AAAA(racer.rrs)...) + } + } + if len(addrs) == 0 && lastErr != nil { + return nil, lastErr + } + return addrs, nil +} + +// goLookupCNAME is the native Go implementation of LookupCNAME. +// Used only if cgoLookupCNAME refuses to handle the request +// (that is, only if cgoLookupCNAME is the stub in cgo_stub.go). +// Normally we let cgo use the C library resolver instead of +// depending on our lookup code, so that Go and C get the same +// answers. +func goLookupCNAME(name string) (cname string, err error) { + _, rr, err := lookup(name, dnsTypeCNAME) + if err != nil { + return + } + cname = rr[0].(*dnsRR_CNAME).Cname + return +} diff --git a/src/net/dnsclient_unix_test.go b/src/net/dnsclient_unix_test.go new file mode 100644 index 000000000..1167c26b3 --- /dev/null +++ b/src/net/dnsclient_unix_test.go @@ -0,0 +1,246 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package net + +import ( + "io" + "io/ioutil" + "os" + "path" + "reflect" + "testing" + "time" +) + +var dnsTransportFallbackTests = []struct { + server string + name string + qtype uint16 + timeout int + rcode int +}{ + // Querying "com." with qtype=255 usually makes an answer + // which requires more than 512 bytes. + {"8.8.8.8:53", "com.", dnsTypeALL, 2, dnsRcodeSuccess}, + {"8.8.4.4:53", "com.", dnsTypeALL, 4, dnsRcodeSuccess}, +} + +func TestDNSTransportFallback(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + for _, tt := range dnsTransportFallbackTests { + timeout := time.Duration(tt.timeout) * time.Second + msg, err := exchange(tt.server, tt.name, tt.qtype, timeout) + if err != nil { + t.Error(err) + continue + } + switch msg.rcode { + case tt.rcode, dnsRcodeServerFailure: + default: + t.Errorf("got %v from %v; want %v", msg.rcode, tt.server, tt.rcode) + continue + } + } +} + +// See RFC 6761 for further information about the reserved, pseudo +// domain names. +var specialDomainNameTests = []struct { + name string + qtype uint16 + rcode int +}{ + // Name resoltion APIs and libraries should not recongnize the + // followings as special. + {"1.0.168.192.in-addr.arpa.", dnsTypePTR, dnsRcodeNameError}, + {"test.", dnsTypeALL, dnsRcodeNameError}, + {"example.com.", dnsTypeALL, dnsRcodeSuccess}, + + // Name resoltion APIs and libraries should recongnize the + // followings as special and should not send any queries. + // Though, we test those names here for verifying nagative + // answers at DNS query-response interaction level. + {"localhost.", dnsTypeALL, dnsRcodeNameError}, + {"invalid.", dnsTypeALL, dnsRcodeNameError}, +} + +func TestSpecialDomainName(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + server := "8.8.8.8:53" + for _, tt := range specialDomainNameTests { + msg, err := exchange(server, tt.name, tt.qtype, 0) + if err != nil { + t.Error(err) + continue + } + switch msg.rcode { + case tt.rcode, dnsRcodeServerFailure: + default: + t.Errorf("got %v from %v; want %v", msg.rcode, server, tt.rcode) + continue + } + } +} + +type resolvConfTest struct { + *testing.T + dir string + path string + started bool + quitc chan chan struct{} +} + +func newResolvConfTest(t *testing.T) *resolvConfTest { + dir, err := ioutil.TempDir("", "resolvConfTest") + if err != nil { + t.Fatalf("could not create temp dir: %v", err) + } + + // Disable the default loadConfig + onceLoadConfig.Do(func() {}) + + r := &resolvConfTest{ + T: t, + dir: dir, + path: path.Join(dir, "resolv.conf"), + quitc: make(chan chan struct{}), + } + + return r +} + +func (r *resolvConfTest) Start() { + loadConfig(r.path, 100*time.Millisecond, r.quitc) + r.started = true +} + +func (r *resolvConfTest) SetConf(s string) { + // Make sure the file mtime will be different once we're done here, + // even on systems with coarse (1s) mtime resolution. + time.Sleep(time.Second) + + f, err := os.OpenFile(r.path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + if err != nil { + r.Fatalf("failed to create temp file %s: %v", r.path, err) + } + if _, err := io.WriteString(f, s); err != nil { + f.Close() + r.Fatalf("failed to write temp file: %v", err) + } + f.Close() + + if r.started { + cfg.ch <- struct{}{} // fill buffer + cfg.ch <- struct{}{} // wait for reload to begin + cfg.ch <- struct{}{} // wait for reload to complete + } +} + +func (r *resolvConfTest) WantServers(want []string) { + cfg.mu.RLock() + defer cfg.mu.RUnlock() + if got := cfg.dnsConfig.servers; !reflect.DeepEqual(got, want) { + r.Fatalf("Unexpected dns server loaded, got %v want %v", got, want) + } +} + +func (r *resolvConfTest) Close() { + resp := make(chan struct{}) + r.quitc <- resp + <-resp + if err := os.RemoveAll(r.dir); err != nil { + r.Logf("failed to remove temp dir %s: %v", r.dir, err) + } +} + +func TestReloadResolvConfFail(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + r := newResolvConfTest(t) + defer r.Close() + + // resolv.conf.tmp does not exist yet + r.Start() + if _, err := goLookupIP("golang.org"); err == nil { + t.Fatal("goLookupIP(missing) succeeded") + } + + r.SetConf("nameserver 8.8.8.8") + if _, err := goLookupIP("golang.org"); err != nil { + t.Fatalf("goLookupIP(missing; good) failed: %v", err) + } + + // Using a bad resolv.conf while we had a good + // one before should not update the config + r.SetConf("") + if _, err := goLookupIP("golang.org"); err != nil { + t.Fatalf("goLookupIP(missing; good; bad) failed: %v", err) + } +} + +func TestReloadResolvConfChange(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + r := newResolvConfTest(t) + defer r.Close() + + r.SetConf("nameserver 8.8.8.8") + r.Start() + + if _, err := goLookupIP("golang.org"); err != nil { + t.Fatalf("goLookupIP(good) failed: %v", err) + } + r.WantServers([]string{"8.8.8.8"}) + + // Using a bad resolv.conf when we had a good one + // before should not update the config + r.SetConf("") + if _, err := goLookupIP("golang.org"); err != nil { + t.Fatalf("goLookupIP(good; bad) failed: %v", err) + } + + // A new good config should get picked up + r.SetConf("nameserver 8.8.4.4") + r.WantServers([]string{"8.8.4.4"}) +} + +func BenchmarkGoLookupIP(b *testing.B) { + for i := 0; i < b.N; i++ { + goLookupIP("www.example.com") + } +} + +func BenchmarkGoLookupIPNoSuchHost(b *testing.B) { + for i := 0; i < b.N; i++ { + goLookupIP("some.nonexistent") + } +} + +func BenchmarkGoLookupIPWithBrokenNameServer(b *testing.B) { + onceLoadConfig.Do(loadDefaultConfig) + if cfg.dnserr != nil || cfg.dnsConfig == nil { + b.Fatalf("loadConfig failed: %v", cfg.dnserr) + } + // This looks ugly but it's safe as long as benchmarks are run + // sequentially in package testing. + orig := cfg.dnsConfig + cfg.dnsConfig.servers = append([]string{"203.0.113.254"}, cfg.dnsConfig.servers...) // use TEST-NET-3 block, see RFC 5737 + for i := 0; i < b.N; i++ { + goLookupIP("www.example.com") + } + cfg.dnsConfig = orig +} diff --git a/src/net/dnsconfig_unix.go b/src/net/dnsconfig_unix.go new file mode 100644 index 000000000..ebb6e673f --- /dev/null +++ b/src/net/dnsconfig_unix.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +// Read system DNS config from /etc/resolv.conf + +package net + +type dnsConfig struct { + servers []string // servers to use + search []string // suffixes to append to local name + ndots int // number of dots in name to trigger absolute lookup + timeout int // seconds before giving up on packet + attempts int // lost packets before giving up on server + rotate bool // round robin among servers +} + +// See resolv.conf(5) on a Linux machine. +// TODO(rsc): Supposed to call uname() and chop the beginning +// of the host name to get the default search domain. +func dnsReadConfig(filename string) (*dnsConfig, error) { + file, err := open(filename) + if err != nil { + return nil, &DNSConfigError{err} + } + defer file.close() + conf := &dnsConfig{ + ndots: 1, + timeout: 5, + attempts: 2, + } + for line, ok := file.readLine(); ok; line, ok = file.readLine() { + f := getFields(line) + if len(f) < 1 { + continue + } + switch f[0] { + case "nameserver": // add one name server + if len(f) > 1 && len(conf.servers) < 3 { // small, but the standard limit + // One more check: make sure server name is + // just an IP address. Otherwise we need DNS + // to look it up. + if parseIPv4(f[1]) != nil { + conf.servers = append(conf.servers, f[1]) + } else if ip, _ := parseIPv6(f[1], true); ip != nil { + conf.servers = append(conf.servers, f[1]) + } + } + + case "domain": // set search path to just this domain + if len(f) > 1 { + conf.search = []string{f[1]} + } + + case "search": // set search path to given servers + conf.search = make([]string, len(f)-1) + for i := 0; i < len(conf.search); i++ { + conf.search[i] = f[i+1] + } + + case "options": // magic options + for i := 1; i < len(f); i++ { + s := f[i] + switch { + case hasPrefix(s, "ndots:"): + n, _, _ := dtoi(s, 6) + if n < 1 { + n = 1 + } + conf.ndots = n + case hasPrefix(s, "timeout:"): + n, _, _ := dtoi(s, 8) + if n < 1 { + n = 1 + } + conf.timeout = n + case hasPrefix(s, "attempts:"): + n, _, _ := dtoi(s, 9) + if n < 1 { + n = 1 + } + conf.attempts = n + case s == "rotate": + conf.rotate = true + } + } + } + } + return conf, nil +} + +func hasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} diff --git a/src/net/dnsconfig_unix_test.go b/src/net/dnsconfig_unix_test.go new file mode 100644 index 000000000..94fb0c32e --- /dev/null +++ b/src/net/dnsconfig_unix_test.go @@ -0,0 +1,69 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package net + +import ( + "reflect" + "testing" +) + +var dnsReadConfigTests = []struct { + name string + conf dnsConfig +}{ + { + name: "testdata/resolv.conf", + conf: dnsConfig{ + servers: []string{"8.8.8.8", "2001:4860:4860::8888", "fe80::1%lo0"}, + search: []string{"localdomain"}, + ndots: 5, + timeout: 10, + attempts: 3, + rotate: true, + }, + }, + { + name: "testdata/domain-resolv.conf", + conf: dnsConfig{ + servers: []string{"8.8.8.8"}, + search: []string{"localdomain"}, + ndots: 1, + timeout: 5, + attempts: 2, + }, + }, + { + name: "testdata/search-resolv.conf", + conf: dnsConfig{ + servers: []string{"8.8.8.8"}, + search: []string{"test", "invalid"}, + ndots: 1, + timeout: 5, + attempts: 2, + }, + }, + { + name: "testdata/empty-resolv.conf", + conf: dnsConfig{ + ndots: 1, + timeout: 5, + attempts: 2, + }, + }, +} + +func TestDNSReadConfig(t *testing.T) { + for _, tt := range dnsReadConfigTests { + conf, err := dnsReadConfig(tt.name) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(conf, &tt.conf) { + t.Errorf("got %v; want %v", conf, &tt.conf) + } + } +} diff --git a/src/net/dnsmsg.go b/src/net/dnsmsg.go new file mode 100644 index 000000000..161afb2a5 --- /dev/null +++ b/src/net/dnsmsg.go @@ -0,0 +1,887 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// DNS packet assembly. See RFC 1035. +// +// This is intended to support name resolution during Dial. +// It doesn't have to be blazing fast. +// +// Each message structure has a Walk method that is used by +// a generic pack/unpack routine. Thus, if in the future we need +// to define new message structs, no new pack/unpack/printing code +// needs to be written. +// +// The first half of this file defines the DNS message formats. +// The second half implements the conversion to and from wire format. +// A few of the structure elements have string tags to aid the +// generic pack/unpack routines. +// +// TODO(rsc): There are enough names defined in this file that they're all +// prefixed with dns. Perhaps put this in its own package later. + +package net + +// Packet formats + +// Wire constants. +const ( + // valid dnsRR_Header.Rrtype and dnsQuestion.qtype + dnsTypeA = 1 + dnsTypeNS = 2 + dnsTypeMD = 3 + dnsTypeMF = 4 + dnsTypeCNAME = 5 + dnsTypeSOA = 6 + dnsTypeMB = 7 + dnsTypeMG = 8 + dnsTypeMR = 9 + dnsTypeNULL = 10 + dnsTypeWKS = 11 + dnsTypePTR = 12 + dnsTypeHINFO = 13 + dnsTypeMINFO = 14 + dnsTypeMX = 15 + dnsTypeTXT = 16 + dnsTypeAAAA = 28 + dnsTypeSRV = 33 + + // valid dnsQuestion.qtype only + dnsTypeAXFR = 252 + dnsTypeMAILB = 253 + dnsTypeMAILA = 254 + dnsTypeALL = 255 + + // valid dnsQuestion.qclass + dnsClassINET = 1 + dnsClassCSNET = 2 + dnsClassCHAOS = 3 + dnsClassHESIOD = 4 + dnsClassANY = 255 + + // dnsMsg.rcode + dnsRcodeSuccess = 0 + dnsRcodeFormatError = 1 + dnsRcodeServerFailure = 2 + dnsRcodeNameError = 3 + dnsRcodeNotImplemented = 4 + dnsRcodeRefused = 5 +) + +// A dnsStruct describes how to iterate over its fields to emulate +// reflective marshalling. +type dnsStruct interface { + // Walk iterates over fields of a structure and calls f + // with a reference to that field, the name of the field + // and a tag ("", "domain", "ipv4", "ipv6") specifying + // particular encodings. Possible concrete types + // for v are *uint16, *uint32, *string, or []byte, and + // *int, *bool in the case of dnsMsgHdr. + // Whenever f returns false, Walk must stop and return + // false, and otherwise return true. + Walk(f func(v interface{}, name, tag string) (ok bool)) (ok bool) +} + +// The wire format for the DNS packet header. +type dnsHeader struct { + Id uint16 + Bits uint16 + Qdcount, Ancount, Nscount, Arcount uint16 +} + +func (h *dnsHeader) Walk(f func(v interface{}, name, tag string) bool) bool { + return f(&h.Id, "Id", "") && + f(&h.Bits, "Bits", "") && + f(&h.Qdcount, "Qdcount", "") && + f(&h.Ancount, "Ancount", "") && + f(&h.Nscount, "Nscount", "") && + f(&h.Arcount, "Arcount", "") +} + +const ( + // dnsHeader.Bits + _QR = 1 << 15 // query/response (response=1) + _AA = 1 << 10 // authoritative + _TC = 1 << 9 // truncated + _RD = 1 << 8 // recursion desired + _RA = 1 << 7 // recursion available +) + +// DNS queries. +type dnsQuestion struct { + Name string `net:"domain-name"` // `net:"domain-name"` specifies encoding; see packers below + Qtype uint16 + Qclass uint16 +} + +func (q *dnsQuestion) Walk(f func(v interface{}, name, tag string) bool) bool { + return f(&q.Name, "Name", "domain") && + f(&q.Qtype, "Qtype", "") && + f(&q.Qclass, "Qclass", "") +} + +// DNS responses (resource records). +// There are many types of messages, +// but they all share the same header. +type dnsRR_Header struct { + Name string `net:"domain-name"` + Rrtype uint16 + Class uint16 + Ttl uint32 + Rdlength uint16 // length of data after header +} + +func (h *dnsRR_Header) Header() *dnsRR_Header { + return h +} + +func (h *dnsRR_Header) Walk(f func(v interface{}, name, tag string) bool) bool { + return f(&h.Name, "Name", "domain") && + f(&h.Rrtype, "Rrtype", "") && + f(&h.Class, "Class", "") && + f(&h.Ttl, "Ttl", "") && + f(&h.Rdlength, "Rdlength", "") +} + +type dnsRR interface { + dnsStruct + Header() *dnsRR_Header +} + +// Specific DNS RR formats for each query type. + +type dnsRR_CNAME struct { + Hdr dnsRR_Header + Cname string `net:"domain-name"` +} + +func (rr *dnsRR_CNAME) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_CNAME) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Cname, "Cname", "domain") +} + +type dnsRR_HINFO struct { + Hdr dnsRR_Header + Cpu string + Os string +} + +func (rr *dnsRR_HINFO) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_HINFO) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Cpu, "Cpu", "") && f(&rr.Os, "Os", "") +} + +type dnsRR_MB struct { + Hdr dnsRR_Header + Mb string `net:"domain-name"` +} + +func (rr *dnsRR_MB) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_MB) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Mb, "Mb", "domain") +} + +type dnsRR_MG struct { + Hdr dnsRR_Header + Mg string `net:"domain-name"` +} + +func (rr *dnsRR_MG) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_MG) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Mg, "Mg", "domain") +} + +type dnsRR_MINFO struct { + Hdr dnsRR_Header + Rmail string `net:"domain-name"` + Email string `net:"domain-name"` +} + +func (rr *dnsRR_MINFO) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_MINFO) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Rmail, "Rmail", "domain") && f(&rr.Email, "Email", "domain") +} + +type dnsRR_MR struct { + Hdr dnsRR_Header + Mr string `net:"domain-name"` +} + +func (rr *dnsRR_MR) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_MR) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Mr, "Mr", "domain") +} + +type dnsRR_MX struct { + Hdr dnsRR_Header + Pref uint16 + Mx string `net:"domain-name"` +} + +func (rr *dnsRR_MX) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_MX) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Pref, "Pref", "") && f(&rr.Mx, "Mx", "domain") +} + +type dnsRR_NS struct { + Hdr dnsRR_Header + Ns string `net:"domain-name"` +} + +func (rr *dnsRR_NS) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_NS) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Ns, "Ns", "domain") +} + +type dnsRR_PTR struct { + Hdr dnsRR_Header + Ptr string `net:"domain-name"` +} + +func (rr *dnsRR_PTR) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_PTR) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Ptr, "Ptr", "domain") +} + +type dnsRR_SOA struct { + Hdr dnsRR_Header + Ns string `net:"domain-name"` + Mbox string `net:"domain-name"` + Serial uint32 + Refresh uint32 + Retry uint32 + Expire uint32 + Minttl uint32 +} + +func (rr *dnsRR_SOA) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_SOA) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && + f(&rr.Ns, "Ns", "domain") && + f(&rr.Mbox, "Mbox", "domain") && + f(&rr.Serial, "Serial", "") && + f(&rr.Refresh, "Refresh", "") && + f(&rr.Retry, "Retry", "") && + f(&rr.Expire, "Expire", "") && + f(&rr.Minttl, "Minttl", "") +} + +type dnsRR_TXT struct { + Hdr dnsRR_Header + Txt string // not domain name +} + +func (rr *dnsRR_TXT) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_TXT) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.Txt, "Txt", "") +} + +type dnsRR_SRV struct { + Hdr dnsRR_Header + Priority uint16 + Weight uint16 + Port uint16 + Target string `net:"domain-name"` +} + +func (rr *dnsRR_SRV) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_SRV) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && + f(&rr.Priority, "Priority", "") && + f(&rr.Weight, "Weight", "") && + f(&rr.Port, "Port", "") && + f(&rr.Target, "Target", "domain") +} + +type dnsRR_A struct { + Hdr dnsRR_Header + A uint32 `net:"ipv4"` +} + +func (rr *dnsRR_A) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_A) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(&rr.A, "A", "ipv4") +} + +type dnsRR_AAAA struct { + Hdr dnsRR_Header + AAAA [16]byte `net:"ipv6"` +} + +func (rr *dnsRR_AAAA) Header() *dnsRR_Header { + return &rr.Hdr +} + +func (rr *dnsRR_AAAA) Walk(f func(v interface{}, name, tag string) bool) bool { + return rr.Hdr.Walk(f) && f(rr.AAAA[:], "AAAA", "ipv6") +} + +// Packing and unpacking. +// +// All the packers and unpackers take a (msg []byte, off int) +// and return (off1 int, ok bool). If they return ok==false, they +// also return off1==len(msg), so that the next unpacker will +// also fail. This lets us avoid checks of ok until the end of a +// packing sequence. + +// Map of constructors for each RR wire type. +var rr_mk = map[int]func() dnsRR{ + dnsTypeCNAME: func() dnsRR { return new(dnsRR_CNAME) }, + dnsTypeHINFO: func() dnsRR { return new(dnsRR_HINFO) }, + dnsTypeMB: func() dnsRR { return new(dnsRR_MB) }, + dnsTypeMG: func() dnsRR { return new(dnsRR_MG) }, + dnsTypeMINFO: func() dnsRR { return new(dnsRR_MINFO) }, + dnsTypeMR: func() dnsRR { return new(dnsRR_MR) }, + dnsTypeMX: func() dnsRR { return new(dnsRR_MX) }, + dnsTypeNS: func() dnsRR { return new(dnsRR_NS) }, + dnsTypePTR: func() dnsRR { return new(dnsRR_PTR) }, + dnsTypeSOA: func() dnsRR { return new(dnsRR_SOA) }, + dnsTypeTXT: func() dnsRR { return new(dnsRR_TXT) }, + dnsTypeSRV: func() dnsRR { return new(dnsRR_SRV) }, + dnsTypeA: func() dnsRR { return new(dnsRR_A) }, + dnsTypeAAAA: func() dnsRR { return new(dnsRR_AAAA) }, +} + +// Pack a domain name s into msg[off:]. +// Domain names are a sequence of counted strings +// split at the dots. They end with a zero-length string. +func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { + // Add trailing dot to canonicalize name. + if n := len(s); n == 0 || s[n-1] != '.' { + s += "." + } + + // Each dot ends a segment of the name. + // We trade each dot byte for a length byte. + // There is also a trailing zero. + // Check that we have all the space we need. + tot := len(s) + 1 + if off+tot > len(msg) { + return len(msg), false + } + + // Emit sequence of counted strings, chopping at dots. + begin := 0 + for i := 0; i < len(s); i++ { + if s[i] == '.' { + if i-begin >= 1<<6 { // top two bits of length must be clear + return len(msg), false + } + msg[off] = byte(i - begin) + off++ + for j := begin; j < i; j++ { + msg[off] = s[j] + off++ + } + begin = i + 1 + } + } + msg[off] = 0 + off++ + return off, true +} + +// Unpack a domain name. +// In addition to the simple sequences of counted strings above, +// domain names are allowed to refer to strings elsewhere in the +// packet, to avoid repeating common suffixes when returning +// many entries in a single domain. The pointers are marked +// by a length byte with the top two bits set. Ignoring those +// two bits, that byte and the next give a 14 bit offset from msg[0] +// where we should pick up the trail. +// Note that if we jump elsewhere in the packet, +// we return off1 == the offset after the first pointer we found, +// which is where the next record will start. +// In theory, the pointers are only allowed to jump backward. +// We let them jump anywhere and stop jumping after a while. +func unpackDomainName(msg []byte, off int) (s string, off1 int, ok bool) { + s = "" + ptr := 0 // number of pointers followed +Loop: + for { + if off >= len(msg) { + return "", len(msg), false + } + c := int(msg[off]) + off++ + switch c & 0xC0 { + case 0x00: + if c == 0x00 { + // end of name + break Loop + } + // literal string + if off+c > len(msg) { + return "", len(msg), false + } + s += string(msg[off:off+c]) + "." + off += c + case 0xC0: + // pointer to somewhere else in msg. + // remember location after first ptr, + // since that's how many bytes we consumed. + // also, don't follow too many pointers -- + // maybe there's a loop. + if off >= len(msg) { + return "", len(msg), false + } + c1 := msg[off] + off++ + if ptr == 0 { + off1 = off + } + if ptr++; ptr > 10 { + return "", len(msg), false + } + off = (c^0xC0)<<8 | int(c1) + default: + // 0x80 and 0x40 are reserved + return "", len(msg), false + } + } + if ptr == 0 { + off1 = off + } + return s, off1, true +} + +// packStruct packs a structure into msg at specified offset off, and +// returns off1 such that msg[off:off1] is the encoded data. +func packStruct(any dnsStruct, msg []byte, off int) (off1 int, ok bool) { + ok = any.Walk(func(field interface{}, name, tag string) bool { + switch fv := field.(type) { + default: + println("net: dns: unknown packing type") + return false + case *uint16: + i := *fv + if off+2 > len(msg) { + return false + } + msg[off] = byte(i >> 8) + msg[off+1] = byte(i) + off += 2 + case *uint32: + i := *fv + msg[off] = byte(i >> 24) + msg[off+1] = byte(i >> 16) + msg[off+2] = byte(i >> 8) + msg[off+3] = byte(i) + off += 4 + case []byte: + n := len(fv) + if off+n > len(msg) { + return false + } + copy(msg[off:off+n], fv) + off += n + case *string: + s := *fv + switch tag { + default: + println("net: dns: unknown string tag", tag) + return false + case "domain": + off, ok = packDomainName(s, msg, off) + if !ok { + return false + } + case "": + // Counted string: 1 byte length. + if len(s) > 255 || off+1+len(s) > len(msg) { + return false + } + msg[off] = byte(len(s)) + off++ + off += copy(msg[off:], s) + } + } + return true + }) + if !ok { + return len(msg), false + } + return off, true +} + +// unpackStruct decodes msg[off:] into the given structure, and +// returns off1 such that msg[off:off1] is the encoded data. +func unpackStruct(any dnsStruct, msg []byte, off int) (off1 int, ok bool) { + ok = any.Walk(func(field interface{}, name, tag string) bool { + switch fv := field.(type) { + default: + println("net: dns: unknown packing type") + return false + case *uint16: + if off+2 > len(msg) { + return false + } + *fv = uint16(msg[off])<<8 | uint16(msg[off+1]) + off += 2 + case *uint32: + if off+4 > len(msg) { + return false + } + *fv = uint32(msg[off])<<24 | uint32(msg[off+1])<<16 | + uint32(msg[off+2])<<8 | uint32(msg[off+3]) + off += 4 + case []byte: + n := len(fv) + if off+n > len(msg) { + return false + } + copy(fv, msg[off:off+n]) + off += n + case *string: + var s string + switch tag { + default: + println("net: dns: unknown string tag", tag) + return false + case "domain": + s, off, ok = unpackDomainName(msg, off) + if !ok { + return false + } + case "": + if off >= len(msg) || off+1+int(msg[off]) > len(msg) { + return false + } + n := int(msg[off]) + off++ + b := make([]byte, n) + for i := 0; i < n; i++ { + b[i] = msg[off+i] + } + off += n + s = string(b) + } + *fv = s + } + return true + }) + if !ok { + return len(msg), false + } + return off, true +} + +// Generic struct printer. Prints fields with tag "ipv4" or "ipv6" +// as IP addresses. +func printStruct(any dnsStruct) string { + s := "{" + i := 0 + any.Walk(func(val interface{}, name, tag string) bool { + i++ + if i > 1 { + s += ", " + } + s += name + "=" + switch tag { + case "ipv4": + i := *val.(*uint32) + s += IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i)).String() + case "ipv6": + i := val.([]byte) + s += IP(i).String() + default: + var i int64 + switch v := val.(type) { + default: + // can't really happen. + s += "<unknown type>" + return true + case *string: + s += *v + return true + case []byte: + s += string(v) + return true + case *bool: + if *v { + s += "true" + } else { + s += "false" + } + return true + case *int: + i = int64(*v) + case *uint: + i = int64(*v) + case *uint8: + i = int64(*v) + case *uint16: + i = int64(*v) + case *uint32: + i = int64(*v) + case *uint64: + i = int64(*v) + case *uintptr: + i = int64(*v) + } + s += itoa(int(i)) + } + return true + }) + s += "}" + return s +} + +// Resource record packer. +func packRR(rr dnsRR, msg []byte, off int) (off2 int, ok bool) { + var off1 int + // pack twice, once to find end of header + // and again to find end of packet. + // a bit inefficient but this doesn't need to be fast. + // off1 is end of header + // off2 is end of rr + off1, ok = packStruct(rr.Header(), msg, off) + off2, ok = packStruct(rr, msg, off) + if !ok { + return len(msg), false + } + // pack a third time; redo header with correct data length + rr.Header().Rdlength = uint16(off2 - off1) + packStruct(rr.Header(), msg, off) + return off2, true +} + +// Resource record unpacker. +func unpackRR(msg []byte, off int) (rr dnsRR, off1 int, ok bool) { + // unpack just the header, to find the rr type and length + var h dnsRR_Header + off0 := off + if off, ok = unpackStruct(&h, msg, off); !ok { + return nil, len(msg), false + } + end := off + int(h.Rdlength) + + // make an rr of that type and re-unpack. + // again inefficient but doesn't need to be fast. + mk, known := rr_mk[int(h.Rrtype)] + if !known { + return &h, end, true + } + rr = mk() + off, ok = unpackStruct(rr, msg, off0) + if off != end { + return &h, end, true + } + return rr, off, ok +} + +// Usable representation of a DNS packet. + +// A manually-unpacked version of (id, bits). +// This is in its own struct for easy printing. +type dnsMsgHdr struct { + id uint16 + response bool + opcode int + authoritative bool + truncated bool + recursion_desired bool + recursion_available bool + rcode int +} + +func (h *dnsMsgHdr) Walk(f func(v interface{}, name, tag string) bool) bool { + return f(&h.id, "id", "") && + f(&h.response, "response", "") && + f(&h.opcode, "opcode", "") && + f(&h.authoritative, "authoritative", "") && + f(&h.truncated, "truncated", "") && + f(&h.recursion_desired, "recursion_desired", "") && + f(&h.recursion_available, "recursion_available", "") && + f(&h.rcode, "rcode", "") +} + +type dnsMsg struct { + dnsMsgHdr + question []dnsQuestion + answer []dnsRR + ns []dnsRR + extra []dnsRR +} + +func (dns *dnsMsg) Pack() (msg []byte, ok bool) { + var dh dnsHeader + + // Convert convenient dnsMsg into wire-like dnsHeader. + dh.Id = dns.id + dh.Bits = uint16(dns.opcode)<<11 | uint16(dns.rcode) + if dns.recursion_available { + dh.Bits |= _RA + } + if dns.recursion_desired { + dh.Bits |= _RD + } + if dns.truncated { + dh.Bits |= _TC + } + if dns.authoritative { + dh.Bits |= _AA + } + if dns.response { + dh.Bits |= _QR + } + + // Prepare variable sized arrays. + question := dns.question + answer := dns.answer + ns := dns.ns + extra := dns.extra + + dh.Qdcount = uint16(len(question)) + dh.Ancount = uint16(len(answer)) + dh.Nscount = uint16(len(ns)) + dh.Arcount = uint16(len(extra)) + + // Could work harder to calculate message size, + // but this is far more than we need and not + // big enough to hurt the allocator. + msg = make([]byte, 2000) + + // Pack it in: header and then the pieces. + off := 0 + off, ok = packStruct(&dh, msg, off) + for i := 0; i < len(question); i++ { + off, ok = packStruct(&question[i], msg, off) + } + for i := 0; i < len(answer); i++ { + off, ok = packRR(answer[i], msg, off) + } + for i := 0; i < len(ns); i++ { + off, ok = packRR(ns[i], msg, off) + } + for i := 0; i < len(extra); i++ { + off, ok = packRR(extra[i], msg, off) + } + if !ok { + return nil, false + } + return msg[0:off], true +} + +func (dns *dnsMsg) Unpack(msg []byte) bool { + // Header. + var dh dnsHeader + off := 0 + var ok bool + if off, ok = unpackStruct(&dh, msg, off); !ok { + return false + } + dns.id = dh.Id + dns.response = (dh.Bits & _QR) != 0 + dns.opcode = int(dh.Bits>>11) & 0xF + dns.authoritative = (dh.Bits & _AA) != 0 + dns.truncated = (dh.Bits & _TC) != 0 + dns.recursion_desired = (dh.Bits & _RD) != 0 + dns.recursion_available = (dh.Bits & _RA) != 0 + dns.rcode = int(dh.Bits & 0xF) + + // Arrays. + dns.question = make([]dnsQuestion, dh.Qdcount) + dns.answer = make([]dnsRR, 0, dh.Ancount) + dns.ns = make([]dnsRR, 0, dh.Nscount) + dns.extra = make([]dnsRR, 0, dh.Arcount) + + var rec dnsRR + + for i := 0; i < len(dns.question); i++ { + off, ok = unpackStruct(&dns.question[i], msg, off) + } + for i := 0; i < int(dh.Ancount); i++ { + rec, off, ok = unpackRR(msg, off) + if !ok { + return false + } + dns.answer = append(dns.answer, rec) + } + for i := 0; i < int(dh.Nscount); i++ { + rec, off, ok = unpackRR(msg, off) + if !ok { + return false + } + dns.ns = append(dns.ns, rec) + } + for i := 0; i < int(dh.Arcount); i++ { + rec, off, ok = unpackRR(msg, off) + if !ok { + return false + } + dns.extra = append(dns.extra, rec) + } + // if off != len(msg) { + // println("extra bytes in dns packet", off, "<", len(msg)); + // } + return true +} + +func (dns *dnsMsg) String() string { + s := "DNS: " + printStruct(&dns.dnsMsgHdr) + "\n" + if len(dns.question) > 0 { + s += "-- Questions\n" + for i := 0; i < len(dns.question); i++ { + s += printStruct(&dns.question[i]) + "\n" + } + } + if len(dns.answer) > 0 { + s += "-- Answers\n" + for i := 0; i < len(dns.answer); i++ { + s += printStruct(dns.answer[i]) + "\n" + } + } + if len(dns.ns) > 0 { + s += "-- Name servers\n" + for i := 0; i < len(dns.ns); i++ { + s += printStruct(dns.ns[i]) + "\n" + } + } + if len(dns.extra) > 0 { + s += "-- Extra\n" + for i := 0; i < len(dns.extra); i++ { + s += printStruct(dns.extra[i]) + "\n" + } + } + return s +} diff --git a/src/net/dnsmsg_test.go b/src/net/dnsmsg_test.go new file mode 100644 index 000000000..c39dbdb04 --- /dev/null +++ b/src/net/dnsmsg_test.go @@ -0,0 +1,113 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "encoding/hex" + "reflect" + "testing" +) + +func TestDNSParseSRVReply(t *testing.T) { + data, err := hex.DecodeString(dnsSRVReply) + if err != nil { + t.Fatal(err) + } + msg := new(dnsMsg) + ok := msg.Unpack(data) + if !ok { + t.Fatalf("unpacking packet failed") + } + msg.String() // exercise this code path + if g, e := len(msg.answer), 5; g != e { + t.Errorf("len(msg.answer) = %d; want %d", g, e) + } + for idx, rr := range msg.answer { + if g, e := rr.Header().Rrtype, uint16(dnsTypeSRV); g != e { + t.Errorf("rr[%d].Header().Rrtype = %d; want %d", idx, g, e) + } + if _, ok := rr.(*dnsRR_SRV); !ok { + t.Errorf("answer[%d] = %T; want *dnsRR_SRV", idx, rr) + } + } + _, addrs, err := answer("_xmpp-server._tcp.google.com.", "foo:53", msg, uint16(dnsTypeSRV)) + if err != nil { + t.Fatalf("answer: %v", err) + } + if g, e := len(addrs), 5; g != e { + t.Errorf("len(addrs) = %d; want %d", g, e) + t.Logf("addrs = %#v", addrs) + } + // repack and unpack. + data2, ok := msg.Pack() + msg2 := new(dnsMsg) + msg2.Unpack(data2) + switch { + case !ok: + t.Errorf("failed to repack message") + case !reflect.DeepEqual(msg, msg2): + t.Errorf("repacked message differs from original") + } +} + +func TestDNSParseCorruptSRVReply(t *testing.T) { + data, err := hex.DecodeString(dnsSRVCorruptReply) + if err != nil { + t.Fatal(err) + } + msg := new(dnsMsg) + ok := msg.Unpack(data) + if !ok { + t.Fatalf("unpacking packet failed") + } + msg.String() // exercise this code path + if g, e := len(msg.answer), 5; g != e { + t.Errorf("len(msg.answer) = %d; want %d", g, e) + } + for idx, rr := range msg.answer { + if g, e := rr.Header().Rrtype, uint16(dnsTypeSRV); g != e { + t.Errorf("rr[%d].Header().Rrtype = %d; want %d", idx, g, e) + } + if idx == 4 { + if _, ok := rr.(*dnsRR_Header); !ok { + t.Errorf("answer[%d] = %T; want *dnsRR_Header", idx, rr) + } + } else { + if _, ok := rr.(*dnsRR_SRV); !ok { + t.Errorf("answer[%d] = %T; want *dnsRR_SRV", idx, rr) + } + } + } + _, addrs, err := answer("_xmpp-server._tcp.google.com.", "foo:53", msg, uint16(dnsTypeSRV)) + if err != nil { + t.Fatalf("answer: %v", err) + } + if g, e := len(addrs), 4; g != e { + t.Errorf("len(addrs) = %d; want %d", g, e) + t.Logf("addrs = %#v", addrs) + } +} + +// Valid DNS SRV reply +const dnsSRVReply = "0901818000010005000000000c5f786d70702d736572766572045f74637006676f6f67" + + "6c6503636f6d0000210001c00c002100010000012c00210014000014950c786d70702d" + + "73657276657234016c06676f6f676c6503636f6d00c00c002100010000012c00210014" + + "000014950c786d70702d73657276657232016c06676f6f676c6503636f6d00c00c0021" + + "00010000012c00210014000014950c786d70702d73657276657233016c06676f6f676c" + + "6503636f6d00c00c002100010000012c00200005000014950b786d70702d7365727665" + + "72016c06676f6f676c6503636f6d00c00c002100010000012c00210014000014950c78" + + "6d70702d73657276657231016c06676f6f676c6503636f6d00" + +// Corrupt DNS SRV reply, with its final RR having a bogus length +// (perhaps it was truncated, or it's malicious) The mutation is the +// capital "FF" below, instead of the proper "21". +const dnsSRVCorruptReply = "0901818000010005000000000c5f786d70702d736572766572045f74637006676f6f67" + + "6c6503636f6d0000210001c00c002100010000012c00210014000014950c786d70702d" + + "73657276657234016c06676f6f676c6503636f6d00c00c002100010000012c00210014" + + "000014950c786d70702d73657276657232016c06676f6f676c6503636f6d00c00c0021" + + "00010000012c00210014000014950c786d70702d73657276657233016c06676f6f676c" + + "6503636f6d00c00c002100010000012c00200005000014950b786d70702d7365727665" + + "72016c06676f6f676c6503636f6d00c00c002100010000012c00FF0014000014950c78" + + "6d70702d73657276657231016c06676f6f676c6503636f6d00" diff --git a/src/net/dnsname_test.go b/src/net/dnsname_test.go new file mode 100644 index 000000000..57dd25fe4 --- /dev/null +++ b/src/net/dnsname_test.go @@ -0,0 +1,83 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "strings" + "testing" +) + +type testCase struct { + name string + result bool +} + +var tests = []testCase{ + // RFC2181, section 11. + {"_xmpp-server._tcp.google.com", true}, + {"foo.com", true}, + {"1foo.com", true}, + {"26.0.0.73.com", true}, + {"fo-o.com", true}, + {"fo1o.com", true}, + {"foo1.com", true}, + {"a.b..com", false}, + {"a.b-.com", false}, + {"a.b.com-", false}, + {"a.b..", false}, + {"b.com.", true}, +} + +func getTestCases(ch chan<- testCase) { + defer close(ch) + var char59 = "" + var char63 = "" + var char64 = "" + for i := 0; i < 59; i++ { + char59 += "a" + } + char63 = char59 + "aaaa" + char64 = char63 + "a" + + for _, tc := range tests { + ch <- tc + } + + ch <- testCase{char63 + ".com", true} + ch <- testCase{char64 + ".com", false} + // 255 char name is fine: + ch <- testCase{char59 + "." + char63 + "." + char63 + "." + + char63 + ".com", + true} + // 256 char name is bad: + ch <- testCase{char59 + "a." + char63 + "." + char63 + "." + + char63 + ".com", + false} +} + +func TestDNSNames(t *testing.T) { + ch := make(chan testCase) + go getTestCases(ch) + for tc := range ch { + if isDomainName(tc.name) != tc.result { + t.Errorf("isDomainName(%v) failed: Should be %v", + tc.name, tc.result) + } + } +} + +func BenchmarkDNSNames(b *testing.B) { + benchmarks := append(tests, []testCase{ + {strings.Repeat("a", 63), true}, + {strings.Repeat("a", 64), false}, + }...) + for n := 0; n < b.N; n++ { + for _, tc := range benchmarks { + if isDomainName(tc.name) != tc.result { + b.Errorf("isDomainName(%q) = %v; want %v", tc.name, !tc.result, tc.result) + } + } + } +} diff --git a/src/net/empty.c b/src/net/empty.c new file mode 100644 index 000000000..a515c2fe2 --- /dev/null +++ b/src/net/empty.c @@ -0,0 +1,8 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is required to prevent compiler errors +// when the package built with CGO_ENABLED=0. +// Otherwise the compiler says: +// pkg/net/fd_poll_runtime.go:15: missing function body diff --git a/src/net/example_test.go b/src/net/example_test.go new file mode 100644 index 000000000..6f2f9074c --- /dev/null +++ b/src/net/example_test.go @@ -0,0 +1,36 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net_test + +import ( + "io" + "log" + "net" +) + +func ExampleListener() { + // Listen on TCP port 2000 on all interfaces. + l, err := net.Listen("tcp", ":2000") + if err != nil { + log.Fatal(err) + } + defer l.Close() + for { + // Wait for a connection. + conn, err := l.Accept() + if err != nil { + log.Fatal(err) + } + // Handle the connection in a new goroutine. + // The loop then returns to accepting, so that + // multiple connections may be served concurrently. + go func(c net.Conn) { + // Echo all incoming data. + io.Copy(c, c) + // Shut down the connection. + c.Close() + }(conn) + } +} diff --git a/src/net/fd_mutex.go b/src/net/fd_mutex.go new file mode 100644 index 000000000..6d5509d7f --- /dev/null +++ b/src/net/fd_mutex.go @@ -0,0 +1,184 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "sync/atomic" + +// fdMutex is a specialized synchronization primitive +// that manages lifetime of an fd and serializes access +// to Read and Write methods on netFD. +type fdMutex struct { + state uint64 + rsema uint32 + wsema uint32 +} + +// fdMutex.state is organized as follows: +// 1 bit - whether netFD is closed, if set all subsequent lock operations will fail. +// 1 bit - lock for read operations. +// 1 bit - lock for write operations. +// 20 bits - total number of references (read+write+misc). +// 20 bits - number of outstanding read waiters. +// 20 bits - number of outstanding write waiters. +const ( + mutexClosed = 1 << 0 + mutexRLock = 1 << 1 + mutexWLock = 1 << 2 + mutexRef = 1 << 3 + mutexRefMask = (1<<20 - 1) << 3 + mutexRWait = 1 << 23 + mutexRMask = (1<<20 - 1) << 23 + mutexWWait = 1 << 43 + mutexWMask = (1<<20 - 1) << 43 +) + +// Read operations must do RWLock(true)/RWUnlock(true). +// Write operations must do RWLock(false)/RWUnlock(false). +// Misc operations must do Incref/Decref. Misc operations include functions like +// setsockopt and setDeadline. They need to use Incref/Decref to ensure that +// they operate on the correct fd in presence of a concurrent Close call +// (otherwise fd can be closed under their feet). +// Close operation must do IncrefAndClose/Decref. + +// RWLock/Incref return whether fd is open. +// RWUnlock/Decref return whether fd is closed and there are no remaining references. + +func (mu *fdMutex) Incref() bool { + for { + old := atomic.LoadUint64(&mu.state) + if old&mutexClosed != 0 { + return false + } + new := old + mutexRef + if new&mutexRefMask == 0 { + panic("net: inconsistent fdMutex") + } + if atomic.CompareAndSwapUint64(&mu.state, old, new) { + return true + } + } +} + +func (mu *fdMutex) IncrefAndClose() bool { + for { + old := atomic.LoadUint64(&mu.state) + if old&mutexClosed != 0 { + return false + } + // Mark as closed and acquire a reference. + new := (old | mutexClosed) + mutexRef + if new&mutexRefMask == 0 { + panic("net: inconsistent fdMutex") + } + // Remove all read and write waiters. + new &^= mutexRMask | mutexWMask + if atomic.CompareAndSwapUint64(&mu.state, old, new) { + // Wake all read and write waiters, + // they will observe closed flag after wakeup. + for old&mutexRMask != 0 { + old -= mutexRWait + runtime_Semrelease(&mu.rsema) + } + for old&mutexWMask != 0 { + old -= mutexWWait + runtime_Semrelease(&mu.wsema) + } + return true + } + } +} + +func (mu *fdMutex) Decref() bool { + for { + old := atomic.LoadUint64(&mu.state) + if old&mutexRefMask == 0 { + panic("net: inconsistent fdMutex") + } + new := old - mutexRef + if atomic.CompareAndSwapUint64(&mu.state, old, new) { + return new&(mutexClosed|mutexRefMask) == mutexClosed + } + } +} + +func (mu *fdMutex) RWLock(read bool) bool { + var mutexBit, mutexWait, mutexMask uint64 + var mutexSema *uint32 + if read { + mutexBit = mutexRLock + mutexWait = mutexRWait + mutexMask = mutexRMask + mutexSema = &mu.rsema + } else { + mutexBit = mutexWLock + mutexWait = mutexWWait + mutexMask = mutexWMask + mutexSema = &mu.wsema + } + for { + old := atomic.LoadUint64(&mu.state) + if old&mutexClosed != 0 { + return false + } + var new uint64 + if old&mutexBit == 0 { + // Lock is free, acquire it. + new = (old | mutexBit) + mutexRef + if new&mutexRefMask == 0 { + panic("net: inconsistent fdMutex") + } + } else { + // Wait for lock. + new = old + mutexWait + if new&mutexMask == 0 { + panic("net: inconsistent fdMutex") + } + } + if atomic.CompareAndSwapUint64(&mu.state, old, new) { + if old&mutexBit == 0 { + return true + } + runtime_Semacquire(mutexSema) + // The signaller has subtracted mutexWait. + } + } +} + +func (mu *fdMutex) RWUnlock(read bool) bool { + var mutexBit, mutexWait, mutexMask uint64 + var mutexSema *uint32 + if read { + mutexBit = mutexRLock + mutexWait = mutexRWait + mutexMask = mutexRMask + mutexSema = &mu.rsema + } else { + mutexBit = mutexWLock + mutexWait = mutexWWait + mutexMask = mutexWMask + mutexSema = &mu.wsema + } + for { + old := atomic.LoadUint64(&mu.state) + if old&mutexBit == 0 || old&mutexRefMask == 0 { + panic("net: inconsistent fdMutex") + } + // Drop lock, drop reference and wake read waiter if present. + new := (old &^ mutexBit) - mutexRef + if old&mutexMask != 0 { + new -= mutexWait + } + if atomic.CompareAndSwapUint64(&mu.state, old, new) { + if old&mutexMask != 0 { + runtime_Semrelease(mutexSema) + } + return new&(mutexClosed|mutexRefMask) == mutexClosed + } + } +} + +// Implemented in runtime package. +func runtime_Semacquire(sema *uint32) +func runtime_Semrelease(sema *uint32) diff --git a/src/net/fd_mutex_test.go b/src/net/fd_mutex_test.go new file mode 100644 index 000000000..c34ec59b9 --- /dev/null +++ b/src/net/fd_mutex_test.go @@ -0,0 +1,195 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "math/rand" + "runtime" + "testing" + "time" +) + +func TestMutexLock(t *testing.T) { + var mu fdMutex + + if !mu.Incref() { + t.Fatal("broken") + } + if mu.Decref() { + t.Fatal("broken") + } + + if !mu.RWLock(true) { + t.Fatal("broken") + } + if mu.RWUnlock(true) { + t.Fatal("broken") + } + + if !mu.RWLock(false) { + t.Fatal("broken") + } + if mu.RWUnlock(false) { + t.Fatal("broken") + } +} + +func TestMutexClose(t *testing.T) { + var mu fdMutex + if !mu.IncrefAndClose() { + t.Fatal("broken") + } + + if mu.Incref() { + t.Fatal("broken") + } + if mu.RWLock(true) { + t.Fatal("broken") + } + if mu.RWLock(false) { + t.Fatal("broken") + } + if mu.IncrefAndClose() { + t.Fatal("broken") + } +} + +func TestMutexCloseUnblock(t *testing.T) { + c := make(chan bool) + var mu fdMutex + mu.RWLock(true) + for i := 0; i < 4; i++ { + go func() { + if mu.RWLock(true) { + t.Error("broken") + return + } + c <- true + }() + } + // Concurrent goroutines must not be able to read lock the mutex. + time.Sleep(time.Millisecond) + select { + case <-c: + t.Fatal("broken") + default: + } + mu.IncrefAndClose() // Must unblock the readers. + for i := 0; i < 4; i++ { + select { + case <-c: + case <-time.After(10 * time.Second): + t.Fatal("broken") + } + } + if mu.Decref() { + t.Fatal("broken") + } + if !mu.RWUnlock(true) { + t.Fatal("broken") + } +} + +func TestMutexPanic(t *testing.T) { + ensurePanics := func(f func()) { + defer func() { + if recover() == nil { + t.Fatal("does not panic") + } + }() + f() + } + + var mu fdMutex + ensurePanics(func() { mu.Decref() }) + ensurePanics(func() { mu.RWUnlock(true) }) + ensurePanics(func() { mu.RWUnlock(false) }) + + ensurePanics(func() { mu.Incref(); mu.Decref(); mu.Decref() }) + ensurePanics(func() { mu.RWLock(true); mu.RWUnlock(true); mu.RWUnlock(true) }) + ensurePanics(func() { mu.RWLock(false); mu.RWUnlock(false); mu.RWUnlock(false) }) + + // ensure that it's still not broken + mu.Incref() + mu.Decref() + mu.RWLock(true) + mu.RWUnlock(true) + mu.RWLock(false) + mu.RWUnlock(false) +} + +func TestMutexStress(t *testing.T) { + P := 8 + N := int(1e6) + if testing.Short() { + P = 4 + N = 1e4 + } + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(P)) + done := make(chan bool) + var mu fdMutex + var readState [2]uint64 + var writeState [2]uint64 + for p := 0; p < P; p++ { + go func() { + r := rand.New(rand.NewSource(rand.Int63())) + for i := 0; i < N; i++ { + switch r.Intn(3) { + case 0: + if !mu.Incref() { + t.Error("broken") + return + } + if mu.Decref() { + t.Error("broken") + return + } + case 1: + if !mu.RWLock(true) { + t.Error("broken") + return + } + // Ensure that it provides mutual exclusion for readers. + if readState[0] != readState[1] { + t.Error("broken") + return + } + readState[0]++ + readState[1]++ + if mu.RWUnlock(true) { + t.Error("broken") + return + } + case 2: + if !mu.RWLock(false) { + t.Error("broken") + return + } + // Ensure that it provides mutual exclusion for writers. + if writeState[0] != writeState[1] { + t.Error("broken") + return + } + writeState[0]++ + writeState[1]++ + if mu.RWUnlock(false) { + t.Error("broken") + return + } + } + } + done <- true + }() + } + for p := 0; p < P; p++ { + <-done + } + if !mu.IncrefAndClose() { + t.Fatal("broken") + } + if !mu.Decref() { + t.Fatal("broken") + } +} diff --git a/src/net/fd_plan9.go b/src/net/fd_plan9.go new file mode 100644 index 000000000..5fe8effc2 --- /dev/null +++ b/src/net/fd_plan9.go @@ -0,0 +1,232 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" + "time" +) + +// Network file descritor. +type netFD struct { + // locking/lifetime of sysfd + serialize access to Read and Write methods + fdmu fdMutex + + // immutable until Close + proto string + n string + dir string + ctl, data *os.File + laddr, raddr Addr +} + +var ( + netdir string // default network +) + +func sysInit() { + netdir = "/net" +} + +func dial(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { + // On plan9, use the relatively inefficient + // goroutine-racing implementation. + return dialChannel(net, ra, dialer, deadline) +} + +func newFD(proto, name string, ctl, data *os.File, laddr, raddr Addr) (*netFD, error) { + return &netFD{proto: proto, n: name, dir: netdir + "/" + proto + "/" + name, ctl: ctl, data: data, laddr: laddr, raddr: raddr}, nil +} + +func (fd *netFD) init() error { + // stub for future fd.pd.Init(fd) + return nil +} + +func (fd *netFD) name() string { + var ls, rs string + if fd.laddr != nil { + ls = fd.laddr.String() + } + if fd.raddr != nil { + rs = fd.raddr.String() + } + return fd.proto + ":" + ls + "->" + rs +} + +func (fd *netFD) ok() bool { return fd != nil && fd.ctl != nil } + +func (fd *netFD) destroy() { + if !fd.ok() { + return + } + err := fd.ctl.Close() + if fd.data != nil { + if err1 := fd.data.Close(); err1 != nil && err == nil { + err = err1 + } + } + fd.ctl = nil + fd.data = nil +} + +// Add a reference to this fd. +// Returns an error if the fd cannot be used. +func (fd *netFD) incref() error { + if !fd.fdmu.Incref() { + return errClosing + } + return nil +} + +// Remove a reference to this FD and close if we've been asked to do so +// (and there are no references left). +func (fd *netFD) decref() { + if fd.fdmu.Decref() { + fd.destroy() + } +} + +// Add a reference to this fd and lock for reading. +// Returns an error if the fd cannot be used. +func (fd *netFD) readLock() error { + if !fd.fdmu.RWLock(true) { + return errClosing + } + return nil +} + +// Unlock for reading and remove a reference to this FD. +func (fd *netFD) readUnlock() { + if fd.fdmu.RWUnlock(true) { + fd.destroy() + } +} + +// Add a reference to this fd and lock for writing. +// Returns an error if the fd cannot be used. +func (fd *netFD) writeLock() error { + if !fd.fdmu.RWLock(false) { + return errClosing + } + return nil +} + +// Unlock for writing and remove a reference to this FD. +func (fd *netFD) writeUnlock() { + if fd.fdmu.RWUnlock(false) { + fd.destroy() + } +} + +func (fd *netFD) Read(b []byte) (n int, err error) { + if !fd.ok() || fd.data == nil { + return 0, syscall.EINVAL + } + if err := fd.readLock(); err != nil { + return 0, err + } + defer fd.readUnlock() + n, err = fd.data.Read(b) + if fd.proto == "udp" && err == io.EOF { + n = 0 + err = nil + } + return +} + +func (fd *netFD) Write(b []byte) (n int, err error) { + if !fd.ok() || fd.data == nil { + return 0, syscall.EINVAL + } + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + return fd.data.Write(b) +} + +func (fd *netFD) closeRead() error { + if !fd.ok() { + return syscall.EINVAL + } + return syscall.EPLAN9 +} + +func (fd *netFD) closeWrite() error { + if !fd.ok() { + return syscall.EINVAL + } + return syscall.EPLAN9 +} + +func (fd *netFD) Close() error { + if !fd.fdmu.IncrefAndClose() { + return errClosing + } + if !fd.ok() { + return syscall.EINVAL + } + err := fd.ctl.Close() + if fd.data != nil { + if err1 := fd.data.Close(); err1 != nil && err == nil { + err = err1 + } + } + fd.ctl = nil + fd.data = nil + return err +} + +// This method is only called via Conn. +func (fd *netFD) dup() (*os.File, error) { + if !fd.ok() || fd.data == nil { + return nil, syscall.EINVAL + } + return fd.file(fd.data, fd.dir+"/data") +} + +func (l *TCPListener) dup() (*os.File, error) { + if !l.fd.ok() { + return nil, syscall.EINVAL + } + return l.fd.file(l.fd.ctl, l.fd.dir+"/ctl") +} + +func (fd *netFD) file(f *os.File, s string) (*os.File, error) { + syscall.ForkLock.RLock() + dfd, err := syscall.Dup(int(f.Fd()), -1) + syscall.ForkLock.RUnlock() + if err != nil { + return nil, &OpError{"dup", s, fd.laddr, err} + } + return os.NewFile(uintptr(dfd), s), nil +} + +func (fd *netFD) setDeadline(t time.Time) error { + return syscall.EPLAN9 +} + +func (fd *netFD) setReadDeadline(t time.Time) error { + return syscall.EPLAN9 +} + +func (fd *netFD) setWriteDeadline(t time.Time) error { + return syscall.EPLAN9 +} + +func setReadBuffer(fd *netFD, bytes int) error { + return syscall.EPLAN9 +} + +func setWriteBuffer(fd *netFD, bytes int) error { + return syscall.EPLAN9 +} + +func skipRawSocketTests() (skip bool, skipmsg string, err error) { + return true, "skipping test on plan9", nil +} diff --git a/src/net/fd_poll_nacl.go b/src/net/fd_poll_nacl.go new file mode 100644 index 000000000..a3701f876 --- /dev/null +++ b/src/net/fd_poll_nacl.go @@ -0,0 +1,94 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "syscall" + "time" +) + +type pollDesc struct { + fd *netFD + closing bool +} + +func (pd *pollDesc) Init(fd *netFD) error { pd.fd = fd; return nil } + +func (pd *pollDesc) Close() {} + +func (pd *pollDesc) Lock() {} + +func (pd *pollDesc) Unlock() {} + +func (pd *pollDesc) Wakeup() {} + +func (pd *pollDesc) Evict() bool { + pd.closing = true + if pd.fd != nil { + syscall.StopIO(pd.fd.sysfd) + } + return false +} + +func (pd *pollDesc) Prepare(mode int) error { + if pd.closing { + return errClosing + } + return nil +} + +func (pd *pollDesc) PrepareRead() error { return pd.Prepare('r') } + +func (pd *pollDesc) PrepareWrite() error { return pd.Prepare('w') } + +func (pd *pollDesc) Wait(mode int) error { + if pd.closing { + return errClosing + } + return errTimeout +} + +func (pd *pollDesc) WaitRead() error { return pd.Wait('r') } + +func (pd *pollDesc) WaitWrite() error { return pd.Wait('w') } + +func (pd *pollDesc) WaitCanceled(mode int) {} + +func (pd *pollDesc) WaitCanceledRead() {} + +func (pd *pollDesc) WaitCanceledWrite() {} + +func (fd *netFD) setDeadline(t time.Time) error { + return setDeadlineImpl(fd, t, 'r'+'w') +} + +func (fd *netFD) setReadDeadline(t time.Time) error { + return setDeadlineImpl(fd, t, 'r') +} + +func (fd *netFD) setWriteDeadline(t time.Time) error { + return setDeadlineImpl(fd, t, 'w') +} + +func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { + d := t.UnixNano() + if t.IsZero() { + d = 0 + } + if err := fd.incref(); err != nil { + return err + } + switch mode { + case 'r': + syscall.SetReadDeadline(fd.sysfd, d) + case 'w': + syscall.SetWriteDeadline(fd.sysfd, d) + case 'r' + 'w': + syscall.SetReadDeadline(fd.sysfd, d) + syscall.SetWriteDeadline(fd.sysfd, d) + } + fd.decref() + return nil +} diff --git a/src/net/fd_poll_runtime.go b/src/net/fd_poll_runtime.go new file mode 100644 index 000000000..2bddc836c --- /dev/null +++ b/src/net/fd_poll_runtime.go @@ -0,0 +1,144 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd windows solaris + +package net + +import ( + "sync" + "syscall" + "time" +) + +// runtimeNano returns the current value of the runtime clock in nanoseconds. +func runtimeNano() int64 + +func runtime_pollServerInit() +func runtime_pollOpen(fd uintptr) (uintptr, int) +func runtime_pollClose(ctx uintptr) +func runtime_pollWait(ctx uintptr, mode int) int +func runtime_pollWaitCanceled(ctx uintptr, mode int) int +func runtime_pollReset(ctx uintptr, mode int) int +func runtime_pollSetDeadline(ctx uintptr, d int64, mode int) +func runtime_pollUnblock(ctx uintptr) + +type pollDesc struct { + runtimeCtx uintptr +} + +var serverInit sync.Once + +func (pd *pollDesc) Init(fd *netFD) error { + serverInit.Do(runtime_pollServerInit) + ctx, errno := runtime_pollOpen(uintptr(fd.sysfd)) + if errno != 0 { + return syscall.Errno(errno) + } + pd.runtimeCtx = ctx + return nil +} + +func (pd *pollDesc) Close() { + if pd.runtimeCtx == 0 { + return + } + runtime_pollClose(pd.runtimeCtx) + pd.runtimeCtx = 0 +} + +func (pd *pollDesc) Lock() { +} + +func (pd *pollDesc) Unlock() { +} + +func (pd *pollDesc) Wakeup() { +} + +// Evict evicts fd from the pending list, unblocking any I/O running on fd. +// Return value is whether the pollServer should be woken up. +func (pd *pollDesc) Evict() bool { + if pd.runtimeCtx == 0 { + return false + } + runtime_pollUnblock(pd.runtimeCtx) + return false +} + +func (pd *pollDesc) Prepare(mode int) error { + res := runtime_pollReset(pd.runtimeCtx, mode) + return convertErr(res) +} + +func (pd *pollDesc) PrepareRead() error { + return pd.Prepare('r') +} + +func (pd *pollDesc) PrepareWrite() error { + return pd.Prepare('w') +} + +func (pd *pollDesc) Wait(mode int) error { + res := runtime_pollWait(pd.runtimeCtx, mode) + return convertErr(res) +} + +func (pd *pollDesc) WaitRead() error { + return pd.Wait('r') +} + +func (pd *pollDesc) WaitWrite() error { + return pd.Wait('w') +} + +func (pd *pollDesc) WaitCanceled(mode int) { + runtime_pollWaitCanceled(pd.runtimeCtx, mode) +} + +func (pd *pollDesc) WaitCanceledRead() { + pd.WaitCanceled('r') +} + +func (pd *pollDesc) WaitCanceledWrite() { + pd.WaitCanceled('w') +} + +func convertErr(res int) error { + switch res { + case 0: + return nil + case 1: + return errClosing + case 2: + return errTimeout + } + println("unreachable: ", res) + panic("unreachable") +} + +func (fd *netFD) setDeadline(t time.Time) error { + return setDeadlineImpl(fd, t, 'r'+'w') +} + +func (fd *netFD) setReadDeadline(t time.Time) error { + return setDeadlineImpl(fd, t, 'r') +} + +func (fd *netFD) setWriteDeadline(t time.Time) error { + return setDeadlineImpl(fd, t, 'w') +} + +func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { + d := runtimeNano() + int64(t.Sub(time.Now())) + if t.IsZero() { + d = 0 + } + if err := fd.incref(); err != nil { + return err + } + runtime_pollSetDeadline(fd.pd.runtimeCtx, d, mode) + fd.decref() + return nil +} diff --git a/src/net/fd_unix.go b/src/net/fd_unix.go new file mode 100644 index 000000000..7fa43f6ab --- /dev/null +++ b/src/net/fd_unix.go @@ -0,0 +1,518 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +package net + +import ( + "io" + "os" + "runtime" + "sync/atomic" + "syscall" + "time" +) + +// Network file descriptor. +type netFD struct { + // locking/lifetime of sysfd + serialize access to Read and Write methods + fdmu fdMutex + + // immutable until Close + sysfd int + family int + sotype int + isConnected bool + net string + laddr Addr + raddr Addr + + // wait server + pd pollDesc +} + +func sysInit() { +} + +func dial(network string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { + return dialer(deadline) +} + +func newFD(sysfd, family, sotype int, net string) (*netFD, error) { + return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net}, nil +} + +func (fd *netFD) init() error { + if err := fd.pd.Init(fd); err != nil { + return err + } + return nil +} + +func (fd *netFD) setAddr(laddr, raddr Addr) { + fd.laddr = laddr + fd.raddr = raddr + runtime.SetFinalizer(fd, (*netFD).Close) +} + +func (fd *netFD) name() string { + var ls, rs string + if fd.laddr != nil { + ls = fd.laddr.String() + } + if fd.raddr != nil { + rs = fd.raddr.String() + } + return fd.net + ":" + ls + "->" + rs +} + +func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time) error { + // Do not need to call fd.writeLock here, + // because fd is not yet accessible to user, + // so no concurrent operations are possible. + switch err := syscall.Connect(fd.sysfd, ra); err { + case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: + case nil, syscall.EISCONN: + if !deadline.IsZero() && deadline.Before(time.Now()) { + return errTimeout + } + if err := fd.init(); err != nil { + return err + } + return nil + case syscall.EINVAL: + // On Solaris we can see EINVAL if the socket has + // already been accepted and closed by the server. + // Treat this as a successful connection--writes to + // the socket will see EOF. For details and a test + // case in C see http://golang.org/issue/6828. + if runtime.GOOS == "solaris" { + return nil + } + fallthrough + default: + return err + } + if err := fd.init(); err != nil { + return err + } + if !deadline.IsZero() { + fd.setWriteDeadline(deadline) + defer fd.setWriteDeadline(noDeadline) + } + for { + // Performing multiple connect system calls on a + // non-blocking socket under Unix variants does not + // necessarily result in earlier errors being + // returned. Instead, once runtime-integrated network + // poller tells us that the socket is ready, get the + // SO_ERROR socket option to see if the connection + // succeeded or failed. See issue 7474 for further + // details. + if err := fd.pd.WaitWrite(); err != nil { + return err + } + nerr, err := syscall.GetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_ERROR) + if err != nil { + return err + } + switch err := syscall.Errno(nerr); err { + case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: + case syscall.Errno(0), syscall.EISCONN: + return nil + default: + return err + } + } +} + +func (fd *netFD) destroy() { + // Poller may want to unregister fd in readiness notification mechanism, + // so this must be executed before closesocket. + fd.pd.Close() + closesocket(fd.sysfd) + fd.sysfd = -1 + runtime.SetFinalizer(fd, nil) +} + +// Add a reference to this fd. +// Returns an error if the fd cannot be used. +func (fd *netFD) incref() error { + if !fd.fdmu.Incref() { + return errClosing + } + return nil +} + +// Remove a reference to this FD and close if we've been asked to do so +// (and there are no references left). +func (fd *netFD) decref() { + if fd.fdmu.Decref() { + fd.destroy() + } +} + +// Add a reference to this fd and lock for reading. +// Returns an error if the fd cannot be used. +func (fd *netFD) readLock() error { + if !fd.fdmu.RWLock(true) { + return errClosing + } + return nil +} + +// Unlock for reading and remove a reference to this FD. +func (fd *netFD) readUnlock() { + if fd.fdmu.RWUnlock(true) { + fd.destroy() + } +} + +// Add a reference to this fd and lock for writing. +// Returns an error if the fd cannot be used. +func (fd *netFD) writeLock() error { + if !fd.fdmu.RWLock(false) { + return errClosing + } + return nil +} + +// Unlock for writing and remove a reference to this FD. +func (fd *netFD) writeUnlock() { + if fd.fdmu.RWUnlock(false) { + fd.destroy() + } +} + +func (fd *netFD) Close() error { + fd.pd.Lock() // needed for both fd.incref(true) and pollDesc.Evict + if !fd.fdmu.IncrefAndClose() { + fd.pd.Unlock() + return errClosing + } + // Unblock any I/O. Once it all unblocks and returns, + // so that it cannot be referring to fd.sysfd anymore, + // the final decref will close fd.sysfd. This should happen + // fairly quickly, since all the I/O is non-blocking, and any + // attempts to block in the pollDesc will return errClosing. + doWakeup := fd.pd.Evict() + fd.pd.Unlock() + fd.decref() + if doWakeup { + fd.pd.Wakeup() + } + return nil +} + +func (fd *netFD) shutdown(how int) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + err := syscall.Shutdown(fd.sysfd, how) + if err != nil { + return &OpError{"shutdown", fd.net, fd.laddr, err} + } + return nil +} + +func (fd *netFD) closeRead() error { + return fd.shutdown(syscall.SHUT_RD) +} + +func (fd *netFD) closeWrite() error { + return fd.shutdown(syscall.SHUT_WR) +} + +func (fd *netFD) Read(p []byte) (n int, err error) { + if err := fd.readLock(); err != nil { + return 0, err + } + defer fd.readUnlock() + if err := fd.pd.PrepareRead(); err != nil { + return 0, &OpError{"read", fd.net, fd.raddr, err} + } + for { + n, err = syscall.Read(int(fd.sysfd), p) + if err != nil { + n = 0 + if err == syscall.EAGAIN { + if err = fd.pd.WaitRead(); err == nil { + continue + } + } + } + err = chkReadErr(n, err, fd) + break + } + if err != nil && err != io.EOF { + err = &OpError{"read", fd.net, fd.raddr, err} + } + return +} + +func (fd *netFD) readFrom(p []byte) (n int, sa syscall.Sockaddr, err error) { + if err := fd.readLock(); err != nil { + return 0, nil, err + } + defer fd.readUnlock() + if err := fd.pd.PrepareRead(); err != nil { + return 0, nil, &OpError{"read", fd.net, fd.laddr, err} + } + for { + n, sa, err = syscall.Recvfrom(fd.sysfd, p, 0) + if err != nil { + n = 0 + if err == syscall.EAGAIN { + if err = fd.pd.WaitRead(); err == nil { + continue + } + } + } + err = chkReadErr(n, err, fd) + break + } + if err != nil && err != io.EOF { + err = &OpError{"read", fd.net, fd.laddr, err} + } + return +} + +func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.Sockaddr, err error) { + if err := fd.readLock(); err != nil { + return 0, 0, 0, nil, err + } + defer fd.readUnlock() + if err := fd.pd.PrepareRead(); err != nil { + return 0, 0, 0, nil, &OpError{"read", fd.net, fd.laddr, err} + } + for { + n, oobn, flags, sa, err = syscall.Recvmsg(fd.sysfd, p, oob, 0) + if err != nil { + // TODO(dfc) should n and oobn be set to 0 + if err == syscall.EAGAIN { + if err = fd.pd.WaitRead(); err == nil { + continue + } + } + } + err = chkReadErr(n, err, fd) + break + } + if err != nil && err != io.EOF { + err = &OpError{"read", fd.net, fd.laddr, err} + } + return +} + +func chkReadErr(n int, err error, fd *netFD) error { + if n == 0 && err == nil && fd.sotype != syscall.SOCK_DGRAM && fd.sotype != syscall.SOCK_RAW { + return io.EOF + } + return err +} + +func (fd *netFD) Write(p []byte) (nn int, err error) { + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + if err := fd.pd.PrepareWrite(); err != nil { + return 0, &OpError{"write", fd.net, fd.raddr, err} + } + for { + var n int + n, err = syscall.Write(int(fd.sysfd), p[nn:]) + if n > 0 { + nn += n + } + if nn == len(p) { + break + } + if err == syscall.EAGAIN { + if err = fd.pd.WaitWrite(); err == nil { + continue + } + } + if err != nil { + n = 0 + break + } + if n == 0 { + err = io.ErrUnexpectedEOF + break + } + } + if err != nil { + err = &OpError{"write", fd.net, fd.raddr, err} + } + return nn, err +} + +func (fd *netFD) writeTo(p []byte, sa syscall.Sockaddr) (n int, err error) { + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + if err := fd.pd.PrepareWrite(); err != nil { + return 0, &OpError{"write", fd.net, fd.raddr, err} + } + for { + err = syscall.Sendto(fd.sysfd, p, 0, sa) + if err == syscall.EAGAIN { + if err = fd.pd.WaitWrite(); err == nil { + continue + } + } + break + } + if err == nil { + n = len(p) + } else { + err = &OpError{"write", fd.net, fd.raddr, err} + } + return +} + +func (fd *netFD) writeMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oobn int, err error) { + if err := fd.writeLock(); err != nil { + return 0, 0, err + } + defer fd.writeUnlock() + if err := fd.pd.PrepareWrite(); err != nil { + return 0, 0, &OpError{"write", fd.net, fd.raddr, err} + } + for { + n, err = syscall.SendmsgN(fd.sysfd, p, oob, sa, 0) + if err == syscall.EAGAIN { + if err = fd.pd.WaitWrite(); err == nil { + continue + } + } + break + } + if err == nil { + oobn = len(oob) + } else { + err = &OpError{"write", fd.net, fd.raddr, err} + } + return +} + +func (fd *netFD) accept() (netfd *netFD, err error) { + if err := fd.readLock(); err != nil { + return nil, err + } + defer fd.readUnlock() + + var s int + var rsa syscall.Sockaddr + if err = fd.pd.PrepareRead(); err != nil { + return nil, &OpError{"accept", fd.net, fd.laddr, err} + } + for { + s, rsa, err = accept(fd.sysfd) + if err != nil { + if err == syscall.EAGAIN { + if err = fd.pd.WaitRead(); err == nil { + continue + } + } else if err == syscall.ECONNABORTED { + // This means that a socket on the listen queue was closed + // before we Accept()ed it; it's a silly error, so try again. + continue + } + return nil, &OpError{"accept", fd.net, fd.laddr, err} + } + break + } + + if netfd, err = newFD(s, fd.family, fd.sotype, fd.net); err != nil { + closesocket(s) + return nil, err + } + if err = netfd.init(); err != nil { + fd.Close() + return nil, err + } + lsa, _ := syscall.Getsockname(netfd.sysfd) + netfd.setAddr(netfd.addrFunc()(lsa), netfd.addrFunc()(rsa)) + return netfd, nil +} + +// tryDupCloexec indicates whether F_DUPFD_CLOEXEC should be used. +// If the kernel doesn't support it, this is set to 0. +var tryDupCloexec = int32(1) + +func dupCloseOnExec(fd int) (newfd int, err error) { + if atomic.LoadInt32(&tryDupCloexec) == 1 { + r0, _, e1 := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_DUPFD_CLOEXEC, 0) + if runtime.GOOS == "darwin" && e1 == syscall.EBADF { + // On OS X 10.6 and below (but we only support + // >= 10.6), F_DUPFD_CLOEXEC is unsupported + // and fcntl there falls back (undocumented) + // to doing an ioctl instead, returning EBADF + // in this case because fd is not of the + // expected device fd type. Treat it as + // EINVAL instead, so we fall back to the + // normal dup path. + // TODO: only do this on 10.6 if we can detect 10.6 + // cheaply. + e1 = syscall.EINVAL + } + switch e1 { + case 0: + return int(r0), nil + case syscall.EINVAL: + // Old kernel. Fall back to the portable way + // from now on. + atomic.StoreInt32(&tryDupCloexec, 0) + default: + return -1, e1 + } + } + return dupCloseOnExecOld(fd) +} + +// dupCloseOnExecUnixOld is the traditional way to dup an fd and +// set its O_CLOEXEC bit, using two system calls. +func dupCloseOnExecOld(fd int) (newfd int, err error) { + syscall.ForkLock.RLock() + defer syscall.ForkLock.RUnlock() + newfd, err = syscall.Dup(fd) + if err != nil { + return -1, err + } + syscall.CloseOnExec(newfd) + return +} + +func (fd *netFD) dup() (f *os.File, err error) { + ns, err := dupCloseOnExec(fd.sysfd) + if err != nil { + return nil, &OpError{"dup", fd.net, fd.laddr, err} + } + + // We want blocking mode for the new fd, hence the double negative. + // This also puts the old fd into blocking mode, meaning that + // I/O will block the thread instead of letting us use the epoll server. + // Everything will still work, just with more threads. + if err = syscall.SetNonblock(ns, false); err != nil { + return nil, &OpError{"setnonblock", fd.net, fd.laddr, err} + } + + return os.NewFile(uintptr(ns), fd.name()), nil +} + +func closesocket(s int) error { + return syscall.Close(s) +} + +func skipRawSocketTests() (skip bool, skipmsg string, err error) { + if os.Getuid() != 0 { + return true, "skipping test; must be root", nil + } + return false, "", nil +} diff --git a/src/net/fd_unix_test.go b/src/net/fd_unix_test.go new file mode 100644 index 000000000..fe8e8ff6a --- /dev/null +++ b/src/net/fd_unix_test.go @@ -0,0 +1,58 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package net + +import ( + "io" + "syscall" + "testing" +) + +var chkReadErrTests = []struct { + n int + err error + fd *netFD + expected error +}{ + + {100, nil, &netFD{sotype: syscall.SOCK_STREAM}, nil}, + {100, io.EOF, &netFD{sotype: syscall.SOCK_STREAM}, io.EOF}, + {100, errClosing, &netFD{sotype: syscall.SOCK_STREAM}, errClosing}, + {0, nil, &netFD{sotype: syscall.SOCK_STREAM}, io.EOF}, + {0, io.EOF, &netFD{sotype: syscall.SOCK_STREAM}, io.EOF}, + {0, errClosing, &netFD{sotype: syscall.SOCK_STREAM}, errClosing}, + + {100, nil, &netFD{sotype: syscall.SOCK_DGRAM}, nil}, + {100, io.EOF, &netFD{sotype: syscall.SOCK_DGRAM}, io.EOF}, + {100, errClosing, &netFD{sotype: syscall.SOCK_DGRAM}, errClosing}, + {0, nil, &netFD{sotype: syscall.SOCK_DGRAM}, nil}, + {0, io.EOF, &netFD{sotype: syscall.SOCK_DGRAM}, io.EOF}, + {0, errClosing, &netFD{sotype: syscall.SOCK_DGRAM}, errClosing}, + + {100, nil, &netFD{sotype: syscall.SOCK_SEQPACKET}, nil}, + {100, io.EOF, &netFD{sotype: syscall.SOCK_SEQPACKET}, io.EOF}, + {100, errClosing, &netFD{sotype: syscall.SOCK_SEQPACKET}, errClosing}, + {0, nil, &netFD{sotype: syscall.SOCK_SEQPACKET}, io.EOF}, + {0, io.EOF, &netFD{sotype: syscall.SOCK_SEQPACKET}, io.EOF}, + {0, errClosing, &netFD{sotype: syscall.SOCK_SEQPACKET}, errClosing}, + + {100, nil, &netFD{sotype: syscall.SOCK_RAW}, nil}, + {100, io.EOF, &netFD{sotype: syscall.SOCK_RAW}, io.EOF}, + {100, errClosing, &netFD{sotype: syscall.SOCK_RAW}, errClosing}, + {0, nil, &netFD{sotype: syscall.SOCK_RAW}, nil}, + {0, io.EOF, &netFD{sotype: syscall.SOCK_RAW}, io.EOF}, + {0, errClosing, &netFD{sotype: syscall.SOCK_RAW}, errClosing}, +} + +func TestChkReadErr(t *testing.T) { + for _, tt := range chkReadErrTests { + actual := chkReadErr(tt.n, tt.err, tt.fd) + if actual != tt.expected { + t.Errorf("chkReadError(%v, %v, %v): expected %v, actual %v", tt.n, tt.err, tt.fd.sotype, tt.expected, actual) + } + } +} diff --git a/src/net/fd_windows.go b/src/net/fd_windows.go new file mode 100644 index 000000000..6d69e0624 --- /dev/null +++ b/src/net/fd_windows.go @@ -0,0 +1,644 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "io" + "os" + "runtime" + "sync" + "syscall" + "time" + "unsafe" +) + +var ( + initErr error + ioSync uint64 +) + +// CancelIo Windows API cancels all outstanding IO for a particular +// socket on current thread. To overcome that limitation, we run +// special goroutine, locked to OS single thread, that both starts +// and cancels IO. It means, there are 2 unavoidable thread switches +// for every IO. +// Some newer versions of Windows has new CancelIoEx API, that does +// not have that limitation and can be used from any thread. This +// package uses CancelIoEx API, if present, otherwise it fallback +// to CancelIo. + +var ( + canCancelIO bool // determines if CancelIoEx API is present + skipSyncNotif bool + hasLoadSetFileCompletionNotificationModes bool +) + +func sysInit() { + var d syscall.WSAData + e := syscall.WSAStartup(uint32(0x202), &d) + if e != nil { + initErr = os.NewSyscallError("WSAStartup", e) + } + canCancelIO = syscall.LoadCancelIoEx() == nil + if syscall.LoadGetAddrInfo() == nil { + lookupPort = newLookupPort + lookupIP = newLookupIP + } + + hasLoadSetFileCompletionNotificationModes = syscall.LoadSetFileCompletionNotificationModes() == nil + if hasLoadSetFileCompletionNotificationModes { + // It's not safe to use FILE_SKIP_COMPLETION_PORT_ON_SUCCESS if non IFS providers are installed: + // http://support.microsoft.com/kb/2568167 + skipSyncNotif = true + protos := [2]int32{syscall.IPPROTO_TCP, 0} + var buf [32]syscall.WSAProtocolInfo + len := uint32(unsafe.Sizeof(buf)) + n, err := syscall.WSAEnumProtocols(&protos[0], &buf[0], &len) + if err != nil { + skipSyncNotif = false + } else { + for i := int32(0); i < n; i++ { + if buf[i].ServiceFlags1&syscall.XP1_IFS_HANDLES == 0 { + skipSyncNotif = false + break + } + } + } + } +} + +func closesocket(s syscall.Handle) error { + return syscall.Closesocket(s) +} + +func canUseConnectEx(net string) bool { + switch net { + case "udp", "udp4", "udp6", "ip", "ip4", "ip6": + // ConnectEx windows API does not support connectionless sockets. + return false + } + return syscall.LoadConnectEx() == nil +} + +func dial(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { + if !canUseConnectEx(net) { + // Use the relatively inefficient goroutine-racing + // implementation of DialTimeout. + return dialChannel(net, ra, dialer, deadline) + } + return dialer(deadline) +} + +// operation contains superset of data necessary to perform all async IO. +type operation struct { + // Used by IOCP interface, it must be first field + // of the struct, as our code rely on it. + o syscall.Overlapped + + // fields used by runtime.netpoll + runtimeCtx uintptr + mode int32 + errno int32 + qty uint32 + + // fields used only by net package + fd *netFD + errc chan error + buf syscall.WSABuf + sa syscall.Sockaddr + rsa *syscall.RawSockaddrAny + rsan int32 + handle syscall.Handle + flags uint32 +} + +func (o *operation) InitBuf(buf []byte) { + o.buf.Len = uint32(len(buf)) + o.buf.Buf = nil + if len(buf) != 0 { + o.buf.Buf = &buf[0] + } +} + +// ioSrv executes net IO requests. +type ioSrv struct { + req chan ioSrvReq +} + +type ioSrvReq struct { + o *operation + submit func(o *operation) error // if nil, cancel the operation +} + +// ProcessRemoteIO will execute submit IO requests on behalf +// of other goroutines, all on a single os thread, so it can +// cancel them later. Results of all operations will be sent +// back to their requesters via channel supplied in request. +// It is used only when the CancelIoEx API is unavailable. +func (s *ioSrv) ProcessRemoteIO() { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + for r := range s.req { + if r.submit != nil { + r.o.errc <- r.submit(r.o) + } else { + r.o.errc <- syscall.CancelIo(r.o.fd.sysfd) + } + } +} + +// ExecIO executes a single IO operation o. It submits and cancels +// IO in the current thread for systems where Windows CancelIoEx API +// is available. Alternatively, it passes the request onto +// runtime netpoll and waits for completion or cancels request. +func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) error) (int, error) { + fd := o.fd + // Notify runtime netpoll about starting IO. + err := fd.pd.Prepare(int(o.mode)) + if err != nil { + return 0, &OpError{name, fd.net, fd.laddr, err} + } + // Start IO. + if canCancelIO { + err = submit(o) + } else { + // Send request to a special dedicated thread, + // so it can stop the IO with CancelIO later. + s.req <- ioSrvReq{o, submit} + err = <-o.errc + } + switch err { + case nil: + // IO completed immediately + if o.fd.skipSyncNotif { + // No completion message will follow, so return immediately. + return int(o.qty), nil + } + // Need to get our completion message anyway. + case syscall.ERROR_IO_PENDING: + // IO started, and we have to wait for its completion. + err = nil + default: + return 0, &OpError{name, fd.net, fd.laddr, err} + } + // Wait for our request to complete. + err = fd.pd.Wait(int(o.mode)) + if err == nil { + // All is good. Extract our IO results and return. + if o.errno != 0 { + err = syscall.Errno(o.errno) + return 0, &OpError{name, fd.net, fd.laddr, err} + } + return int(o.qty), nil + } + // IO is interrupted by "close" or "timeout" + netpollErr := err + switch netpollErr { + case errClosing, errTimeout: + // will deal with those. + default: + panic("net: unexpected runtime.netpoll error: " + netpollErr.Error()) + } + // Cancel our request. + if canCancelIO { + err := syscall.CancelIoEx(fd.sysfd, &o.o) + // Assuming ERROR_NOT_FOUND is returned, if IO is completed. + if err != nil && err != syscall.ERROR_NOT_FOUND { + // TODO(brainman): maybe do something else, but panic. + panic(err) + } + } else { + s.req <- ioSrvReq{o, nil} + <-o.errc + } + // Wait for cancellation to complete. + fd.pd.WaitCanceled(int(o.mode)) + if o.errno != 0 { + err = syscall.Errno(o.errno) + if err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled + err = netpollErr + } + return 0, &OpError{name, fd.net, fd.laddr, err} + } + // We issued cancellation request. But, it seems, IO operation succeeded + // before cancellation request run. We need to treat IO operation as + // succeeded (the bytes are actually sent/recv from network). + return int(o.qty), nil +} + +// Start helper goroutines. +var rsrv, wsrv *ioSrv +var onceStartServer sync.Once + +func startServer() { + rsrv = new(ioSrv) + wsrv = new(ioSrv) + if !canCancelIO { + // Only CancelIo API is available. Lets start two special goroutines + // locked to an OS thread, that both starts and cancels IO. One will + // process read requests, while other will do writes. + rsrv.req = make(chan ioSrvReq) + go rsrv.ProcessRemoteIO() + wsrv.req = make(chan ioSrvReq) + go wsrv.ProcessRemoteIO() + } +} + +// Network file descriptor. +type netFD struct { + // locking/lifetime of sysfd + serialize access to Read and Write methods + fdmu fdMutex + + // immutable until Close + sysfd syscall.Handle + family int + sotype int + isConnected bool + skipSyncNotif bool + net string + laddr Addr + raddr Addr + + rop operation // read operation + wop operation // write operation + + // wait server + pd pollDesc +} + +func newFD(sysfd syscall.Handle, family, sotype int, net string) (*netFD, error) { + if initErr != nil { + return nil, initErr + } + onceStartServer.Do(startServer) + return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net}, nil +} + +func (fd *netFD) init() error { + if err := fd.pd.Init(fd); err != nil { + return err + } + if hasLoadSetFileCompletionNotificationModes { + // We do not use events, so we can skip them always. + flags := uint8(syscall.FILE_SKIP_SET_EVENT_ON_HANDLE) + // It's not safe to skip completion notifications for UDP: + // http://blogs.technet.com/b/winserverperformance/archive/2008/06/26/designing-applications-for-high-performance-part-iii.aspx + if skipSyncNotif && fd.net == "tcp" { + flags |= syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS + } + err := syscall.SetFileCompletionNotificationModes(fd.sysfd, flags) + if err == nil && flags&syscall.FILE_SKIP_COMPLETION_PORT_ON_SUCCESS != 0 { + fd.skipSyncNotif = true + } + } + fd.rop.mode = 'r' + fd.wop.mode = 'w' + fd.rop.fd = fd + fd.wop.fd = fd + fd.rop.runtimeCtx = fd.pd.runtimeCtx + fd.wop.runtimeCtx = fd.pd.runtimeCtx + if !canCancelIO { + fd.rop.errc = make(chan error) + fd.wop.errc = make(chan error) + } + return nil +} + +func (fd *netFD) setAddr(laddr, raddr Addr) { + fd.laddr = laddr + fd.raddr = raddr + runtime.SetFinalizer(fd, (*netFD).Close) +} + +func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time) error { + // Do not need to call fd.writeLock here, + // because fd is not yet accessible to user, + // so no concurrent operations are possible. + if err := fd.init(); err != nil { + return err + } + if !deadline.IsZero() { + fd.setWriteDeadline(deadline) + defer fd.setWriteDeadline(noDeadline) + } + if !canUseConnectEx(fd.net) { + return syscall.Connect(fd.sysfd, ra) + } + // ConnectEx windows API requires an unconnected, previously bound socket. + if la == nil { + switch ra.(type) { + case *syscall.SockaddrInet4: + la = &syscall.SockaddrInet4{} + case *syscall.SockaddrInet6: + la = &syscall.SockaddrInet6{} + default: + panic("unexpected type in connect") + } + if err := syscall.Bind(fd.sysfd, la); err != nil { + return err + } + } + // Call ConnectEx API. + o := &fd.wop + o.sa = ra + _, err := wsrv.ExecIO(o, "ConnectEx", func(o *operation) error { + return syscall.ConnectEx(o.fd.sysfd, o.sa, nil, 0, nil, &o.o) + }) + if err != nil { + return err + } + // Refresh socket properties. + return syscall.Setsockopt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_UPDATE_CONNECT_CONTEXT, (*byte)(unsafe.Pointer(&fd.sysfd)), int32(unsafe.Sizeof(fd.sysfd))) +} + +func (fd *netFD) destroy() { + if fd.sysfd == syscall.InvalidHandle { + return + } + // Poller may want to unregister fd in readiness notification mechanism, + // so this must be executed before closesocket. + fd.pd.Close() + closesocket(fd.sysfd) + fd.sysfd = syscall.InvalidHandle + // no need for a finalizer anymore + runtime.SetFinalizer(fd, nil) +} + +// Add a reference to this fd. +// Returns an error if the fd cannot be used. +func (fd *netFD) incref() error { + if !fd.fdmu.Incref() { + return errClosing + } + return nil +} + +// Remove a reference to this FD and close if we've been asked to do so +// (and there are no references left). +func (fd *netFD) decref() { + if fd.fdmu.Decref() { + fd.destroy() + } +} + +// Add a reference to this fd and lock for reading. +// Returns an error if the fd cannot be used. +func (fd *netFD) readLock() error { + if !fd.fdmu.RWLock(true) { + return errClosing + } + return nil +} + +// Unlock for reading and remove a reference to this FD. +func (fd *netFD) readUnlock() { + if fd.fdmu.RWUnlock(true) { + fd.destroy() + } +} + +// Add a reference to this fd and lock for writing. +// Returns an error if the fd cannot be used. +func (fd *netFD) writeLock() error { + if !fd.fdmu.RWLock(false) { + return errClosing + } + return nil +} + +// Unlock for writing and remove a reference to this FD. +func (fd *netFD) writeUnlock() { + if fd.fdmu.RWUnlock(false) { + fd.destroy() + } +} + +func (fd *netFD) Close() error { + if !fd.fdmu.IncrefAndClose() { + return errClosing + } + // unblock pending reader and writer + fd.pd.Evict() + fd.decref() + return nil +} + +func (fd *netFD) shutdown(how int) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + err := syscall.Shutdown(fd.sysfd, how) + if err != nil { + return &OpError{"shutdown", fd.net, fd.laddr, err} + } + return nil +} + +func (fd *netFD) closeRead() error { + return fd.shutdown(syscall.SHUT_RD) +} + +func (fd *netFD) closeWrite() error { + return fd.shutdown(syscall.SHUT_WR) +} + +func (fd *netFD) Read(buf []byte) (int, error) { + if err := fd.readLock(); err != nil { + return 0, err + } + defer fd.readUnlock() + o := &fd.rop + o.InitBuf(buf) + n, err := rsrv.ExecIO(o, "WSARecv", func(o *operation) error { + return syscall.WSARecv(o.fd.sysfd, &o.buf, 1, &o.qty, &o.flags, &o.o, nil) + }) + if err == nil && n == 0 { + err = io.EOF + } + if raceenabled { + raceAcquire(unsafe.Pointer(&ioSync)) + } + return n, err +} + +func (fd *netFD) readFrom(buf []byte) (n int, sa syscall.Sockaddr, err error) { + if len(buf) == 0 { + return 0, nil, nil + } + if err := fd.readLock(); err != nil { + return 0, nil, err + } + defer fd.readUnlock() + o := &fd.rop + o.InitBuf(buf) + n, err = rsrv.ExecIO(o, "WSARecvFrom", func(o *operation) error { + if o.rsa == nil { + o.rsa = new(syscall.RawSockaddrAny) + } + o.rsan = int32(unsafe.Sizeof(*o.rsa)) + return syscall.WSARecvFrom(o.fd.sysfd, &o.buf, 1, &o.qty, &o.flags, o.rsa, &o.rsan, &o.o, nil) + }) + if err != nil { + return 0, nil, err + } + sa, _ = o.rsa.Sockaddr() + return +} + +func (fd *netFD) Write(buf []byte) (int, error) { + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + if raceenabled { + raceReleaseMerge(unsafe.Pointer(&ioSync)) + } + o := &fd.wop + o.InitBuf(buf) + return wsrv.ExecIO(o, "WSASend", func(o *operation) error { + return syscall.WSASend(o.fd.sysfd, &o.buf, 1, &o.qty, 0, &o.o, nil) + }) +} + +func (fd *netFD) writeTo(buf []byte, sa syscall.Sockaddr) (int, error) { + if len(buf) == 0 { + return 0, nil + } + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + o := &fd.wop + o.InitBuf(buf) + o.sa = sa + return wsrv.ExecIO(o, "WSASendto", func(o *operation) error { + return syscall.WSASendto(o.fd.sysfd, &o.buf, 1, &o.qty, 0, o.sa, &o.o, nil) + }) +} + +func (fd *netFD) acceptOne(rawsa []syscall.RawSockaddrAny, o *operation) (*netFD, error) { + // Get new socket. + s, err := sysSocket(fd.family, fd.sotype, 0) + if err != nil { + return nil, &OpError{"socket", fd.net, fd.laddr, err} + } + + // Associate our new socket with IOCP. + netfd, err := newFD(s, fd.family, fd.sotype, fd.net) + if err != nil { + closesocket(s) + return nil, &OpError{"accept", fd.net, fd.laddr, err} + } + if err := netfd.init(); err != nil { + fd.Close() + return nil, err + } + + // Submit accept request. + o.handle = s + o.rsan = int32(unsafe.Sizeof(rawsa[0])) + _, err = rsrv.ExecIO(o, "AcceptEx", func(o *operation) error { + return syscall.AcceptEx(o.fd.sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) + }) + if err != nil { + netfd.Close() + return nil, err + } + + // Inherit properties of the listening socket. + err = syscall.Setsockopt(s, syscall.SOL_SOCKET, syscall.SO_UPDATE_ACCEPT_CONTEXT, (*byte)(unsafe.Pointer(&fd.sysfd)), int32(unsafe.Sizeof(fd.sysfd))) + if err != nil { + netfd.Close() + return nil, &OpError{"Setsockopt", fd.net, fd.laddr, err} + } + + return netfd, nil +} + +func (fd *netFD) accept() (*netFD, error) { + if err := fd.readLock(); err != nil { + return nil, err + } + defer fd.readUnlock() + + o := &fd.rop + var netfd *netFD + var err error + var rawsa [2]syscall.RawSockaddrAny + for { + netfd, err = fd.acceptOne(rawsa[:], o) + if err == nil { + break + } + // Sometimes we see WSAECONNRESET and ERROR_NETNAME_DELETED is + // returned here. These happen if connection reset is received + // before AcceptEx could complete. These errors relate to new + // connection, not to AcceptEx, so ignore broken connection and + // try AcceptEx again for more connections. + operr, ok := err.(*OpError) + if !ok { + return nil, err + } + errno, ok := operr.Err.(syscall.Errno) + if !ok { + return nil, err + } + switch errno { + case syscall.ERROR_NETNAME_DELETED, syscall.WSAECONNRESET: + // ignore these and try again + default: + return nil, err + } + } + + // Get local and peer addr out of AcceptEx buffer. + var lrsa, rrsa *syscall.RawSockaddrAny + var llen, rlen int32 + syscall.GetAcceptExSockaddrs((*byte)(unsafe.Pointer(&rawsa[0])), + 0, uint32(o.rsan), uint32(o.rsan), &lrsa, &llen, &rrsa, &rlen) + lsa, _ := lrsa.Sockaddr() + rsa, _ := rrsa.Sockaddr() + + netfd.setAddr(netfd.addrFunc()(lsa), netfd.addrFunc()(rsa)) + return netfd, nil +} + +func skipRawSocketTests() (skip bool, skipmsg string, err error) { + // From http://msdn.microsoft.com/en-us/library/windows/desktop/ms740548.aspx: + // Note: To use a socket of type SOCK_RAW requires administrative privileges. + // Users running Winsock applications that use raw sockets must be a member of + // the Administrators group on the local computer, otherwise raw socket calls + // will fail with an error code of WSAEACCES. On Windows Vista and later, access + // for raw sockets is enforced at socket creation. In earlier versions of Windows, + // access for raw sockets is enforced during other socket operations. + s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, 0) + if err == syscall.WSAEACCES { + return true, "skipping test; no access to raw socket allowed", nil + } + if err != nil { + return true, "", err + } + defer syscall.Closesocket(s) + return false, "", nil +} + +// Unimplemented functions. + +func (fd *netFD) dup() (*os.File, error) { + // TODO: Implement this + return nil, os.NewSyscallError("dup", syscall.EWINDOWS) +} + +var errNoSupport = errors.New("address family not supported") + +func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.Sockaddr, err error) { + return 0, 0, 0, nil, errNoSupport +} + +func (fd *netFD) writeMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oobn int, err error) { + return 0, 0, errNoSupport +} diff --git a/src/net/file_plan9.go b/src/net/file_plan9.go new file mode 100644 index 000000000..068f0881d --- /dev/null +++ b/src/net/file_plan9.go @@ -0,0 +1,157 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "io" + "os" + "syscall" +) + +func (fd *netFD) status(ln int) (string, error) { + if !fd.ok() { + return "", syscall.EINVAL + } + + status, err := os.Open(fd.dir + "/status") + if err != nil { + return "", err + } + defer status.Close() + buf := make([]byte, ln) + n, err := io.ReadFull(status, buf[:]) + if err != nil { + return "", err + } + return string(buf[:n]), nil +} + +func newFileFD(f *os.File) (net *netFD, err error) { + var ctl *os.File + close := func(fd int) { + if err != nil { + syscall.Close(fd) + } + } + + path, err := syscall.Fd2path(int(f.Fd())) + if err != nil { + return nil, os.NewSyscallError("fd2path", err) + } + comp := splitAtBytes(path, "/") + n := len(comp) + if n < 3 || comp[0][0:3] != "net" { + return nil, syscall.EPLAN9 + } + + name := comp[2] + switch file := comp[n-1]; file { + case "ctl", "clone": + syscall.ForkLock.RLock() + fd, err := syscall.Dup(int(f.Fd()), -1) + syscall.ForkLock.RUnlock() + if err != nil { + return nil, os.NewSyscallError("dup", err) + } + defer close(fd) + + dir := netdir + "/" + comp[n-2] + ctl = os.NewFile(uintptr(fd), dir+"/"+file) + ctl.Seek(0, 0) + var buf [16]byte + n, err := ctl.Read(buf[:]) + if err != nil { + return nil, err + } + name = string(buf[:n]) + default: + if len(comp) < 4 { + return nil, errors.New("could not find control file for connection") + } + dir := netdir + "/" + comp[1] + "/" + name + ctl, err = os.OpenFile(dir+"/ctl", os.O_RDWR, 0) + if err != nil { + return nil, err + } + defer close(int(ctl.Fd())) + } + dir := netdir + "/" + comp[1] + "/" + name + laddr, err := readPlan9Addr(comp[1], dir+"/local") + if err != nil { + return nil, err + } + return newFD(comp[1], name, ctl, nil, laddr, nil) +} + +func newFileConn(f *os.File) (c Conn, err error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + if !fd.ok() { + return nil, syscall.EINVAL + } + + fd.data, err = os.OpenFile(fd.dir+"/data", os.O_RDWR, 0) + if err != nil { + return nil, err + } + + switch fd.laddr.(type) { + case *TCPAddr: + return newTCPConn(fd), nil + case *UDPAddr: + return newUDPConn(fd), nil + } + return nil, syscall.EPLAN9 +} + +func newFileListener(f *os.File) (l Listener, err error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch fd.laddr.(type) { + case *TCPAddr: + default: + return nil, syscall.EPLAN9 + } + + // check that file corresponds to a listener + s, err := fd.status(len("Listen")) + if err != nil { + return nil, err + } + if s != "Listen" { + return nil, errors.New("file does not represent a listener") + } + + return &TCPListener{fd}, nil +} + +// FileConn returns a copy of the network connection corresponding to +// the open file f. It is the caller's responsibility to close f when +// finished. Closing c does not affect f, and closing f does not +// affect c. +func FileConn(f *os.File) (c Conn, err error) { + return newFileConn(f) +} + +// FileListener returns a copy of the network listener corresponding +// to the open file f. It is the caller's responsibility to close l +// when finished. Closing l does not affect f, and closing f does not +// affect l. +func FileListener(f *os.File) (l Listener, err error) { + return newFileListener(f) +} + +// FilePacketConn returns a copy of the packet network connection +// corresponding to the open file f. It is the caller's +// responsibility to close f when finished. Closing c does not affect +// f, and closing f does not affect c. +func FilePacketConn(f *os.File) (c PacketConn, err error) { + return nil, syscall.EPLAN9 +} diff --git a/src/net/file_test.go b/src/net/file_test.go new file mode 100644 index 000000000..d81bca782 --- /dev/null +++ b/src/net/file_test.go @@ -0,0 +1,205 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "reflect" + "runtime" + "testing" +) + +type listenerFile interface { + Listener + File() (f *os.File, err error) +} + +type packetConnFile interface { + PacketConn + File() (f *os.File, err error) +} + +type connFile interface { + Conn + File() (f *os.File, err error) +} + +func testFileListener(t *testing.T, net, laddr string) { + switch net { + case "tcp", "tcp4", "tcp6": + laddr += ":0" // any available port + } + l, err := Listen(net, laddr) + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer l.Close() + lf := l.(listenerFile) + f, err := lf.File() + if err != nil { + t.Fatalf("File failed: %v", err) + } + c, err := FileListener(f) + if err != nil { + t.Fatalf("FileListener failed: %v", err) + } + if !reflect.DeepEqual(l.Addr(), c.Addr()) { + t.Fatalf("Addrs not equal: %#v != %#v", l.Addr(), c.Addr()) + } + if err := c.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } +} + +var fileListenerTests = []struct { + net string + laddr string + ipv6 bool // test with underlying AF_INET6 socket + linux bool // test with abstract unix domain socket, a Linux-ism +}{ + {net: "tcp", laddr: ""}, + {net: "tcp", laddr: "0.0.0.0"}, + {net: "tcp", laddr: "[::ffff:0.0.0.0]"}, + {net: "tcp", laddr: "[::]", ipv6: true}, + + {net: "tcp", laddr: "127.0.0.1"}, + {net: "tcp", laddr: "[::ffff:127.0.0.1]"}, + {net: "tcp", laddr: "[::1]", ipv6: true}, + + {net: "tcp4", laddr: ""}, + {net: "tcp4", laddr: "0.0.0.0"}, + {net: "tcp4", laddr: "[::ffff:0.0.0.0]"}, + + {net: "tcp4", laddr: "127.0.0.1"}, + {net: "tcp4", laddr: "[::ffff:127.0.0.1]"}, + + {net: "tcp6", laddr: "", ipv6: true}, + {net: "tcp6", laddr: "[::]", ipv6: true}, + + {net: "tcp6", laddr: "[::1]", ipv6: true}, + + {net: "unix", laddr: "@gotest/net", linux: true}, + {net: "unixpacket", laddr: "@gotest/net", linux: true}, +} + +func TestFileListener(t *testing.T) { + switch runtime.GOOS { + case "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + for _, tt := range fileListenerTests { + if skipServerTest(tt.net, "unix", tt.laddr, tt.ipv6, false, tt.linux) { + continue + } + if skipServerTest(tt.net, "unixpacket", tt.laddr, tt.ipv6, false, tt.linux) { + continue + } + testFileListener(t, tt.net, tt.laddr) + } +} + +func testFilePacketConn(t *testing.T, pcf packetConnFile, listen bool) { + f, err := pcf.File() + if err != nil { + t.Fatalf("File failed: %v", err) + } + c, err := FilePacketConn(f) + if err != nil { + t.Fatalf("FilePacketConn failed: %v", err) + } + if !reflect.DeepEqual(pcf.LocalAddr(), c.LocalAddr()) { + t.Fatalf("LocalAddrs not equal: %#v != %#v", pcf.LocalAddr(), c.LocalAddr()) + } + if listen { + if _, err := c.WriteTo([]byte{}, c.LocalAddr()); err != nil { + t.Fatalf("WriteTo failed: %v", err) + } + } + if err := c.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } +} + +func testFilePacketConnListen(t *testing.T, net, laddr string) { + switch net { + case "udp", "udp4", "udp6": + laddr += ":0" // any available port + } + l, err := ListenPacket(net, laddr) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + testFilePacketConn(t, l.(packetConnFile), true) + if err := l.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } +} + +func testFilePacketConnDial(t *testing.T, net, raddr string) { + switch net { + case "udp", "udp4", "udp6": + raddr += ":12345" + } + c, err := Dial(net, raddr) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + testFilePacketConn(t, c.(packetConnFile), false) + if err := c.Close(); err != nil { + t.Fatalf("Close failed: %v", err) + } +} + +var filePacketConnTests = []struct { + net string + addr string + ipv6 bool // test with underlying AF_INET6 socket + linux bool // test with abstract unix domain socket, a Linux-ism +}{ + {net: "udp", addr: "127.0.0.1"}, + {net: "udp", addr: "[::ffff:127.0.0.1]"}, + {net: "udp", addr: "[::1]", ipv6: true}, + + {net: "udp4", addr: "127.0.0.1"}, + {net: "udp4", addr: "[::ffff:127.0.0.1]"}, + + {net: "udp6", addr: "[::1]", ipv6: true}, + + {net: "ip4:icmp", addr: "127.0.0.1"}, + + {net: "unixgram", addr: "@gotest3/net", linux: true}, +} + +func TestFilePacketConn(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + for _, tt := range filePacketConnTests { + if skipServerTest(tt.net, "unixgram", tt.addr, tt.ipv6, false, tt.linux) { + continue + } + if os.Getuid() != 0 && tt.net == "ip4:icmp" { + t.Log("skipping test; must be root") + continue + } + testFilePacketConnListen(t, tt.net, tt.addr) + switch tt.addr { + case "", "0.0.0.0", "[::ffff:0.0.0.0]", "[::]": + default: + if tt.net != "unixgram" { + testFilePacketConnDial(t, tt.net, tt.addr) + } + } + } +} diff --git a/src/net/file_unix.go b/src/net/file_unix.go new file mode 100644 index 000000000..07b3ecf62 --- /dev/null +++ b/src/net/file_unix.go @@ -0,0 +1,139 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +package net + +import ( + "os" + "syscall" +) + +func newFileFD(f *os.File) (*netFD, error) { + fd, err := dupCloseOnExec(int(f.Fd())) + if err != nil { + return nil, os.NewSyscallError("dup", err) + } + + if err = syscall.SetNonblock(fd, true); err != nil { + closesocket(fd) + return nil, err + } + + sotype, err := syscall.GetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_TYPE) + if err != nil { + closesocket(fd) + return nil, os.NewSyscallError("getsockopt", err) + } + + family := syscall.AF_UNSPEC + toAddr := sockaddrToTCP + lsa, _ := syscall.Getsockname(fd) + switch lsa.(type) { + default: + closesocket(fd) + return nil, syscall.EINVAL + case *syscall.SockaddrInet4: + family = syscall.AF_INET + if sotype == syscall.SOCK_DGRAM { + toAddr = sockaddrToUDP + } else if sotype == syscall.SOCK_RAW { + toAddr = sockaddrToIP + } + case *syscall.SockaddrInet6: + family = syscall.AF_INET6 + if sotype == syscall.SOCK_DGRAM { + toAddr = sockaddrToUDP + } else if sotype == syscall.SOCK_RAW { + toAddr = sockaddrToIP + } + case *syscall.SockaddrUnix: + family = syscall.AF_UNIX + toAddr = sockaddrToUnix + if sotype == syscall.SOCK_DGRAM { + toAddr = sockaddrToUnixgram + } else if sotype == syscall.SOCK_SEQPACKET { + toAddr = sockaddrToUnixpacket + } + } + laddr := toAddr(lsa) + rsa, _ := syscall.Getpeername(fd) + raddr := toAddr(rsa) + + netfd, err := newFD(fd, family, sotype, laddr.Network()) + if err != nil { + closesocket(fd) + return nil, err + } + if err := netfd.init(); err != nil { + netfd.Close() + return nil, err + } + netfd.setAddr(laddr, raddr) + return netfd, nil +} + +// FileConn returns a copy of the network connection corresponding to +// the open file f. It is the caller's responsibility to close f when +// finished. Closing c does not affect f, and closing f does not +// affect c. +func FileConn(f *os.File) (c Conn, err error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch fd.laddr.(type) { + case *TCPAddr: + return newTCPConn(fd), nil + case *UDPAddr: + return newUDPConn(fd), nil + case *IPAddr: + return newIPConn(fd), nil + case *UnixAddr: + return newUnixConn(fd), nil + } + fd.Close() + return nil, syscall.EINVAL +} + +// FileListener returns a copy of the network listener corresponding +// to the open file f. It is the caller's responsibility to close l +// when finished. Closing l does not affect f, and closing f does not +// affect l. +func FileListener(f *os.File) (l Listener, err error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch laddr := fd.laddr.(type) { + case *TCPAddr: + return &TCPListener{fd}, nil + case *UnixAddr: + return &UnixListener{fd, laddr.Name}, nil + } + fd.Close() + return nil, syscall.EINVAL +} + +// FilePacketConn returns a copy of the packet network connection +// corresponding to the open file f. It is the caller's +// responsibility to close f when finished. Closing c does not affect +// f, and closing f does not affect c. +func FilePacketConn(f *os.File) (c PacketConn, err error) { + fd, err := newFileFD(f) + if err != nil { + return nil, err + } + switch fd.laddr.(type) { + case *UDPAddr: + return newUDPConn(fd), nil + case *IPAddr: + return newIPConn(fd), nil + case *UnixAddr: + return newUnixConn(fd), nil + } + fd.Close() + return nil, syscall.EINVAL +} diff --git a/src/net/file_windows.go b/src/net/file_windows.go new file mode 100644 index 000000000..ca2b9b226 --- /dev/null +++ b/src/net/file_windows.go @@ -0,0 +1,37 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +// FileConn returns a copy of the network connection corresponding to +// the open file f. It is the caller's responsibility to close f when +// finished. Closing c does not affect f, and closing f does not +// affect c. +func FileConn(f *os.File) (c Conn, err error) { + // TODO: Implement this + return nil, os.NewSyscallError("FileConn", syscall.EWINDOWS) +} + +// FileListener returns a copy of the network listener corresponding +// to the open file f. It is the caller's responsibility to close l +// when finished. Closing l does not affect f, and closing f does not +// affect l. +func FileListener(f *os.File) (l Listener, err error) { + // TODO: Implement this + return nil, os.NewSyscallError("FileListener", syscall.EWINDOWS) +} + +// FilePacketConn returns a copy of the packet network connection +// corresponding to the open file f. It is the caller's +// responsibility to close f when finished. Closing c does not affect +// f, and closing f does not affect c. +func FilePacketConn(f *os.File) (c PacketConn, err error) { + // TODO: Implement this + return nil, os.NewSyscallError("FilePacketConn", syscall.EWINDOWS) +} diff --git a/src/net/hosts.go b/src/net/hosts.go new file mode 100644 index 000000000..9400503e4 --- /dev/null +++ b/src/net/hosts.go @@ -0,0 +1,86 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Read static host/IP entries from /etc/hosts. + +package net + +import ( + "sync" + "time" +) + +const cacheMaxAge = 5 * time.Minute + +// hostsPath points to the file with static IP/address entries. +var hostsPath = "/etc/hosts" + +// Simple cache. +var hosts struct { + sync.Mutex + byName map[string][]string + byAddr map[string][]string + expire time.Time + path string +} + +func readHosts() { + now := time.Now() + hp := hostsPath + if len(hosts.byName) == 0 || now.After(hosts.expire) || hosts.path != hp { + hs := make(map[string][]string) + is := make(map[string][]string) + var file *file + if file, _ = open(hp); file == nil { + return + } + for line, ok := file.readLine(); ok; line, ok = file.readLine() { + if i := byteIndex(line, '#'); i >= 0 { + // Discard comments. + line = line[0:i] + } + f := getFields(line) + if len(f) < 2 || ParseIP(f[0]) == nil { + continue + } + for i := 1; i < len(f); i++ { + h := f[i] + hs[h] = append(hs[h], f[0]) + is[f[0]] = append(is[f[0]], h) + } + } + // Update the data cache. + hosts.expire = now.Add(cacheMaxAge) + hosts.path = hp + hosts.byName = hs + hosts.byAddr = is + file.close() + } +} + +// lookupStaticHost looks up the addresses for the given host from /etc/hosts. +func lookupStaticHost(host string) []string { + hosts.Lock() + defer hosts.Unlock() + readHosts() + if len(hosts.byName) != 0 { + if ips, ok := hosts.byName[host]; ok { + return ips + } + } + return nil +} + +// lookupStaticAddr looks up the hosts for the given address from /etc/hosts. +func lookupStaticAddr(addr string) []string { + hosts.Lock() + defer hosts.Unlock() + readHosts() + if len(hosts.byAddr) != 0 { + if hosts, ok := hosts.byAddr[addr]; ok { + return hosts + } + } + return nil +} diff --git a/src/net/hosts_test.go b/src/net/hosts_test.go new file mode 100644 index 000000000..2fe358e07 --- /dev/null +++ b/src/net/hosts_test.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "sort" + "testing" +) + +type hostTest struct { + host string + ips []IP +} + +var hosttests = []hostTest{ + {"odin", []IP{ + IPv4(127, 0, 0, 2), + IPv4(127, 0, 0, 3), + ParseIP("::2"), + }}, + {"thor", []IP{ + IPv4(127, 1, 1, 1), + }}, + {"loki", []IP{}}, + {"ullr", []IP{ + IPv4(127, 1, 1, 2), + }}, + {"ullrhost", []IP{ + IPv4(127, 1, 1, 2), + }}, +} + +func TestLookupStaticHost(t *testing.T) { + p := hostsPath + hostsPath = "testdata/hosts" + for i := 0; i < len(hosttests); i++ { + tt := hosttests[i] + ips := lookupStaticHost(tt.host) + if len(ips) != len(tt.ips) { + t.Errorf("# of hosts = %v; want %v", + len(ips), len(tt.ips)) + continue + } + for k, v := range ips { + if tt.ips[k].String() != v { + t.Errorf("lookupStaticHost(%q) = %v; want %v", + tt.host, v, tt.ips[k]) + } + } + } + hostsPath = p +} + +// https://code.google.com/p/go/issues/detail?id=6646 +func TestSingleLineHostsFile(t *testing.T) { + p := hostsPath + hostsPath = "testdata/hosts_singleline" + + ips := lookupStaticHost("odin") + if len(ips) != 1 || ips[0] != "127.0.0.2" { + t.Errorf("lookupStaticHost = %v, want %v", ips, []string{"127.0.0.2"}) + } + + hostsPath = p +} + +func TestLookupHost(t *testing.T) { + // Can't depend on this to return anything in particular, + // but if it does return something, make sure it doesn't + // duplicate addresses (a common bug due to the way + // getaddrinfo works). + addrs, _ := LookupHost("localhost") + sort.Strings(addrs) + for i := 0; i+1 < len(addrs); i++ { + if addrs[i] == addrs[i+1] { + t.Fatalf("LookupHost(\"localhost\") = %v, has duplicate addresses", addrs) + } + } +} diff --git a/src/net/http/cgi/child.go b/src/net/http/cgi/child.go new file mode 100644 index 000000000..45fc2e57c --- /dev/null +++ b/src/net/http/cgi/child.go @@ -0,0 +1,206 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements CGI from the perspective of a child +// process. + +package cgi + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "strconv" + "strings" +) + +// Request returns the HTTP request as represented in the current +// environment. This assumes the current program is being run +// by a web server in a CGI environment. +// The returned Request's Body is populated, if applicable. +func Request() (*http.Request, error) { + r, err := RequestFromMap(envMap(os.Environ())) + if err != nil { + return nil, err + } + if r.ContentLength > 0 { + r.Body = ioutil.NopCloser(io.LimitReader(os.Stdin, r.ContentLength)) + } + return r, nil +} + +func envMap(env []string) map[string]string { + m := make(map[string]string) + for _, kv := range env { + if idx := strings.Index(kv, "="); idx != -1 { + m[kv[:idx]] = kv[idx+1:] + } + } + return m +} + +// RequestFromMap creates an http.Request from CGI variables. +// The returned Request's Body field is not populated. +func RequestFromMap(params map[string]string) (*http.Request, error) { + r := new(http.Request) + r.Method = params["REQUEST_METHOD"] + if r.Method == "" { + return nil, errors.New("cgi: no REQUEST_METHOD in environment") + } + + r.Proto = params["SERVER_PROTOCOL"] + var ok bool + r.ProtoMajor, r.ProtoMinor, ok = http.ParseHTTPVersion(r.Proto) + if !ok { + return nil, errors.New("cgi: invalid SERVER_PROTOCOL version") + } + + r.Close = true + r.Trailer = http.Header{} + r.Header = http.Header{} + + r.Host = params["HTTP_HOST"] + + if lenstr := params["CONTENT_LENGTH"]; lenstr != "" { + clen, err := strconv.ParseInt(lenstr, 10, 64) + if err != nil { + return nil, errors.New("cgi: bad CONTENT_LENGTH in environment: " + lenstr) + } + r.ContentLength = clen + } + + if ct := params["CONTENT_TYPE"]; ct != "" { + r.Header.Set("Content-Type", ct) + } + + // Copy "HTTP_FOO_BAR" variables to "Foo-Bar" Headers + for k, v := range params { + if !strings.HasPrefix(k, "HTTP_") || k == "HTTP_HOST" { + continue + } + r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v) + } + + // TODO: cookies. parsing them isn't exported, though. + + uriStr := params["REQUEST_URI"] + if uriStr == "" { + // Fallback to SCRIPT_NAME, PATH_INFO and QUERY_STRING. + uriStr = params["SCRIPT_NAME"] + params["PATH_INFO"] + s := params["QUERY_STRING"] + if s != "" { + uriStr += "?" + s + } + } + + // There's apparently a de-facto standard for this. + // http://docstore.mik.ua/orelly/linux/cgi/ch03_02.htm#ch03-35636 + if s := params["HTTPS"]; s == "on" || s == "ON" || s == "1" { + r.TLS = &tls.ConnectionState{HandshakeComplete: true} + } + + if r.Host != "" { + // Hostname is provided, so we can reasonably construct a URL. + rawurl := r.Host + uriStr + if r.TLS == nil { + rawurl = "http://" + rawurl + } else { + rawurl = "https://" + rawurl + } + url, err := url.Parse(rawurl) + if err != nil { + return nil, errors.New("cgi: failed to parse host and REQUEST_URI into a URL: " + rawurl) + } + r.URL = url + } + // Fallback logic if we don't have a Host header or the URL + // failed to parse + if r.URL == nil { + url, err := url.Parse(uriStr) + if err != nil { + return nil, errors.New("cgi: failed to parse REQUEST_URI into a URL: " + uriStr) + } + r.URL = url + } + + // Request.RemoteAddr has its port set by Go's standard http + // server, so we do here too. We don't have one, though, so we + // use a dummy one. + r.RemoteAddr = net.JoinHostPort(params["REMOTE_ADDR"], "0") + + return r, nil +} + +// Serve executes the provided Handler on the currently active CGI +// request, if any. If there's no current CGI environment +// an error is returned. The provided handler may be nil to use +// http.DefaultServeMux. +func Serve(handler http.Handler) error { + req, err := Request() + if err != nil { + return err + } + if handler == nil { + handler = http.DefaultServeMux + } + rw := &response{ + req: req, + header: make(http.Header), + bufw: bufio.NewWriter(os.Stdout), + } + handler.ServeHTTP(rw, req) + rw.Write(nil) // make sure a response is sent + if err = rw.bufw.Flush(); err != nil { + return err + } + return nil +} + +type response struct { + req *http.Request + header http.Header + bufw *bufio.Writer + headerSent bool +} + +func (r *response) Flush() { + r.bufw.Flush() +} + +func (r *response) Header() http.Header { + return r.header +} + +func (r *response) Write(p []byte) (n int, err error) { + if !r.headerSent { + r.WriteHeader(http.StatusOK) + } + return r.bufw.Write(p) +} + +func (r *response) WriteHeader(code int) { + if r.headerSent { + // Note: explicitly using Stderr, as Stdout is our HTTP output. + fmt.Fprintf(os.Stderr, "CGI attempted to write header twice on request for %s", r.req.URL) + return + } + r.headerSent = true + fmt.Fprintf(r.bufw, "Status: %d %s\r\n", code, http.StatusText(code)) + + // Set a default Content-Type + if _, hasType := r.header["Content-Type"]; !hasType { + r.header.Add("Content-Type", "text/html; charset=utf-8") + } + + r.header.Write(r.bufw) + r.bufw.WriteString("\r\n") + r.bufw.Flush() +} diff --git a/src/net/http/cgi/child_test.go b/src/net/http/cgi/child_test.go new file mode 100644 index 000000000..075d8411b --- /dev/null +++ b/src/net/http/cgi/child_test.go @@ -0,0 +1,131 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for CGI (the child process perspective) + +package cgi + +import ( + "testing" +) + +func TestRequest(t *testing.T) { + env := map[string]string{ + "SERVER_PROTOCOL": "HTTP/1.1", + "REQUEST_METHOD": "GET", + "HTTP_HOST": "example.com", + "HTTP_REFERER": "elsewhere", + "HTTP_USER_AGENT": "goclient", + "HTTP_FOO_BAR": "baz", + "REQUEST_URI": "/path?a=b", + "CONTENT_LENGTH": "123", + "CONTENT_TYPE": "text/xml", + "REMOTE_ADDR": "5.6.7.8", + } + req, err := RequestFromMap(env) + if err != nil { + t.Fatalf("RequestFromMap: %v", err) + } + if g, e := req.UserAgent(), "goclient"; e != g { + t.Errorf("expected UserAgent %q; got %q", e, g) + } + if g, e := req.Method, "GET"; e != g { + t.Errorf("expected Method %q; got %q", e, g) + } + if g, e := req.Header.Get("Content-Type"), "text/xml"; e != g { + t.Errorf("expected Content-Type %q; got %q", e, g) + } + if g, e := req.ContentLength, int64(123); e != g { + t.Errorf("expected ContentLength %d; got %d", e, g) + } + if g, e := req.Referer(), "elsewhere"; e != g { + t.Errorf("expected Referer %q; got %q", e, g) + } + if req.Header == nil { + t.Fatalf("unexpected nil Header") + } + if g, e := req.Header.Get("Foo-Bar"), "baz"; e != g { + t.Errorf("expected Foo-Bar %q; got %q", e, g) + } + if g, e := req.URL.String(), "http://example.com/path?a=b"; e != g { + t.Errorf("expected URL %q; got %q", e, g) + } + if g, e := req.FormValue("a"), "b"; e != g { + t.Errorf("expected FormValue(a) %q; got %q", e, g) + } + if req.Trailer == nil { + t.Errorf("unexpected nil Trailer") + } + if req.TLS != nil { + t.Errorf("expected nil TLS") + } + if e, g := "5.6.7.8:0", req.RemoteAddr; e != g { + t.Errorf("RemoteAddr: got %q; want %q", g, e) + } +} + +func TestRequestWithTLS(t *testing.T) { + env := map[string]string{ + "SERVER_PROTOCOL": "HTTP/1.1", + "REQUEST_METHOD": "GET", + "HTTP_HOST": "example.com", + "HTTP_REFERER": "elsewhere", + "REQUEST_URI": "/path?a=b", + "CONTENT_TYPE": "text/xml", + "HTTPS": "1", + "REMOTE_ADDR": "5.6.7.8", + } + req, err := RequestFromMap(env) + if err != nil { + t.Fatalf("RequestFromMap: %v", err) + } + if g, e := req.URL.String(), "https://example.com/path?a=b"; e != g { + t.Errorf("expected URL %q; got %q", e, g) + } + if req.TLS == nil { + t.Errorf("expected non-nil TLS") + } +} + +func TestRequestWithoutHost(t *testing.T) { + env := map[string]string{ + "SERVER_PROTOCOL": "HTTP/1.1", + "HTTP_HOST": "", + "REQUEST_METHOD": "GET", + "REQUEST_URI": "/path?a=b", + "CONTENT_LENGTH": "123", + } + req, err := RequestFromMap(env) + if err != nil { + t.Fatalf("RequestFromMap: %v", err) + } + if req.URL == nil { + t.Fatalf("unexpected nil URL") + } + if g, e := req.URL.String(), "/path?a=b"; e != g { + t.Errorf("URL = %q; want %q", g, e) + } +} + +func TestRequestWithoutRequestURI(t *testing.T) { + env := map[string]string{ + "SERVER_PROTOCOL": "HTTP/1.1", + "HTTP_HOST": "example.com", + "REQUEST_METHOD": "GET", + "SCRIPT_NAME": "/dir/scriptname", + "PATH_INFO": "/p1/p2", + "QUERY_STRING": "a=1&b=2", + "CONTENT_LENGTH": "123", + } + req, err := RequestFromMap(env) + if err != nil { + t.Fatalf("RequestFromMap: %v", err) + } + if req.URL == nil { + t.Fatalf("unexpected nil URL") + } + if g, e := req.URL.String(), "http://example.com/dir/scriptname/p1/p2?a=1&b=2"; e != g { + t.Errorf("URL = %q; want %q", g, e) + } +} diff --git a/src/net/http/cgi/host.go b/src/net/http/cgi/host.go new file mode 100644 index 000000000..ec95a972c --- /dev/null +++ b/src/net/http/cgi/host.go @@ -0,0 +1,377 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements the host side of CGI (being the webserver +// parent process). + +// Package cgi implements CGI (Common Gateway Interface) as specified +// in RFC 3875. +// +// Note that using CGI means starting a new process to handle each +// request, which is typically less efficient than using a +// long-running server. This package is intended primarily for +// compatibility with existing systems. +package cgi + +import ( + "bufio" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" +) + +var trailingPort = regexp.MustCompile(`:([0-9]+)$`) + +var osDefaultInheritEnv = map[string][]string{ + "darwin": {"DYLD_LIBRARY_PATH"}, + "freebsd": {"LD_LIBRARY_PATH"}, + "hpux": {"LD_LIBRARY_PATH", "SHLIB_PATH"}, + "irix": {"LD_LIBRARY_PATH", "LD_LIBRARYN32_PATH", "LD_LIBRARY64_PATH"}, + "linux": {"LD_LIBRARY_PATH"}, + "openbsd": {"LD_LIBRARY_PATH"}, + "solaris": {"LD_LIBRARY_PATH", "LD_LIBRARY_PATH_32", "LD_LIBRARY_PATH_64"}, + "windows": {"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"}, +} + +// Handler runs an executable in a subprocess with a CGI environment. +type Handler struct { + Path string // path to the CGI executable + Root string // root URI prefix of handler or empty for "/" + + // Dir specifies the CGI executable's working directory. + // If Dir is empty, the base directory of Path is used. + // If Path has no base directory, the current working + // directory is used. + Dir string + + Env []string // extra environment variables to set, if any, as "key=value" + InheritEnv []string // environment variables to inherit from host, as "key" + Logger *log.Logger // optional log for errors or nil to use log.Print + Args []string // optional arguments to pass to child process + + // PathLocationHandler specifies the root http Handler that + // should handle internal redirects when the CGI process + // returns a Location header value starting with a "/", as + // specified in RFC 3875 § 6.3.2. This will likely be + // http.DefaultServeMux. + // + // If nil, a CGI response with a local URI path is instead sent + // back to the client and not redirected internally. + PathLocationHandler http.Handler +} + +// removeLeadingDuplicates remove leading duplicate in environments. +// It's possible to override environment like following. +// cgi.Handler{ +// ... +// Env: []string{"SCRIPT_FILENAME=foo.php"}, +// } +func removeLeadingDuplicates(env []string) (ret []string) { + n := len(env) + for i := 0; i < n; i++ { + e := env[i] + s := strings.SplitN(e, "=", 2)[0] + found := false + for j := i + 1; j < n; j++ { + if s == strings.SplitN(env[j], "=", 2)[0] { + found = true + break + } + } + if !found { + ret = append(ret, e) + } + } + return +} + +func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + root := h.Root + if root == "" { + root = "/" + } + + if len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" { + rw.WriteHeader(http.StatusBadRequest) + rw.Write([]byte("Chunked request bodies are not supported by CGI.")) + return + } + + pathInfo := req.URL.Path + if root != "/" && strings.HasPrefix(pathInfo, root) { + pathInfo = pathInfo[len(root):] + } + + port := "80" + if matches := trailingPort.FindStringSubmatch(req.Host); len(matches) != 0 { + port = matches[1] + } + + env := []string{ + "SERVER_SOFTWARE=go", + "SERVER_NAME=" + req.Host, + "SERVER_PROTOCOL=HTTP/1.1", + "HTTP_HOST=" + req.Host, + "GATEWAY_INTERFACE=CGI/1.1", + "REQUEST_METHOD=" + req.Method, + "QUERY_STRING=" + req.URL.RawQuery, + "REQUEST_URI=" + req.URL.RequestURI(), + "PATH_INFO=" + pathInfo, + "SCRIPT_NAME=" + root, + "SCRIPT_FILENAME=" + h.Path, + "REMOTE_ADDR=" + req.RemoteAddr, + "REMOTE_HOST=" + req.RemoteAddr, + "SERVER_PORT=" + port, + } + + if req.TLS != nil { + env = append(env, "HTTPS=on") + } + + for k, v := range req.Header { + k = strings.Map(upperCaseAndUnderscore, k) + joinStr := ", " + if k == "COOKIE" { + joinStr = "; " + } + env = append(env, "HTTP_"+k+"="+strings.Join(v, joinStr)) + } + + if req.ContentLength > 0 { + env = append(env, fmt.Sprintf("CONTENT_LENGTH=%d", req.ContentLength)) + } + if ctype := req.Header.Get("Content-Type"); ctype != "" { + env = append(env, "CONTENT_TYPE="+ctype) + } + + if h.Env != nil { + env = append(env, h.Env...) + } + + envPath := os.Getenv("PATH") + if envPath == "" { + envPath = "/bin:/usr/bin:/usr/ucb:/usr/bsd:/usr/local/bin" + } + env = append(env, "PATH="+envPath) + + for _, e := range h.InheritEnv { + if v := os.Getenv(e); v != "" { + env = append(env, e+"="+v) + } + } + + for _, e := range osDefaultInheritEnv[runtime.GOOS] { + if v := os.Getenv(e); v != "" { + env = append(env, e+"="+v) + } + } + + env = removeLeadingDuplicates(env) + + var cwd, path string + if h.Dir != "" { + path = h.Path + cwd = h.Dir + } else { + cwd, path = filepath.Split(h.Path) + } + if cwd == "" { + cwd = "." + } + + internalError := func(err error) { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("CGI error: %v", err) + } + + cmd := &exec.Cmd{ + Path: path, + Args: append([]string{h.Path}, h.Args...), + Dir: cwd, + Env: env, + Stderr: os.Stderr, // for now + } + if req.ContentLength != 0 { + cmd.Stdin = req.Body + } + stdoutRead, err := cmd.StdoutPipe() + if err != nil { + internalError(err) + return + } + + err = cmd.Start() + if err != nil { + internalError(err) + return + } + if hook := testHookStartProcess; hook != nil { + hook(cmd.Process) + } + defer cmd.Wait() + defer stdoutRead.Close() + + linebody := bufio.NewReaderSize(stdoutRead, 1024) + headers := make(http.Header) + statusCode := 0 + headerLines := 0 + sawBlankLine := false + for { + line, isPrefix, err := linebody.ReadLine() + if isPrefix { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("cgi: long header line from subprocess.") + return + } + if err == io.EOF { + break + } + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("cgi: error reading headers: %v", err) + return + } + if len(line) == 0 { + sawBlankLine = true + break + } + headerLines++ + parts := strings.SplitN(string(line), ":", 2) + if len(parts) < 2 { + h.printf("cgi: bogus header line: %s", string(line)) + continue + } + header, val := parts[0], parts[1] + header = strings.TrimSpace(header) + val = strings.TrimSpace(val) + switch { + case header == "Status": + if len(val) < 3 { + h.printf("cgi: bogus status (short): %q", val) + return + } + code, err := strconv.Atoi(val[0:3]) + if err != nil { + h.printf("cgi: bogus status: %q", val) + h.printf("cgi: line was %q", line) + return + } + statusCode = code + default: + headers.Add(header, val) + } + } + if headerLines == 0 || !sawBlankLine { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("cgi: no headers") + return + } + + if loc := headers.Get("Location"); loc != "" { + if strings.HasPrefix(loc, "/") && h.PathLocationHandler != nil { + h.handleInternalRedirect(rw, req, loc) + return + } + if statusCode == 0 { + statusCode = http.StatusFound + } + } + + if statusCode == 0 && headers.Get("Content-Type") == "" { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("cgi: missing required Content-Type in headers") + return + } + + if statusCode == 0 { + statusCode = http.StatusOK + } + + // Copy headers to rw's headers, after we've decided not to + // go into handleInternalRedirect, which won't want its rw + // headers to have been touched. + for k, vv := range headers { + for _, v := range vv { + rw.Header().Add(k, v) + } + } + + rw.WriteHeader(statusCode) + + _, err = io.Copy(rw, linebody) + if err != nil { + h.printf("cgi: copy error: %v", err) + // And kill the child CGI process so we don't hang on + // the deferred cmd.Wait above if the error was just + // the client (rw) going away. If it was a read error + // (because the child died itself), then the extra + // kill of an already-dead process is harmless (the PID + // won't be reused until the Wait above). + cmd.Process.Kill() + } +} + +func (h *Handler) printf(format string, v ...interface{}) { + if h.Logger != nil { + h.Logger.Printf(format, v...) + } else { + log.Printf(format, v...) + } +} + +func (h *Handler) handleInternalRedirect(rw http.ResponseWriter, req *http.Request, path string) { + url, err := req.URL.Parse(path) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + h.printf("cgi: error resolving local URI path %q: %v", path, err) + return + } + // TODO: RFC 3875 isn't clear if only GET is supported, but it + // suggests so: "Note that any message-body attached to the + // request (such as for a POST request) may not be available + // to the resource that is the target of the redirect." We + // should do some tests against Apache to see how it handles + // POST, HEAD, etc. Does the internal redirect get the same + // method or just GET? What about incoming headers? + // (e.g. Cookies) Which headers, if any, are copied into the + // second request? + newReq := &http.Request{ + Method: "GET", + URL: url, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Host: url.Host, + RemoteAddr: req.RemoteAddr, + TLS: req.TLS, + } + h.PathLocationHandler.ServeHTTP(rw, newReq) +} + +func upperCaseAndUnderscore(r rune) rune { + switch { + case r >= 'a' && r <= 'z': + return r - ('a' - 'A') + case r == '-': + return '_' + case r == '=': + // Maybe not part of the CGI 'spec' but would mess up + // the environment in any case, as Go represents the + // environment as a slice of "key=value" strings. + return '_' + } + // TODO: other transformations in spec or practice? + return r +} + +var testHookStartProcess func(*os.Process) // nil except for some tests diff --git a/src/net/http/cgi/host_test.go b/src/net/http/cgi/host_test.go new file mode 100644 index 000000000..8c16e6897 --- /dev/null +++ b/src/net/http/cgi/host_test.go @@ -0,0 +1,461 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for package cgi + +package cgi + +import ( + "bufio" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +func newRequest(httpreq string) *http.Request { + buf := bufio.NewReader(strings.NewReader(httpreq)) + req, err := http.ReadRequest(buf) + if err != nil { + panic("cgi: bogus http request in test: " + httpreq) + } + req.RemoteAddr = "1.2.3.4" + return req +} + +func runCgiTest(t *testing.T, h *Handler, httpreq string, expectedMap map[string]string) *httptest.ResponseRecorder { + rw := httptest.NewRecorder() + req := newRequest(httpreq) + h.ServeHTTP(rw, req) + + // Make a map to hold the test map that the CGI returns. + m := make(map[string]string) + m["_body"] = rw.Body.String() + linesRead := 0 +readlines: + for { + line, err := rw.Body.ReadString('\n') + switch { + case err == io.EOF: + break readlines + case err != nil: + t.Fatalf("unexpected error reading from CGI: %v", err) + } + linesRead++ + trimmedLine := strings.TrimRight(line, "\r\n") + split := strings.SplitN(trimmedLine, "=", 2) + if len(split) != 2 { + t.Fatalf("Unexpected %d parts from invalid line number %v: %q; existing map=%v", + len(split), linesRead, line, m) + } + m[split[0]] = split[1] + } + + for key, expected := range expectedMap { + got := m[key] + if key == "cwd" { + // For Windows. golang.org/issue/4645. + fi1, _ := os.Stat(got) + fi2, _ := os.Stat(expected) + if os.SameFile(fi1, fi2) { + got = expected + } + } + if got != expected { + t.Errorf("for key %q got %q; expected %q", key, got, expected) + } + } + return rw +} + +var cgiTested, cgiWorks bool + +func check(t *testing.T) { + if !cgiTested { + cgiTested = true + cgiWorks = exec.Command("./testdata/test.cgi").Run() == nil + } + if !cgiWorks { + // No Perl on Windows, needed by test.cgi + // TODO: make the child process be Go, not Perl. + t.Skip("Skipping test: test.cgi failed.") + } +} + +func TestCGIBasicGet(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "test": "Hello CGI", + "param-a": "b", + "param-foo": "bar", + "env-GATEWAY_INTERFACE": "CGI/1.1", + "env-HTTP_HOST": "example.com", + "env-PATH_INFO": "", + "env-QUERY_STRING": "foo=bar&a=b", + "env-REMOTE_ADDR": "1.2.3.4", + "env-REMOTE_HOST": "1.2.3.4", + "env-REQUEST_METHOD": "GET", + "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/test.cgi", + "env-SERVER_NAME": "example.com", + "env-SERVER_PORT": "80", + "env-SERVER_SOFTWARE": "go", + } + replay := runCgiTest(t, h, "GET /test.cgi?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) + + if expected, got := "text/html", replay.Header().Get("Content-Type"); got != expected { + t.Errorf("got a Content-Type of %q; expected %q", got, expected) + } + if expected, got := "X-Test-Value", replay.Header().Get("X-Test-Header"); got != expected { + t.Errorf("got a X-Test-Header of %q; expected %q", got, expected) + } +} + +func TestCGIBasicGetAbsPath(t *testing.T) { + check(t) + pwd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd error: %v", err) + } + h := &Handler{ + Path: pwd + "/testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", + "env-SCRIPT_FILENAME": pwd + "/testdata/test.cgi", + "env-SCRIPT_NAME": "/test.cgi", + } + runCgiTest(t, h, "GET /test.cgi?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestPathInfo(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "param-a": "b", + "env-PATH_INFO": "/extrapath", + "env-QUERY_STRING": "a=b", + "env-REQUEST_URI": "/test.cgi/extrapath?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/test.cgi", + } + runCgiTest(t, h, "GET /test.cgi/extrapath?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestPathInfoDirRoot(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/myscript/", + } + expectedMap := map[string]string{ + "env-PATH_INFO": "bar", + "env-QUERY_STRING": "a=b", + "env-REQUEST_URI": "/myscript/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/myscript/", + } + runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestDupHeaders(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + } + expectedMap := map[string]string{ + "env-REQUEST_URI": "/myscript/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-HTTP_COOKIE": "nom=NOM; yum=YUM", + "env-HTTP_X_FOO": "val1, val2", + } + runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\n"+ + "Cookie: nom=NOM\n"+ + "Cookie: yum=YUM\n"+ + "X-Foo: val1\n"+ + "X-Foo: val2\n"+ + "Host: example.com\n\n", + expectedMap) +} + +func TestPathInfoNoRoot(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + Root: "", + } + expectedMap := map[string]string{ + "env-PATH_INFO": "/bar", + "env-QUERY_STRING": "a=b", + "env-REQUEST_URI": "/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_NAME": "/", + } + runCgiTest(t, h, "GET /bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestCGIBasicPost(t *testing.T) { + check(t) + postReq := `POST /test.cgi?a=b HTTP/1.0 +Host: example.com +Content-Type: application/x-www-form-urlencoded +Content-Length: 15 + +postfoo=postbar` + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{ + "test": "Hello CGI", + "param-postfoo": "postbar", + "env-REQUEST_METHOD": "POST", + "env-CONTENT_LENGTH": "15", + "env-REQUEST_URI": "/test.cgi?a=b", + } + runCgiTest(t, h, postReq, expectedMap) +} + +func chunk(s string) string { + return fmt.Sprintf("%x\r\n%s\r\n", len(s), s) +} + +// The CGI spec doesn't allow chunked requests. +func TestCGIPostChunked(t *testing.T) { + check(t) + postReq := `POST /test.cgi?a=b HTTP/1.1 +Host: example.com +Content-Type: application/x-www-form-urlencoded +Transfer-Encoding: chunked + +` + chunk("postfoo") + chunk("=") + chunk("postbar") + chunk("") + + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap := map[string]string{} + resp := runCgiTest(t, h, postReq, expectedMap) + if got, expected := resp.Code, http.StatusBadRequest; got != expected { + t.Fatalf("Expected %v response code from chunked request body; got %d", + expected, got) + } +} + +func TestRedirect(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + rec := runCgiTest(t, h, "GET /test.cgi?loc=http://foo.com/ HTTP/1.0\nHost: example.com\n\n", nil) + if e, g := 302, rec.Code; e != g { + t.Errorf("expected status code %d; got %d", e, g) + } + if e, g := "http://foo.com/", rec.Header().Get("Location"); e != g { + t.Errorf("expected Location header of %q; got %q", e, g) + } +} + +func TestInternalRedirect(t *testing.T) { + check(t) + baseHandler := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + fmt.Fprintf(rw, "basepath=%s\n", req.URL.Path) + fmt.Fprintf(rw, "remoteaddr=%s\n", req.RemoteAddr) + }) + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + PathLocationHandler: baseHandler, + } + expectedMap := map[string]string{ + "basepath": "/foo", + "remoteaddr": "1.2.3.4", + } + runCgiTest(t, h, "GET /test.cgi?loc=/foo HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +// TestCopyError tests that we kill the process if there's an error copying +// its output. (for example, from the client having gone away) +func TestCopyError(t *testing.T) { + check(t) + if runtime.GOOS == "windows" { + t.Skipf("skipping test on %q", runtime.GOOS) + } + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + ts := httptest.NewServer(h) + defer ts.Close() + + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + req, _ := http.NewRequest("GET", "http://example.com/test.cgi?bigresponse=1", nil) + err = req.Write(conn) + if err != nil { + t.Fatalf("Write: %v", err) + } + + res, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + t.Fatalf("ReadResponse: %v", err) + } + + pidstr := res.Header.Get("X-CGI-Pid") + if pidstr == "" { + t.Fatalf("expected an X-CGI-Pid header in response") + } + pid, err := strconv.Atoi(pidstr) + if err != nil { + t.Fatalf("invalid X-CGI-Pid value") + } + + var buf [5000]byte + n, err := io.ReadFull(res.Body, buf[:]) + if err != nil { + t.Fatalf("ReadFull: %d bytes, %v", n, err) + } + + childRunning := func() bool { + return isProcessRunning(t, pid) + } + + if !childRunning() { + t.Fatalf("pre-conn.Close, expected child to be running") + } + conn.Close() + + tries := 0 + for tries < 25 && childRunning() { + time.Sleep(50 * time.Millisecond * time.Duration(tries)) + tries++ + } + if childRunning() { + t.Fatalf("post-conn.Close, expected child to be gone") + } +} + +func TestDirUnix(t *testing.T) { + check(t) + if runtime.GOOS == "windows" { + t.Skipf("skipping test on %q", runtime.GOOS) + } + cwd, _ := os.Getwd() + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + Dir: cwd, + } + expectedMap := map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) + + cwd, _ = os.Getwd() + cwd = filepath.Join(cwd, "testdata") + h = &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + } + expectedMap = map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestDirWindows(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("Skipping windows specific test.") + } + + cgifile, _ := filepath.Abs("testdata/test.cgi") + + var perl string + var err error + perl, err = exec.LookPath("perl") + if err != nil { + t.Skip("Skipping test: perl not found.") + } + perl, _ = filepath.Abs(perl) + + cwd, _ := os.Getwd() + h := &Handler{ + Path: perl, + Root: "/test.cgi", + Dir: cwd, + Args: []string{cgifile}, + Env: []string{"SCRIPT_FILENAME=" + cgifile}, + } + expectedMap := map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) + + // If not specify Dir on windows, working directory should be + // base directory of perl. + cwd, _ = filepath.Split(perl) + if cwd != "" && cwd[len(cwd)-1] == filepath.Separator { + cwd = cwd[:len(cwd)-1] + } + h = &Handler{ + Path: perl, + Root: "/test.cgi", + Args: []string{cgifile}, + Env: []string{"SCRIPT_FILENAME=" + cgifile}, + } + expectedMap = map[string]string{ + "cwd": cwd, + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) +} + +func TestEnvOverride(t *testing.T) { + cgifile, _ := filepath.Abs("testdata/test.cgi") + + var perl string + var err error + perl, err = exec.LookPath("perl") + if err != nil { + t.Skipf("Skipping test: perl not found.") + } + perl, _ = filepath.Abs(perl) + + cwd, _ := os.Getwd() + h := &Handler{ + Path: perl, + Root: "/test.cgi", + Dir: cwd, + Args: []string{cgifile}, + Env: []string{ + "SCRIPT_FILENAME=" + cgifile, + "REQUEST_URI=/foo/bar"}, + } + expectedMap := map[string]string{ + "cwd": cwd, + "env-SCRIPT_FILENAME": cgifile, + "env-REQUEST_URI": "/foo/bar", + } + runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) +} diff --git a/src/net/http/cgi/matryoshka_test.go b/src/net/http/cgi/matryoshka_test.go new file mode 100644 index 000000000..18c4803e7 --- /dev/null +++ b/src/net/http/cgi/matryoshka_test.go @@ -0,0 +1,228 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests a Go CGI program running under a Go CGI host process. +// Further, the two programs are the same binary, just checking +// their environment to figure out what mode to run in. + +package cgi + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "runtime" + "testing" + "time" +) + +// This test is a CGI host (testing host.go) that runs its own binary +// as a child process testing the other half of CGI (child.go). +func TestHostingOurselves(t *testing.T) { + if runtime.GOOS == "nacl" { + t.Skip("skipping on nacl") + } + + h := &Handler{ + Path: os.Args[0], + Root: "/test.go", + Args: []string{"-test.run=TestBeChildCGIProcess"}, + } + expectedMap := map[string]string{ + "test": "Hello CGI-in-CGI", + "param-a": "b", + "param-foo": "bar", + "env-GATEWAY_INTERFACE": "CGI/1.1", + "env-HTTP_HOST": "example.com", + "env-PATH_INFO": "", + "env-QUERY_STRING": "foo=bar&a=b", + "env-REMOTE_ADDR": "1.2.3.4", + "env-REMOTE_HOST": "1.2.3.4", + "env-REQUEST_METHOD": "GET", + "env-REQUEST_URI": "/test.go?foo=bar&a=b", + "env-SCRIPT_FILENAME": os.Args[0], + "env-SCRIPT_NAME": "/test.go", + "env-SERVER_NAME": "example.com", + "env-SERVER_PORT": "80", + "env-SERVER_SOFTWARE": "go", + } + replay := runCgiTest(t, h, "GET /test.go?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) + + if expected, got := "text/html; charset=utf-8", replay.Header().Get("Content-Type"); got != expected { + t.Errorf("got a Content-Type of %q; expected %q", got, expected) + } + if expected, got := "X-Test-Value", replay.Header().Get("X-Test-Header"); got != expected { + t.Errorf("got a X-Test-Header of %q; expected %q", got, expected) + } +} + +type customWriterRecorder struct { + w io.Writer + *httptest.ResponseRecorder +} + +func (r *customWriterRecorder) Write(p []byte) (n int, err error) { + return r.w.Write(p) +} + +type limitWriter struct { + w io.Writer + n int +} + +func (w *limitWriter) Write(p []byte) (n int, err error) { + if len(p) > w.n { + p = p[:w.n] + } + if len(p) > 0 { + n, err = w.w.Write(p) + w.n -= n + } + if w.n == 0 { + err = errors.New("past write limit") + } + return +} + +// If there's an error copying the child's output to the parent, test +// that we kill the child. +func TestKillChildAfterCopyError(t *testing.T) { + if runtime.GOOS == "nacl" { + t.Skip("skipping on nacl") + } + + defer func() { testHookStartProcess = nil }() + proc := make(chan *os.Process, 1) + testHookStartProcess = func(p *os.Process) { + proc <- p + } + + h := &Handler{ + Path: os.Args[0], + Root: "/test.go", + Args: []string{"-test.run=TestBeChildCGIProcess"}, + } + req, _ := http.NewRequest("GET", "http://example.com/test.cgi?write-forever=1", nil) + rec := httptest.NewRecorder() + var out bytes.Buffer + const writeLen = 50 << 10 + rw := &customWriterRecorder{&limitWriter{&out, writeLen}, rec} + + donec := make(chan bool, 1) + go func() { + h.ServeHTTP(rw, req) + donec <- true + }() + + select { + case <-donec: + if out.Len() != writeLen || out.Bytes()[0] != 'a' { + t.Errorf("unexpected output: %q", out.Bytes()) + } + case <-time.After(5 * time.Second): + t.Errorf("timeout. ServeHTTP hung and didn't kill the child process?") + select { + case p := <-proc: + p.Kill() + t.Logf("killed process") + default: + t.Logf("didn't kill process") + } + } +} + +// Test that a child handler writing only headers works. +// golang.org/issue/7196 +func TestChildOnlyHeaders(t *testing.T) { + if runtime.GOOS == "nacl" { + t.Skip("skipping on nacl") + } + + h := &Handler{ + Path: os.Args[0], + Root: "/test.go", + Args: []string{"-test.run=TestBeChildCGIProcess"}, + } + expectedMap := map[string]string{ + "_body": "", + } + replay := runCgiTest(t, h, "GET /test.go?no-body=1 HTTP/1.0\nHost: example.com\n\n", expectedMap) + if expected, got := "X-Test-Value", replay.Header().Get("X-Test-Header"); got != expected { + t.Errorf("got a X-Test-Header of %q; expected %q", got, expected) + } +} + +// golang.org/issue/7198 +func Test500WithNoHeaders(t *testing.T) { want500Test(t, "/immediate-disconnect") } +func Test500WithNoContentType(t *testing.T) { want500Test(t, "/no-content-type") } +func Test500WithEmptyHeaders(t *testing.T) { want500Test(t, "/empty-headers") } + +func want500Test(t *testing.T, path string) { + h := &Handler{ + Path: os.Args[0], + Root: "/test.go", + Args: []string{"-test.run=TestBeChildCGIProcess"}, + } + expectedMap := map[string]string{ + "_body": "", + } + replay := runCgiTest(t, h, "GET "+path+" HTTP/1.0\nHost: example.com\n\n", expectedMap) + if replay.Code != 500 { + t.Errorf("Got code %d; want 500", replay.Code) + } +} + +type neverEnding byte + +func (b neverEnding) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = byte(b) + } + return len(p), nil +} + +// Note: not actually a test. +func TestBeChildCGIProcess(t *testing.T) { + if os.Getenv("REQUEST_METHOD") == "" { + // Not in a CGI environment; skipping test. + return + } + switch os.Getenv("REQUEST_URI") { + case "/immediate-disconnect": + os.Exit(0) + case "/no-content-type": + fmt.Printf("Content-Length: 6\n\nHello\n") + os.Exit(0) + case "/empty-headers": + fmt.Printf("\nHello") + os.Exit(0) + } + Serve(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("X-Test-Header", "X-Test-Value") + req.ParseForm() + if req.FormValue("no-body") == "1" { + return + } + if req.FormValue("write-forever") == "1" { + io.Copy(rw, neverEnding('a')) + for { + time.Sleep(5 * time.Second) // hang forever, until killed + } + } + fmt.Fprintf(rw, "test=Hello CGI-in-CGI\n") + for k, vv := range req.Form { + for _, v := range vv { + fmt.Fprintf(rw, "param-%s=%s\n", k, v) + } + } + for _, kv := range os.Environ() { + fmt.Fprintf(rw, "env-%s\n", kv) + } + })) + os.Exit(0) +} diff --git a/src/net/http/cgi/plan9_test.go b/src/net/http/cgi/plan9_test.go new file mode 100644 index 000000000..c8235831b --- /dev/null +++ b/src/net/http/cgi/plan9_test.go @@ -0,0 +1,18 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build plan9 + +package cgi + +import ( + "os" + "strconv" + "testing" +) + +func isProcessRunning(t *testing.T, pid int) bool { + _, err := os.Stat("/proc/" + strconv.Itoa(pid)) + return err == nil +} diff --git a/src/net/http/cgi/posix_test.go b/src/net/http/cgi/posix_test.go new file mode 100644 index 000000000..5ff9e7d5e --- /dev/null +++ b/src/net/http/cgi/posix_test.go @@ -0,0 +1,21 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9 + +package cgi + +import ( + "os" + "syscall" + "testing" +) + +func isProcessRunning(t *testing.T, pid int) bool { + p, err := os.FindProcess(pid) + if err != nil { + return false + } + return p.Signal(syscall.Signal(0)) == nil +} diff --git a/src/net/http/cgi/testdata/test.cgi b/src/net/http/cgi/testdata/test.cgi new file mode 100755 index 000000000..3214df6f0 --- /dev/null +++ b/src/net/http/cgi/testdata/test.cgi @@ -0,0 +1,91 @@ +#!/usr/bin/perl +# Copyright 2011 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. +# +# Test script run as a child process under cgi_test.go + +use strict; +use Cwd; + +binmode STDOUT; + +my $q = MiniCGI->new; +my $params = $q->Vars; + +if ($params->{"loc"}) { + print "Location: $params->{loc}\r\n\r\n"; + exit(0); +} + +print "Content-Type: text/html\r\n"; +print "X-CGI-Pid: $$\r\n"; +print "X-Test-Header: X-Test-Value\r\n"; +print "\r\n"; + +if ($params->{"bigresponse"}) { + # 17 MB, for OS X: golang.org/issue/4958 + for (1..(17 * 1024)) { + print "A" x 1024, "\r\n"; + } + exit 0; +} + +print "test=Hello CGI\r\n"; + +foreach my $k (sort keys %$params) { + print "param-$k=$params->{$k}\r\n"; +} + +foreach my $k (sort keys %ENV) { + my $clean_env = $ENV{$k}; + $clean_env =~ s/[\n\r]//g; + print "env-$k=$clean_env\r\n"; +} + +# NOTE: msys perl returns /c/go/src/... not C:\go\.... +my $dir = getcwd(); +if ($^O eq 'MSWin32' || $^O eq 'msys') { + if ($dir =~ /^.:/) { + $dir =~ s!/!\\!g; + } else { + my $cmd = $ENV{'COMSPEC'} || 'c:\\windows\\system32\\cmd.exe'; + $cmd =~ s!\\!/!g; + $dir = `$cmd /c cd`; + chomp $dir; + } +} +print "cwd=$dir\r\n"; + +# A minimal version of CGI.pm, for people without the perl-modules +# package installed. (CGI.pm used to be part of the Perl core, but +# some distros now bundle perl-base and perl-modules separately...) +package MiniCGI; + +sub new { + my $class = shift; + return bless {}, $class; +} + +sub Vars { + my $self = shift; + my $pairs; + if ($ENV{CONTENT_LENGTH}) { + $pairs = do { local $/; <STDIN> }; + } else { + $pairs = $ENV{QUERY_STRING}; + } + my $vars = {}; + foreach my $kv (split(/&/, $pairs)) { + my ($k, $v) = split(/=/, $kv, 2); + $vars->{_urldecode($k)} = _urldecode($v); + } + return $vars; +} + +sub _urldecode { + my $v = shift; + $v =~ tr/+/ /; + $v =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; + return $v; +} diff --git a/src/net/http/client.go b/src/net/http/client.go new file mode 100644 index 000000000..a5a3abe61 --- /dev/null +++ b/src/net/http/client.go @@ -0,0 +1,487 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP client. See RFC 2616. +// +// This is the high-level Client interface. +// The low-level implementation is in transport.go. + +package http + +import ( + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/url" + "strings" + "sync" + "time" +) + +// A Client is an HTTP client. Its zero value (DefaultClient) is a +// usable client that uses DefaultTransport. +// +// The Client's Transport typically has internal state (cached TCP +// connections), so Clients should be reused instead of created as +// needed. Clients are safe for concurrent use by multiple goroutines. +// +// A Client is higher-level than a RoundTripper (such as Transport) +// and additionally handles HTTP details such as cookies and +// redirects. +type Client struct { + // Transport specifies the mechanism by which individual + // HTTP requests are made. + // If nil, DefaultTransport is used. + Transport RoundTripper + + // CheckRedirect specifies the policy for handling redirects. + // If CheckRedirect is not nil, the client calls it before + // following an HTTP redirect. The arguments req and via are + // the upcoming request and the requests made already, oldest + // first. If CheckRedirect returns an error, the Client's Get + // method returns both the previous Response and + // CheckRedirect's error (wrapped in a url.Error) instead of + // issuing the Request req. + // + // If CheckRedirect is nil, the Client uses its default policy, + // which is to stop after 10 consecutive requests. + CheckRedirect func(req *Request, via []*Request) error + + // Jar specifies the cookie jar. + // If Jar is nil, cookies are not sent in requests and ignored + // in responses. + Jar CookieJar + + // Timeout specifies a time limit for requests made by this + // Client. The timeout includes connection time, any + // redirects, and reading the response body. The timer remains + // running after Get, Head, Post, or Do return and will + // interrupt reading of the Response.Body. + // + // A Timeout of zero means no timeout. + // + // The Client's Transport must support the CancelRequest + // method or Client will return errors when attempting to make + // a request with Get, Head, Post, or Do. Client's default + // Transport (DefaultTransport) supports CancelRequest. + Timeout time.Duration +} + +// DefaultClient is the default Client and is used by Get, Head, and Post. +var DefaultClient = &Client{} + +// RoundTripper is an interface representing the ability to execute a +// single HTTP transaction, obtaining the Response for a given Request. +// +// A RoundTripper must be safe for concurrent use by multiple +// goroutines. +type RoundTripper interface { + // RoundTrip executes a single HTTP transaction, returning + // the Response for the request req. RoundTrip should not + // attempt to interpret the response. In particular, + // RoundTrip must return err == nil if it obtained a response, + // regardless of the response's HTTP status code. A non-nil + // err should be reserved for failure to obtain a response. + // Similarly, RoundTrip should not attempt to handle + // higher-level protocol details such as redirects, + // authentication, or cookies. + // + // RoundTrip should not modify the request, except for + // consuming and closing the Body, including on errors. The + // request's URL and Header fields are guaranteed to be + // initialized. + RoundTrip(*Request) (*Response, error) +} + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +// Used in Send to implement io.ReadCloser by bundling together the +// bufio.Reader through which we read the response, and the underlying +// network connection. +type readClose struct { + io.Reader + io.Closer +} + +func (c *Client) send(req *Request) (*Response, error) { + if c.Jar != nil { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + } + resp, err := send(req, c.transport()) + if err != nil { + return nil, err + } + if c.Jar != nil { + if rc := resp.Cookies(); len(rc) > 0 { + c.Jar.SetCookies(req.URL, rc) + } + } + return resp, err +} + +// Do sends an HTTP request and returns an HTTP response, following +// policy (e.g. redirects, cookies, auth) as configured on the client. +// +// An error is returned if caused by client policy (such as +// CheckRedirect), or if there was an HTTP protocol error. +// A non-2xx response doesn't cause an error. +// +// When err is nil, resp always contains a non-nil resp.Body. +// +// Callers should close resp.Body when done reading from it. If +// resp.Body is not closed, the Client's underlying RoundTripper +// (typically Transport) may not be able to re-use a persistent TCP +// connection to the server for a subsequent "keep-alive" request. +// +// The request Body, if non-nil, will be closed by the underlying +// Transport, even on errors. +// +// Generally Get, Post, or PostForm will be used instead of Do. +func (c *Client) Do(req *Request) (resp *Response, err error) { + if req.Method == "GET" || req.Method == "HEAD" { + return c.doFollowingRedirects(req, shouldRedirectGet) + } + if req.Method == "POST" || req.Method == "PUT" { + return c.doFollowingRedirects(req, shouldRedirectPost) + } + return c.send(req) +} + +func (c *Client) transport() RoundTripper { + if c.Transport != nil { + return c.Transport + } + return DefaultTransport +} + +// send issues an HTTP request. +// Caller should close resp.Body when done reading from it. +func send(req *Request, t RoundTripper) (resp *Response, err error) { + if t == nil { + req.closeBody() + return nil, errors.New("http: no Client.Transport or DefaultTransport") + } + + if req.URL == nil { + req.closeBody() + return nil, errors.New("http: nil Request.URL") + } + + if req.RequestURI != "" { + req.closeBody() + return nil, errors.New("http: Request.RequestURI can't be set in client requests.") + } + + // Most the callers of send (Get, Post, et al) don't need + // Headers, leaving it uninitialized. We guarantee to the + // Transport that this has been initialized, though. + if req.Header == nil { + req.Header = make(Header) + } + + if u := req.URL.User; u != nil { + username := u.Username() + password, _ := u.Password() + req.Header.Set("Authorization", "Basic "+basicAuth(username, password)) + } + resp, err = t.RoundTrip(req) + if err != nil { + if resp != nil { + log.Printf("RoundTripper returned a response & error; ignoring response") + } + return nil, err + } + return resp, nil +} + +// See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt +// "To receive authorization, the client sends the userid and password, +// separated by a single colon (":") character, within a base64 +// encoded string in the credentials." +// It is not meant to be urlencoded. +func basicAuth(username, password string) string { + auth := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(auth)) +} + +// True if the specified HTTP status code is one for which the Get utility should +// automatically redirect. +func shouldRedirectGet(statusCode int) bool { + switch statusCode { + case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect: + return true + } + return false +} + +// True if the specified HTTP status code is one for which the Post utility should +// automatically redirect. +func shouldRedirectPost(statusCode int) bool { + switch statusCode { + case StatusFound, StatusSeeOther: + return true + } + return false +} + +// Get issues a GET to the specified URL. If the response is one of the following +// redirect codes, Get follows the redirect, up to a maximum of 10 redirects: +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// +// An error is returned if there were too many redirects or if there +// was an HTTP protocol error. A non-2xx response doesn't cause an +// error. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +// +// Get is a wrapper around DefaultClient.Get. +func Get(url string) (resp *Response, err error) { + return DefaultClient.Get(url) +} + +// Get issues a GET to the specified URL. If the response is one of the +// following redirect codes, Get follows the redirect after calling the +// Client's CheckRedirect function. +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// +// An error is returned if the Client's CheckRedirect function fails +// or if there was an HTTP protocol error. A non-2xx response doesn't +// cause an error. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +func (c *Client) Get(url string) (resp *Response, err error) { + req, err := NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + return c.doFollowingRedirects(req, shouldRedirectGet) +} + +func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) { + var base *url.URL + redirectChecker := c.CheckRedirect + if redirectChecker == nil { + redirectChecker = defaultCheckRedirect + } + var via []*Request + + if ireq.URL == nil { + ireq.closeBody() + return nil, errors.New("http: nil Request.URL") + } + + var reqmu sync.Mutex // guards req + req := ireq + + var timer *time.Timer + if c.Timeout > 0 { + type canceler interface { + CancelRequest(*Request) + } + tr, ok := c.transport().(canceler) + if !ok { + return nil, fmt.Errorf("net/http: Client Transport of type %T doesn't support CancelRequest; Timeout not supported", c.transport()) + } + timer = time.AfterFunc(c.Timeout, func() { + reqmu.Lock() + defer reqmu.Unlock() + tr.CancelRequest(req) + }) + } + + urlStr := "" // next relative or absolute URL to fetch (after first request) + redirectFailed := false + for redirect := 0; ; redirect++ { + if redirect != 0 { + nreq := new(Request) + nreq.Method = ireq.Method + if ireq.Method == "POST" || ireq.Method == "PUT" { + nreq.Method = "GET" + } + nreq.Header = make(Header) + nreq.URL, err = base.Parse(urlStr) + if err != nil { + break + } + if len(via) > 0 { + // Add the Referer header. + lastReq := via[len(via)-1] + if lastReq.URL.Scheme != "https" { + nreq.Header.Set("Referer", lastReq.URL.String()) + } + + err = redirectChecker(nreq, via) + if err != nil { + redirectFailed = true + break + } + } + reqmu.Lock() + req = nreq + reqmu.Unlock() + } + + urlStr = req.URL.String() + if resp, err = c.send(req); err != nil { + break + } + + if shouldRedirect(resp.StatusCode) { + // Read the body if small so underlying TCP connection will be re-used. + // No need to check for errors: if it fails, Transport won't reuse it anyway. + const maxBodySlurpSize = 2 << 10 + if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { + io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize) + } + resp.Body.Close() + if urlStr = resp.Header.Get("Location"); urlStr == "" { + err = errors.New(fmt.Sprintf("%d response missing Location header", resp.StatusCode)) + break + } + base = req.URL + via = append(via, req) + continue + } + if timer != nil { + resp.Body = &cancelTimerBody{timer, resp.Body} + } + return resp, nil + } + + method := ireq.Method + urlErr := &url.Error{ + Op: method[0:1] + strings.ToLower(method[1:]), + URL: urlStr, + Err: err, + } + + if redirectFailed { + // Special case for Go 1 compatibility: return both the response + // and an error if the CheckRedirect function failed. + // See http://golang.org/issue/3795 + return resp, urlErr + } + + if resp != nil { + resp.Body.Close() + } + return nil, urlErr +} + +func defaultCheckRedirect(req *Request, via []*Request) error { + if len(via) >= 10 { + return errors.New("stopped after 10 redirects") + } + return nil +} + +// Post issues a POST to the specified URL. +// +// Caller should close resp.Body when done reading from it. +// +// Post is a wrapper around DefaultClient.Post +func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { + return DefaultClient.Post(url, bodyType, body) +} + +// Post issues a POST to the specified URL. +// +// Caller should close resp.Body when done reading from it. +// +// If the provided body is also an io.Closer, it is closed after the +// request. +func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { + req, err := NewRequest("POST", url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", bodyType) + return c.doFollowingRedirects(req, shouldRedirectPost) +} + +// PostForm issues a POST to the specified URL, with data's keys and +// values URL-encoded as the request body. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +// +// PostForm is a wrapper around DefaultClient.PostForm +func PostForm(url string, data url.Values) (resp *Response, err error) { + return DefaultClient.PostForm(url, data) +} + +// PostForm issues a POST to the specified URL, +// with data's keys and values urlencoded as the request body. +// +// When err is nil, resp always contains a non-nil resp.Body. +// Caller should close resp.Body when done reading from it. +func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { + return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) +} + +// Head issues a HEAD to the specified URL. If the response is one of the +// following redirect codes, Head follows the redirect after calling the +// Client's CheckRedirect function. +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +// +// Head is a wrapper around DefaultClient.Head +func Head(url string) (resp *Response, err error) { + return DefaultClient.Head(url) +} + +// Head issues a HEAD to the specified URL. If the response is one of the +// following redirect codes, Head follows the redirect after calling the +// Client's CheckRedirect function. +// +// 301 (Moved Permanently) +// 302 (Found) +// 303 (See Other) +// 307 (Temporary Redirect) +func (c *Client) Head(url string) (resp *Response, err error) { + req, err := NewRequest("HEAD", url, nil) + if err != nil { + return nil, err + } + return c.doFollowingRedirects(req, shouldRedirectGet) +} + +type cancelTimerBody struct { + t *time.Timer + rc io.ReadCloser +} + +func (b *cancelTimerBody) Read(p []byte) (n int, err error) { + n, err = b.rc.Read(p) + if err == io.EOF { + b.t.Stop() + } + return +} + +func (b *cancelTimerBody) Close() error { + err := b.rc.Close() + b.t.Stop() + return err +} diff --git a/src/net/http/client_test.go b/src/net/http/client_test.go new file mode 100644 index 000000000..6392c1baf --- /dev/null +++ b/src/net/http/client_test.go @@ -0,0 +1,1038 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for client.go + +package http_test + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net" + . "net/http" + "net/http/httptest" + "net/url" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +var robotsTxtHandler = HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Last-Modified", "sometime") + fmt.Fprintf(w, "User-agent: go\nDisallow: /something/") +}) + +// pedanticReadAll works like ioutil.ReadAll but additionally +// verifies that r obeys the documented io.Reader contract. +func pedanticReadAll(r io.Reader) (b []byte, err error) { + var bufa [64]byte + buf := bufa[:] + for { + n, err := r.Read(buf) + if n == 0 && err == nil { + return nil, fmt.Errorf("Read: n=0 with err=nil") + } + b = append(b, buf[:n]...) + if err == io.EOF { + n, err := r.Read(buf) + if n != 0 || err != io.EOF { + return nil, fmt.Errorf("Read: n=%d err=%#v after EOF", n, err) + } + return b, nil + } + if err != nil { + return b, err + } + } +} + +type chanWriter chan string + +func (w chanWriter) Write(p []byte) (n int, err error) { + w <- string(p) + return len(p), nil +} + +func TestClient(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(robotsTxtHandler) + defer ts.Close() + + r, err := Get(ts.URL) + var b []byte + if err == nil { + b, err = pedanticReadAll(r.Body) + r.Body.Close() + } + if err != nil { + t.Error(err) + } else if s := string(b); !strings.HasPrefix(s, "User-agent:") { + t.Errorf("Incorrect page body (did not begin with User-agent): %q", s) + } +} + +func TestClientHead(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(robotsTxtHandler) + defer ts.Close() + + r, err := Head(ts.URL) + if err != nil { + t.Fatal(err) + } + if _, ok := r.Header["Last-Modified"]; !ok { + t.Error("Last-Modified header not found.") + } +} + +type recordingTransport struct { + req *Request +} + +func (t *recordingTransport) RoundTrip(req *Request) (resp *Response, err error) { + t.req = req + return nil, errors.New("dummy impl") +} + +func TestGetRequestFormat(t *testing.T) { + defer afterTest(t) + tr := &recordingTransport{} + client := &Client{Transport: tr} + url := "http://dummy.faketld/" + client.Get(url) // Note: doesn't hit network + if tr.req.Method != "GET" { + t.Errorf("expected method %q; got %q", "GET", tr.req.Method) + } + if tr.req.URL.String() != url { + t.Errorf("expected URL %q; got %q", url, tr.req.URL.String()) + } + if tr.req.Header == nil { + t.Errorf("expected non-nil request Header") + } +} + +func TestPostRequestFormat(t *testing.T) { + defer afterTest(t) + tr := &recordingTransport{} + client := &Client{Transport: tr} + + url := "http://dummy.faketld/" + json := `{"key":"value"}` + b := strings.NewReader(json) + client.Post(url, "application/json", b) // Note: doesn't hit network + + if tr.req.Method != "POST" { + t.Errorf("got method %q, want %q", tr.req.Method, "POST") + } + if tr.req.URL.String() != url { + t.Errorf("got URL %q, want %q", tr.req.URL.String(), url) + } + if tr.req.Header == nil { + t.Fatalf("expected non-nil request Header") + } + if tr.req.Close { + t.Error("got Close true, want false") + } + if g, e := tr.req.ContentLength, int64(len(json)); g != e { + t.Errorf("got ContentLength %d, want %d", g, e) + } +} + +func TestPostFormRequestFormat(t *testing.T) { + defer afterTest(t) + tr := &recordingTransport{} + client := &Client{Transport: tr} + + urlStr := "http://dummy.faketld/" + form := make(url.Values) + form.Set("foo", "bar") + form.Add("foo", "bar2") + form.Set("bar", "baz") + client.PostForm(urlStr, form) // Note: doesn't hit network + + if tr.req.Method != "POST" { + t.Errorf("got method %q, want %q", tr.req.Method, "POST") + } + if tr.req.URL.String() != urlStr { + t.Errorf("got URL %q, want %q", tr.req.URL.String(), urlStr) + } + if tr.req.Header == nil { + t.Fatalf("expected non-nil request Header") + } + if g, e := tr.req.Header.Get("Content-Type"), "application/x-www-form-urlencoded"; g != e { + t.Errorf("got Content-Type %q, want %q", g, e) + } + if tr.req.Close { + t.Error("got Close true, want false") + } + // Depending on map iteration, body can be either of these. + expectedBody := "foo=bar&foo=bar2&bar=baz" + expectedBody1 := "bar=baz&foo=bar&foo=bar2" + if g, e := tr.req.ContentLength, int64(len(expectedBody)); g != e { + t.Errorf("got ContentLength %d, want %d", g, e) + } + bodyb, err := ioutil.ReadAll(tr.req.Body) + if err != nil { + t.Fatalf("ReadAll on req.Body: %v", err) + } + if g := string(bodyb); g != expectedBody && g != expectedBody1 { + t.Errorf("got body %q, want %q or %q", g, expectedBody, expectedBody1) + } +} + +func TestClientRedirects(t *testing.T) { + defer afterTest(t) + var ts *httptest.Server + ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + n, _ := strconv.Atoi(r.FormValue("n")) + // Test Referer header. (7 is arbitrary position to test at) + if n == 7 { + if g, e := r.Referer(), ts.URL+"/?n=6"; e != g { + t.Errorf("on request ?n=7, expected referer of %q; got %q", e, g) + } + } + if n < 15 { + Redirect(w, r, fmt.Sprintf("/?n=%d", n+1), StatusFound) + return + } + fmt.Fprintf(w, "n=%d", n) + })) + defer ts.Close() + + c := &Client{} + _, err := c.Get(ts.URL) + if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + t.Errorf("with default client Get, expected error %q, got %q", e, g) + } + + // HEAD request should also have the ability to follow redirects. + _, err = c.Head(ts.URL) + if e, g := "Head /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + t.Errorf("with default client Head, expected error %q, got %q", e, g) + } + + // Do should also follow redirects. + greq, _ := NewRequest("GET", ts.URL, nil) + _, err = c.Do(greq) + if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { + t.Errorf("with default client Do, expected error %q, got %q", e, g) + } + + var checkErr error + var lastVia []*Request + c = &Client{CheckRedirect: func(_ *Request, via []*Request) error { + lastVia = via + return checkErr + }} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatalf("Get error: %v", err) + } + res.Body.Close() + finalUrl := res.Request.URL.String() + if e, g := "<nil>", fmt.Sprintf("%v", err); e != g { + t.Errorf("with custom client, expected error %q, got %q", e, g) + } + if !strings.HasSuffix(finalUrl, "/?n=15") { + t.Errorf("expected final url to end in /?n=15; got url %q", finalUrl) + } + if e, g := 15, len(lastVia); e != g { + t.Errorf("expected lastVia to have contained %d elements; got %d", e, g) + } + + checkErr = errors.New("no redirects allowed") + res, err = c.Get(ts.URL) + if urlError, ok := err.(*url.Error); !ok || urlError.Err != checkErr { + t.Errorf("with redirects forbidden, expected a *url.Error with our 'no redirects allowed' error inside; got %#v (%q)", err, err) + } + if res == nil { + t.Fatalf("Expected a non-nil Response on CheckRedirect failure (http://golang.org/issue/3795)") + } + res.Body.Close() + if res.Header.Get("Location") == "" { + t.Errorf("no Location header in Response") + } +} + +func TestPostRedirects(t *testing.T) { + defer afterTest(t) + var log struct { + sync.Mutex + bytes.Buffer + } + var ts *httptest.Server + ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + log.Lock() + fmt.Fprintf(&log.Buffer, "%s %s ", r.Method, r.RequestURI) + log.Unlock() + if v := r.URL.Query().Get("code"); v != "" { + code, _ := strconv.Atoi(v) + if code/100 == 3 { + w.Header().Set("Location", ts.URL) + } + w.WriteHeader(code) + } + })) + defer ts.Close() + tests := []struct { + suffix string + want int // response code + }{ + {"/", 200}, + {"/?code=301", 301}, + {"/?code=302", 200}, + {"/?code=303", 200}, + {"/?code=404", 404}, + } + for _, tt := range tests { + res, err := Post(ts.URL+tt.suffix, "text/plain", strings.NewReader("Some content")) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != tt.want { + t.Errorf("POST %s: status code = %d; want %d", tt.suffix, res.StatusCode, tt.want) + } + } + log.Lock() + got := log.String() + log.Unlock() + want := "POST / POST /?code=301 POST /?code=302 GET / POST /?code=303 GET / POST /?code=404 " + if got != want { + t.Errorf("Log differs.\n Got: %q\nWant: %q", got, want) + } +} + +var expectedCookies = []*Cookie{ + {Name: "ChocolateChip", Value: "tasty"}, + {Name: "First", Value: "Hit"}, + {Name: "Second", Value: "Hit"}, +} + +var echoCookiesRedirectHandler = HandlerFunc(func(w ResponseWriter, r *Request) { + for _, cookie := range r.Cookies() { + SetCookie(w, cookie) + } + if r.URL.Path == "/" { + SetCookie(w, expectedCookies[1]) + Redirect(w, r, "/second", StatusMovedPermanently) + } else { + SetCookie(w, expectedCookies[2]) + w.Write([]byte("hello")) + } +}) + +func TestClientSendsCookieFromJar(t *testing.T) { + tr := &recordingTransport{} + client := &Client{Transport: tr} + client.Jar = &TestJar{perURL: make(map[string][]*Cookie)} + us := "http://dummy.faketld/" + u, _ := url.Parse(us) + client.Jar.SetCookies(u, expectedCookies) + + client.Get(us) // Note: doesn't hit network + matchReturnedCookies(t, expectedCookies, tr.req.Cookies()) + + client.Head(us) // Note: doesn't hit network + matchReturnedCookies(t, expectedCookies, tr.req.Cookies()) + + client.Post(us, "text/plain", strings.NewReader("body")) // Note: doesn't hit network + matchReturnedCookies(t, expectedCookies, tr.req.Cookies()) + + client.PostForm(us, url.Values{}) // Note: doesn't hit network + matchReturnedCookies(t, expectedCookies, tr.req.Cookies()) + + req, _ := NewRequest("GET", us, nil) + client.Do(req) // Note: doesn't hit network + matchReturnedCookies(t, expectedCookies, tr.req.Cookies()) + + req, _ = NewRequest("POST", us, nil) + client.Do(req) // Note: doesn't hit network + matchReturnedCookies(t, expectedCookies, tr.req.Cookies()) +} + +// Just enough correctness for our redirect tests. Uses the URL.Host as the +// scope of all cookies. +type TestJar struct { + m sync.Mutex + perURL map[string][]*Cookie +} + +func (j *TestJar) SetCookies(u *url.URL, cookies []*Cookie) { + j.m.Lock() + defer j.m.Unlock() + if j.perURL == nil { + j.perURL = make(map[string][]*Cookie) + } + j.perURL[u.Host] = cookies +} + +func (j *TestJar) Cookies(u *url.URL) []*Cookie { + j.m.Lock() + defer j.m.Unlock() + return j.perURL[u.Host] +} + +func TestRedirectCookiesJar(t *testing.T) { + defer afterTest(t) + var ts *httptest.Server + ts = httptest.NewServer(echoCookiesRedirectHandler) + defer ts.Close() + c := &Client{ + Jar: new(TestJar), + } + u, _ := url.Parse(ts.URL) + c.Jar.SetCookies(u, []*Cookie{expectedCookies[0]}) + resp, err := c.Get(ts.URL) + if err != nil { + t.Fatalf("Get: %v", err) + } + resp.Body.Close() + matchReturnedCookies(t, expectedCookies, resp.Cookies()) +} + +func matchReturnedCookies(t *testing.T, expected, given []*Cookie) { + if len(given) != len(expected) { + t.Logf("Received cookies: %v", given) + t.Errorf("Expected %d cookies, got %d", len(expected), len(given)) + } + for _, ec := range expected { + foundC := false + for _, c := range given { + if ec.Name == c.Name && ec.Value == c.Value { + foundC = true + break + } + } + if !foundC { + t.Errorf("Missing cookie %v", ec) + } + } +} + +func TestJarCalls(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + pathSuffix := r.RequestURI[1:] + if r.RequestURI == "/nosetcookie" { + return // dont set cookies for this path + } + SetCookie(w, &Cookie{Name: "name" + pathSuffix, Value: "val" + pathSuffix}) + if r.RequestURI == "/" { + Redirect(w, r, "http://secondhost.fake/secondpath", 302) + } + })) + defer ts.Close() + jar := new(RecordingJar) + c := &Client{ + Jar: jar, + Transport: &Transport{ + Dial: func(_ string, _ string) (net.Conn, error) { + return net.Dial("tcp", ts.Listener.Addr().String()) + }, + }, + } + _, err := c.Get("http://firsthost.fake/") + if err != nil { + t.Fatal(err) + } + _, err = c.Get("http://firsthost.fake/nosetcookie") + if err != nil { + t.Fatal(err) + } + got := jar.log.String() + want := `Cookies("http://firsthost.fake/") +SetCookie("http://firsthost.fake/", [name=val]) +Cookies("http://secondhost.fake/secondpath") +SetCookie("http://secondhost.fake/secondpath", [namesecondpath=valsecondpath]) +Cookies("http://firsthost.fake/nosetcookie") +` + if got != want { + t.Errorf("Got Jar calls:\n%s\nWant:\n%s", got, want) + } +} + +// RecordingJar keeps a log of calls made to it, without +// tracking any cookies. +type RecordingJar struct { + mu sync.Mutex + log bytes.Buffer +} + +func (j *RecordingJar) SetCookies(u *url.URL, cookies []*Cookie) { + j.logf("SetCookie(%q, %v)\n", u, cookies) +} + +func (j *RecordingJar) Cookies(u *url.URL) []*Cookie { + j.logf("Cookies(%q)\n", u) + return nil +} + +func (j *RecordingJar) logf(format string, args ...interface{}) { + j.mu.Lock() + defer j.mu.Unlock() + fmt.Fprintf(&j.log, format, args...) +} + +func TestStreamingGet(t *testing.T) { + defer afterTest(t) + say := make(chan string) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.(Flusher).Flush() + for str := range say { + w.Write([]byte(str)) + w.(Flusher).Flush() + } + })) + defer ts.Close() + + c := &Client{} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + var buf [10]byte + for _, str := range []string{"i", "am", "also", "known", "as", "comet"} { + say <- str + n, err := io.ReadFull(res.Body, buf[0:len(str)]) + if err != nil { + t.Fatalf("ReadFull on %q: %v", str, err) + } + if n != len(str) { + t.Fatalf("Receiving %q, only read %d bytes", str, n) + } + got := string(buf[0:n]) + if got != str { + t.Fatalf("Expected %q, got %q", str, got) + } + } + close(say) + _, err = io.ReadFull(res.Body, buf[0:1]) + if err != io.EOF { + t.Fatalf("at end expected EOF, got %v", err) + } +} + +type writeCountingConn struct { + net.Conn + count *int +} + +func (c *writeCountingConn) Write(p []byte) (int, error) { + *c.count++ + return c.Conn.Write(p) +} + +// TestClientWrites verifies that client requests are buffered and we +// don't send a TCP packet per line of the http request + body. +func TestClientWrites(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + })) + defer ts.Close() + + writes := 0 + dialer := func(netz string, addr string) (net.Conn, error) { + c, err := net.Dial(netz, addr) + if err == nil { + c = &writeCountingConn{c, &writes} + } + return c, err + } + c := &Client{Transport: &Transport{Dial: dialer}} + + _, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Errorf("Get request did %d Write calls, want 1", writes) + } + + writes = 0 + _, err = c.PostForm(ts.URL, url.Values{"foo": {"bar"}}) + if err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Errorf("Post request did %d Write calls, want 1", writes) + } +} + +func TestClientInsecureTransport(t *testing.T) { + defer afterTest(t) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Write([]byte("Hello")) + })) + errc := make(chanWriter, 10) // but only expecting 1 + ts.Config.ErrorLog = log.New(errc, "", 0) + defer ts.Close() + + // TODO(bradfitz): add tests for skipping hostname checks too? + // would require a new cert for testing, and probably + // redundant with these tests. + for _, insecure := range []bool{true, false} { + tr := &Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecure, + }, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + res, err := c.Get(ts.URL) + if (err == nil) != insecure { + t.Errorf("insecure=%v: got unexpected err=%v", insecure, err) + } + if res != nil { + res.Body.Close() + } + } + + select { + case v := <-errc: + if !strings.Contains(v, "TLS handshake error") { + t.Errorf("expected an error log message containing 'TLS handshake error'; got %q", v) + } + case <-time.After(5 * time.Second): + t.Errorf("timeout waiting for logged error") + } + +} + +func TestClientErrorWithRequestURI(t *testing.T) { + defer afterTest(t) + req, _ := NewRequest("GET", "http://localhost:1234/", nil) + req.RequestURI = "/this/field/is/illegal/and/should/error/" + _, err := DefaultClient.Do(req) + if err == nil { + t.Fatalf("expected an error") + } + if !strings.Contains(err.Error(), "RequestURI") { + t.Errorf("wanted error mentioning RequestURI; got error: %v", err) + } +} + +func newTLSTransport(t *testing.T, ts *httptest.Server) *Transport { + certs := x509.NewCertPool() + for _, c := range ts.TLS.Certificates { + roots, err := x509.ParseCertificates(c.Certificate[len(c.Certificate)-1]) + if err != nil { + t.Fatalf("error parsing server's root cert: %v", err) + } + for _, root := range roots { + certs.AddCert(root) + } + } + return &Transport{ + TLSClientConfig: &tls.Config{RootCAs: certs}, + } +} + +func TestClientWithCorrectTLSServerName(t *testing.T) { + defer afterTest(t) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.TLS.ServerName != "127.0.0.1" { + t.Errorf("expected client to set ServerName 127.0.0.1, got: %q", r.TLS.ServerName) + } + })) + defer ts.Close() + + c := &Client{Transport: newTLSTransport(t, ts)} + if _, err := c.Get(ts.URL); err != nil { + t.Fatalf("expected successful TLS connection, got error: %v", err) + } +} + +func TestClientWithIncorrectTLSServerName(t *testing.T) { + defer afterTest(t) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + defer ts.Close() + errc := make(chanWriter, 10) // but only expecting 1 + ts.Config.ErrorLog = log.New(errc, "", 0) + + trans := newTLSTransport(t, ts) + trans.TLSClientConfig.ServerName = "badserver" + c := &Client{Transport: trans} + _, err := c.Get(ts.URL) + if err == nil { + t.Fatalf("expected an error") + } + if !strings.Contains(err.Error(), "127.0.0.1") || !strings.Contains(err.Error(), "badserver") { + t.Errorf("wanted error mentioning 127.0.0.1 and badserver; got error: %v", err) + } + select { + case v := <-errc: + if !strings.Contains(v, "TLS handshake error") { + t.Errorf("expected an error log message containing 'TLS handshake error'; got %q", v) + } + case <-time.After(5 * time.Second): + t.Errorf("timeout waiting for logged error") + } +} + +// Test for golang.org/issue/5829; the Transport should respect TLSClientConfig.ServerName +// when not empty. +// +// tls.Config.ServerName (non-empty, set to "example.com") takes +// precedence over "some-other-host.tld" which previously incorrectly +// took precedence. We don't actually connect to (or even resolve) +// "some-other-host.tld", though, because of the Transport.Dial hook. +// +// The httptest.Server has a cert with "example.com" as its name. +func TestTransportUsesTLSConfigServerName(t *testing.T) { + defer afterTest(t) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Write([]byte("Hello")) + })) + defer ts.Close() + + tr := newTLSTransport(t, ts) + tr.TLSClientConfig.ServerName = "example.com" // one of httptest's Server cert names + tr.Dial = func(netw, addr string) (net.Conn, error) { + return net.Dial(netw, ts.Listener.Addr().String()) + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + res, err := c.Get("https://some-other-host.tld/") + if err != nil { + t.Fatal(err) + } + res.Body.Close() +} + +func TestResponseSetsTLSConnectionState(t *testing.T) { + defer afterTest(t) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Write([]byte("Hello")) + })) + defer ts.Close() + + tr := newTLSTransport(t, ts) + tr.TLSClientConfig.CipherSuites = []uint16{tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA} + tr.Dial = func(netw, addr string) (net.Conn, error) { + return net.Dial(netw, ts.Listener.Addr().String()) + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + res, err := c.Get("https://example.com/") + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.TLS == nil { + t.Fatal("Response didn't set TLS Connection State.") + } + if got, want := res.TLS.CipherSuite, tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA; got != want { + t.Errorf("TLS Cipher Suite = %d; want %d", got, want) + } +} + +// Verify Response.ContentLength is populated. http://golang.org/issue/4126 +func TestClientHeadContentLength(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if v := r.FormValue("cl"); v != "" { + w.Header().Set("Content-Length", v) + } + })) + defer ts.Close() + tests := []struct { + suffix string + want int64 + }{ + {"/?cl=1234", 1234}, + {"/?cl=0", 0}, + {"", -1}, + } + for _, tt := range tests { + req, _ := NewRequest("HEAD", ts.URL+tt.suffix, nil) + res, err := DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if res.ContentLength != tt.want { + t.Errorf("Content-Length = %d; want %d", res.ContentLength, tt.want) + } + bs, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if len(bs) != 0 { + t.Errorf("Unexpected content: %q", bs) + } + } +} + +func TestEmptyPasswordAuth(t *testing.T) { + defer afterTest(t) + gopher := "gopher" + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + auth := r.Header.Get("Authorization") + if strings.HasPrefix(auth, "Basic ") { + encoded := auth[6:] + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + expected := gopher + ":" + s := string(decoded) + if expected != s { + t.Errorf("Invalid Authorization header. Got %q, wanted %q", s, expected) + } + } else { + t.Errorf("Invalid auth %q", auth) + } + })) + defer ts.Close() + c := &Client{} + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + t.Fatal(err) + } + req.URL.User = url.User(gopher) + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() +} + +func TestBasicAuth(t *testing.T) { + defer afterTest(t) + tr := &recordingTransport{} + client := &Client{Transport: tr} + + url := "http://My%20User:My%20Pass@dummy.faketld/" + expected := "My User:My Pass" + client.Get(url) + + if tr.req.Method != "GET" { + t.Errorf("got method %q, want %q", tr.req.Method, "GET") + } + if tr.req.URL.String() != url { + t.Errorf("got URL %q, want %q", tr.req.URL.String(), url) + } + if tr.req.Header == nil { + t.Fatalf("expected non-nil request Header") + } + auth := tr.req.Header.Get("Authorization") + if strings.HasPrefix(auth, "Basic ") { + encoded := auth[6:] + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + t.Fatal(err) + } + s := string(decoded) + if expected != s { + t.Errorf("Invalid Authorization header. Got %q, wanted %q", s, expected) + } + } else { + t.Errorf("Invalid auth %q", auth) + } +} + +func TestClientTimeout(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + defer afterTest(t) + sawRoot := make(chan bool, 1) + sawSlow := make(chan bool, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/" { + sawRoot <- true + Redirect(w, r, "/slow", StatusFound) + return + } + if r.URL.Path == "/slow" { + w.Write([]byte("Hello")) + w.(Flusher).Flush() + sawSlow <- true + time.Sleep(2 * time.Second) + return + } + })) + defer ts.Close() + const timeout = 500 * time.Millisecond + c := &Client{ + Timeout: timeout, + } + + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + + select { + case <-sawRoot: + // good. + default: + t.Fatal("handler never got / request") + } + + select { + case <-sawSlow: + // good. + default: + t.Fatal("handler never got /slow request") + } + + errc := make(chan error, 1) + go func() { + _, err := ioutil.ReadAll(res.Body) + errc <- err + res.Body.Close() + }() + + const failTime = timeout * 2 + select { + case err := <-errc: + if err == nil { + t.Error("expected error from ReadAll") + } + // Expected error. + case <-time.After(failTime): + t.Errorf("timeout after %v waiting for timeout of %v", failTime, timeout) + } +} + +func TestClientRedirectEatsBody(t *testing.T) { + defer afterTest(t) + saw := make(chan string, 2) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + saw <- r.RemoteAddr + if r.URL.Path == "/" { + Redirect(w, r, "/foo", StatusFound) // which includes a body + } + })) + defer ts.Close() + + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + _, err = ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + + var first string + select { + case first = <-saw: + default: + t.Fatal("server didn't see a request") + } + + var second string + select { + case second = <-saw: + default: + t.Fatal("server didn't see a second request") + } + + if first != second { + t.Fatal("server saw different client ports before & after the redirect") + } +} + +// eofReaderFunc is an io.Reader that runs itself, and then returns io.EOF. +type eofReaderFunc func() + +func (f eofReaderFunc) Read(p []byte) (n int, err error) { + f() + return 0, io.EOF +} + +func TestClientTrailers(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + w.Header().Set("Trailer", "Server-Trailer-A, Server-Trailer-B") + w.Header().Add("Trailer", "Server-Trailer-C") + + var decl []string + for k := range r.Trailer { + decl = append(decl, k) + } + sort.Strings(decl) + + slurp, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Errorf("Server reading request body: %v", err) + } + if string(slurp) != "foo" { + t.Errorf("Server read request body %q; want foo", slurp) + } + if r.Trailer == nil { + io.WriteString(w, "nil Trailer") + } else { + fmt.Fprintf(w, "decl: %v, vals: %s, %s", + decl, + r.Trailer.Get("Client-Trailer-A"), + r.Trailer.Get("Client-Trailer-B")) + } + + // TODO: golang.org/issue/7759: there's no way yet for + // the server to set trailers without hijacking, so do + // that for now, just to test the client. Later, in + // Go 1.4, it should be implicit that any mutations + // to w.Header() after the initial write are the + // trailers to be sent, if and only if they were + // previously declared with w.Header().Set("Trailer", + // ..keys..) + w.(Flusher).Flush() + conn, buf, _ := w.(Hijacker).Hijack() + t := Header{} + t.Set("Server-Trailer-A", "valuea") + t.Set("Server-Trailer-C", "valuec") // skipping B + buf.WriteString("0\r\n") // eof + t.Write(buf) + buf.WriteString("\r\n") // end of trailers + buf.Flush() + conn.Close() + })) + defer ts.Close() + + var req *Request + req, _ = NewRequest("POST", ts.URL, io.MultiReader( + eofReaderFunc(func() { + req.Trailer["Client-Trailer-A"] = []string{"valuea"} + }), + strings.NewReader("foo"), + eofReaderFunc(func() { + req.Trailer["Client-Trailer-B"] = []string{"valueb"} + }), + )) + req.Trailer = Header{ + "Client-Trailer-A": nil, // to be set later + "Client-Trailer-B": nil, // to be set later + } + req.ContentLength = -1 + res, err := DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if err := wantBody(res, err, "decl: [Client-Trailer-A Client-Trailer-B], vals: valuea, valueb"); err != nil { + t.Error(err) + } + want := Header{ + "Server-Trailer-A": []string{"valuea"}, + "Server-Trailer-B": nil, + "Server-Trailer-C": []string{"valuec"}, + } + if !reflect.DeepEqual(res.Trailer, want) { + t.Errorf("Response trailers = %#v; want %#v", res.Trailer, want) + } +} diff --git a/src/net/http/cookie.go b/src/net/http/cookie.go new file mode 100644 index 000000000..dc60ba87f --- /dev/null +++ b/src/net/http/cookie.go @@ -0,0 +1,363 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "fmt" + "log" + "net" + "strconv" + "strings" + "time" +) + +// This implementation is done according to RFC 6265: +// +// http://tools.ietf.org/html/rfc6265 + +// A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an +// HTTP response or the Cookie header of an HTTP request. +type Cookie struct { + Name string + Value string + Path string + Domain string + Expires time.Time + RawExpires string + + // MaxAge=0 means no 'Max-Age' attribute specified. + // MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0' + // MaxAge>0 means Max-Age attribute present and given in seconds + MaxAge int + Secure bool + HttpOnly bool + Raw string + Unparsed []string // Raw text of unparsed attribute-value pairs +} + +// readSetCookies parses all "Set-Cookie" values from +// the header h and returns the successfully parsed Cookies. +func readSetCookies(h Header) []*Cookie { + cookies := []*Cookie{} + for _, line := range h["Set-Cookie"] { + parts := strings.Split(strings.TrimSpace(line), ";") + if len(parts) == 1 && parts[0] == "" { + continue + } + parts[0] = strings.TrimSpace(parts[0]) + j := strings.Index(parts[0], "=") + if j < 0 { + continue + } + name, value := parts[0][:j], parts[0][j+1:] + if !isCookieNameValid(name) { + continue + } + value, success := parseCookieValue(value) + if !success { + continue + } + c := &Cookie{ + Name: name, + Value: value, + Raw: line, + } + for i := 1; i < len(parts); i++ { + parts[i] = strings.TrimSpace(parts[i]) + if len(parts[i]) == 0 { + continue + } + + attr, val := parts[i], "" + if j := strings.Index(attr, "="); j >= 0 { + attr, val = attr[:j], attr[j+1:] + } + lowerAttr := strings.ToLower(attr) + val, success = parseCookieValue(val) + if !success { + c.Unparsed = append(c.Unparsed, parts[i]) + continue + } + switch lowerAttr { + case "secure": + c.Secure = true + continue + case "httponly": + c.HttpOnly = true + continue + case "domain": + c.Domain = val + continue + case "max-age": + secs, err := strconv.Atoi(val) + if err != nil || secs != 0 && val[0] == '0' { + break + } + if secs <= 0 { + c.MaxAge = -1 + } else { + c.MaxAge = secs + } + continue + case "expires": + c.RawExpires = val + exptime, err := time.Parse(time.RFC1123, val) + if err != nil { + exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", val) + if err != nil { + c.Expires = time.Time{} + break + } + } + c.Expires = exptime.UTC() + continue + case "path": + c.Path = val + continue + } + c.Unparsed = append(c.Unparsed, parts[i]) + } + cookies = append(cookies, c) + } + return cookies +} + +// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers. +func SetCookie(w ResponseWriter, cookie *Cookie) { + w.Header().Add("Set-Cookie", cookie.String()) +} + +// String returns the serialization of the cookie for use in a Cookie +// header (if only Name and Value are set) or a Set-Cookie response +// header (if other fields are set). +func (c *Cookie) String() string { + var b bytes.Buffer + fmt.Fprintf(&b, "%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) + if len(c.Path) > 0 { + fmt.Fprintf(&b, "; Path=%s", sanitizeCookiePath(c.Path)) + } + if len(c.Domain) > 0 { + if validCookieDomain(c.Domain) { + // A c.Domain containing illegal characters is not + // sanitized but simply dropped which turns the cookie + // into a host-only cookie. A leading dot is okay + // but won't be sent. + d := c.Domain + if d[0] == '.' { + d = d[1:] + } + fmt.Fprintf(&b, "; Domain=%s", d) + } else { + log.Printf("net/http: invalid Cookie.Domain %q; dropping domain attribute", + c.Domain) + } + } + if c.Expires.Unix() > 0 { + fmt.Fprintf(&b, "; Expires=%s", c.Expires.UTC().Format(time.RFC1123)) + } + if c.MaxAge > 0 { + fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge) + } else if c.MaxAge < 0 { + fmt.Fprintf(&b, "; Max-Age=0") + } + if c.HttpOnly { + fmt.Fprintf(&b, "; HttpOnly") + } + if c.Secure { + fmt.Fprintf(&b, "; Secure") + } + return b.String() +} + +// readCookies parses all "Cookie" values from the header h and +// returns the successfully parsed Cookies. +// +// if filter isn't empty, only cookies of that name are returned +func readCookies(h Header, filter string) []*Cookie { + cookies := []*Cookie{} + lines, ok := h["Cookie"] + if !ok { + return cookies + } + + for _, line := range lines { + parts := strings.Split(strings.TrimSpace(line), ";") + if len(parts) == 1 && parts[0] == "" { + continue + } + // Per-line attributes + parsedPairs := 0 + for i := 0; i < len(parts); i++ { + parts[i] = strings.TrimSpace(parts[i]) + if len(parts[i]) == 0 { + continue + } + name, val := parts[i], "" + if j := strings.Index(name, "="); j >= 0 { + name, val = name[:j], name[j+1:] + } + if !isCookieNameValid(name) { + continue + } + if filter != "" && filter != name { + continue + } + val, success := parseCookieValue(val) + if !success { + continue + } + cookies = append(cookies, &Cookie{Name: name, Value: val}) + parsedPairs++ + } + } + return cookies +} + +// validCookieDomain returns wheter v is a valid cookie domain-value. +func validCookieDomain(v string) bool { + if isCookieDomainName(v) { + return true + } + if net.ParseIP(v) != nil && !strings.Contains(v, ":") { + return true + } + return false +} + +// isCookieDomainName returns whether s is a valid domain name or a valid +// domain name with a leading dot '.'. It is almost a direct copy of +// package net's isDomainName. +func isCookieDomainName(s string) bool { + if len(s) == 0 { + return false + } + if len(s) > 255 { + return false + } + + if s[0] == '.' { + // A cookie a domain attribute may start with a leading dot. + s = s[1:] + } + last := byte('.') + ok := false // Ok once we've seen a letter. + partlen := 0 + for i := 0; i < len(s); i++ { + c := s[i] + switch { + default: + return false + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': + // No '_' allowed here (in contrast to package net). + ok = true + partlen++ + case '0' <= c && c <= '9': + // fine + partlen++ + case c == '-': + // Byte before dash cannot be dot. + if last == '.' { + return false + } + partlen++ + case c == '.': + // Byte before dot cannot be dot, dash. + if last == '.' || last == '-' { + return false + } + if partlen > 63 || partlen == 0 { + return false + } + partlen = 0 + } + last = c + } + if last == '-' || partlen > 63 { + return false + } + + return ok +} + +var cookieNameSanitizer = strings.NewReplacer("\n", "-", "\r", "-") + +func sanitizeCookieName(n string) string { + return cookieNameSanitizer.Replace(n) +} + +// http://tools.ietf.org/html/rfc6265#section-4.1.1 +// cookie-value = *cookie-octet / ( DQUOTE *cookie-octet DQUOTE ) +// cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E +// ; US-ASCII characters excluding CTLs, +// ; whitespace DQUOTE, comma, semicolon, +// ; and backslash +// We loosen this as spaces and commas are common in cookie values +// but we produce a quoted cookie-value in when value starts or ends +// with a comma or space. +// See http://golang.org/issue/7243 for the discussion. +func sanitizeCookieValue(v string) string { + v = sanitizeOrWarn("Cookie.Value", validCookieValueByte, v) + if len(v) == 0 { + return v + } + if v[0] == ' ' || v[0] == ',' || v[len(v)-1] == ' ' || v[len(v)-1] == ',' { + return `"` + v + `"` + } + return v +} + +func validCookieValueByte(b byte) bool { + return 0x20 <= b && b < 0x7f && b != '"' && b != ';' && b != '\\' +} + +// path-av = "Path=" path-value +// path-value = <any CHAR except CTLs or ";"> +func sanitizeCookiePath(v string) string { + return sanitizeOrWarn("Cookie.Path", validCookiePathByte, v) +} + +func validCookiePathByte(b byte) bool { + return 0x20 <= b && b < 0x7f && b != ';' +} + +func sanitizeOrWarn(fieldName string, valid func(byte) bool, v string) string { + ok := true + for i := 0; i < len(v); i++ { + if valid(v[i]) { + continue + } + log.Printf("net/http: invalid byte %q in %s; dropping invalid bytes", v[i], fieldName) + ok = false + break + } + if ok { + return v + } + buf := make([]byte, 0, len(v)) + for i := 0; i < len(v); i++ { + if b := v[i]; valid(b) { + buf = append(buf, b) + } + } + return string(buf) +} + +func parseCookieValue(raw string) (string, bool) { + // Strip the quotes, if present. + if len(raw) > 1 && raw[0] == '"' && raw[len(raw)-1] == '"' { + raw = raw[1 : len(raw)-1] + } + for i := 0; i < len(raw); i++ { + if !validCookieValueByte(raw[i]) { + return "", false + } + } + return raw, true +} + +func isCookieNameValid(raw string) bool { + return strings.IndexFunc(raw, isNotToken) < 0 +} diff --git a/src/net/http/cookie_test.go b/src/net/http/cookie_test.go new file mode 100644 index 000000000..f78f37299 --- /dev/null +++ b/src/net/http/cookie_test.go @@ -0,0 +1,380 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "os" + "reflect" + "strings" + "testing" + "time" +) + +var writeSetCookiesTests = []struct { + Cookie *Cookie + Raw string +}{ + { + &Cookie{Name: "cookie-1", Value: "v$1"}, + "cookie-1=v$1", + }, + { + &Cookie{Name: "cookie-2", Value: "two", MaxAge: 3600}, + "cookie-2=two; Max-Age=3600", + }, + { + &Cookie{Name: "cookie-3", Value: "three", Domain: ".example.com"}, + "cookie-3=three; Domain=example.com", + }, + { + &Cookie{Name: "cookie-4", Value: "four", Path: "/restricted/"}, + "cookie-4=four; Path=/restricted/", + }, + { + &Cookie{Name: "cookie-5", Value: "five", Domain: "wrong;bad.abc"}, + "cookie-5=five", + }, + { + &Cookie{Name: "cookie-6", Value: "six", Domain: "bad-.abc"}, + "cookie-6=six", + }, + { + &Cookie{Name: "cookie-7", Value: "seven", Domain: "127.0.0.1"}, + "cookie-7=seven; Domain=127.0.0.1", + }, + { + &Cookie{Name: "cookie-8", Value: "eight", Domain: "::1"}, + "cookie-8=eight", + }, + // The "special" cookies have values containing commas or spaces which + // are disallowed by RFC 6265 but are common in the wild. + { + &Cookie{Name: "special-1", Value: "a z"}, + `special-1=a z`, + }, + { + &Cookie{Name: "special-2", Value: " z"}, + `special-2=" z"`, + }, + { + &Cookie{Name: "special-3", Value: "a "}, + `special-3="a "`, + }, + { + &Cookie{Name: "special-4", Value: " "}, + `special-4=" "`, + }, + { + &Cookie{Name: "special-5", Value: "a,z"}, + `special-5=a,z`, + }, + { + &Cookie{Name: "special-6", Value: ",z"}, + `special-6=",z"`, + }, + { + &Cookie{Name: "special-7", Value: "a,"}, + `special-7="a,"`, + }, + { + &Cookie{Name: "special-8", Value: ","}, + `special-8=","`, + }, + { + &Cookie{Name: "empty-value", Value: ""}, + `empty-value=`, + }, +} + +func TestWriteSetCookies(t *testing.T) { + defer log.SetOutput(os.Stderr) + var logbuf bytes.Buffer + log.SetOutput(&logbuf) + + for i, tt := range writeSetCookiesTests { + if g, e := tt.Cookie.String(), tt.Raw; g != e { + t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, e, g) + continue + } + } + + if got, sub := logbuf.String(), "dropping domain attribute"; !strings.Contains(got, sub) { + t.Errorf("Expected substring %q in log output. Got:\n%s", sub, got) + } +} + +type headerOnlyResponseWriter Header + +func (ho headerOnlyResponseWriter) Header() Header { + return Header(ho) +} + +func (ho headerOnlyResponseWriter) Write([]byte) (int, error) { + panic("NOIMPL") +} + +func (ho headerOnlyResponseWriter) WriteHeader(int) { + panic("NOIMPL") +} + +func TestSetCookie(t *testing.T) { + m := make(Header) + SetCookie(headerOnlyResponseWriter(m), &Cookie{Name: "cookie-1", Value: "one", Path: "/restricted/"}) + SetCookie(headerOnlyResponseWriter(m), &Cookie{Name: "cookie-2", Value: "two", MaxAge: 3600}) + if l := len(m["Set-Cookie"]); l != 2 { + t.Fatalf("expected %d cookies, got %d", 2, l) + } + if g, e := m["Set-Cookie"][0], "cookie-1=one; Path=/restricted/"; g != e { + t.Errorf("cookie #1: want %q, got %q", e, g) + } + if g, e := m["Set-Cookie"][1], "cookie-2=two; Max-Age=3600"; g != e { + t.Errorf("cookie #2: want %q, got %q", e, g) + } +} + +var addCookieTests = []struct { + Cookies []*Cookie + Raw string +}{ + { + []*Cookie{}, + "", + }, + { + []*Cookie{{Name: "cookie-1", Value: "v$1"}}, + "cookie-1=v$1", + }, + { + []*Cookie{ + {Name: "cookie-1", Value: "v$1"}, + {Name: "cookie-2", Value: "v$2"}, + {Name: "cookie-3", Value: "v$3"}, + }, + "cookie-1=v$1; cookie-2=v$2; cookie-3=v$3", + }, +} + +func TestAddCookie(t *testing.T) { + for i, tt := range addCookieTests { + req, _ := NewRequest("GET", "http://example.com/", nil) + for _, c := range tt.Cookies { + req.AddCookie(c) + } + if g := req.Header.Get("Cookie"); g != tt.Raw { + t.Errorf("Test %d:\nwant: %s\n got: %s\n", i, tt.Raw, g) + continue + } + } +} + +var readSetCookiesTests = []struct { + Header Header + Cookies []*Cookie +}{ + { + Header{"Set-Cookie": {"Cookie-1=v$1"}}, + []*Cookie{{Name: "Cookie-1", Value: "v$1", Raw: "Cookie-1=v$1"}}, + }, + { + Header{"Set-Cookie": {"NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly"}}, + []*Cookie{{ + Name: "NID", + Value: "99=YsDT5i3E-CXax-", + Path: "/", + Domain: ".google.ch", + HttpOnly: true, + Expires: time.Date(2011, 11, 23, 1, 5, 3, 0, time.UTC), + RawExpires: "Wed, 23-Nov-2011 01:05:03 GMT", + Raw: "NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly", + }}, + }, + { + Header{"Set-Cookie": {".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly"}}, + []*Cookie{{ + Name: ".ASPXAUTH", + Value: "7E3AA", + Path: "/", + Expires: time.Date(2012, 3, 7, 14, 25, 6, 0, time.UTC), + RawExpires: "Wed, 07-Mar-2012 14:25:06 GMT", + HttpOnly: true, + Raw: ".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly", + }}, + }, + { + Header{"Set-Cookie": {"ASP.NET_SessionId=foo; path=/; HttpOnly"}}, + []*Cookie{{ + Name: "ASP.NET_SessionId", + Value: "foo", + Path: "/", + HttpOnly: true, + Raw: "ASP.NET_SessionId=foo; path=/; HttpOnly", + }}, + }, + // Make sure we can properly read back the Set-Cookie headers we create + // for values containing spaces or commas: + { + Header{"Set-Cookie": {`special-1=a z`}}, + []*Cookie{{Name: "special-1", Value: "a z", Raw: `special-1=a z`}}, + }, + { + Header{"Set-Cookie": {`special-2=" z"`}}, + []*Cookie{{Name: "special-2", Value: " z", Raw: `special-2=" z"`}}, + }, + { + Header{"Set-Cookie": {`special-3="a "`}}, + []*Cookie{{Name: "special-3", Value: "a ", Raw: `special-3="a "`}}, + }, + { + Header{"Set-Cookie": {`special-4=" "`}}, + []*Cookie{{Name: "special-4", Value: " ", Raw: `special-4=" "`}}, + }, + { + Header{"Set-Cookie": {`special-5=a,z`}}, + []*Cookie{{Name: "special-5", Value: "a,z", Raw: `special-5=a,z`}}, + }, + { + Header{"Set-Cookie": {`special-6=",z"`}}, + []*Cookie{{Name: "special-6", Value: ",z", Raw: `special-6=",z"`}}, + }, + { + Header{"Set-Cookie": {`special-7=a,`}}, + []*Cookie{{Name: "special-7", Value: "a,", Raw: `special-7=a,`}}, + }, + { + Header{"Set-Cookie": {`special-8=","`}}, + []*Cookie{{Name: "special-8", Value: ",", Raw: `special-8=","`}}, + }, + + // TODO(bradfitz): users have reported seeing this in the + // wild, but do browsers handle it? RFC 6265 just says "don't + // do that" (section 3) and then never mentions header folding + // again. + // Header{"Set-Cookie": {"ASP.NET_SessionId=foo; path=/; HttpOnly, .ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly"}}, +} + +func toJSON(v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + return fmt.Sprintf("%#v", v) + } + return string(b) +} + +func TestReadSetCookies(t *testing.T) { + for i, tt := range readSetCookiesTests { + for n := 0; n < 2; n++ { // to verify readSetCookies doesn't mutate its input + c := readSetCookies(tt.Header) + if !reflect.DeepEqual(c, tt.Cookies) { + t.Errorf("#%d readSetCookies: have\n%s\nwant\n%s\n", i, toJSON(c), toJSON(tt.Cookies)) + continue + } + } + } +} + +var readCookiesTests = []struct { + Header Header + Filter string + Cookies []*Cookie +}{ + { + Header{"Cookie": {"Cookie-1=v$1", "c2=v2"}}, + "", + []*Cookie{ + {Name: "Cookie-1", Value: "v$1"}, + {Name: "c2", Value: "v2"}, + }, + }, + { + Header{"Cookie": {"Cookie-1=v$1", "c2=v2"}}, + "c2", + []*Cookie{ + {Name: "c2", Value: "v2"}, + }, + }, + { + Header{"Cookie": {"Cookie-1=v$1; c2=v2"}}, + "", + []*Cookie{ + {Name: "Cookie-1", Value: "v$1"}, + {Name: "c2", Value: "v2"}, + }, + }, + { + Header{"Cookie": {"Cookie-1=v$1; c2=v2"}}, + "c2", + []*Cookie{ + {Name: "c2", Value: "v2"}, + }, + }, +} + +func TestReadCookies(t *testing.T) { + for i, tt := range readCookiesTests { + for n := 0; n < 2; n++ { // to verify readCookies doesn't mutate its input + c := readCookies(tt.Header, tt.Filter) + if !reflect.DeepEqual(c, tt.Cookies) { + t.Errorf("#%d readCookies:\nhave: %s\nwant: %s\n", i, toJSON(c), toJSON(tt.Cookies)) + continue + } + } + } +} + +func TestCookieSanitizeValue(t *testing.T) { + defer log.SetOutput(os.Stderr) + var logbuf bytes.Buffer + log.SetOutput(&logbuf) + + tests := []struct { + in, want string + }{ + {"foo", "foo"}, + {"foo;bar", "foobar"}, + {"foo\\bar", "foobar"}, + {"foo\"bar", "foobar"}, + {"\x00\x7e\x7f\x80", "\x7e"}, + {`"withquotes"`, "withquotes"}, + {"a z", "a z"}, + {" z", `" z"`}, + {"a ", `"a "`}, + } + for _, tt := range tests { + if got := sanitizeCookieValue(tt.in); got != tt.want { + t.Errorf("sanitizeCookieValue(%q) = %q; want %q", tt.in, got, tt.want) + } + } + + if got, sub := logbuf.String(), "dropping invalid bytes"; !strings.Contains(got, sub) { + t.Errorf("Expected substring %q in log output. Got:\n%s", sub, got) + } +} + +func TestCookieSanitizePath(t *testing.T) { + defer log.SetOutput(os.Stderr) + var logbuf bytes.Buffer + log.SetOutput(&logbuf) + + tests := []struct { + in, want string + }{ + {"/path", "/path"}, + {"/path with space/", "/path with space/"}, + {"/just;no;semicolon\x00orstuff/", "/justnosemicolonorstuff/"}, + } + for _, tt := range tests { + if got := sanitizeCookiePath(tt.in); got != tt.want { + t.Errorf("sanitizeCookiePath(%q) = %q; want %q", tt.in, got, tt.want) + } + } + + if got, sub := logbuf.String(), "dropping invalid bytes"; !strings.Contains(got, sub) { + t.Errorf("Expected substring %q in log output. Got:\n%s", sub, got) + } +} diff --git a/src/net/http/cookiejar/jar.go b/src/net/http/cookiejar/jar.go new file mode 100644 index 000000000..389ab58e4 --- /dev/null +++ b/src/net/http/cookiejar/jar.go @@ -0,0 +1,497 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cookiejar implements an in-memory RFC 6265-compliant http.CookieJar. +package cookiejar + +import ( + "errors" + "fmt" + "net" + "net/http" + "net/url" + "sort" + "strings" + "sync" + "time" +) + +// PublicSuffixList provides the public suffix of a domain. For example: +// - the public suffix of "example.com" is "com", +// - the public suffix of "foo1.foo2.foo3.co.uk" is "co.uk", and +// - the public suffix of "bar.pvt.k12.ma.us" is "pvt.k12.ma.us". +// +// Implementations of PublicSuffixList must be safe for concurrent use by +// multiple goroutines. +// +// An implementation that always returns "" is valid and may be useful for +// testing but it is not secure: it means that the HTTP server for foo.com can +// set a cookie for bar.com. +// +// A public suffix list implementation is in the package +// code.google.com/p/go.net/publicsuffix. +type PublicSuffixList interface { + // PublicSuffix returns the public suffix of domain. + // + // TODO: specify which of the caller and callee is responsible for IP + // addresses, for leading and trailing dots, for case sensitivity, and + // for IDN/Punycode. + PublicSuffix(domain string) string + + // String returns a description of the source of this public suffix + // list. The description will typically contain something like a time + // stamp or version number. + String() string +} + +// Options are the options for creating a new Jar. +type Options struct { + // PublicSuffixList is the public suffix list that determines whether + // an HTTP server can set a cookie for a domain. + // + // A nil value is valid and may be useful for testing but it is not + // secure: it means that the HTTP server for foo.co.uk can set a cookie + // for bar.co.uk. + PublicSuffixList PublicSuffixList +} + +// Jar implements the http.CookieJar interface from the net/http package. +type Jar struct { + psList PublicSuffixList + + // mu locks the remaining fields. + mu sync.Mutex + + // entries is a set of entries, keyed by their eTLD+1 and subkeyed by + // their name/domain/path. + entries map[string]map[string]entry + + // nextSeqNum is the next sequence number assigned to a new cookie + // created SetCookies. + nextSeqNum uint64 +} + +// New returns a new cookie jar. A nil *Options is equivalent to a zero +// Options. +func New(o *Options) (*Jar, error) { + jar := &Jar{ + entries: make(map[string]map[string]entry), + } + if o != nil { + jar.psList = o.PublicSuffixList + } + return jar, nil +} + +// entry is the internal representation of a cookie. +// +// This struct type is not used outside of this package per se, but the exported +// fields are those of RFC 6265. +type entry struct { + Name string + Value string + Domain string + Path string + Secure bool + HttpOnly bool + Persistent bool + HostOnly bool + Expires time.Time + Creation time.Time + LastAccess time.Time + + // seqNum is a sequence number so that Cookies returns cookies in a + // deterministic order, even for cookies that have equal Path length and + // equal Creation time. This simplifies testing. + seqNum uint64 +} + +// Id returns the domain;path;name triple of e as an id. +func (e *entry) id() string { + return fmt.Sprintf("%s;%s;%s", e.Domain, e.Path, e.Name) +} + +// shouldSend determines whether e's cookie qualifies to be included in a +// request to host/path. It is the caller's responsibility to check if the +// cookie is expired. +func (e *entry) shouldSend(https bool, host, path string) bool { + return e.domainMatch(host) && e.pathMatch(path) && (https || !e.Secure) +} + +// domainMatch implements "domain-match" of RFC 6265 section 5.1.3. +func (e *entry) domainMatch(host string) bool { + if e.Domain == host { + return true + } + return !e.HostOnly && hasDotSuffix(host, e.Domain) +} + +// pathMatch implements "path-match" according to RFC 6265 section 5.1.4. +func (e *entry) pathMatch(requestPath string) bool { + if requestPath == e.Path { + return true + } + if strings.HasPrefix(requestPath, e.Path) { + if e.Path[len(e.Path)-1] == '/' { + return true // The "/any/" matches "/any/path" case. + } else if requestPath[len(e.Path)] == '/' { + return true // The "/any" matches "/any/path" case. + } + } + return false +} + +// hasDotSuffix reports whether s ends in "."+suffix. +func hasDotSuffix(s, suffix string) bool { + return len(s) > len(suffix) && s[len(s)-len(suffix)-1] == '.' && s[len(s)-len(suffix):] == suffix +} + +// byPathLength is a []entry sort.Interface that sorts according to RFC 6265 +// section 5.4 point 2: by longest path and then by earliest creation time. +type byPathLength []entry + +func (s byPathLength) Len() int { return len(s) } + +func (s byPathLength) Less(i, j int) bool { + if len(s[i].Path) != len(s[j].Path) { + return len(s[i].Path) > len(s[j].Path) + } + if !s[i].Creation.Equal(s[j].Creation) { + return s[i].Creation.Before(s[j].Creation) + } + return s[i].seqNum < s[j].seqNum +} + +func (s byPathLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Cookies implements the Cookies method of the http.CookieJar interface. +// +// It returns an empty slice if the URL's scheme is not HTTP or HTTPS. +func (j *Jar) Cookies(u *url.URL) (cookies []*http.Cookie) { + return j.cookies(u, time.Now()) +} + +// cookies is like Cookies but takes the current time as a parameter. +func (j *Jar) cookies(u *url.URL, now time.Time) (cookies []*http.Cookie) { + if u.Scheme != "http" && u.Scheme != "https" { + return cookies + } + host, err := canonicalHost(u.Host) + if err != nil { + return cookies + } + key := jarKey(host, j.psList) + + j.mu.Lock() + defer j.mu.Unlock() + + submap := j.entries[key] + if submap == nil { + return cookies + } + + https := u.Scheme == "https" + path := u.Path + if path == "" { + path = "/" + } + + modified := false + var selected []entry + for id, e := range submap { + if e.Persistent && !e.Expires.After(now) { + delete(submap, id) + modified = true + continue + } + if !e.shouldSend(https, host, path) { + continue + } + e.LastAccess = now + submap[id] = e + selected = append(selected, e) + modified = true + } + if modified { + if len(submap) == 0 { + delete(j.entries, key) + } else { + j.entries[key] = submap + } + } + + sort.Sort(byPathLength(selected)) + for _, e := range selected { + cookies = append(cookies, &http.Cookie{Name: e.Name, Value: e.Value}) + } + + return cookies +} + +// SetCookies implements the SetCookies method of the http.CookieJar interface. +// +// It does nothing if the URL's scheme is not HTTP or HTTPS. +func (j *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) { + j.setCookies(u, cookies, time.Now()) +} + +// setCookies is like SetCookies but takes the current time as parameter. +func (j *Jar) setCookies(u *url.URL, cookies []*http.Cookie, now time.Time) { + if len(cookies) == 0 { + return + } + if u.Scheme != "http" && u.Scheme != "https" { + return + } + host, err := canonicalHost(u.Host) + if err != nil { + return + } + key := jarKey(host, j.psList) + defPath := defaultPath(u.Path) + + j.mu.Lock() + defer j.mu.Unlock() + + submap := j.entries[key] + + modified := false + for _, cookie := range cookies { + e, remove, err := j.newEntry(cookie, now, defPath, host) + if err != nil { + continue + } + id := e.id() + if remove { + if submap != nil { + if _, ok := submap[id]; ok { + delete(submap, id) + modified = true + } + } + continue + } + if submap == nil { + submap = make(map[string]entry) + } + + if old, ok := submap[id]; ok { + e.Creation = old.Creation + e.seqNum = old.seqNum + } else { + e.Creation = now + e.seqNum = j.nextSeqNum + j.nextSeqNum++ + } + e.LastAccess = now + submap[id] = e + modified = true + } + + if modified { + if len(submap) == 0 { + delete(j.entries, key) + } else { + j.entries[key] = submap + } + } +} + +// canonicalHost strips port from host if present and returns the canonicalized +// host name. +func canonicalHost(host string) (string, error) { + var err error + host = strings.ToLower(host) + if hasPort(host) { + host, _, err = net.SplitHostPort(host) + if err != nil { + return "", err + } + } + if strings.HasSuffix(host, ".") { + // Strip trailing dot from fully qualified domain names. + host = host[:len(host)-1] + } + return toASCII(host) +} + +// hasPort reports whether host contains a port number. host may be a host +// name, an IPv4 or an IPv6 address. +func hasPort(host string) bool { + colons := strings.Count(host, ":") + if colons == 0 { + return false + } + if colons == 1 { + return true + } + return host[0] == '[' && strings.Contains(host, "]:") +} + +// jarKey returns the key to use for a jar. +func jarKey(host string, psl PublicSuffixList) string { + if isIP(host) { + return host + } + + var i int + if psl == nil { + i = strings.LastIndex(host, ".") + if i == -1 { + return host + } + } else { + suffix := psl.PublicSuffix(host) + if suffix == host { + return host + } + i = len(host) - len(suffix) + if i <= 0 || host[i-1] != '.' { + // The provided public suffix list psl is broken. + // Storing cookies under host is a safe stopgap. + return host + } + } + prevDot := strings.LastIndex(host[:i-1], ".") + return host[prevDot+1:] +} + +// isIP reports whether host is an IP address. +func isIP(host string) bool { + return net.ParseIP(host) != nil +} + +// defaultPath returns the directory part of an URL's path according to +// RFC 6265 section 5.1.4. +func defaultPath(path string) string { + if len(path) == 0 || path[0] != '/' { + return "/" // Path is empty or malformed. + } + + i := strings.LastIndex(path, "/") // Path starts with "/", so i != -1. + if i == 0 { + return "/" // Path has the form "/abc". + } + return path[:i] // Path is either of form "/abc/xyz" or "/abc/xyz/". +} + +// newEntry creates an entry from a http.Cookie c. now is the current time and +// is compared to c.Expires to determine deletion of c. defPath and host are the +// default-path and the canonical host name of the URL c was received from. +// +// remove records whether the jar should delete this cookie, as it has already +// expired with respect to now. In this case, e may be incomplete, but it will +// be valid to call e.id (which depends on e's Name, Domain and Path). +// +// A malformed c.Domain will result in an error. +func (j *Jar) newEntry(c *http.Cookie, now time.Time, defPath, host string) (e entry, remove bool, err error) { + e.Name = c.Name + + if c.Path == "" || c.Path[0] != '/' { + e.Path = defPath + } else { + e.Path = c.Path + } + + e.Domain, e.HostOnly, err = j.domainAndType(host, c.Domain) + if err != nil { + return e, false, err + } + + // MaxAge takes precedence over Expires. + if c.MaxAge < 0 { + return e, true, nil + } else if c.MaxAge > 0 { + e.Expires = now.Add(time.Duration(c.MaxAge) * time.Second) + e.Persistent = true + } else { + if c.Expires.IsZero() { + e.Expires = endOfTime + e.Persistent = false + } else { + if !c.Expires.After(now) { + return e, true, nil + } + e.Expires = c.Expires + e.Persistent = true + } + } + + e.Value = c.Value + e.Secure = c.Secure + e.HttpOnly = c.HttpOnly + + return e, false, nil +} + +var ( + errIllegalDomain = errors.New("cookiejar: illegal cookie domain attribute") + errMalformedDomain = errors.New("cookiejar: malformed cookie domain attribute") + errNoHostname = errors.New("cookiejar: no host name available (IP only)") +) + +// endOfTime is the time when session (non-persistent) cookies expire. +// This instant is representable in most date/time formats (not just +// Go's time.Time) and should be far enough in the future. +var endOfTime = time.Date(9999, 12, 31, 23, 59, 59, 0, time.UTC) + +// domainAndType determines the cookie's domain and hostOnly attribute. +func (j *Jar) domainAndType(host, domain string) (string, bool, error) { + if domain == "" { + // No domain attribute in the SetCookie header indicates a + // host cookie. + return host, true, nil + } + + if isIP(host) { + // According to RFC 6265 domain-matching includes not being + // an IP address. + // TODO: This might be relaxed as in common browsers. + return "", false, errNoHostname + } + + // From here on: If the cookie is valid, it is a domain cookie (with + // the one exception of a public suffix below). + // See RFC 6265 section 5.2.3. + if domain[0] == '.' { + domain = domain[1:] + } + + if len(domain) == 0 || domain[0] == '.' { + // Received either "Domain=." or "Domain=..some.thing", + // both are illegal. + return "", false, errMalformedDomain + } + domain = strings.ToLower(domain) + + if domain[len(domain)-1] == '.' { + // We received stuff like "Domain=www.example.com.". + // Browsers do handle such stuff (actually differently) but + // RFC 6265 seems to be clear here (e.g. section 4.1.2.3) in + // requiring a reject. 4.1.2.3 is not normative, but + // "Domain Matching" (5.1.3) and "Canonicalized Host Names" + // (5.1.2) are. + return "", false, errMalformedDomain + } + + // See RFC 6265 section 5.3 #5. + if j.psList != nil { + if ps := j.psList.PublicSuffix(domain); ps != "" && !hasDotSuffix(domain, ps) { + if host == domain { + // This is the one exception in which a cookie + // with a domain attribute is a host cookie. + return host, true, nil + } + return "", false, errIllegalDomain + } + } + + // The domain must domain-match host: www.mycompany.com cannot + // set cookies for .ourcompetitors.com. + if host != domain && !hasDotSuffix(host, domain) { + return "", false, errIllegalDomain + } + + return domain, false, nil +} diff --git a/src/net/http/cookiejar/jar_test.go b/src/net/http/cookiejar/jar_test.go new file mode 100644 index 000000000..3aa601586 --- /dev/null +++ b/src/net/http/cookiejar/jar_test.go @@ -0,0 +1,1267 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cookiejar + +import ( + "fmt" + "net/http" + "net/url" + "sort" + "strings" + "testing" + "time" +) + +// tNow is the synthetic current time used as now during testing. +var tNow = time.Date(2013, 1, 1, 12, 0, 0, 0, time.UTC) + +// testPSL implements PublicSuffixList with just two rules: "co.uk" +// and the default rule "*". +type testPSL struct{} + +func (testPSL) String() string { + return "testPSL" +} +func (testPSL) PublicSuffix(d string) string { + if d == "co.uk" || strings.HasSuffix(d, ".co.uk") { + return "co.uk" + } + return d[strings.LastIndex(d, ".")+1:] +} + +// newTestJar creates an empty Jar with testPSL as the public suffix list. +func newTestJar() *Jar { + jar, err := New(&Options{PublicSuffixList: testPSL{}}) + if err != nil { + panic(err) + } + return jar +} + +var hasDotSuffixTests = [...]struct { + s, suffix string +}{ + {"", ""}, + {"", "."}, + {"", "x"}, + {".", ""}, + {".", "."}, + {".", ".."}, + {".", "x"}, + {".", "x."}, + {".", ".x"}, + {".", ".x."}, + {"x", ""}, + {"x", "."}, + {"x", ".."}, + {"x", "x"}, + {"x", "x."}, + {"x", ".x"}, + {"x", ".x."}, + {".x", ""}, + {".x", "."}, + {".x", ".."}, + {".x", "x"}, + {".x", "x."}, + {".x", ".x"}, + {".x", ".x."}, + {"x.", ""}, + {"x.", "."}, + {"x.", ".."}, + {"x.", "x"}, + {"x.", "x."}, + {"x.", ".x"}, + {"x.", ".x."}, + {"com", ""}, + {"com", "m"}, + {"com", "om"}, + {"com", "com"}, + {"com", ".com"}, + {"com", "x.com"}, + {"com", "xcom"}, + {"com", "xorg"}, + {"com", "org"}, + {"com", "rg"}, + {"foo.com", ""}, + {"foo.com", "m"}, + {"foo.com", "om"}, + {"foo.com", "com"}, + {"foo.com", ".com"}, + {"foo.com", "o.com"}, + {"foo.com", "oo.com"}, + {"foo.com", "foo.com"}, + {"foo.com", ".foo.com"}, + {"foo.com", "x.foo.com"}, + {"foo.com", "xfoo.com"}, + {"foo.com", "xfoo.org"}, + {"foo.com", "foo.org"}, + {"foo.com", "oo.org"}, + {"foo.com", "o.org"}, + {"foo.com", ".org"}, + {"foo.com", "org"}, + {"foo.com", "rg"}, +} + +func TestHasDotSuffix(t *testing.T) { + for _, tc := range hasDotSuffixTests { + got := hasDotSuffix(tc.s, tc.suffix) + want := strings.HasSuffix(tc.s, "."+tc.suffix) + if got != want { + t.Errorf("s=%q, suffix=%q: got %v, want %v", tc.s, tc.suffix, got, want) + } + } +} + +var canonicalHostTests = map[string]string{ + "www.example.com": "www.example.com", + "WWW.EXAMPLE.COM": "www.example.com", + "wWw.eXAmple.CoM": "www.example.com", + "www.example.com:80": "www.example.com", + "192.168.0.10": "192.168.0.10", + "192.168.0.5:8080": "192.168.0.5", + "2001:4860:0:2001::68": "2001:4860:0:2001::68", + "[2001:4860:0:::68]:8080": "2001:4860:0:::68", + "www.bücher.de": "www.xn--bcher-kva.de", + "www.example.com.": "www.example.com", + "[bad.unmatched.bracket:": "error", +} + +func TestCanonicalHost(t *testing.T) { + for h, want := range canonicalHostTests { + got, err := canonicalHost(h) + if want == "error" { + if err == nil { + t.Errorf("%q: got nil error, want non-nil", h) + } + continue + } + if err != nil { + t.Errorf("%q: %v", h, err) + continue + } + if got != want { + t.Errorf("%q: got %q, want %q", h, got, want) + continue + } + } +} + +var hasPortTests = map[string]bool{ + "www.example.com": false, + "www.example.com:80": true, + "127.0.0.1": false, + "127.0.0.1:8080": true, + "2001:4860:0:2001::68": false, + "[2001::0:::68]:80": true, +} + +func TestHasPort(t *testing.T) { + for host, want := range hasPortTests { + if got := hasPort(host); got != want { + t.Errorf("%q: got %t, want %t", host, got, want) + } + } +} + +var jarKeyTests = map[string]string{ + "foo.www.example.com": "example.com", + "www.example.com": "example.com", + "example.com": "example.com", + "com": "com", + "foo.www.bbc.co.uk": "bbc.co.uk", + "www.bbc.co.uk": "bbc.co.uk", + "bbc.co.uk": "bbc.co.uk", + "co.uk": "co.uk", + "uk": "uk", + "192.168.0.5": "192.168.0.5", +} + +func TestJarKey(t *testing.T) { + for host, want := range jarKeyTests { + if got := jarKey(host, testPSL{}); got != want { + t.Errorf("%q: got %q, want %q", host, got, want) + } + } +} + +var jarKeyNilPSLTests = map[string]string{ + "foo.www.example.com": "example.com", + "www.example.com": "example.com", + "example.com": "example.com", + "com": "com", + "foo.www.bbc.co.uk": "co.uk", + "www.bbc.co.uk": "co.uk", + "bbc.co.uk": "co.uk", + "co.uk": "co.uk", + "uk": "uk", + "192.168.0.5": "192.168.0.5", +} + +func TestJarKeyNilPSL(t *testing.T) { + for host, want := range jarKeyNilPSLTests { + if got := jarKey(host, nil); got != want { + t.Errorf("%q: got %q, want %q", host, got, want) + } + } +} + +var isIPTests = map[string]bool{ + "127.0.0.1": true, + "1.2.3.4": true, + "2001:4860:0:2001::68": true, + "example.com": false, + "1.1.1.300": false, + "www.foo.bar.net": false, + "123.foo.bar.net": false, +} + +func TestIsIP(t *testing.T) { + for host, want := range isIPTests { + if got := isIP(host); got != want { + t.Errorf("%q: got %t, want %t", host, got, want) + } + } +} + +var defaultPathTests = map[string]string{ + "/": "/", + "/abc": "/", + "/abc/": "/abc", + "/abc/xyz": "/abc", + "/abc/xyz/": "/abc/xyz", + "/a/b/c.html": "/a/b", + "": "/", + "strange": "/", + "//": "/", + "/a//b": "/a/", + "/a/./b": "/a/.", + "/a/../b": "/a/..", +} + +func TestDefaultPath(t *testing.T) { + for path, want := range defaultPathTests { + if got := defaultPath(path); got != want { + t.Errorf("%q: got %q, want %q", path, got, want) + } + } +} + +var domainAndTypeTests = [...]struct { + host string // host Set-Cookie header was received from + domain string // domain attribute in Set-Cookie header + wantDomain string // expected domain of cookie + wantHostOnly bool // expected host-cookie flag + wantErr error // expected error +}{ + {"www.example.com", "", "www.example.com", true, nil}, + {"127.0.0.1", "", "127.0.0.1", true, nil}, + {"2001:4860:0:2001::68", "", "2001:4860:0:2001::68", true, nil}, + {"www.example.com", "example.com", "example.com", false, nil}, + {"www.example.com", ".example.com", "example.com", false, nil}, + {"www.example.com", "www.example.com", "www.example.com", false, nil}, + {"www.example.com", ".www.example.com", "www.example.com", false, nil}, + {"foo.sso.example.com", "sso.example.com", "sso.example.com", false, nil}, + {"bar.co.uk", "bar.co.uk", "bar.co.uk", false, nil}, + {"foo.bar.co.uk", ".bar.co.uk", "bar.co.uk", false, nil}, + {"127.0.0.1", "127.0.0.1", "", false, errNoHostname}, + {"2001:4860:0:2001::68", "2001:4860:0:2001::68", "2001:4860:0:2001::68", false, errNoHostname}, + {"www.example.com", ".", "", false, errMalformedDomain}, + {"www.example.com", "..", "", false, errMalformedDomain}, + {"www.example.com", "other.com", "", false, errIllegalDomain}, + {"www.example.com", "com", "", false, errIllegalDomain}, + {"www.example.com", ".com", "", false, errIllegalDomain}, + {"foo.bar.co.uk", ".co.uk", "", false, errIllegalDomain}, + {"127.www.0.0.1", "127.0.0.1", "", false, errIllegalDomain}, + {"com", "", "com", true, nil}, + {"com", "com", "com", true, nil}, + {"com", ".com", "com", true, nil}, + {"co.uk", "", "co.uk", true, nil}, + {"co.uk", "co.uk", "co.uk", true, nil}, + {"co.uk", ".co.uk", "co.uk", true, nil}, +} + +func TestDomainAndType(t *testing.T) { + jar := newTestJar() + for _, tc := range domainAndTypeTests { + domain, hostOnly, err := jar.domainAndType(tc.host, tc.domain) + if err != tc.wantErr { + t.Errorf("%q/%q: got %q error, want %q", + tc.host, tc.domain, err, tc.wantErr) + continue + } + if err != nil { + continue + } + if domain != tc.wantDomain || hostOnly != tc.wantHostOnly { + t.Errorf("%q/%q: got %q/%t want %q/%t", + tc.host, tc.domain, domain, hostOnly, + tc.wantDomain, tc.wantHostOnly) + } + } +} + +// expiresIn creates an expires attribute delta seconds from tNow. +func expiresIn(delta int) string { + t := tNow.Add(time.Duration(delta) * time.Second) + return "expires=" + t.Format(time.RFC1123) +} + +// mustParseURL parses s to an URL and panics on error. +func mustParseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil || u.Scheme == "" || u.Host == "" { + panic(fmt.Sprintf("Unable to parse URL %s.", s)) + } + return u +} + +// jarTest encapsulates the following actions on a jar: +// 1. Perform SetCookies with fromURL and the cookies from setCookies. +// (Done at time tNow + 0 ms.) +// 2. Check that the entries in the jar matches content. +// (Done at time tNow + 1001 ms.) +// 3. For each query in tests: Check that Cookies with toURL yields the +// cookies in want. +// (Query n done at tNow + (n+2)*1001 ms.) +type jarTest struct { + description string // The description of what this test is supposed to test + fromURL string // The full URL of the request from which Set-Cookie headers where received + setCookies []string // All the cookies received from fromURL + content string // The whole (non-expired) content of the jar + queries []query // Queries to test the Jar.Cookies method +} + +// query contains one test of the cookies returned from Jar.Cookies. +type query struct { + toURL string // the URL in the Cookies call + want string // the expected list of cookies (order matters) +} + +// run runs the jarTest. +func (test jarTest) run(t *testing.T, jar *Jar) { + now := tNow + + // Populate jar with cookies. + setCookies := make([]*http.Cookie, len(test.setCookies)) + for i, cs := range test.setCookies { + cookies := (&http.Response{Header: http.Header{"Set-Cookie": {cs}}}).Cookies() + if len(cookies) != 1 { + panic(fmt.Sprintf("Wrong cookie line %q: %#v", cs, cookies)) + } + setCookies[i] = cookies[0] + } + jar.setCookies(mustParseURL(test.fromURL), setCookies, now) + now = now.Add(1001 * time.Millisecond) + + // Serialize non-expired entries in the form "name1=val1 name2=val2". + var cs []string + for _, submap := range jar.entries { + for _, cookie := range submap { + if !cookie.Expires.After(now) { + continue + } + cs = append(cs, cookie.Name+"="+cookie.Value) + } + } + sort.Strings(cs) + got := strings.Join(cs, " ") + + // Make sure jar content matches our expectations. + if got != test.content { + t.Errorf("Test %q Content\ngot %q\nwant %q", + test.description, got, test.content) + } + + // Test different calls to Cookies. + for i, query := range test.queries { + now = now.Add(1001 * time.Millisecond) + var s []string + for _, c := range jar.cookies(mustParseURL(query.toURL), now) { + s = append(s, c.Name+"="+c.Value) + } + if got := strings.Join(s, " "); got != query.want { + t.Errorf("Test %q #%d\ngot %q\nwant %q", test.description, i, got, query.want) + } + } +} + +// basicsTests contains fundamental tests. Each jarTest has to be performed on +// a fresh, empty Jar. +var basicsTests = [...]jarTest{ + { + "Retrieval of a plain host cookie.", + "http://www.host.test/", + []string{"A=a"}, + "A=a", + []query{ + {"http://www.host.test", "A=a"}, + {"http://www.host.test/", "A=a"}, + {"http://www.host.test/some/path", "A=a"}, + {"https://www.host.test", "A=a"}, + {"https://www.host.test/", "A=a"}, + {"https://www.host.test/some/path", "A=a"}, + {"ftp://www.host.test", ""}, + {"ftp://www.host.test/", ""}, + {"ftp://www.host.test/some/path", ""}, + {"http://www.other.org", ""}, + {"http://sibling.host.test", ""}, + {"http://deep.www.host.test", ""}, + }, + }, + { + "Secure cookies are not returned to http.", + "http://www.host.test/", + []string{"A=a; secure"}, + "A=a", + []query{ + {"http://www.host.test", ""}, + {"http://www.host.test/", ""}, + {"http://www.host.test/some/path", ""}, + {"https://www.host.test", "A=a"}, + {"https://www.host.test/", "A=a"}, + {"https://www.host.test/some/path", "A=a"}, + }, + }, + { + "Explicit path.", + "http://www.host.test/", + []string{"A=a; path=/some/path"}, + "A=a", + []query{ + {"http://www.host.test", ""}, + {"http://www.host.test/", ""}, + {"http://www.host.test/some", ""}, + {"http://www.host.test/some/", ""}, + {"http://www.host.test/some/path", "A=a"}, + {"http://www.host.test/some/paths", ""}, + {"http://www.host.test/some/path/foo", "A=a"}, + {"http://www.host.test/some/path/foo/", "A=a"}, + }, + }, + { + "Implicit path #1: path is a directory.", + "http://www.host.test/some/path/", + []string{"A=a"}, + "A=a", + []query{ + {"http://www.host.test", ""}, + {"http://www.host.test/", ""}, + {"http://www.host.test/some", ""}, + {"http://www.host.test/some/", ""}, + {"http://www.host.test/some/path", "A=a"}, + {"http://www.host.test/some/paths", ""}, + {"http://www.host.test/some/path/foo", "A=a"}, + {"http://www.host.test/some/path/foo/", "A=a"}, + }, + }, + { + "Implicit path #2: path is not a directory.", + "http://www.host.test/some/path/index.html", + []string{"A=a"}, + "A=a", + []query{ + {"http://www.host.test", ""}, + {"http://www.host.test/", ""}, + {"http://www.host.test/some", ""}, + {"http://www.host.test/some/", ""}, + {"http://www.host.test/some/path", "A=a"}, + {"http://www.host.test/some/paths", ""}, + {"http://www.host.test/some/path/foo", "A=a"}, + {"http://www.host.test/some/path/foo/", "A=a"}, + }, + }, + { + "Implicit path #3: no path in URL at all.", + "http://www.host.test", + []string{"A=a"}, + "A=a", + []query{ + {"http://www.host.test", "A=a"}, + {"http://www.host.test/", "A=a"}, + {"http://www.host.test/some/path", "A=a"}, + }, + }, + { + "Cookies are sorted by path length.", + "http://www.host.test/", + []string{ + "A=a; path=/foo/bar", + "B=b; path=/foo/bar/baz/qux", + "C=c; path=/foo/bar/baz", + "D=d; path=/foo"}, + "A=a B=b C=c D=d", + []query{ + {"http://www.host.test/foo/bar/baz/qux", "B=b C=c A=a D=d"}, + {"http://www.host.test/foo/bar/baz/", "C=c A=a D=d"}, + {"http://www.host.test/foo/bar", "A=a D=d"}, + }, + }, + { + "Creation time determines sorting on same length paths.", + "http://www.host.test/", + []string{ + "A=a; path=/foo/bar", + "X=x; path=/foo/bar", + "Y=y; path=/foo/bar/baz/qux", + "B=b; path=/foo/bar/baz/qux", + "C=c; path=/foo/bar/baz", + "W=w; path=/foo/bar/baz", + "Z=z; path=/foo", + "D=d; path=/foo"}, + "A=a B=b C=c D=d W=w X=x Y=y Z=z", + []query{ + {"http://www.host.test/foo/bar/baz/qux", "Y=y B=b C=c W=w A=a X=x Z=z D=d"}, + {"http://www.host.test/foo/bar/baz/", "C=c W=w A=a X=x Z=z D=d"}, + {"http://www.host.test/foo/bar", "A=a X=x Z=z D=d"}, + }, + }, + { + "Sorting of same-name cookies.", + "http://www.host.test/", + []string{ + "A=1; path=/", + "A=2; path=/path", + "A=3; path=/quux", + "A=4; path=/path/foo", + "A=5; domain=.host.test; path=/path", + "A=6; domain=.host.test; path=/quux", + "A=7; domain=.host.test; path=/path/foo", + }, + "A=1 A=2 A=3 A=4 A=5 A=6 A=7", + []query{ + {"http://www.host.test/path", "A=2 A=5 A=1"}, + {"http://www.host.test/path/foo", "A=4 A=7 A=2 A=5 A=1"}, + }, + }, + { + "Disallow domain cookie on public suffix.", + "http://www.bbc.co.uk", + []string{ + "a=1", + "b=2; domain=co.uk", + }, + "a=1", + []query{{"http://www.bbc.co.uk", "a=1"}}, + }, + { + "Host cookie on IP.", + "http://192.168.0.10", + []string{"a=1"}, + "a=1", + []query{{"http://192.168.0.10", "a=1"}}, + }, + { + "Port is ignored #1.", + "http://www.host.test/", + []string{"a=1"}, + "a=1", + []query{ + {"http://www.host.test", "a=1"}, + {"http://www.host.test:8080/", "a=1"}, + }, + }, + { + "Port is ignored #2.", + "http://www.host.test:8080/", + []string{"a=1"}, + "a=1", + []query{ + {"http://www.host.test", "a=1"}, + {"http://www.host.test:8080/", "a=1"}, + {"http://www.host.test:1234/", "a=1"}, + }, + }, +} + +func TestBasics(t *testing.T) { + for _, test := range basicsTests { + jar := newTestJar() + test.run(t, jar) + } +} + +// updateAndDeleteTests contains jarTests which must be performed on the same +// Jar. +var updateAndDeleteTests = [...]jarTest{ + { + "Set initial cookies.", + "http://www.host.test", + []string{ + "a=1", + "b=2; secure", + "c=3; httponly", + "d=4; secure; httponly"}, + "a=1 b=2 c=3 d=4", + []query{ + {"http://www.host.test", "a=1 c=3"}, + {"https://www.host.test", "a=1 b=2 c=3 d=4"}, + }, + }, + { + "Update value via http.", + "http://www.host.test", + []string{ + "a=w", + "b=x; secure", + "c=y; httponly", + "d=z; secure; httponly"}, + "a=w b=x c=y d=z", + []query{ + {"http://www.host.test", "a=w c=y"}, + {"https://www.host.test", "a=w b=x c=y d=z"}, + }, + }, + { + "Clear Secure flag from a http.", + "http://www.host.test/", + []string{ + "b=xx", + "d=zz; httponly"}, + "a=w b=xx c=y d=zz", + []query{{"http://www.host.test", "a=w b=xx c=y d=zz"}}, + }, + { + "Delete all.", + "http://www.host.test/", + []string{ + "a=1; max-Age=-1", // delete via MaxAge + "b=2; " + expiresIn(-10), // delete via Expires + "c=2; max-age=-1; " + expiresIn(-10), // delete via both + "d=4; max-age=-1; " + expiresIn(10)}, // MaxAge takes precedence + "", + []query{{"http://www.host.test", ""}}, + }, + { + "Refill #1.", + "http://www.host.test", + []string{ + "A=1", + "A=2; path=/foo", + "A=3; domain=.host.test", + "A=4; path=/foo; domain=.host.test"}, + "A=1 A=2 A=3 A=4", + []query{{"http://www.host.test/foo", "A=2 A=4 A=1 A=3"}}, + }, + { + "Refill #2.", + "http://www.google.com", + []string{ + "A=6", + "A=7; path=/foo", + "A=8; domain=.google.com", + "A=9; path=/foo; domain=.google.com"}, + "A=1 A=2 A=3 A=4 A=6 A=7 A=8 A=9", + []query{ + {"http://www.host.test/foo", "A=2 A=4 A=1 A=3"}, + {"http://www.google.com/foo", "A=7 A=9 A=6 A=8"}, + }, + }, + { + "Delete A7.", + "http://www.google.com", + []string{"A=; path=/foo; max-age=-1"}, + "A=1 A=2 A=3 A=4 A=6 A=8 A=9", + []query{ + {"http://www.host.test/foo", "A=2 A=4 A=1 A=3"}, + {"http://www.google.com/foo", "A=9 A=6 A=8"}, + }, + }, + { + "Delete A4.", + "http://www.host.test", + []string{"A=; path=/foo; domain=host.test; max-age=-1"}, + "A=1 A=2 A=3 A=6 A=8 A=9", + []query{ + {"http://www.host.test/foo", "A=2 A=1 A=3"}, + {"http://www.google.com/foo", "A=9 A=6 A=8"}, + }, + }, + { + "Delete A6.", + "http://www.google.com", + []string{"A=; max-age=-1"}, + "A=1 A=2 A=3 A=8 A=9", + []query{ + {"http://www.host.test/foo", "A=2 A=1 A=3"}, + {"http://www.google.com/foo", "A=9 A=8"}, + }, + }, + { + "Delete A3.", + "http://www.host.test", + []string{"A=; domain=host.test; max-age=-1"}, + "A=1 A=2 A=8 A=9", + []query{ + {"http://www.host.test/foo", "A=2 A=1"}, + {"http://www.google.com/foo", "A=9 A=8"}, + }, + }, + { + "No cross-domain delete.", + "http://www.host.test", + []string{ + "A=; domain=google.com; max-age=-1", + "A=; path=/foo; domain=google.com; max-age=-1"}, + "A=1 A=2 A=8 A=9", + []query{ + {"http://www.host.test/foo", "A=2 A=1"}, + {"http://www.google.com/foo", "A=9 A=8"}, + }, + }, + { + "Delete A8 and A9.", + "http://www.google.com", + []string{ + "A=; domain=google.com; max-age=-1", + "A=; path=/foo; domain=google.com; max-age=-1"}, + "A=1 A=2", + []query{ + {"http://www.host.test/foo", "A=2 A=1"}, + {"http://www.google.com/foo", ""}, + }, + }, +} + +func TestUpdateAndDelete(t *testing.T) { + jar := newTestJar() + for _, test := range updateAndDeleteTests { + test.run(t, jar) + } +} + +func TestExpiration(t *testing.T) { + jar := newTestJar() + jarTest{ + "Expiration.", + "http://www.host.test", + []string{ + "a=1", + "b=2; max-age=3", + "c=3; " + expiresIn(3), + "d=4; max-age=5", + "e=5; " + expiresIn(5), + "f=6; max-age=100", + }, + "a=1 b=2 c=3 d=4 e=5 f=6", // executed at t0 + 1001 ms + []query{ + {"http://www.host.test", "a=1 b=2 c=3 d=4 e=5 f=6"}, // t0 + 2002 ms + {"http://www.host.test", "a=1 d=4 e=5 f=6"}, // t0 + 3003 ms + {"http://www.host.test", "a=1 d=4 e=5 f=6"}, // t0 + 4004 ms + {"http://www.host.test", "a=1 f=6"}, // t0 + 5005 ms + {"http://www.host.test", "a=1 f=6"}, // t0 + 6006 ms + }, + }.run(t, jar) +} + +// +// Tests derived from Chromium's cookie_store_unittest.h. +// + +// See http://src.chromium.org/viewvc/chrome/trunk/src/net/cookies/cookie_store_unittest.h?revision=159685&content-type=text/plain +// Some of the original tests are in a bad condition (e.g. +// DomainWithTrailingDotTest) or are not RFC 6265 conforming (e.g. +// TestNonDottedAndTLD #1 and #6) and have not been ported. + +// chromiumBasicsTests contains fundamental tests. Each jarTest has to be +// performed on a fresh, empty Jar. +var chromiumBasicsTests = [...]jarTest{ + { + "DomainWithTrailingDotTest.", + "http://www.google.com/", + []string{ + "a=1; domain=.www.google.com.", + "b=2; domain=.www.google.com.."}, + "", + []query{ + {"http://www.google.com", ""}, + }, + }, + { + "ValidSubdomainTest #1.", + "http://a.b.c.d.com", + []string{ + "a=1; domain=.a.b.c.d.com", + "b=2; domain=.b.c.d.com", + "c=3; domain=.c.d.com", + "d=4; domain=.d.com"}, + "a=1 b=2 c=3 d=4", + []query{ + {"http://a.b.c.d.com", "a=1 b=2 c=3 d=4"}, + {"http://b.c.d.com", "b=2 c=3 d=4"}, + {"http://c.d.com", "c=3 d=4"}, + {"http://d.com", "d=4"}, + }, + }, + { + "ValidSubdomainTest #2.", + "http://a.b.c.d.com", + []string{ + "a=1; domain=.a.b.c.d.com", + "b=2; domain=.b.c.d.com", + "c=3; domain=.c.d.com", + "d=4; domain=.d.com", + "X=bcd; domain=.b.c.d.com", + "X=cd; domain=.c.d.com"}, + "X=bcd X=cd a=1 b=2 c=3 d=4", + []query{ + {"http://b.c.d.com", "b=2 c=3 d=4 X=bcd X=cd"}, + {"http://c.d.com", "c=3 d=4 X=cd"}, + }, + }, + { + "InvalidDomainTest #1.", + "http://foo.bar.com", + []string{ + "a=1; domain=.yo.foo.bar.com", + "b=2; domain=.foo.com", + "c=3; domain=.bar.foo.com", + "d=4; domain=.foo.bar.com.net", + "e=5; domain=ar.com", + "f=6; domain=.", + "g=7; domain=/", + "h=8; domain=http://foo.bar.com", + "i=9; domain=..foo.bar.com", + "j=10; domain=..bar.com", + "k=11; domain=.foo.bar.com?blah", + "l=12; domain=.foo.bar.com/blah", + "m=12; domain=.foo.bar.com:80", + "n=14; domain=.foo.bar.com:", + "o=15; domain=.foo.bar.com#sup", + }, + "", // Jar is empty. + []query{{"http://foo.bar.com", ""}}, + }, + { + "InvalidDomainTest #2.", + "http://foo.com.com", + []string{"a=1; domain=.foo.com.com.com"}, + "", + []query{{"http://foo.bar.com", ""}}, + }, + { + "DomainWithoutLeadingDotTest #1.", + "http://manage.hosted.filefront.com", + []string{"a=1; domain=filefront.com"}, + "a=1", + []query{{"http://www.filefront.com", "a=1"}}, + }, + { + "DomainWithoutLeadingDotTest #2.", + "http://www.google.com", + []string{"a=1; domain=www.google.com"}, + "a=1", + []query{ + {"http://www.google.com", "a=1"}, + {"http://sub.www.google.com", "a=1"}, + {"http://something-else.com", ""}, + }, + }, + { + "CaseInsensitiveDomainTest.", + "http://www.google.com", + []string{ + "a=1; domain=.GOOGLE.COM", + "b=2; domain=.www.gOOgLE.coM"}, + "a=1 b=2", + []query{{"http://www.google.com", "a=1 b=2"}}, + }, + { + "TestIpAddress #1.", + "http://1.2.3.4/foo", + []string{"a=1; path=/"}, + "a=1", + []query{{"http://1.2.3.4/foo", "a=1"}}, + }, + { + "TestIpAddress #2.", + "http://1.2.3.4/foo", + []string{ + "a=1; domain=.1.2.3.4", + "b=2; domain=.3.4"}, + "", + []query{{"http://1.2.3.4/foo", ""}}, + }, + { + "TestIpAddress #3.", + "http://1.2.3.4/foo", + []string{"a=1; domain=1.2.3.4"}, + "", + []query{{"http://1.2.3.4/foo", ""}}, + }, + { + "TestNonDottedAndTLD #2.", + "http://com./index.html", + []string{"a=1"}, + "a=1", + []query{ + {"http://com./index.html", "a=1"}, + {"http://no-cookies.com./index.html", ""}, + }, + }, + { + "TestNonDottedAndTLD #3.", + "http://a.b", + []string{ + "a=1; domain=.b", + "b=2; domain=b"}, + "", + []query{{"http://bar.foo", ""}}, + }, + { + "TestNonDottedAndTLD #4.", + "http://google.com", + []string{ + "a=1; domain=.com", + "b=2; domain=com"}, + "", + []query{{"http://google.com", ""}}, + }, + { + "TestNonDottedAndTLD #5.", + "http://google.co.uk", + []string{ + "a=1; domain=.co.uk", + "b=2; domain=.uk"}, + "", + []query{ + {"http://google.co.uk", ""}, + {"http://else.co.com", ""}, + {"http://else.uk", ""}, + }, + }, + { + "TestHostEndsWithDot.", + "http://www.google.com", + []string{ + "a=1", + "b=2; domain=.www.google.com."}, + "a=1", + []query{{"http://www.google.com", "a=1"}}, + }, + { + "PathTest", + "http://www.google.izzle", + []string{"a=1; path=/wee"}, + "a=1", + []query{ + {"http://www.google.izzle/wee", "a=1"}, + {"http://www.google.izzle/wee/", "a=1"}, + {"http://www.google.izzle/wee/war", "a=1"}, + {"http://www.google.izzle/wee/war/more/more", "a=1"}, + {"http://www.google.izzle/weehee", ""}, + {"http://www.google.izzle/", ""}, + }, + }, +} + +func TestChromiumBasics(t *testing.T) { + for _, test := range chromiumBasicsTests { + jar := newTestJar() + test.run(t, jar) + } +} + +// chromiumDomainTests contains jarTests which must be executed all on the +// same Jar. +var chromiumDomainTests = [...]jarTest{ + { + "Fill #1.", + "http://www.google.izzle", + []string{"A=B"}, + "A=B", + []query{{"http://www.google.izzle", "A=B"}}, + }, + { + "Fill #2.", + "http://www.google.izzle", + []string{"C=D; domain=.google.izzle"}, + "A=B C=D", + []query{{"http://www.google.izzle", "A=B C=D"}}, + }, + { + "Verify A is a host cookie and not accessible from subdomain.", + "http://unused.nil", + []string{}, + "A=B C=D", + []query{{"http://foo.www.google.izzle", "C=D"}}, + }, + { + "Verify domain cookies are found on proper domain.", + "http://www.google.izzle", + []string{"E=F; domain=.www.google.izzle"}, + "A=B C=D E=F", + []query{{"http://www.google.izzle", "A=B C=D E=F"}}, + }, + { + "Leading dots in domain attributes are optional.", + "http://www.google.izzle", + []string{"G=H; domain=www.google.izzle"}, + "A=B C=D E=F G=H", + []query{{"http://www.google.izzle", "A=B C=D E=F G=H"}}, + }, + { + "Verify domain enforcement works #1.", + "http://www.google.izzle", + []string{"K=L; domain=.bar.www.google.izzle"}, + "A=B C=D E=F G=H", + []query{{"http://bar.www.google.izzle", "C=D E=F G=H"}}, + }, + { + "Verify domain enforcement works #2.", + "http://unused.nil", + []string{}, + "A=B C=D E=F G=H", + []query{{"http://www.google.izzle", "A=B C=D E=F G=H"}}, + }, +} + +func TestChromiumDomain(t *testing.T) { + jar := newTestJar() + for _, test := range chromiumDomainTests { + test.run(t, jar) + } + +} + +// chromiumDeletionTests must be performed all on the same Jar. +var chromiumDeletionTests = [...]jarTest{ + { + "Create session cookie a1.", + "http://www.google.com", + []string{"a=1"}, + "a=1", + []query{{"http://www.google.com", "a=1"}}, + }, + { + "Delete sc a1 via MaxAge.", + "http://www.google.com", + []string{"a=1; max-age=-1"}, + "", + []query{{"http://www.google.com", ""}}, + }, + { + "Create session cookie b2.", + "http://www.google.com", + []string{"b=2"}, + "b=2", + []query{{"http://www.google.com", "b=2"}}, + }, + { + "Delete sc b2 via Expires.", + "http://www.google.com", + []string{"b=2; " + expiresIn(-10)}, + "", + []query{{"http://www.google.com", ""}}, + }, + { + "Create persistent cookie c3.", + "http://www.google.com", + []string{"c=3; max-age=3600"}, + "c=3", + []query{{"http://www.google.com", "c=3"}}, + }, + { + "Delete pc c3 via MaxAge.", + "http://www.google.com", + []string{"c=3; max-age=-1"}, + "", + []query{{"http://www.google.com", ""}}, + }, + { + "Create persistent cookie d4.", + "http://www.google.com", + []string{"d=4; max-age=3600"}, + "d=4", + []query{{"http://www.google.com", "d=4"}}, + }, + { + "Delete pc d4 via Expires.", + "http://www.google.com", + []string{"d=4; " + expiresIn(-10)}, + "", + []query{{"http://www.google.com", ""}}, + }, +} + +func TestChromiumDeletion(t *testing.T) { + jar := newTestJar() + for _, test := range chromiumDeletionTests { + test.run(t, jar) + } +} + +// domainHandlingTests tests and documents the rules for domain handling. +// Each test must be performed on an empty new Jar. +var domainHandlingTests = [...]jarTest{ + { + "Host cookie", + "http://www.host.test", + []string{"a=1"}, + "a=1", + []query{ + {"http://www.host.test", "a=1"}, + {"http://host.test", ""}, + {"http://bar.host.test", ""}, + {"http://foo.www.host.test", ""}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Domain cookie #1", + "http://www.host.test", + []string{"a=1; domain=host.test"}, + "a=1", + []query{ + {"http://www.host.test", "a=1"}, + {"http://host.test", "a=1"}, + {"http://bar.host.test", "a=1"}, + {"http://foo.www.host.test", "a=1"}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Domain cookie #2", + "http://www.host.test", + []string{"a=1; domain=.host.test"}, + "a=1", + []query{ + {"http://www.host.test", "a=1"}, + {"http://host.test", "a=1"}, + {"http://bar.host.test", "a=1"}, + {"http://foo.www.host.test", "a=1"}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Host cookie on IDNA domain #1", + "http://www.bücher.test", + []string{"a=1"}, + "a=1", + []query{ + {"http://www.bücher.test", "a=1"}, + {"http://www.xn--bcher-kva.test", "a=1"}, + {"http://bücher.test", ""}, + {"http://xn--bcher-kva.test", ""}, + {"http://bar.bücher.test", ""}, + {"http://bar.xn--bcher-kva.test", ""}, + {"http://foo.www.bücher.test", ""}, + {"http://foo.www.xn--bcher-kva.test", ""}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Host cookie on IDNA domain #2", + "http://www.xn--bcher-kva.test", + []string{"a=1"}, + "a=1", + []query{ + {"http://www.bücher.test", "a=1"}, + {"http://www.xn--bcher-kva.test", "a=1"}, + {"http://bücher.test", ""}, + {"http://xn--bcher-kva.test", ""}, + {"http://bar.bücher.test", ""}, + {"http://bar.xn--bcher-kva.test", ""}, + {"http://foo.www.bücher.test", ""}, + {"http://foo.www.xn--bcher-kva.test", ""}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Domain cookie on IDNA domain #1", + "http://www.bücher.test", + []string{"a=1; domain=xn--bcher-kva.test"}, + "a=1", + []query{ + {"http://www.bücher.test", "a=1"}, + {"http://www.xn--bcher-kva.test", "a=1"}, + {"http://bücher.test", "a=1"}, + {"http://xn--bcher-kva.test", "a=1"}, + {"http://bar.bücher.test", "a=1"}, + {"http://bar.xn--bcher-kva.test", "a=1"}, + {"http://foo.www.bücher.test", "a=1"}, + {"http://foo.www.xn--bcher-kva.test", "a=1"}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Domain cookie on IDNA domain #2", + "http://www.xn--bcher-kva.test", + []string{"a=1; domain=xn--bcher-kva.test"}, + "a=1", + []query{ + {"http://www.bücher.test", "a=1"}, + {"http://www.xn--bcher-kva.test", "a=1"}, + {"http://bücher.test", "a=1"}, + {"http://xn--bcher-kva.test", "a=1"}, + {"http://bar.bücher.test", "a=1"}, + {"http://bar.xn--bcher-kva.test", "a=1"}, + {"http://foo.www.bücher.test", "a=1"}, + {"http://foo.www.xn--bcher-kva.test", "a=1"}, + {"http://other.test", ""}, + {"http://test", ""}, + }, + }, + { + "Host cookie on TLD.", + "http://com", + []string{"a=1"}, + "a=1", + []query{ + {"http://com", "a=1"}, + {"http://any.com", ""}, + {"http://any.test", ""}, + }, + }, + { + "Domain cookie on TLD becomes a host cookie.", + "http://com", + []string{"a=1; domain=com"}, + "a=1", + []query{ + {"http://com", "a=1"}, + {"http://any.com", ""}, + {"http://any.test", ""}, + }, + }, + { + "Host cookie on public suffix.", + "http://co.uk", + []string{"a=1"}, + "a=1", + []query{ + {"http://co.uk", "a=1"}, + {"http://uk", ""}, + {"http://some.co.uk", ""}, + {"http://foo.some.co.uk", ""}, + {"http://any.uk", ""}, + }, + }, + { + "Domain cookie on public suffix is ignored.", + "http://some.co.uk", + []string{"a=1; domain=co.uk"}, + "", + []query{ + {"http://co.uk", ""}, + {"http://uk", ""}, + {"http://some.co.uk", ""}, + {"http://foo.some.co.uk", ""}, + {"http://any.uk", ""}, + }, + }, +} + +func TestDomainHandling(t *testing.T) { + for _, test := range domainHandlingTests { + jar := newTestJar() + test.run(t, jar) + } +} diff --git a/src/net/http/cookiejar/punycode.go b/src/net/http/cookiejar/punycode.go new file mode 100644 index 000000000..ea7ceb5ef --- /dev/null +++ b/src/net/http/cookiejar/punycode.go @@ -0,0 +1,159 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cookiejar + +// This file implements the Punycode algorithm from RFC 3492. + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// These parameter values are specified in section 5. +// +// All computation is done with int32s, so that overflow behavior is identical +// regardless of whether int is 32-bit or 64-bit. +const ( + base int32 = 36 + damp int32 = 700 + initialBias int32 = 72 + initialN int32 = 128 + skew int32 = 38 + tmax int32 = 26 + tmin int32 = 1 +) + +// encode encodes a string as specified in section 6.3 and prepends prefix to +// the result. +// +// The "while h < length(input)" line in the specification becomes "for +// remaining != 0" in the Go code, because len(s) in Go is in bytes, not runes. +func encode(prefix, s string) (string, error) { + output := make([]byte, len(prefix), len(prefix)+1+2*len(s)) + copy(output, prefix) + delta, n, bias := int32(0), initialN, initialBias + b, remaining := int32(0), int32(0) + for _, r := range s { + if r < 0x80 { + b++ + output = append(output, byte(r)) + } else { + remaining++ + } + } + h := b + if b > 0 { + output = append(output, '-') + } + for remaining != 0 { + m := int32(0x7fffffff) + for _, r := range s { + if m > r && r >= n { + m = r + } + } + delta += (m - n) * (h + 1) + if delta < 0 { + return "", fmt.Errorf("cookiejar: invalid label %q", s) + } + n = m + for _, r := range s { + if r < n { + delta++ + if delta < 0 { + return "", fmt.Errorf("cookiejar: invalid label %q", s) + } + continue + } + if r > n { + continue + } + q := delta + for k := base; ; k += base { + t := k - bias + if t < tmin { + t = tmin + } else if t > tmax { + t = tmax + } + if q < t { + break + } + output = append(output, encodeDigit(t+(q-t)%(base-t))) + q = (q - t) / (base - t) + } + output = append(output, encodeDigit(q)) + bias = adapt(delta, h+1, h == b) + delta = 0 + h++ + remaining-- + } + delta++ + n++ + } + return string(output), nil +} + +func encodeDigit(digit int32) byte { + switch { + case 0 <= digit && digit < 26: + return byte(digit + 'a') + case 26 <= digit && digit < 36: + return byte(digit + ('0' - 26)) + } + panic("cookiejar: internal error in punycode encoding") +} + +// adapt is the bias adaptation function specified in section 6.1. +func adapt(delta, numPoints int32, firstTime bool) int32 { + if firstTime { + delta /= damp + } else { + delta /= 2 + } + delta += delta / numPoints + k := int32(0) + for delta > ((base-tmin)*tmax)/2 { + delta /= base - tmin + k += base + } + return k + (base-tmin+1)*delta/(delta+skew) +} + +// Strictly speaking, the remaining code below deals with IDNA (RFC 5890 and +// friends) and not Punycode (RFC 3492) per se. + +// acePrefix is the ASCII Compatible Encoding prefix. +const acePrefix = "xn--" + +// toASCII converts a domain or domain label to its ASCII form. For example, +// toASCII("bücher.example.com") is "xn--bcher-kva.example.com", and +// toASCII("golang") is "golang". +func toASCII(s string) (string, error) { + if ascii(s) { + return s, nil + } + labels := strings.Split(s, ".") + for i, label := range labels { + if !ascii(label) { + a, err := encode(acePrefix, label) + if err != nil { + return "", err + } + labels[i] = a + } + } + return strings.Join(labels, "."), nil +} + +func ascii(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} diff --git a/src/net/http/cookiejar/punycode_test.go b/src/net/http/cookiejar/punycode_test.go new file mode 100644 index 000000000..0301de14e --- /dev/null +++ b/src/net/http/cookiejar/punycode_test.go @@ -0,0 +1,161 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cookiejar + +import ( + "testing" +) + +var punycodeTestCases = [...]struct { + s, encoded string +}{ + {"", ""}, + {"-", "--"}, + {"-a", "-a-"}, + {"-a-", "-a--"}, + {"a", "a-"}, + {"a-", "a--"}, + {"a-b", "a-b-"}, + {"books", "books-"}, + {"bücher", "bcher-kva"}, + {"Hello世界", "Hello-ck1hg65u"}, + {"ü", "tda"}, + {"üý", "tdac"}, + + // The test cases below come from RFC 3492 section 7.1 with Errata 3026. + { + // (A) Arabic (Egyptian). + "\u0644\u064A\u0647\u0645\u0627\u0628\u062A\u0643\u0644" + + "\u0645\u0648\u0634\u0639\u0631\u0628\u064A\u061F", + "egbpdaj6bu4bxfgehfvwxn", + }, + { + // (B) Chinese (simplified). + "\u4ED6\u4EEC\u4E3A\u4EC0\u4E48\u4E0D\u8BF4\u4E2D\u6587", + "ihqwcrb4cv8a8dqg056pqjye", + }, + { + // (C) Chinese (traditional). + "\u4ED6\u5011\u7232\u4EC0\u9EBD\u4E0D\u8AAA\u4E2D\u6587", + "ihqwctvzc91f659drss3x8bo0yb", + }, + { + // (D) Czech. + "\u0050\u0072\u006F\u010D\u0070\u0072\u006F\u0073\u0074" + + "\u011B\u006E\u0065\u006D\u006C\u0075\u0076\u00ED\u010D" + + "\u0065\u0073\u006B\u0079", + "Proprostnemluvesky-uyb24dma41a", + }, + { + // (E) Hebrew. + "\u05DC\u05DE\u05D4\u05D4\u05DD\u05E4\u05E9\u05D5\u05D8" + + "\u05DC\u05D0\u05DE\u05D3\u05D1\u05E8\u05D9\u05DD\u05E2" + + "\u05D1\u05E8\u05D9\u05EA", + "4dbcagdahymbxekheh6e0a7fei0b", + }, + { + // (F) Hindi (Devanagari). + "\u092F\u0939\u0932\u094B\u0917\u0939\u093F\u0928\u094D" + + "\u0926\u0940\u0915\u094D\u092F\u094B\u0902\u0928\u0939" + + "\u0940\u0902\u092C\u094B\u0932\u0938\u0915\u0924\u0947" + + "\u0939\u0948\u0902", + "i1baa7eci9glrd9b2ae1bj0hfcgg6iyaf8o0a1dig0cd", + }, + { + // (G) Japanese (kanji and hiragana). + "\u306A\u305C\u307F\u3093\u306A\u65E5\u672C\u8A9E\u3092" + + "\u8A71\u3057\u3066\u304F\u308C\u306A\u3044\u306E\u304B", + "n8jok5ay5dzabd5bym9f0cm5685rrjetr6pdxa", + }, + { + // (H) Korean (Hangul syllables). + "\uC138\uACC4\uC758\uBAA8\uB4E0\uC0AC\uB78C\uB4E4\uC774" + + "\uD55C\uAD6D\uC5B4\uB97C\uC774\uD574\uD55C\uB2E4\uBA74" + + "\uC5BC\uB9C8\uB098\uC88B\uC744\uAE4C", + "989aomsvi5e83db1d2a355cv1e0vak1dwrv93d5xbh15a0dt30a5j" + + "psd879ccm6fea98c", + }, + { + // (I) Russian (Cyrillic). + "\u043F\u043E\u0447\u0435\u043C\u0443\u0436\u0435\u043E" + + "\u043D\u0438\u043D\u0435\u0433\u043E\u0432\u043E\u0440" + + "\u044F\u0442\u043F\u043E\u0440\u0443\u0441\u0441\u043A" + + "\u0438", + "b1abfaaepdrnnbgefbadotcwatmq2g4l", + }, + { + // (J) Spanish. + "\u0050\u006F\u0072\u0071\u0075\u00E9\u006E\u006F\u0070" + + "\u0075\u0065\u0064\u0065\u006E\u0073\u0069\u006D\u0070" + + "\u006C\u0065\u006D\u0065\u006E\u0074\u0065\u0068\u0061" + + "\u0062\u006C\u0061\u0072\u0065\u006E\u0045\u0073\u0070" + + "\u0061\u00F1\u006F\u006C", + "PorqunopuedensimplementehablarenEspaol-fmd56a", + }, + { + // (K) Vietnamese. + "\u0054\u1EA1\u0069\u0073\u0061\u006F\u0068\u1ECD\u006B" + + "\u0068\u00F4\u006E\u0067\u0074\u0068\u1EC3\u0063\u0068" + + "\u1EC9\u006E\u00F3\u0069\u0074\u0069\u1EBF\u006E\u0067" + + "\u0056\u0069\u1EC7\u0074", + "TisaohkhngthchnitingVit-kjcr8268qyxafd2f1b9g", + }, + { + // (L) 3<nen>B<gumi><kinpachi><sensei>. + "\u0033\u5E74\u0042\u7D44\u91D1\u516B\u5148\u751F", + "3B-ww4c5e180e575a65lsy2b", + }, + { + // (M) <amuro><namie>-with-SUPER-MONKEYS. + "\u5B89\u5BA4\u5948\u7F8E\u6075\u002D\u0077\u0069\u0074" + + "\u0068\u002D\u0053\u0055\u0050\u0045\u0052\u002D\u004D" + + "\u004F\u004E\u004B\u0045\u0059\u0053", + "-with-SUPER-MONKEYS-pc58ag80a8qai00g7n9n", + }, + { + // (N) Hello-Another-Way-<sorezore><no><basho>. + "\u0048\u0065\u006C\u006C\u006F\u002D\u0041\u006E\u006F" + + "\u0074\u0068\u0065\u0072\u002D\u0057\u0061\u0079\u002D" + + "\u305D\u308C\u305E\u308C\u306E\u5834\u6240", + "Hello-Another-Way--fc4qua05auwb3674vfr0b", + }, + { + // (O) <hitotsu><yane><no><shita>2. + "\u3072\u3068\u3064\u5C4B\u6839\u306E\u4E0B\u0032", + "2-u9tlzr9756bt3uc0v", + }, + { + // (P) Maji<de>Koi<suru>5<byou><mae> + "\u004D\u0061\u006A\u0069\u3067\u004B\u006F\u0069\u3059" + + "\u308B\u0035\u79D2\u524D", + "MajiKoi5-783gue6qz075azm5e", + }, + { + // (Q) <pafii>de<runba> + "\u30D1\u30D5\u30A3\u30FC\u0064\u0065\u30EB\u30F3\u30D0", + "de-jg4avhby1noc0d", + }, + { + // (R) <sono><supiido><de> + "\u305D\u306E\u30B9\u30D4\u30FC\u30C9\u3067", + "d9juau41awczczp", + }, + { + // (S) -> $1.00 <- + "\u002D\u003E\u0020\u0024\u0031\u002E\u0030\u0030\u0020" + + "\u003C\u002D", + "-> $1.00 <--", + }, +} + +func TestPunycode(t *testing.T) { + for _, tc := range punycodeTestCases { + if got, err := encode("", tc.s); err != nil { + t.Errorf(`encode("", %q): %v`, tc.s, err) + } else if got != tc.encoded { + t.Errorf(`encode("", %q): got %q, want %q`, tc.s, got, tc.encoded) + } + } +} diff --git a/src/net/http/doc.go b/src/net/http/doc.go new file mode 100644 index 000000000..b1216e8da --- /dev/null +++ b/src/net/http/doc.go @@ -0,0 +1,80 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package http provides HTTP client and server implementations. + +Get, Head, Post, and PostForm make HTTP (or HTTPS) requests: + + resp, err := http.Get("http://example.com/") + ... + resp, err := http.Post("http://example.com/upload", "image/jpeg", &buf) + ... + resp, err := http.PostForm("http://example.com/form", + url.Values{"key": {"Value"}, "id": {"123"}}) + +The client must close the response body when finished with it: + + resp, err := http.Get("http://example.com/") + if err != nil { + // handle error + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + // ... + +For control over HTTP client headers, redirect policy, and other +settings, create a Client: + + client := &http.Client{ + CheckRedirect: redirectPolicyFunc, + } + + resp, err := client.Get("http://example.com") + // ... + + req, err := http.NewRequest("GET", "http://example.com", nil) + // ... + req.Header.Add("If-None-Match", `W/"wyzzy"`) + resp, err := client.Do(req) + // ... + +For control over proxies, TLS configuration, keep-alives, +compression, and other settings, create a Transport: + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: pool}, + DisableCompression: true, + } + client := &http.Client{Transport: tr} + resp, err := client.Get("https://example.com") + +Clients and Transports are safe for concurrent use by multiple +goroutines and for efficiency should only be created once and re-used. + +ListenAndServe starts an HTTP server with a given address and handler. +The handler is usually nil, which means to use DefaultServeMux. +Handle and HandleFunc add handlers to DefaultServeMux: + + http.Handle("/foo", fooHandler) + + http.HandleFunc("/bar", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, %q", html.EscapeString(r.URL.Path)) + }) + + log.Fatal(http.ListenAndServe(":8080", nil)) + +More control over the server's behavior is available by creating a +custom Server: + + s := &http.Server{ + Addr: ":8080", + Handler: myHandler, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + MaxHeaderBytes: 1 << 20, + } + log.Fatal(s.ListenAndServe()) +*/ +package http diff --git a/src/net/http/example_test.go b/src/net/http/example_test.go new file mode 100644 index 000000000..88b97d9e3 --- /dev/null +++ b/src/net/http/example_test.go @@ -0,0 +1,88 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" +) + +func ExampleHijacker() { + http.HandleFunc("/hijack", func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "webserver doesn't support hijacking", http.StatusInternalServerError) + return + } + conn, bufrw, err := hj.Hijack() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // Don't forget to close the connection: + defer conn.Close() + bufrw.WriteString("Now we're speaking raw TCP. Say hi: ") + bufrw.Flush() + s, err := bufrw.ReadString('\n') + if err != nil { + log.Printf("error reading string: %v", err) + return + } + fmt.Fprintf(bufrw, "You said: %q\nBye.\n", s) + bufrw.Flush() + }) +} + +func ExampleGet() { + res, err := http.Get("http://www.google.com/robots.txt") + if err != nil { + log.Fatal(err) + } + robots, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + fmt.Printf("%s", robots) +} + +func ExampleFileServer() { + // Simple static webserver: + log.Fatal(http.ListenAndServe(":8080", http.FileServer(http.Dir("/usr/share/doc")))) +} + +func ExampleFileServer_stripPrefix() { + // To serve a directory on disk (/tmp) under an alternate URL + // path (/tmpfiles/), use StripPrefix to modify the request + // URL's path before the FileServer sees it: + http.Handle("/tmpfiles/", http.StripPrefix("/tmpfiles/", http.FileServer(http.Dir("/tmp")))) +} + +func ExampleStripPrefix() { + // To serve a directory on disk (/tmp) under an alternate URL + // path (/tmpfiles/), use StripPrefix to modify the request + // URL's path before the FileServer sees it: + http.Handle("/tmpfiles/", http.StripPrefix("/tmpfiles/", http.FileServer(http.Dir("/tmp")))) +} + +type apiHandler struct{} + +func (apiHandler) ServeHTTP(http.ResponseWriter, *http.Request) {} + +func ExampleServeMux_Handle() { + mux := http.NewServeMux() + mux.Handle("/api/", apiHandler{}) + mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + // The "/" pattern matches everything, so we need to check + // that we're at the root here. + if req.URL.Path != "/" { + http.NotFound(w, req) + return + } + fmt.Fprintf(w, "Welcome to the home page!") + }) +} diff --git a/src/net/http/export_test.go b/src/net/http/export_test.go new file mode 100644 index 000000000..2c8735355 --- /dev/null +++ b/src/net/http/export_test.go @@ -0,0 +1,78 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Bridge package to expose http internals to tests in the http_test +// package. + +package http + +import ( + "net" + "time" +) + +func NewLoggingConn(baseName string, c net.Conn) net.Conn { + return newLoggingConn(baseName, c) +} + +var ExportAppendTime = appendTime + +func (t *Transport) NumPendingRequestsForTesting() int { + t.reqMu.Lock() + defer t.reqMu.Unlock() + return len(t.reqCanceler) +} + +func (t *Transport) IdleConnKeysForTesting() (keys []string) { + keys = make([]string, 0) + t.idleMu.Lock() + defer t.idleMu.Unlock() + if t.idleConn == nil { + return + } + for key := range t.idleConn { + keys = append(keys, key.String()) + } + return +} + +func (t *Transport) IdleConnCountForTesting(cacheKey string) int { + t.idleMu.Lock() + defer t.idleMu.Unlock() + if t.idleConn == nil { + return 0 + } + for k, conns := range t.idleConn { + if k.String() == cacheKey { + return len(conns) + } + } + return 0 +} + +func (t *Transport) IdleConnChMapSizeForTesting() int { + t.idleMu.Lock() + defer t.idleMu.Unlock() + return len(t.idleConnCh) +} + +func NewTestTimeoutHandler(handler Handler, ch <-chan time.Time) Handler { + f := func() <-chan time.Time { + return ch + } + return &timeoutHandler{handler, f, ""} +} + +func ResetCachedEnvironment() { + httpProxyEnv.reset() + noProxyEnv.reset() +} + +var DefaultUserAgent = defaultUserAgent + +// SetPendingDialHooks sets the hooks that run before and after handling +// pending dials. +func SetPendingDialHooks(before, after func()) { + prePendingDial, postPendingDial = before, after +} diff --git a/src/net/http/fcgi/child.go b/src/net/http/fcgi/child.go new file mode 100644 index 000000000..a3beaa33a --- /dev/null +++ b/src/net/http/fcgi/child.go @@ -0,0 +1,305 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fcgi + +// This file implements FastCGI from the perspective of a child process. + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/cgi" + "os" + "strings" + "sync" + "time" +) + +// request holds the state for an in-progress request. As soon as it's complete, +// it's converted to an http.Request. +type request struct { + pw *io.PipeWriter + reqId uint16 + params map[string]string + buf [1024]byte + rawParams []byte + keepConn bool +} + +func newRequest(reqId uint16, flags uint8) *request { + r := &request{ + reqId: reqId, + params: map[string]string{}, + keepConn: flags&flagKeepConn != 0, + } + r.rawParams = r.buf[:0] + return r +} + +// parseParams reads an encoded []byte into Params. +func (r *request) parseParams() { + text := r.rawParams + r.rawParams = nil + for len(text) > 0 { + keyLen, n := readSize(text) + if n == 0 { + return + } + text = text[n:] + valLen, n := readSize(text) + if n == 0 { + return + } + text = text[n:] + key := readString(text, keyLen) + text = text[keyLen:] + val := readString(text, valLen) + text = text[valLen:] + r.params[key] = val + } +} + +// response implements http.ResponseWriter. +type response struct { + req *request + header http.Header + w *bufWriter + wroteHeader bool +} + +func newResponse(c *child, req *request) *response { + return &response{ + req: req, + header: http.Header{}, + w: newWriter(c.conn, typeStdout, req.reqId), + } +} + +func (r *response) Header() http.Header { + return r.header +} + +func (r *response) Write(data []byte) (int, error) { + if !r.wroteHeader { + r.WriteHeader(http.StatusOK) + } + return r.w.Write(data) +} + +func (r *response) WriteHeader(code int) { + if r.wroteHeader { + return + } + r.wroteHeader = true + if code == http.StatusNotModified { + // Must not have body. + r.header.Del("Content-Type") + r.header.Del("Content-Length") + r.header.Del("Transfer-Encoding") + } else if r.header.Get("Content-Type") == "" { + r.header.Set("Content-Type", "text/html; charset=utf-8") + } + + if r.header.Get("Date") == "" { + r.header.Set("Date", time.Now().UTC().Format(http.TimeFormat)) + } + + fmt.Fprintf(r.w, "Status: %d %s\r\n", code, http.StatusText(code)) + r.header.Write(r.w) + r.w.WriteString("\r\n") +} + +func (r *response) Flush() { + if !r.wroteHeader { + r.WriteHeader(http.StatusOK) + } + r.w.Flush() +} + +func (r *response) Close() error { + r.Flush() + return r.w.Close() +} + +type child struct { + conn *conn + handler http.Handler + + mu sync.Mutex // protects requests: + requests map[uint16]*request // keyed by request ID +} + +func newChild(rwc io.ReadWriteCloser, handler http.Handler) *child { + return &child{ + conn: newConn(rwc), + handler: handler, + requests: make(map[uint16]*request), + } +} + +func (c *child) serve() { + defer c.conn.Close() + var rec record + for { + if err := rec.read(c.conn.rwc); err != nil { + return + } + if err := c.handleRecord(&rec); err != nil { + return + } + } +} + +var errCloseConn = errors.New("fcgi: connection should be closed") + +var emptyBody = ioutil.NopCloser(strings.NewReader("")) + +func (c *child) handleRecord(rec *record) error { + c.mu.Lock() + req, ok := c.requests[rec.h.Id] + c.mu.Unlock() + if !ok && rec.h.Type != typeBeginRequest && rec.h.Type != typeGetValues { + // The spec says to ignore unknown request IDs. + return nil + } + + switch rec.h.Type { + case typeBeginRequest: + if req != nil { + // The server is trying to begin a request with the same ID + // as an in-progress request. This is an error. + return errors.New("fcgi: received ID that is already in-flight") + } + + var br beginRequest + if err := br.read(rec.content()); err != nil { + return err + } + if br.role != roleResponder { + c.conn.writeEndRequest(rec.h.Id, 0, statusUnknownRole) + return nil + } + req = newRequest(rec.h.Id, br.flags) + c.mu.Lock() + c.requests[rec.h.Id] = req + c.mu.Unlock() + return nil + case typeParams: + // NOTE(eds): Technically a key-value pair can straddle the boundary + // between two packets. We buffer until we've received all parameters. + if len(rec.content()) > 0 { + req.rawParams = append(req.rawParams, rec.content()...) + return nil + } + req.parseParams() + return nil + case typeStdin: + content := rec.content() + if req.pw == nil { + var body io.ReadCloser + if len(content) > 0 { + // body could be an io.LimitReader, but it shouldn't matter + // as long as both sides are behaving. + body, req.pw = io.Pipe() + } else { + body = emptyBody + } + go c.serveRequest(req, body) + } + if len(content) > 0 { + // TODO(eds): This blocks until the handler reads from the pipe. + // If the handler takes a long time, it might be a problem. + req.pw.Write(content) + } else if req.pw != nil { + req.pw.Close() + } + return nil + case typeGetValues: + values := map[string]string{"FCGI_MPXS_CONNS": "1"} + c.conn.writePairs(typeGetValuesResult, 0, values) + return nil + case typeData: + // If the filter role is implemented, read the data stream here. + return nil + case typeAbortRequest: + println("abort") + c.mu.Lock() + delete(c.requests, rec.h.Id) + c.mu.Unlock() + c.conn.writeEndRequest(rec.h.Id, 0, statusRequestComplete) + if !req.keepConn { + // connection will close upon return + return errCloseConn + } + return nil + default: + b := make([]byte, 8) + b[0] = byte(rec.h.Type) + c.conn.writeRecord(typeUnknownType, 0, b) + return nil + } +} + +func (c *child) serveRequest(req *request, body io.ReadCloser) { + r := newResponse(c, req) + httpReq, err := cgi.RequestFromMap(req.params) + if err != nil { + // there was an error reading the request + r.WriteHeader(http.StatusInternalServerError) + c.conn.writeRecord(typeStderr, req.reqId, []byte(err.Error())) + } else { + httpReq.Body = body + c.handler.ServeHTTP(r, httpReq) + } + r.Close() + c.mu.Lock() + delete(c.requests, req.reqId) + c.mu.Unlock() + c.conn.writeEndRequest(req.reqId, 0, statusRequestComplete) + + // Consume the entire body, so the host isn't still writing to + // us when we close the socket below in the !keepConn case, + // otherwise we'd send a RST. (golang.org/issue/4183) + // TODO(bradfitz): also bound this copy in time. Or send + // some sort of abort request to the host, so the host + // can properly cut off the client sending all the data. + // For now just bound it a little and + io.CopyN(ioutil.Discard, body, 100<<20) + body.Close() + + if !req.keepConn { + c.conn.Close() + } +} + +// Serve accepts incoming FastCGI connections on the listener l, creating a new +// goroutine for each. The goroutine reads requests and then calls handler +// to reply to them. +// If l is nil, Serve accepts connections from os.Stdin. +// If handler is nil, http.DefaultServeMux is used. +func Serve(l net.Listener, handler http.Handler) error { + if l == nil { + var err error + l, err = net.FileListener(os.Stdin) + if err != nil { + return err + } + defer l.Close() + } + if handler == nil { + handler = http.DefaultServeMux + } + for { + rw, err := l.Accept() + if err != nil { + return err + } + c := newChild(rw, handler) + go c.serve() + } +} diff --git a/src/net/http/fcgi/fcgi.go b/src/net/http/fcgi/fcgi.go new file mode 100644 index 000000000..06bba0488 --- /dev/null +++ b/src/net/http/fcgi/fcgi.go @@ -0,0 +1,274 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fcgi implements the FastCGI protocol. +// Currently only the responder role is supported. +// The protocol is defined at http://www.fastcgi.com/drupal/node/6?q=node/22 +package fcgi + +// This file defines the raw protocol and some utilities used by the child and +// the host. + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "io" + "sync" +) + +// recType is a record type, as defined by +// http://www.fastcgi.com/devkit/doc/fcgi-spec.html#S8 +type recType uint8 + +const ( + typeBeginRequest recType = 1 + typeAbortRequest recType = 2 + typeEndRequest recType = 3 + typeParams recType = 4 + typeStdin recType = 5 + typeStdout recType = 6 + typeStderr recType = 7 + typeData recType = 8 + typeGetValues recType = 9 + typeGetValuesResult recType = 10 + typeUnknownType recType = 11 +) + +// keep the connection between web-server and responder open after request +const flagKeepConn = 1 + +const ( + maxWrite = 65535 // maximum record body + maxPad = 255 +) + +const ( + roleResponder = iota + 1 // only Responders are implemented. + roleAuthorizer + roleFilter +) + +const ( + statusRequestComplete = iota + statusCantMultiplex + statusOverloaded + statusUnknownRole +) + +const headerLen = 8 + +type header struct { + Version uint8 + Type recType + Id uint16 + ContentLength uint16 + PaddingLength uint8 + Reserved uint8 +} + +type beginRequest struct { + role uint16 + flags uint8 + reserved [5]uint8 +} + +func (br *beginRequest) read(content []byte) error { + if len(content) != 8 { + return errors.New("fcgi: invalid begin request record") + } + br.role = binary.BigEndian.Uint16(content) + br.flags = content[2] + return nil +} + +// for padding so we don't have to allocate all the time +// not synchronized because we don't care what the contents are +var pad [maxPad]byte + +func (h *header) init(recType recType, reqId uint16, contentLength int) { + h.Version = 1 + h.Type = recType + h.Id = reqId + h.ContentLength = uint16(contentLength) + h.PaddingLength = uint8(-contentLength & 7) +} + +// conn sends records over rwc +type conn struct { + mutex sync.Mutex + rwc io.ReadWriteCloser + + // to avoid allocations + buf bytes.Buffer + h header +} + +func newConn(rwc io.ReadWriteCloser) *conn { + return &conn{rwc: rwc} +} + +func (c *conn) Close() error { + c.mutex.Lock() + defer c.mutex.Unlock() + return c.rwc.Close() +} + +type record struct { + h header + buf [maxWrite + maxPad]byte +} + +func (rec *record) read(r io.Reader) (err error) { + if err = binary.Read(r, binary.BigEndian, &rec.h); err != nil { + return err + } + if rec.h.Version != 1 { + return errors.New("fcgi: invalid header version") + } + n := int(rec.h.ContentLength) + int(rec.h.PaddingLength) + if _, err = io.ReadFull(r, rec.buf[:n]); err != nil { + return err + } + return nil +} + +func (r *record) content() []byte { + return r.buf[:r.h.ContentLength] +} + +// writeRecord writes and sends a single record. +func (c *conn) writeRecord(recType recType, reqId uint16, b []byte) error { + c.mutex.Lock() + defer c.mutex.Unlock() + c.buf.Reset() + c.h.init(recType, reqId, len(b)) + if err := binary.Write(&c.buf, binary.BigEndian, c.h); err != nil { + return err + } + if _, err := c.buf.Write(b); err != nil { + return err + } + if _, err := c.buf.Write(pad[:c.h.PaddingLength]); err != nil { + return err + } + _, err := c.rwc.Write(c.buf.Bytes()) + return err +} + +func (c *conn) writeBeginRequest(reqId uint16, role uint16, flags uint8) error { + b := [8]byte{byte(role >> 8), byte(role), flags} + return c.writeRecord(typeBeginRequest, reqId, b[:]) +} + +func (c *conn) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error { + b := make([]byte, 8) + binary.BigEndian.PutUint32(b, uint32(appStatus)) + b[4] = protocolStatus + return c.writeRecord(typeEndRequest, reqId, b) +} + +func (c *conn) writePairs(recType recType, reqId uint16, pairs map[string]string) error { + w := newWriter(c, recType, reqId) + b := make([]byte, 8) + for k, v := range pairs { + n := encodeSize(b, uint32(len(k))) + n += encodeSize(b[n:], uint32(len(v))) + if _, err := w.Write(b[:n]); err != nil { + return err + } + if _, err := w.WriteString(k); err != nil { + return err + } + if _, err := w.WriteString(v); err != nil { + return err + } + } + w.Close() + return nil +} + +func readSize(s []byte) (uint32, int) { + if len(s) == 0 { + return 0, 0 + } + size, n := uint32(s[0]), 1 + if size&(1<<7) != 0 { + if len(s) < 4 { + return 0, 0 + } + n = 4 + size = binary.BigEndian.Uint32(s) + size &^= 1 << 31 + } + return size, n +} + +func readString(s []byte, size uint32) string { + if size > uint32(len(s)) { + return "" + } + return string(s[:size]) +} + +func encodeSize(b []byte, size uint32) int { + if size > 127 { + size |= 1 << 31 + binary.BigEndian.PutUint32(b, size) + return 4 + } + b[0] = byte(size) + return 1 +} + +// bufWriter encapsulates bufio.Writer but also closes the underlying stream when +// Closed. +type bufWriter struct { + closer io.Closer + *bufio.Writer +} + +func (w *bufWriter) Close() error { + if err := w.Writer.Flush(); err != nil { + w.closer.Close() + return err + } + return w.closer.Close() +} + +func newWriter(c *conn, recType recType, reqId uint16) *bufWriter { + s := &streamWriter{c: c, recType: recType, reqId: reqId} + w := bufio.NewWriterSize(s, maxWrite) + return &bufWriter{s, w} +} + +// streamWriter abstracts out the separation of a stream into discrete records. +// It only writes maxWrite bytes at a time. +type streamWriter struct { + c *conn + recType recType + reqId uint16 +} + +func (w *streamWriter) Write(p []byte) (int, error) { + nn := 0 + for len(p) > 0 { + n := len(p) + if n > maxWrite { + n = maxWrite + } + if err := w.c.writeRecord(w.recType, w.reqId, p[:n]); err != nil { + return nn, err + } + nn += n + p = p[n:] + } + return nn, nil +} + +func (w *streamWriter) Close() error { + // send empty record to close the stream + return w.c.writeRecord(w.recType, w.reqId, nil) +} diff --git a/src/net/http/fcgi/fcgi_test.go b/src/net/http/fcgi/fcgi_test.go new file mode 100644 index 000000000..6c7e1a9ce --- /dev/null +++ b/src/net/http/fcgi/fcgi_test.go @@ -0,0 +1,150 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fcgi + +import ( + "bytes" + "errors" + "io" + "testing" +) + +var sizeTests = []struct { + size uint32 + bytes []byte +}{ + {0, []byte{0x00}}, + {127, []byte{0x7F}}, + {128, []byte{0x80, 0x00, 0x00, 0x80}}, + {1000, []byte{0x80, 0x00, 0x03, 0xE8}}, + {33554431, []byte{0x81, 0xFF, 0xFF, 0xFF}}, +} + +func TestSize(t *testing.T) { + b := make([]byte, 4) + for i, test := range sizeTests { + n := encodeSize(b, test.size) + if !bytes.Equal(b[:n], test.bytes) { + t.Errorf("%d expected %x, encoded %x", i, test.bytes, b) + } + size, n := readSize(test.bytes) + if size != test.size { + t.Errorf("%d expected %d, read %d", i, test.size, size) + } + if len(test.bytes) != n { + t.Errorf("%d did not consume all the bytes", i) + } + } +} + +var streamTests = []struct { + desc string + recType recType + reqId uint16 + content []byte + raw []byte +}{ + {"single record", typeStdout, 1, nil, + []byte{1, byte(typeStdout), 0, 1, 0, 0, 0, 0}, + }, + // this data will have to be split into two records + {"two records", typeStdin, 300, make([]byte, 66000), + bytes.Join([][]byte{ + // header for the first record + {1, byte(typeStdin), 0x01, 0x2C, 0xFF, 0xFF, 1, 0}, + make([]byte, 65536), + // header for the second + {1, byte(typeStdin), 0x01, 0x2C, 0x01, 0xD1, 7, 0}, + make([]byte, 472), + // header for the empty record + {1, byte(typeStdin), 0x01, 0x2C, 0, 0, 0, 0}, + }, + nil), + }, +} + +type nilCloser struct { + io.ReadWriter +} + +func (c *nilCloser) Close() error { return nil } + +func TestStreams(t *testing.T) { + var rec record +outer: + for _, test := range streamTests { + buf := bytes.NewBuffer(test.raw) + var content []byte + for buf.Len() > 0 { + if err := rec.read(buf); err != nil { + t.Errorf("%s: error reading record: %v", test.desc, err) + continue outer + } + content = append(content, rec.content()...) + } + if rec.h.Type != test.recType { + t.Errorf("%s: got type %d expected %d", test.desc, rec.h.Type, test.recType) + continue + } + if rec.h.Id != test.reqId { + t.Errorf("%s: got request ID %d expected %d", test.desc, rec.h.Id, test.reqId) + continue + } + if !bytes.Equal(content, test.content) { + t.Errorf("%s: read wrong content", test.desc) + continue + } + buf.Reset() + c := newConn(&nilCloser{buf}) + w := newWriter(c, test.recType, test.reqId) + if _, err := w.Write(test.content); err != nil { + t.Errorf("%s: error writing record: %v", test.desc, err) + continue + } + if err := w.Close(); err != nil { + t.Errorf("%s: error closing stream: %v", test.desc, err) + continue + } + if !bytes.Equal(buf.Bytes(), test.raw) { + t.Errorf("%s: wrote wrong content", test.desc) + } + } +} + +type writeOnlyConn struct { + buf []byte +} + +func (c *writeOnlyConn) Write(p []byte) (int, error) { + c.buf = append(c.buf, p...) + return len(p), nil +} + +func (c *writeOnlyConn) Read(p []byte) (int, error) { + return 0, errors.New("conn is write-only") +} + +func (c *writeOnlyConn) Close() error { + return nil +} + +func TestGetValues(t *testing.T) { + var rec record + rec.h.Type = typeGetValues + + wc := new(writeOnlyConn) + c := newChild(wc, nil) + err := c.handleRecord(&rec) + if err != nil { + t.Fatalf("handleRecord: %v", err) + } + + const want = "\x01\n\x00\x00\x00\x12\x06\x00" + + "\x0f\x01FCGI_MPXS_CONNS1" + + "\x00\x00\x00\x00\x00\x00\x01\n\x00\x00\x00\x00\x00\x00" + if got := string(wc.buf); got != want { + t.Errorf(" got: %q\nwant: %q\n", got, want) + } +} diff --git a/src/net/http/filetransport.go b/src/net/http/filetransport.go new file mode 100644 index 000000000..821787e0c --- /dev/null +++ b/src/net/http/filetransport.go @@ -0,0 +1,123 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "fmt" + "io" +) + +// fileTransport implements RoundTripper for the 'file' protocol. +type fileTransport struct { + fh fileHandler +} + +// NewFileTransport returns a new RoundTripper, serving the provided +// FileSystem. The returned RoundTripper ignores the URL host in its +// incoming requests, as well as most other properties of the +// request. +// +// The typical use case for NewFileTransport is to register the "file" +// protocol with a Transport, as in: +// +// t := &http.Transport{} +// t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) +// c := &http.Client{Transport: t} +// res, err := c.Get("file:///etc/passwd") +// ... +func NewFileTransport(fs FileSystem) RoundTripper { + return fileTransport{fileHandler{fs}} +} + +func (t fileTransport) RoundTrip(req *Request) (resp *Response, err error) { + // We start ServeHTTP in a goroutine, which may take a long + // time if the file is large. The newPopulateResponseWriter + // call returns a channel which either ServeHTTP or finish() + // sends our *Response on, once the *Response itself has been + // populated (even if the body itself is still being + // written to the res.Body, a pipe) + rw, resc := newPopulateResponseWriter() + go func() { + t.fh.ServeHTTP(rw, req) + rw.finish() + }() + return <-resc, nil +} + +func newPopulateResponseWriter() (*populateResponse, <-chan *Response) { + pr, pw := io.Pipe() + rw := &populateResponse{ + ch: make(chan *Response), + pw: pw, + res: &Response{ + Proto: "HTTP/1.0", + ProtoMajor: 1, + Header: make(Header), + Close: true, + Body: pr, + }, + } + return rw, rw.ch +} + +// populateResponse is a ResponseWriter that populates the *Response +// in res, and writes its body to a pipe connected to the response +// body. Once writes begin or finish() is called, the response is sent +// on ch. +type populateResponse struct { + res *Response + ch chan *Response + wroteHeader bool + hasContent bool + sentResponse bool + pw *io.PipeWriter +} + +func (pr *populateResponse) finish() { + if !pr.wroteHeader { + pr.WriteHeader(500) + } + if !pr.sentResponse { + pr.sendResponse() + } + pr.pw.Close() +} + +func (pr *populateResponse) sendResponse() { + if pr.sentResponse { + return + } + pr.sentResponse = true + + if pr.hasContent { + pr.res.ContentLength = -1 + } + pr.ch <- pr.res +} + +func (pr *populateResponse) Header() Header { + return pr.res.Header +} + +func (pr *populateResponse) WriteHeader(code int) { + if pr.wroteHeader { + return + } + pr.wroteHeader = true + + pr.res.StatusCode = code + pr.res.Status = fmt.Sprintf("%d %s", code, StatusText(code)) +} + +func (pr *populateResponse) Write(p []byte) (n int, err error) { + if !pr.wroteHeader { + pr.WriteHeader(StatusOK) + } + pr.hasContent = true + if !pr.sentResponse { + pr.sendResponse() + } + return pr.pw.Write(p) +} diff --git a/src/net/http/filetransport_test.go b/src/net/http/filetransport_test.go new file mode 100644 index 000000000..6f1a537e2 --- /dev/null +++ b/src/net/http/filetransport_test.go @@ -0,0 +1,65 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +func checker(t *testing.T) func(string, error) { + return func(call string, err error) { + if err == nil { + return + } + t.Fatalf("%s: %v", call, err) + } +} + +func TestFileTransport(t *testing.T) { + check := checker(t) + + dname, err := ioutil.TempDir("", "") + check("TempDir", err) + fname := filepath.Join(dname, "foo.txt") + err = ioutil.WriteFile(fname, []byte("Bar"), 0644) + check("WriteFile", err) + defer os.Remove(dname) + defer os.Remove(fname) + + tr := &Transport{} + tr.RegisterProtocol("file", NewFileTransport(Dir(dname))) + c := &Client{Transport: tr} + + fooURLs := []string{"file:///foo.txt", "file://../foo.txt"} + for _, urlstr := range fooURLs { + res, err := c.Get(urlstr) + check("Get "+urlstr, err) + if res.StatusCode != 200 { + t.Errorf("for %s, StatusCode = %d, want 200", urlstr, res.StatusCode) + } + if res.ContentLength != -1 { + t.Errorf("for %s, ContentLength = %d, want -1", urlstr, res.ContentLength) + } + if res.Body == nil { + t.Fatalf("for %s, nil Body", urlstr) + } + slurp, err := ioutil.ReadAll(res.Body) + check("ReadAll "+urlstr, err) + if string(slurp) != "Bar" { + t.Errorf("for %s, got content %q, want %q", urlstr, string(slurp), "Bar") + } + } + + const badURL = "file://../no-exist.txt" + res, err := c.Get(badURL) + check("Get "+badURL, err) + if res.StatusCode != 404 { + t.Errorf("for %s, StatusCode = %d, want 404", badURL, res.StatusCode) + } + res.Body.Close() +} diff --git a/src/net/http/fs.go b/src/net/http/fs.go new file mode 100644 index 000000000..bae902cd2 --- /dev/null +++ b/src/net/http/fs.go @@ -0,0 +1,552 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP file system request handler + +package http + +import ( + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" +) + +// A Dir implements http.FileSystem using the native file +// system restricted to a specific directory tree. +// +// An empty Dir is treated as ".". +type Dir string + +func (d Dir) Open(name string) (File, error) { + if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + strings.Contains(name, "\x00") { + return nil, errors.New("http: invalid character in file path") + } + dir := string(d) + if dir == "" { + dir = "." + } + f, err := os.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name)))) + if err != nil { + return nil, err + } + return f, nil +} + +// A FileSystem implements access to a collection of named files. +// The elements in a file path are separated by slash ('/', U+002F) +// characters, regardless of host operating system convention. +type FileSystem interface { + Open(name string) (File, error) +} + +// A File is returned by a FileSystem's Open method and can be +// served by the FileServer implementation. +// +// The methods should behave the same as those on an *os.File. +type File interface { + io.Closer + io.Reader + Readdir(count int) ([]os.FileInfo, error) + Seek(offset int64, whence int) (int64, error) + Stat() (os.FileInfo, error) +} + +func dirList(w ResponseWriter, f File) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + fmt.Fprintf(w, "<pre>\n") + for { + dirs, err := f.Readdir(100) + if err != nil || len(dirs) == 0 { + break + } + for _, d := range dirs { + name := d.Name() + if d.IsDir() { + name += "/" + } + // name may contain '?' or '#', which must be escaped to remain + // part of the URL path, and not indicate the start of a query + // string or fragment. + url := url.URL{Path: name} + fmt.Fprintf(w, "<a href=\"%s\">%s</a>\n", url.String(), htmlReplacer.Replace(name)) + } + } + fmt.Fprintf(w, "</pre>\n") +} + +// ServeContent replies to the request using the content in the +// provided ReadSeeker. The main benefit of ServeContent over io.Copy +// is that it handles Range requests properly, sets the MIME type, and +// handles If-Modified-Since requests. +// +// If the response's Content-Type header is not set, ServeContent +// first tries to deduce the type from name's file extension and, +// if that fails, falls back to reading the first block of the content +// and passing it to DetectContentType. +// The name is otherwise unused; in particular it can be empty and is +// never sent in the response. +// +// If modtime is not the zero time, ServeContent includes it in a +// Last-Modified header in the response. If the request includes an +// If-Modified-Since header, ServeContent uses modtime to decide +// whether the content needs to be sent at all. +// +// The content's Seek method must work: ServeContent uses +// a seek to the end of the content to determine its size. +// +// If the caller has set w's ETag header, ServeContent uses it to +// handle requests using If-Range and If-None-Match. +// +// Note that *os.File implements the io.ReadSeeker interface. +func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) { + sizeFunc := func() (int64, error) { + size, err := content.Seek(0, os.SEEK_END) + if err != nil { + return 0, errSeeker + } + _, err = content.Seek(0, os.SEEK_SET) + if err != nil { + return 0, errSeeker + } + return size, nil + } + serveContent(w, req, name, modtime, sizeFunc, content) +} + +// errSeeker is returned by ServeContent's sizeFunc when the content +// doesn't seek properly. The underlying Seeker's error text isn't +// included in the sizeFunc reply so it's not sent over HTTP to end +// users. +var errSeeker = errors.New("seeker can't seek") + +// if name is empty, filename is unknown. (used for mime type, before sniffing) +// if modtime.IsZero(), modtime is unknown. +// content must be seeked to the beginning of the file. +// The sizeFunc is called at most once. Its error, if any, is sent in the HTTP response. +func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker) { + if checkLastModified(w, r, modtime) { + return + } + rangeReq, done := checkETag(w, r, modtime) + if done { + return + } + + code := StatusOK + + // If Content-Type isn't set, use the file's extension to find it, but + // if the Content-Type is unset explicitly, do not sniff the type. + ctypes, haveType := w.Header()["Content-Type"] + var ctype string + if !haveType { + ctype = mime.TypeByExtension(filepath.Ext(name)) + if ctype == "" { + // read a chunk to decide between utf-8 text and binary + var buf [sniffLen]byte + n, _ := io.ReadFull(content, buf[:]) + ctype = DetectContentType(buf[:n]) + _, err := content.Seek(0, os.SEEK_SET) // rewind to output whole file + if err != nil { + Error(w, "seeker can't seek", StatusInternalServerError) + return + } + } + w.Header().Set("Content-Type", ctype) + } else if len(ctypes) > 0 { + ctype = ctypes[0] + } + + size, err := sizeFunc() + if err != nil { + Error(w, err.Error(), StatusInternalServerError) + return + } + + // handle Content-Range header. + sendSize := size + var sendContent io.Reader = content + if size >= 0 { + ranges, err := parseRange(rangeReq, size) + if err != nil { + Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) + return + } + if sumRangesSize(ranges) > size { + // The total number of bytes in all the ranges + // is larger than the size of the file by + // itself, so this is probably an attack, or a + // dumb client. Ignore the range request. + ranges = nil + } + switch { + case len(ranges) == 1: + // RFC 2616, Section 14.16: + // "When an HTTP message includes the content of a single + // range (for example, a response to a request for a + // single range, or to a request for a set of ranges + // that overlap without any holes), this content is + // transmitted with a Content-Range header, and a + // Content-Length header showing the number of bytes + // actually transferred. + // ... + // A response to a request for a single range MUST NOT + // be sent using the multipart/byteranges media type." + ra := ranges[0] + if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { + Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) + return + } + sendSize = ra.length + code = StatusPartialContent + w.Header().Set("Content-Range", ra.contentRange(size)) + case len(ranges) > 1: + sendSize = rangesMIMESize(ranges, ctype, size) + code = StatusPartialContent + + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) + sendContent = pr + defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. + go func() { + for _, ra := range ranges { + part, err := mw.CreatePart(ra.mimeHeader(ctype, size)) + if err != nil { + pw.CloseWithError(err) + return + } + if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { + pw.CloseWithError(err) + return + } + if _, err := io.CopyN(part, content, ra.length); err != nil { + pw.CloseWithError(err) + return + } + } + mw.Close() + pw.Close() + }() + } + + w.Header().Set("Accept-Ranges", "bytes") + if w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + } + } + + w.WriteHeader(code) + + if r.Method != "HEAD" { + io.CopyN(w, sendContent, sendSize) + } +} + +// modtime is the modification time of the resource to be served, or IsZero(). +// return value is whether this request is now complete. +func checkLastModified(w ResponseWriter, r *Request, modtime time.Time) bool { + if modtime.IsZero() { + return false + } + + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) { + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + w.WriteHeader(StatusNotModified) + return true + } + w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) + return false +} + +// checkETag implements If-None-Match and If-Range checks. +// +// The ETag or modtime must have been previously set in the +// ResponseWriter's headers. The modtime is only compared at second +// granularity and may be the zero value to mean unknown. +// +// The return value is the effective request "Range" header to use and +// whether this request is now considered done. +func checkETag(w ResponseWriter, r *Request, modtime time.Time) (rangeReq string, done bool) { + etag := w.Header().get("Etag") + rangeReq = r.Header.get("Range") + + // Invalidate the range request if the entity doesn't match the one + // the client was expecting. + // "If-Range: version" means "ignore the Range: header unless version matches the + // current file." + // We only support ETag versions. + // The caller must have set the ETag on the response already. + if ir := r.Header.get("If-Range"); ir != "" && ir != etag { + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + timeMatches := false + if !modtime.IsZero() { + if t, err := ParseTime(ir); err == nil && t.Unix() == modtime.Unix() { + timeMatches = true + } + } + if !timeMatches { + rangeReq = "" + } + } + + if inm := r.Header.get("If-None-Match"); inm != "" { + // Must know ETag. + if etag == "" { + return rangeReq, false + } + + // TODO(bradfitz): non-GET/HEAD requests require more work: + // sending a different status code on matches, and + // also can't use weak cache validators (those with a "W/ + // prefix). But most users of ServeContent will be using + // it on GET or HEAD, so only support those for now. + if r.Method != "GET" && r.Method != "HEAD" { + return rangeReq, false + } + + // TODO(bradfitz): deal with comma-separated or multiple-valued + // list of If-None-match values. For now just handle the common + // case of a single item. + if inm == etag || inm == "*" { + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + w.WriteHeader(StatusNotModified) + return "", true + } + } + return rangeReq, false +} + +// name is '/'-separated, not filepath.Separator. +func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirect bool) { + const indexPage = "/index.html" + + // redirect .../index.html to .../ + // can't use Redirect() because that would make the path absolute, + // which would be a problem running under StripPrefix + if strings.HasSuffix(r.URL.Path, indexPage) { + localRedirect(w, r, "./") + return + } + + f, err := fs.Open(name) + if err != nil { + // TODO expose actual error? + NotFound(w, r) + return + } + defer f.Close() + + d, err1 := f.Stat() + if err1 != nil { + // TODO expose actual error? + NotFound(w, r) + return + } + + if redirect { + // redirect to canonical path: / at end of directory url + // r.URL.Path always begins with / + url := r.URL.Path + if d.IsDir() { + if url[len(url)-1] != '/' { + localRedirect(w, r, path.Base(url)+"/") + return + } + } else { + if url[len(url)-1] == '/' { + localRedirect(w, r, "../"+path.Base(url)) + return + } + } + } + + // use contents of index.html for directory, if present + if d.IsDir() { + index := name + indexPage + ff, err := fs.Open(index) + if err == nil { + defer ff.Close() + dd, err := ff.Stat() + if err == nil { + name = index + d = dd + f = ff + } + } + } + + // Still a directory? (we didn't find an index.html file) + if d.IsDir() { + if checkLastModified(w, r, d.ModTime()) { + return + } + dirList(w, f) + return + } + + // serveContent will check modification time + sizeFunc := func() (int64, error) { return d.Size(), nil } + serveContent(w, r, d.Name(), d.ModTime(), sizeFunc, f) +} + +// localRedirect gives a Moved Permanently response. +// It does not convert relative paths to absolute paths like Redirect does. +func localRedirect(w ResponseWriter, r *Request, newPath string) { + if q := r.URL.RawQuery; q != "" { + newPath += "?" + q + } + w.Header().Set("Location", newPath) + w.WriteHeader(StatusMovedPermanently) +} + +// ServeFile replies to the request with the contents of the named file or directory. +func ServeFile(w ResponseWriter, r *Request, name string) { + dir, file := filepath.Split(name) + serveFile(w, r, Dir(dir), file, false) +} + +type fileHandler struct { + root FileSystem +} + +// FileServer returns a handler that serves HTTP requests +// with the contents of the file system rooted at root. +// +// To use the operating system's file system implementation, +// use http.Dir: +// +// http.Handle("/", http.FileServer(http.Dir("/tmp"))) +func FileServer(root FileSystem) Handler { + return &fileHandler{root} +} + +func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { + upath := r.URL.Path + if !strings.HasPrefix(upath, "/") { + upath = "/" + upath + r.URL.Path = upath + } + serveFile(w, r, f.root, path.Clean(upath), true) +} + +// httpRange specifies the byte range to be sent to the client. +type httpRange struct { + start, length int64 +} + +func (r httpRange) contentRange(size int64) string { + return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, size) +} + +func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHeader { + return textproto.MIMEHeader{ + "Content-Range": {r.contentRange(size)}, + "Content-Type": {contentType}, + } +} + +// parseRange parses a Range header string as per RFC 2616. +func parseRange(s string, size int64) ([]httpRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []httpRange + for _, ra := range strings.Split(s[len(b):], ",") { + ra = strings.TrimSpace(ra) + if ra == "" { + continue + } + i := strings.Index(ra, "-") + if i < 0 { + return nil, errors.New("invalid range") + } + start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) + var r httpRange + if start == "" { + // If no start is specified, end specifies the + // range start relative to the end of the file. + i, err := strconv.ParseInt(end, 10, 64) + if err != nil { + return nil, errors.New("invalid range") + } + if i > size { + i = size + } + r.start = size - i + r.length = size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i > size || i < 0 { + return nil, errors.New("invalid range") + } + r.start = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.length = size - r.start + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > i { + return nil, errors.New("invalid range") + } + if i >= size { + i = size - 1 + } + r.length = i - r.start + 1 + } + } + ranges = append(ranges, r) + } + return ranges, nil +} + +// countingWriter counts how many bytes have been written to it. +type countingWriter int64 + +func (w *countingWriter) Write(p []byte) (n int, err error) { + *w += countingWriter(len(p)) + return len(p), nil +} + +// rangesMIMESize returns the number of bytes it takes to encode the +// provided ranges as a multipart response. +func rangesMIMESize(ranges []httpRange, contentType string, contentSize int64) (encSize int64) { + var w countingWriter + mw := multipart.NewWriter(&w) + for _, ra := range ranges { + mw.CreatePart(ra.mimeHeader(contentType, contentSize)) + encSize += ra.length + } + mw.Close() + encSize += int64(w) + return +} + +func sumRangesSize(ranges []httpRange) (size int64) { + for _, ra := range ranges { + size += ra.length + } + return +} diff --git a/src/net/http/fs_test.go b/src/net/http/fs_test.go new file mode 100644 index 000000000..a6f33cc42 --- /dev/null +++ b/src/net/http/fs_test.go @@ -0,0 +1,880 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "mime" + "mime/multipart" + "net" + . "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "testing" + "time" +) + +const ( + testFile = "testdata/file" + testFileLen = 11 +) + +type wantRange struct { + start, end int64 // range [start,end) +} + +var itoa = strconv.Itoa + +var ServeFileRangeTests = []struct { + r string + code int + ranges []wantRange +}{ + {r: "", code: StatusOK}, + {r: "bytes=0-4", code: StatusPartialContent, ranges: []wantRange{{0, 5}}}, + {r: "bytes=2-", code: StatusPartialContent, ranges: []wantRange{{2, testFileLen}}}, + {r: "bytes=-5", code: StatusPartialContent, ranges: []wantRange{{testFileLen - 5, testFileLen}}}, + {r: "bytes=3-7", code: StatusPartialContent, ranges: []wantRange{{3, 8}}}, + {r: "bytes=20-", code: StatusRequestedRangeNotSatisfiable}, + {r: "bytes=0-0,-2", code: StatusPartialContent, ranges: []wantRange{{0, 1}, {testFileLen - 2, testFileLen}}}, + {r: "bytes=0-1,5-8", code: StatusPartialContent, ranges: []wantRange{{0, 2}, {5, 9}}}, + {r: "bytes=0-1,5-", code: StatusPartialContent, ranges: []wantRange{{0, 2}, {5, testFileLen}}}, + {r: "bytes=5-1000", code: StatusPartialContent, ranges: []wantRange{{5, testFileLen}}}, + {r: "bytes=0-,1-,2-,3-,4-", code: StatusOK}, // ignore wasteful range request + {r: "bytes=0-" + itoa(testFileLen-2), code: StatusPartialContent, ranges: []wantRange{{0, testFileLen - 1}}}, + {r: "bytes=0-" + itoa(testFileLen-1), code: StatusPartialContent, ranges: []wantRange{{0, testFileLen}}}, + {r: "bytes=0-" + itoa(testFileLen), code: StatusPartialContent, ranges: []wantRange{{0, testFileLen}}}, +} + +func TestServeFile(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, "testdata/file") + })) + defer ts.Close() + + var err error + + file, err := ioutil.ReadFile(testFile) + if err != nil { + t.Fatal("reading file:", err) + } + + // set up the Request (re-used for all tests) + var req Request + req.Header = make(Header) + if req.URL, err = url.Parse(ts.URL); err != nil { + t.Fatal("ParseURL:", err) + } + req.Method = "GET" + + // straight GET + _, body := getBody(t, "straight get", req) + if !bytes.Equal(body, file) { + t.Fatalf("body mismatch: got %q, want %q", body, file) + } + + // Range tests +Cases: + for _, rt := range ServeFileRangeTests { + if rt.r != "" { + req.Header.Set("Range", rt.r) + } + resp, body := getBody(t, fmt.Sprintf("range test %q", rt.r), req) + if resp.StatusCode != rt.code { + t.Errorf("range=%q: StatusCode=%d, want %d", rt.r, resp.StatusCode, rt.code) + } + if rt.code == StatusRequestedRangeNotSatisfiable { + continue + } + wantContentRange := "" + if len(rt.ranges) == 1 { + rng := rt.ranges[0] + wantContentRange = fmt.Sprintf("bytes %d-%d/%d", rng.start, rng.end-1, testFileLen) + } + cr := resp.Header.Get("Content-Range") + if cr != wantContentRange { + t.Errorf("range=%q: Content-Range = %q, want %q", rt.r, cr, wantContentRange) + } + ct := resp.Header.Get("Content-Type") + if len(rt.ranges) == 1 { + rng := rt.ranges[0] + wantBody := file[rng.start:rng.end] + if !bytes.Equal(body, wantBody) { + t.Errorf("range=%q: body = %q, want %q", rt.r, body, wantBody) + } + if strings.HasPrefix(ct, "multipart/byteranges") { + t.Errorf("range=%q content-type = %q; unexpected multipart/byteranges", rt.r, ct) + } + } + if len(rt.ranges) > 1 { + typ, params, err := mime.ParseMediaType(ct) + if err != nil { + t.Errorf("range=%q content-type = %q; %v", rt.r, ct, err) + continue + } + if typ != "multipart/byteranges" { + t.Errorf("range=%q content-type = %q; want multipart/byteranges", rt.r, typ) + continue + } + if params["boundary"] == "" { + t.Errorf("range=%q content-type = %q; lacks boundary", rt.r, ct) + continue + } + if g, w := resp.ContentLength, int64(len(body)); g != w { + t.Errorf("range=%q Content-Length = %d; want %d", rt.r, g, w) + continue + } + mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + for ri, rng := range rt.ranges { + part, err := mr.NextPart() + if err != nil { + t.Errorf("range=%q, reading part index %d: %v", rt.r, ri, err) + continue Cases + } + wantContentRange = fmt.Sprintf("bytes %d-%d/%d", rng.start, rng.end-1, testFileLen) + if g, w := part.Header.Get("Content-Range"), wantContentRange; g != w { + t.Errorf("range=%q: part Content-Range = %q; want %q", rt.r, g, w) + } + body, err := ioutil.ReadAll(part) + if err != nil { + t.Errorf("range=%q, reading part index %d body: %v", rt.r, ri, err) + continue Cases + } + wantBody := file[rng.start:rng.end] + if !bytes.Equal(body, wantBody) { + t.Errorf("range=%q: body = %q, want %q", rt.r, body, wantBody) + } + } + _, err = mr.NextPart() + if err != io.EOF { + t.Errorf("range=%q; expected final error io.EOF; got %v", rt.r, err) + } + } + } +} + +var fsRedirectTestData = []struct { + original, redirect string +}{ + {"/test/index.html", "/test/"}, + {"/test/testdata", "/test/testdata/"}, + {"/test/testdata/file/", "/test/testdata/file"}, +} + +func TestFSRedirect(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(StripPrefix("/test", FileServer(Dir(".")))) + defer ts.Close() + + for _, data := range fsRedirectTestData { + res, err := Get(ts.URL + data.original) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if g, e := res.Request.URL.Path, data.redirect; g != e { + t.Errorf("redirect from %s: got %s, want %s", data.original, g, e) + } + } +} + +type testFileSystem struct { + open func(name string) (File, error) +} + +func (fs *testFileSystem) Open(name string) (File, error) { + return fs.open(name) +} + +func TestFileServerCleans(t *testing.T) { + defer afterTest(t) + ch := make(chan string, 1) + fs := FileServer(&testFileSystem{func(name string) (File, error) { + ch <- name + return nil, errors.New("file does not exist") + }}) + tests := []struct { + reqPath, openArg string + }{ + {"/foo.txt", "/foo.txt"}, + {"//foo.txt", "/foo.txt"}, + {"/../foo.txt", "/foo.txt"}, + } + req, _ := NewRequest("GET", "http://example.com", nil) + for n, test := range tests { + rec := httptest.NewRecorder() + req.URL.Path = test.reqPath + fs.ServeHTTP(rec, req) + if got := <-ch; got != test.openArg { + t.Errorf("test %d: got %q, want %q", n, got, test.openArg) + } + } +} + +func TestFileServerEscapesNames(t *testing.T) { + defer afterTest(t) + const dirListPrefix = "<pre>\n" + const dirListSuffix = "\n</pre>\n" + tests := []struct { + name, escaped string + }{ + {`simple_name`, `<a href="simple_name">simple_name</a>`}, + {`"'<>&`, `<a href="%22%27%3C%3E&">"'<>&</a>`}, + {`?foo=bar#baz`, `<a href="%3Ffoo=bar%23baz">?foo=bar#baz</a>`}, + {`<combo>?foo`, `<a href="%3Ccombo%3E%3Ffoo"><combo>?foo</a>`}, + } + + // We put each test file in its own directory in the fakeFS so we can look at it in isolation. + fs := make(fakeFS) + for i, test := range tests { + testFile := &fakeFileInfo{basename: test.name} + fs[fmt.Sprintf("/%d", i)] = &fakeFileInfo{ + dir: true, + modtime: time.Unix(1000000000, 0).UTC(), + ents: []*fakeFileInfo{testFile}, + } + fs[fmt.Sprintf("/%d/%s", i, test.name)] = testFile + } + + ts := httptest.NewServer(FileServer(&fs)) + defer ts.Close() + for i, test := range tests { + url := fmt.Sprintf("%s/%d", ts.URL, i) + res, err := Get(url) + if err != nil { + t.Fatalf("test %q: Get: %v", test.name, err) + } + b, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("test %q: read Body: %v", test.name, err) + } + s := string(b) + if !strings.HasPrefix(s, dirListPrefix) || !strings.HasSuffix(s, dirListSuffix) { + t.Errorf("test %q: listing dir, full output is %q, want prefix %q and suffix %q", test.name, s, dirListPrefix, dirListSuffix) + } + if trimmed := strings.TrimSuffix(strings.TrimPrefix(s, dirListPrefix), dirListSuffix); trimmed != test.escaped { + t.Errorf("test %q: listing dir, filename escaped to %q, want %q", test.name, trimmed, test.escaped) + } + res.Body.Close() + } +} + +func mustRemoveAll(dir string) { + err := os.RemoveAll(dir) + if err != nil { + panic(err) + } +} + +func TestFileServerImplicitLeadingSlash(t *testing.T) { + defer afterTest(t) + tempDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("TempDir: %v", err) + } + defer mustRemoveAll(tempDir) + if err := ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("Hello world"), 0644); err != nil { + t.Fatalf("WriteFile: %v", err) + } + ts := httptest.NewServer(StripPrefix("/bar/", FileServer(Dir(tempDir)))) + defer ts.Close() + get := func(suffix string) string { + res, err := Get(ts.URL + suffix) + if err != nil { + t.Fatalf("Get %s: %v", suffix, err) + } + b, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("ReadAll %s: %v", suffix, err) + } + res.Body.Close() + return string(b) + } + if s := get("/bar/"); !strings.Contains(s, ">foo.txt<") { + t.Logf("expected a directory listing with foo.txt, got %q", s) + } + if s := get("/bar/foo.txt"); s != "Hello world" { + t.Logf("expected %q, got %q", "Hello world", s) + } +} + +func TestDirJoin(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping test on windows") + } + wfi, err := os.Stat("/etc/hosts") + if err != nil { + t.Skip("skipping test; no /etc/hosts file") + } + test := func(d Dir, name string) { + f, err := d.Open(name) + if err != nil { + t.Fatalf("open of %s: %v", name, err) + } + defer f.Close() + gfi, err := f.Stat() + if err != nil { + t.Fatalf("stat of %s: %v", name, err) + } + if !os.SameFile(gfi, wfi) { + t.Errorf("%s got different file", name) + } + } + test(Dir("/etc/"), "/hosts") + test(Dir("/etc/"), "hosts") + test(Dir("/etc/"), "../../../../hosts") + test(Dir("/etc"), "/hosts") + test(Dir("/etc"), "hosts") + test(Dir("/etc"), "../../../../hosts") + + // Not really directories, but since we use this trick in + // ServeFile, test it: + test(Dir("/etc/hosts"), "") + test(Dir("/etc/hosts"), "/") + test(Dir("/etc/hosts"), "../") +} + +func TestEmptyDirOpenCWD(t *testing.T) { + test := func(d Dir) { + name := "fs_test.go" + f, err := d.Open(name) + if err != nil { + t.Fatalf("open of %s: %v", name, err) + } + defer f.Close() + } + test(Dir("")) + test(Dir(".")) + test(Dir("./")) +} + +func TestServeFileContentType(t *testing.T) { + defer afterTest(t) + const ctype = "icecream/chocolate" + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + switch r.FormValue("override") { + case "1": + w.Header().Set("Content-Type", ctype) + case "2": + // Explicitly inhibit sniffing. + w.Header()["Content-Type"] = []string{} + } + ServeFile(w, r, "testdata/file") + })) + defer ts.Close() + get := func(override string, want []string) { + resp, err := Get(ts.URL + "?override=" + override) + if err != nil { + t.Fatal(err) + } + if h := resp.Header["Content-Type"]; !reflect.DeepEqual(h, want) { + t.Errorf("Content-Type mismatch: got %v, want %v", h, want) + } + resp.Body.Close() + } + get("0", []string{"text/plain; charset=utf-8"}) + get("1", []string{ctype}) + get("2", nil) +} + +func TestServeFileMimeType(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, "testdata/style.css") + })) + defer ts.Close() + resp, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + want := "text/css; charset=utf-8" + if h := resp.Header.Get("Content-Type"); h != want { + t.Errorf("Content-Type mismatch: got %q, want %q", h, want) + } +} + +func TestServeFileFromCWD(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, "fs_test.go") + })) + defer ts.Close() + r, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + r.Body.Close() + if r.StatusCode != 200 { + t.Fatalf("expected 200 OK, got %s", r.Status) + } +} + +func TestServeFileWithContentEncoding(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", "foo") + ServeFile(w, r, "testdata/file") + })) + defer ts.Close() + resp, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if g, e := resp.ContentLength, int64(-1); g != e { + t.Errorf("Content-Length mismatch: got %d, want %d", g, e) + } +} + +func TestServeIndexHtml(t *testing.T) { + defer afterTest(t) + const want = "index.html says hello\n" + ts := httptest.NewServer(FileServer(Dir("."))) + defer ts.Close() + + for _, path := range []string{"/testdata/", "/testdata/index.html"} { + res, err := Get(ts.URL + path) + if err != nil { + t.Fatal(err) + } + b, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal("reading Body:", err) + } + if s := string(b); s != want { + t.Errorf("for path %q got %q, want %q", path, s, want) + } + res.Body.Close() + } +} + +func TestFileServerZeroByte(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(FileServer(Dir("."))) + defer ts.Close() + + res, err := Get(ts.URL + "/..\x00") + if err != nil { + t.Fatal(err) + } + b, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal("reading Body:", err) + } + if res.StatusCode == 200 { + t.Errorf("got status 200; want an error. Body is:\n%s", string(b)) + } +} + +type fakeFileInfo struct { + dir bool + basename string + modtime time.Time + ents []*fakeFileInfo + contents string +} + +func (f *fakeFileInfo) Name() string { return f.basename } +func (f *fakeFileInfo) Sys() interface{} { return nil } +func (f *fakeFileInfo) ModTime() time.Time { return f.modtime } +func (f *fakeFileInfo) IsDir() bool { return f.dir } +func (f *fakeFileInfo) Size() int64 { return int64(len(f.contents)) } +func (f *fakeFileInfo) Mode() os.FileMode { + if f.dir { + return 0755 | os.ModeDir + } + return 0644 +} + +type fakeFile struct { + io.ReadSeeker + fi *fakeFileInfo + path string // as opened + entpos int +} + +func (f *fakeFile) Close() error { return nil } +func (f *fakeFile) Stat() (os.FileInfo, error) { return f.fi, nil } +func (f *fakeFile) Readdir(count int) ([]os.FileInfo, error) { + if !f.fi.dir { + return nil, os.ErrInvalid + } + var fis []os.FileInfo + + limit := f.entpos + count + if count <= 0 || limit > len(f.fi.ents) { + limit = len(f.fi.ents) + } + for ; f.entpos < limit; f.entpos++ { + fis = append(fis, f.fi.ents[f.entpos]) + } + + if len(fis) == 0 && count > 0 { + return fis, io.EOF + } else { + return fis, nil + } +} + +type fakeFS map[string]*fakeFileInfo + +func (fs fakeFS) Open(name string) (File, error) { + name = path.Clean(name) + f, ok := fs[name] + if !ok { + return nil, os.ErrNotExist + } + return &fakeFile{ReadSeeker: strings.NewReader(f.contents), fi: f, path: name}, nil +} + +func TestDirectoryIfNotModified(t *testing.T) { + defer afterTest(t) + const indexContents = "I am a fake index.html file" + fileMod := time.Unix(1000000000, 0).UTC() + fileModStr := fileMod.Format(TimeFormat) + dirMod := time.Unix(123, 0).UTC() + indexFile := &fakeFileInfo{ + basename: "index.html", + modtime: fileMod, + contents: indexContents, + } + fs := fakeFS{ + "/": &fakeFileInfo{ + dir: true, + modtime: dirMod, + ents: []*fakeFileInfo{indexFile}, + }, + "/index.html": indexFile, + } + + ts := httptest.NewServer(FileServer(fs)) + defer ts.Close() + + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + b, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if string(b) != indexContents { + t.Fatalf("Got body %q; want %q", b, indexContents) + } + res.Body.Close() + + lastMod := res.Header.Get("Last-Modified") + if lastMod != fileModStr { + t.Fatalf("initial Last-Modified = %q; want %q", lastMod, fileModStr) + } + + req, _ := NewRequest("GET", ts.URL, nil) + req.Header.Set("If-Modified-Since", lastMod) + + res, err = DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 304 { + t.Fatalf("Code after If-Modified-Since request = %v; want 304", res.StatusCode) + } + res.Body.Close() + + // Advance the index.html file's modtime, but not the directory's. + indexFile.modtime = indexFile.modtime.Add(1 * time.Hour) + + res, err = DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Fatalf("Code after second If-Modified-Since request = %v; want 200; res is %#v", res.StatusCode, res) + } + res.Body.Close() +} + +func mustStat(t *testing.T, fileName string) os.FileInfo { + fi, err := os.Stat(fileName) + if err != nil { + t.Fatal(err) + } + return fi +} + +func TestServeContent(t *testing.T) { + defer afterTest(t) + type serveParam struct { + name string + modtime time.Time + content io.ReadSeeker + contentType string + etag string + } + servec := make(chan serveParam, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + p := <-servec + if p.etag != "" { + w.Header().Set("ETag", p.etag) + } + if p.contentType != "" { + w.Header().Set("Content-Type", p.contentType) + } + ServeContent(w, r, p.name, p.modtime, p.content) + })) + defer ts.Close() + + type testCase struct { + // One of file or content must be set: + file string + content io.ReadSeeker + + modtime time.Time + serveETag string // optional + serveContentType string // optional + reqHeader map[string]string + wantLastMod string + wantContentType string + wantStatus int + } + htmlModTime := mustStat(t, "testdata/index.html").ModTime() + tests := map[string]testCase{ + "no_last_modified": { + file: "testdata/style.css", + wantContentType: "text/css; charset=utf-8", + wantStatus: 200, + }, + "with_last_modified": { + file: "testdata/index.html", + wantContentType: "text/html; charset=utf-8", + modtime: htmlModTime, + wantLastMod: htmlModTime.UTC().Format(TimeFormat), + wantStatus: 200, + }, + "not_modified_modtime": { + file: "testdata/style.css", + modtime: htmlModTime, + reqHeader: map[string]string{ + "If-Modified-Since": htmlModTime.UTC().Format(TimeFormat), + }, + wantStatus: 304, + }, + "not_modified_modtime_with_contenttype": { + file: "testdata/style.css", + serveContentType: "text/css", // explicit content type + modtime: htmlModTime, + reqHeader: map[string]string{ + "If-Modified-Since": htmlModTime.UTC().Format(TimeFormat), + }, + wantStatus: 304, + }, + "not_modified_etag": { + file: "testdata/style.css", + serveETag: `"foo"`, + reqHeader: map[string]string{ + "If-None-Match": `"foo"`, + }, + wantStatus: 304, + }, + "not_modified_etag_no_seek": { + content: panicOnSeek{nil}, // should never be called + serveETag: `"foo"`, + reqHeader: map[string]string{ + "If-None-Match": `"foo"`, + }, + wantStatus: 304, + }, + "range_good": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "Range": "bytes=0-4", + }, + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + }, + // An If-Range resource for entity "A", but entity "B" is now current. + // The Range request should be ignored. + "range_no_match": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "Range": "bytes=0-4", + "If-Range": `"B"`, + }, + wantStatus: 200, + wantContentType: "text/css; charset=utf-8", + }, + "range_with_modtime": { + file: "testdata/style.css", + modtime: time.Date(2014, 6, 25, 17, 12, 18, 0 /* nanos */, time.UTC), + reqHeader: map[string]string{ + "Range": "bytes=0-4", + "If-Range": "Wed, 25 Jun 2014 17:12:18 GMT", + }, + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + wantLastMod: "Wed, 25 Jun 2014 17:12:18 GMT", + }, + "range_with_modtime_nanos": { + file: "testdata/style.css", + modtime: time.Date(2014, 6, 25, 17, 12, 18, 123 /* nanos */, time.UTC), + reqHeader: map[string]string{ + "Range": "bytes=0-4", + "If-Range": "Wed, 25 Jun 2014 17:12:18 GMT", + }, + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + wantLastMod: "Wed, 25 Jun 2014 17:12:18 GMT", + }, + } + for testName, tt := range tests { + var content io.ReadSeeker + if tt.file != "" { + f, err := os.Open(tt.file) + if err != nil { + t.Fatalf("test %q: %v", testName, err) + } + defer f.Close() + content = f + } else { + content = tt.content + } + + servec <- serveParam{ + name: filepath.Base(tt.file), + content: content, + modtime: tt.modtime, + etag: tt.serveETag, + contentType: tt.serveContentType, + } + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + t.Fatal(err) + } + for k, v := range tt.reqHeader { + req.Header.Set(k, v) + } + res, err := DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + io.Copy(ioutil.Discard, res.Body) + res.Body.Close() + if res.StatusCode != tt.wantStatus { + t.Errorf("test %q: status = %d; want %d", testName, res.StatusCode, tt.wantStatus) + } + if g, e := res.Header.Get("Content-Type"), tt.wantContentType; g != e { + t.Errorf("test %q: content-type = %q, want %q", testName, g, e) + } + if g, e := res.Header.Get("Last-Modified"), tt.wantLastMod; g != e { + t.Errorf("test %q: last-modified = %q, want %q", testName, g, e) + } + } +} + +// verifies that sendfile is being used on Linux +func TestLinuxSendfile(t *testing.T) { + defer afterTest(t) + if runtime.GOOS != "linux" { + t.Skip("skipping; linux-only test") + } + if _, err := exec.LookPath("strace"); err != nil { + t.Skip("skipping; strace not found in path") + } + + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + lnf, err := ln.(*net.TCPListener).File() + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + var buf bytes.Buffer + child := exec.Command("strace", "-f", "-q", "-e", "trace=sendfile,sendfile64", os.Args[0], "-test.run=TestLinuxSendfileChild") + child.ExtraFiles = append(child.ExtraFiles, lnf) + child.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) + child.Stdout = &buf + child.Stderr = &buf + if err := child.Start(); err != nil { + t.Skipf("skipping; failed to start straced child: %v", err) + } + + res, err := Get(fmt.Sprintf("http://%s/", ln.Addr())) + if err != nil { + t.Fatalf("http client error: %v", err) + } + _, err = io.Copy(ioutil.Discard, res.Body) + if err != nil { + t.Fatalf("client body read error: %v", err) + } + res.Body.Close() + + // Force child to exit cleanly. + Get(fmt.Sprintf("http://%s/quit", ln.Addr())) + child.Wait() + + rx := regexp.MustCompile(`sendfile(64)?\(\d+,\s*\d+,\s*NULL,\s*\d+\)\s*=\s*\d+\s*\n`) + rxResume := regexp.MustCompile(`<\.\.\. sendfile(64)? resumed> \)\s*=\s*\d+\s*\n`) + out := buf.String() + if !rx.MatchString(out) && !rxResume.MatchString(out) { + t.Errorf("no sendfile system call found in:\n%s", out) + } +} + +func getBody(t *testing.T, testName string, req Request) (*Response, []byte) { + r, err := DefaultClient.Do(&req) + if err != nil { + t.Fatalf("%s: for URL %q, send error: %v", testName, req.URL.String(), err) + } + b, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("%s: for URL %q, reading body: %v", testName, req.URL.String(), err) + } + return r, b +} + +// TestLinuxSendfileChild isn't a real test. It's used as a helper process +// for TestLinuxSendfile. +func TestLinuxSendfileChild(*testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + defer os.Exit(0) + fd3 := os.NewFile(3, "ephemeral-port-listener") + ln, err := net.FileListener(fd3) + if err != nil { + panic(err) + } + mux := NewServeMux() + mux.Handle("/", FileServer(Dir("testdata"))) + mux.HandleFunc("/quit", func(ResponseWriter, *Request) { + os.Exit(0) + }) + s := &Server{Handler: mux} + err = s.Serve(ln) + if err != nil { + panic(err) + } +} + +type panicOnSeek struct{ io.ReadSeeker } diff --git a/src/net/http/header.go b/src/net/http/header.go new file mode 100644 index 000000000..153b94370 --- /dev/null +++ b/src/net/http/header.go @@ -0,0 +1,211 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "io" + "net/textproto" + "sort" + "strings" + "sync" + "time" +) + +var raceEnabled = false // set by race.go + +// A Header represents the key-value pairs in an HTTP header. +type Header map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +func (h Header) Add(key, value string) { + textproto.MIMEHeader(h).Add(key, value) +} + +// Set sets the header entries associated with key to +// the single element value. It replaces any existing +// values associated with key. +func (h Header) Set(key, value string) { + textproto.MIMEHeader(h).Set(key, value) +} + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +// To access multiple values of a key, access the map directly +// with CanonicalHeaderKey. +func (h Header) Get(key string) string { + return textproto.MIMEHeader(h).Get(key) +} + +// get is like Get, but key must already be in CanonicalHeaderKey form. +func (h Header) get(key string) string { + if v := h[key]; len(v) > 0 { + return v[0] + } + return "" +} + +// Del deletes the values associated with key. +func (h Header) Del(key string) { + textproto.MIMEHeader(h).Del(key) +} + +// Write writes a header in wire format. +func (h Header) Write(w io.Writer) error { + return h.WriteSubset(w, nil) +} + +func (h Header) clone() Header { + h2 := make(Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 +} + +var timeFormats = []string{ + TimeFormat, + time.RFC850, + time.ANSIC, +} + +// ParseTime parses a time header (such as the Date: header), +// trying each of the three formats allowed by HTTP/1.1: +// TimeFormat, time.RFC850, and time.ANSIC. +func ParseTime(text string) (t time.Time, err error) { + for _, layout := range timeFormats { + t, err = time.Parse(layout, text) + if err == nil { + return + } + } + return +} + +var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ") + +type writeStringer interface { + WriteString(string) (int, error) +} + +// stringWriter implements WriteString on a Writer. +type stringWriter struct { + w io.Writer +} + +func (w stringWriter) WriteString(s string) (n int, err error) { + return w.w.Write([]byte(s)) +} + +type keyValues struct { + key string + values []string +} + +// A headerSorter implements sort.Interface by sorting a []keyValues +// by key. It's used as a pointer, so it can fit in a sort.Interface +// interface value without allocation. +type headerSorter struct { + kvs []keyValues +} + +func (s *headerSorter) Len() int { return len(s.kvs) } +func (s *headerSorter) Swap(i, j int) { s.kvs[i], s.kvs[j] = s.kvs[j], s.kvs[i] } +func (s *headerSorter) Less(i, j int) bool { return s.kvs[i].key < s.kvs[j].key } + +var headerSorterPool = sync.Pool{ + New: func() interface{} { return new(headerSorter) }, +} + +// sortedKeyValues returns h's keys sorted in the returned kvs +// slice. The headerSorter used to sort is also returned, for possible +// return to headerSorterCache. +func (h Header) sortedKeyValues(exclude map[string]bool) (kvs []keyValues, hs *headerSorter) { + hs = headerSorterPool.Get().(*headerSorter) + if cap(hs.kvs) < len(h) { + hs.kvs = make([]keyValues, 0, len(h)) + } + kvs = hs.kvs[:0] + for k, vv := range h { + if !exclude[k] { + kvs = append(kvs, keyValues{k, vv}) + } + } + hs.kvs = kvs + sort.Sort(hs) + return kvs, hs +} + +// WriteSubset writes a header in wire format. +// If exclude is not nil, keys where exclude[key] == true are not written. +func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) error { + ws, ok := w.(writeStringer) + if !ok { + ws = stringWriter{w} + } + kvs, sorter := h.sortedKeyValues(exclude) + for _, kv := range kvs { + for _, v := range kv.values { + v = headerNewlineToSpace.Replace(v) + v = textproto.TrimString(v) + for _, s := range []string{kv.key, ": ", v, "\r\n"} { + if _, err := ws.WriteString(s); err != nil { + return err + } + } + } + } + headerSorterPool.Put(sorter) + return nil +} + +// CanonicalHeaderKey returns the canonical format of the +// header key s. The canonicalization converts the first +// letter and any letter following a hyphen to upper case; +// the rest are converted to lowercase. For example, the +// canonical key for "accept-encoding" is "Accept-Encoding". +func CanonicalHeaderKey(s string) string { return textproto.CanonicalMIMEHeaderKey(s) } + +// hasToken reports whether token appears with v, ASCII +// case-insensitive, with space or comma boundaries. +// token must be all lowercase. +// v may contain mixed cased. +func hasToken(v, token string) bool { + if len(token) > len(v) || token == "" { + return false + } + if v == token { + return true + } + for sp := 0; sp <= len(v)-len(token); sp++ { + // Check that first character is good. + // The token is ASCII, so checking only a single byte + // is sufficient. We skip this potential starting + // position if both the first byte and its potential + // ASCII uppercase equivalent (b|0x20) don't match. + // False positives ('^' => '~') are caught by EqualFold. + if b := v[sp]; b != token[0] && b|0x20 != token[0] { + continue + } + // Check that start pos is on a valid token boundary. + if sp > 0 && !isTokenBoundary(v[sp-1]) { + continue + } + // Check that end pos is on a valid token boundary. + if endPos := sp + len(token); endPos != len(v) && !isTokenBoundary(v[endPos]) { + continue + } + if strings.EqualFold(v[sp:sp+len(token)], token) { + return true + } + } + return false +} + +func isTokenBoundary(b byte) bool { + return b == ' ' || b == ',' || b == '\t' +} diff --git a/src/net/http/header_test.go b/src/net/http/header_test.go new file mode 100644 index 000000000..9dcd591fa --- /dev/null +++ b/src/net/http/header_test.go @@ -0,0 +1,212 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "runtime" + "testing" + "time" +) + +var headerWriteTests = []struct { + h Header + exclude map[string]bool + expected string +}{ + {Header{}, nil, ""}, + { + Header{ + "Content-Type": {"text/html; charset=UTF-8"}, + "Content-Length": {"0"}, + }, + nil, + "Content-Length: 0\r\nContent-Type: text/html; charset=UTF-8\r\n", + }, + { + Header{ + "Content-Length": {"0", "1", "2"}, + }, + nil, + "Content-Length: 0\r\nContent-Length: 1\r\nContent-Length: 2\r\n", + }, + { + Header{ + "Expires": {"-1"}, + "Content-Length": {"0"}, + "Content-Encoding": {"gzip"}, + }, + map[string]bool{"Content-Length": true}, + "Content-Encoding: gzip\r\nExpires: -1\r\n", + }, + { + Header{ + "Expires": {"-1"}, + "Content-Length": {"0", "1", "2"}, + "Content-Encoding": {"gzip"}, + }, + map[string]bool{"Content-Length": true}, + "Content-Encoding: gzip\r\nExpires: -1\r\n", + }, + { + Header{ + "Expires": {"-1"}, + "Content-Length": {"0"}, + "Content-Encoding": {"gzip"}, + }, + map[string]bool{"Content-Length": true, "Expires": true, "Content-Encoding": true}, + "", + }, + { + Header{ + "Nil": nil, + "Empty": {}, + "Blank": {""}, + "Double-Blank": {"", ""}, + }, + nil, + "Blank: \r\nDouble-Blank: \r\nDouble-Blank: \r\n", + }, + // Tests header sorting when over the insertion sort threshold side: + { + Header{ + "k1": {"1a", "1b"}, + "k2": {"2a", "2b"}, + "k3": {"3a", "3b"}, + "k4": {"4a", "4b"}, + "k5": {"5a", "5b"}, + "k6": {"6a", "6b"}, + "k7": {"7a", "7b"}, + "k8": {"8a", "8b"}, + "k9": {"9a", "9b"}, + }, + map[string]bool{"k5": true}, + "k1: 1a\r\nk1: 1b\r\nk2: 2a\r\nk2: 2b\r\nk3: 3a\r\nk3: 3b\r\n" + + "k4: 4a\r\nk4: 4b\r\nk6: 6a\r\nk6: 6b\r\n" + + "k7: 7a\r\nk7: 7b\r\nk8: 8a\r\nk8: 8b\r\nk9: 9a\r\nk9: 9b\r\n", + }, +} + +func TestHeaderWrite(t *testing.T) { + var buf bytes.Buffer + for i, test := range headerWriteTests { + test.h.WriteSubset(&buf, test.exclude) + if buf.String() != test.expected { + t.Errorf("#%d:\n got: %q\nwant: %q", i, buf.String(), test.expected) + } + buf.Reset() + } +} + +var parseTimeTests = []struct { + h Header + err bool +}{ + {Header{"Date": {""}}, true}, + {Header{"Date": {"invalid"}}, true}, + {Header{"Date": {"1994-11-06T08:49:37Z00:00"}}, true}, + {Header{"Date": {"Sun, 06 Nov 1994 08:49:37 GMT"}}, false}, + {Header{"Date": {"Sunday, 06-Nov-94 08:49:37 GMT"}}, false}, + {Header{"Date": {"Sun Nov 6 08:49:37 1994"}}, false}, +} + +func TestParseTime(t *testing.T) { + expect := time.Date(1994, 11, 6, 8, 49, 37, 0, time.UTC) + for i, test := range parseTimeTests { + d, err := ParseTime(test.h.Get("Date")) + if err != nil { + if !test.err { + t.Errorf("#%d:\n got err: %v", i, err) + } + continue + } + if test.err { + t.Errorf("#%d:\n should err", i) + continue + } + if !expect.Equal(d) { + t.Errorf("#%d:\n got: %v\nwant: %v", i, d, expect) + } + } +} + +type hasTokenTest struct { + header string + token string + want bool +} + +var hasTokenTests = []hasTokenTest{ + {"", "", false}, + {"", "foo", false}, + {"foo", "foo", true}, + {"foo ", "foo", true}, + {" foo", "foo", true}, + {" foo ", "foo", true}, + {"foo,bar", "foo", true}, + {"bar,foo", "foo", true}, + {"bar, foo", "foo", true}, + {"bar,foo, baz", "foo", true}, + {"bar, foo,baz", "foo", true}, + {"bar,foo, baz", "foo", true}, + {"bar, foo, baz", "foo", true}, + {"FOO", "foo", true}, + {"FOO ", "foo", true}, + {" FOO", "foo", true}, + {" FOO ", "foo", true}, + {"FOO,BAR", "foo", true}, + {"BAR,FOO", "foo", true}, + {"BAR, FOO", "foo", true}, + {"BAR,FOO, baz", "foo", true}, + {"BAR, FOO,BAZ", "foo", true}, + {"BAR,FOO, BAZ", "foo", true}, + {"BAR, FOO, BAZ", "foo", true}, + {"foobar", "foo", false}, + {"barfoo ", "foo", false}, +} + +func TestHasToken(t *testing.T) { + for _, tt := range hasTokenTests { + if hasToken(tt.header, tt.token) != tt.want { + t.Errorf("hasToken(%q, %q) = %v; want %v", tt.header, tt.token, !tt.want, tt.want) + } + } +} + +var testHeader = Header{ + "Content-Length": {"123"}, + "Content-Type": {"text/plain"}, + "Date": {"some date at some time Z"}, + "Server": {DefaultUserAgent}, +} + +var buf bytes.Buffer + +func BenchmarkHeaderWriteSubset(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + buf.Reset() + testHeader.WriteSubset(&buf, nil) + } +} + +func TestHeaderWriteSubsetAllocs(t *testing.T) { + if testing.Short() { + t.Skip("skipping alloc test in short mode") + } + if raceEnabled { + t.Skip("skipping test under race detector") + } + if runtime.GOMAXPROCS(0) > 1 { + t.Skip("skipping; GOMAXPROCS>1") + } + n := testing.AllocsPerRun(100, func() { + buf.Reset() + testHeader.WriteSubset(&buf, nil) + }) + if n > 0 { + t.Errorf("allocs = %g; want 0", n) + } +} diff --git a/src/net/http/httptest/example_test.go b/src/net/http/httptest/example_test.go new file mode 100644 index 000000000..42a0ec953 --- /dev/null +++ b/src/net/http/httptest/example_test.go @@ -0,0 +1,50 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptest_test + +import ( + "fmt" + "io/ioutil" + "log" + "net/http" + "net/http/httptest" +) + +func ExampleResponseRecorder() { + handler := func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "something failed", http.StatusInternalServerError) + } + + req, err := http.NewRequest("GET", "http://example.com/foo", nil) + if err != nil { + log.Fatal(err) + } + + w := httptest.NewRecorder() + handler(w, req) + + fmt.Printf("%d - %s", w.Code, w.Body.String()) + // Output: 500 - something failed +} + +func ExampleServer() { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "Hello, client") + })) + defer ts.Close() + + res, err := http.Get(ts.URL) + if err != nil { + log.Fatal(err) + } + greeting, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s", greeting) + // Output: Hello, client +} diff --git a/src/net/http/httptest/recorder.go b/src/net/http/httptest/recorder.go new file mode 100644 index 000000000..5451f5423 --- /dev/null +++ b/src/net/http/httptest/recorder.go @@ -0,0 +1,72 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httptest provides utilities for HTTP testing. +package httptest + +import ( + "bytes" + "net/http" +) + +// ResponseRecorder is an implementation of http.ResponseWriter that +// records its mutations for later inspection in tests. +type ResponseRecorder struct { + Code int // the HTTP response code from WriteHeader + HeaderMap http.Header // the HTTP response headers + Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to + Flushed bool + + wroteHeader bool +} + +// NewRecorder returns an initialized ResponseRecorder. +func NewRecorder() *ResponseRecorder { + return &ResponseRecorder{ + HeaderMap: make(http.Header), + Body: new(bytes.Buffer), + Code: 200, + } +} + +// DefaultRemoteAddr is the default remote address to return in RemoteAddr if +// an explicit DefaultRemoteAddr isn't set on ResponseRecorder. +const DefaultRemoteAddr = "1.2.3.4" + +// Header returns the response headers. +func (rw *ResponseRecorder) Header() http.Header { + m := rw.HeaderMap + if m == nil { + m = make(http.Header) + rw.HeaderMap = m + } + return m +} + +// Write always succeeds and writes to rw.Body, if not nil. +func (rw *ResponseRecorder) Write(buf []byte) (int, error) { + if !rw.wroteHeader { + rw.WriteHeader(200) + } + if rw.Body != nil { + rw.Body.Write(buf) + } + return len(buf), nil +} + +// WriteHeader sets rw.Code. +func (rw *ResponseRecorder) WriteHeader(code int) { + if !rw.wroteHeader { + rw.Code = code + } + rw.wroteHeader = true +} + +// Flush sets rw.Flushed to true. +func (rw *ResponseRecorder) Flush() { + if !rw.wroteHeader { + rw.WriteHeader(200) + } + rw.Flushed = true +} diff --git a/src/net/http/httptest/recorder_test.go b/src/net/http/httptest/recorder_test.go new file mode 100644 index 000000000..2b563260c --- /dev/null +++ b/src/net/http/httptest/recorder_test.go @@ -0,0 +1,90 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptest + +import ( + "fmt" + "net/http" + "testing" +) + +func TestRecorder(t *testing.T) { + type checkFunc func(*ResponseRecorder) error + check := func(fns ...checkFunc) []checkFunc { return fns } + + hasStatus := func(wantCode int) checkFunc { + return func(rec *ResponseRecorder) error { + if rec.Code != wantCode { + return fmt.Errorf("Status = %d; want %d", rec.Code, wantCode) + } + return nil + } + } + hasContents := func(want string) checkFunc { + return func(rec *ResponseRecorder) error { + if rec.Body.String() != want { + return fmt.Errorf("wrote = %q; want %q", rec.Body.String(), want) + } + return nil + } + } + hasFlush := func(want bool) checkFunc { + return func(rec *ResponseRecorder) error { + if rec.Flushed != want { + return fmt.Errorf("Flushed = %v; want %v", rec.Flushed, want) + } + return nil + } + } + + tests := []struct { + name string + h func(w http.ResponseWriter, r *http.Request) + checks []checkFunc + }{ + { + "200 default", + func(w http.ResponseWriter, r *http.Request) {}, + check(hasStatus(200), hasContents("")), + }, + { + "first code only", + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(201) + w.WriteHeader(202) + w.Write([]byte("hi")) + }, + check(hasStatus(201), hasContents("hi")), + }, + { + "write sends 200", + func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hi first")) + w.WriteHeader(201) + w.WriteHeader(202) + }, + check(hasStatus(200), hasContents("hi first"), hasFlush(false)), + }, + { + "flush", + func(w http.ResponseWriter, r *http.Request) { + w.(http.Flusher).Flush() // also sends a 200 + w.WriteHeader(201) + }, + check(hasStatus(200), hasFlush(true)), + }, + } + r, _ := http.NewRequest("GET", "http://foo.com/", nil) + for _, tt := range tests { + h := http.HandlerFunc(tt.h) + rec := NewRecorder() + h.ServeHTTP(rec, r) + for _, check := range tt.checks { + if err := check(rec); err != nil { + t.Errorf("%s: %v", tt.name, err) + } + } + } +} diff --git a/src/net/http/httptest/server.go b/src/net/http/httptest/server.go new file mode 100644 index 000000000..789e7bf41 --- /dev/null +++ b/src/net/http/httptest/server.go @@ -0,0 +1,228 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implementation of Server + +package httptest + +import ( + "crypto/tls" + "flag" + "fmt" + "net" + "net/http" + "os" + "sync" +) + +// A Server is an HTTP server listening on a system-chosen port on the +// local loopback interface, for use in end-to-end HTTP tests. +type Server struct { + URL string // base URL of form http://ipaddr:port with no trailing slash + Listener net.Listener + + // TLS is the optional TLS configuration, populated with a new config + // after TLS is started. If set on an unstarted server before StartTLS + // is called, existing fields are copied into the new config. + TLS *tls.Config + + // Config may be changed after calling NewUnstartedServer and + // before Start or StartTLS. + Config *http.Server + + // wg counts the number of outstanding HTTP requests on this server. + // Close blocks until all requests are finished. + wg sync.WaitGroup +} + +// historyListener keeps track of all connections that it's ever +// accepted. +type historyListener struct { + net.Listener + sync.Mutex // protects history + history []net.Conn +} + +func (hs *historyListener) Accept() (c net.Conn, err error) { + c, err = hs.Listener.Accept() + if err == nil { + hs.Lock() + hs.history = append(hs.history, c) + hs.Unlock() + } + return +} + +func newLocalListener() net.Listener { + if *serve != "" { + l, err := net.Listen("tcp", *serve) + if err != nil { + panic(fmt.Sprintf("httptest: failed to listen on %v: %v", *serve, err)) + } + return l + } + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) + } + } + return l +} + +// When debugging a particular http server-based test, +// this flag lets you run +// go test -run=BrokenTest -httptest.serve=127.0.0.1:8000 +// to start the broken server so you can interact with it manually. +var serve = flag.String("httptest.serve", "", "if non-empty, httptest.NewServer serves on this address and blocks") + +// NewServer starts and returns a new Server. +// The caller should call Close when finished, to shut it down. +func NewServer(handler http.Handler) *Server { + ts := NewUnstartedServer(handler) + ts.Start() + return ts +} + +// NewUnstartedServer returns a new Server but doesn't start it. +// +// After changing its configuration, the caller should call Start or +// StartTLS. +// +// The caller should call Close when finished, to shut it down. +func NewUnstartedServer(handler http.Handler) *Server { + return &Server{ + Listener: newLocalListener(), + Config: &http.Server{Handler: handler}, + } +} + +// Start starts a server from NewUnstartedServer. +func (s *Server) Start() { + if s.URL != "" { + panic("Server already started") + } + s.Listener = &historyListener{Listener: s.Listener} + s.URL = "http://" + s.Listener.Addr().String() + s.wrapHandler() + go s.Config.Serve(s.Listener) + if *serve != "" { + fmt.Fprintln(os.Stderr, "httptest: serving on", s.URL) + select {} + } +} + +// StartTLS starts TLS on a server from NewUnstartedServer. +func (s *Server) StartTLS() { + if s.URL != "" { + panic("Server already started") + } + cert, err := tls.X509KeyPair(localhostCert, localhostKey) + if err != nil { + panic(fmt.Sprintf("httptest: NewTLSServer: %v", err)) + } + + existingConfig := s.TLS + s.TLS = new(tls.Config) + if existingConfig != nil { + *s.TLS = *existingConfig + } + if s.TLS.NextProtos == nil { + s.TLS.NextProtos = []string{"http/1.1"} + } + if len(s.TLS.Certificates) == 0 { + s.TLS.Certificates = []tls.Certificate{cert} + } + tlsListener := tls.NewListener(s.Listener, s.TLS) + + s.Listener = &historyListener{Listener: tlsListener} + s.URL = "https://" + s.Listener.Addr().String() + s.wrapHandler() + go s.Config.Serve(s.Listener) +} + +func (s *Server) wrapHandler() { + h := s.Config.Handler + if h == nil { + h = http.DefaultServeMux + } + s.Config.Handler = &waitGroupHandler{ + s: s, + h: h, + } +} + +// NewTLSServer starts and returns a new Server using TLS. +// The caller should call Close when finished, to shut it down. +func NewTLSServer(handler http.Handler) *Server { + ts := NewUnstartedServer(handler) + ts.StartTLS() + return ts +} + +// Close shuts down the server and blocks until all outstanding +// requests on this server have completed. +func (s *Server) Close() { + s.Listener.Close() + s.wg.Wait() + s.CloseClientConnections() + if t, ok := http.DefaultTransport.(*http.Transport); ok { + t.CloseIdleConnections() + } +} + +// CloseClientConnections closes any currently open HTTP connections +// to the test Server. +func (s *Server) CloseClientConnections() { + hl, ok := s.Listener.(*historyListener) + if !ok { + return + } + hl.Lock() + for _, conn := range hl.history { + conn.Close() + } + hl.Unlock() +} + +// waitGroupHandler wraps a handler, incrementing and decrementing a +// sync.WaitGroup on each request, to enable Server.Close to block +// until outstanding requests are finished. +type waitGroupHandler struct { + s *Server + h http.Handler // non-nil +} + +func (h *waitGroupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.s.wg.Add(1) + defer h.s.wg.Done() // a defer, in case ServeHTTP below panics + h.h.ServeHTTP(w, r) +} + +// localhostCert is a PEM-encoded TLS cert with SAN IPs +// "127.0.0.1" and "[::1]", expiring at the last second of 2049 (the end +// of ASN.1 time). +// generated from src/crypto/tls: +// go run generate_cert.go --rsa-bits 512 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h +var localhostCert = []byte(`-----BEGIN CERTIFICATE----- +MIIBdzCCASOgAwIBAgIBADALBgkqhkiG9w0BAQUwEjEQMA4GA1UEChMHQWNtZSBD +bzAeFw03MDAxMDEwMDAwMDBaFw00OTEyMzEyMzU5NTlaMBIxEDAOBgNVBAoTB0Fj +bWUgQ28wWjALBgkqhkiG9w0BAQEDSwAwSAJBAN55NcYKZeInyTuhcCwFMhDHCmwa +IUSdtXdcbItRB/yfXGBhiex00IaLXQnSU+QZPRZWYqeTEbFSgihqi1PUDy8CAwEA +AaNoMGYwDgYDVR0PAQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud +EwEB/wQFMAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAA +AAAAAAAAAAAAAAEwCwYJKoZIhvcNAQEFA0EAAoQn/ytgqpiLcZu9XKbCJsJcvkgk +Se6AbGXgSlq+ZCEVo0qIwSgeBqmsJxUu7NCSOwVJLYNEBO2DtIxoYVk+MA== +-----END CERTIFICATE-----`) + +// localhostKey is the private key for localhostCert. +var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIBPAIBAAJBAN55NcYKZeInyTuhcCwFMhDHCmwaIUSdtXdcbItRB/yfXGBhiex0 +0IaLXQnSU+QZPRZWYqeTEbFSgihqi1PUDy8CAwEAAQJBAQdUx66rfh8sYsgfdcvV +NoafYpnEcB5s4m/vSVe6SU7dCK6eYec9f9wpT353ljhDUHq3EbmE4foNzJngh35d +AekCIQDhRQG5Li0Wj8TM4obOnnXUXf1jRv0UkzE9AHWLG5q3AwIhAPzSjpYUDjVW +MCUXgckTpKCuGwbJk7424Nb8bLzf3kllAiA5mUBgjfr/WtFSJdWcPQ4Zt9KTMNKD +EUO0ukpTwEIl6wIhAMbGqZK3zAAFdq8DD2jPx+UJXnh0rnOkZBzDtJ6/iN69AiEA +1Aq8MJgTaYsDQWyU/hDq5YkDJc9e9DSCvUIzqxQWMQE= +-----END RSA PRIVATE KEY-----`) diff --git a/src/net/http/httptest/server_test.go b/src/net/http/httptest/server_test.go new file mode 100644 index 000000000..500a9f0b8 --- /dev/null +++ b/src/net/http/httptest/server_test.go @@ -0,0 +1,29 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptest + +import ( + "io/ioutil" + "net/http" + "testing" +) + +func TestServer(t *testing.T) { + ts := NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello")) + })) + defer ts.Close() + res, err := http.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + got, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if string(got) != "hello" { + t.Errorf("got %q, want hello", string(got)) + } +} diff --git a/src/net/http/httputil/dump.go b/src/net/http/httputil/dump.go new file mode 100644 index 000000000..2a7a413d0 --- /dev/null +++ b/src/net/http/httputil/dump.go @@ -0,0 +1,276 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httputil + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +// One of the copies, say from b to r2, could be avoided by using a more +// elaborate trick where the other copy is made during Request/Response.Write. +// This would complicate things too much, given that these functions are for +// debugging only. +func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) { + var buf bytes.Buffer + if _, err = buf.ReadFrom(b); err != nil { + return nil, nil, err + } + if err = b.Close(); err != nil { + return nil, nil, err + } + return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewReader(buf.Bytes())), nil +} + +// dumpConn is a net.Conn which writes to Writer and reads from Reader +type dumpConn struct { + io.Writer + io.Reader +} + +func (c *dumpConn) Close() error { return nil } +func (c *dumpConn) LocalAddr() net.Addr { return nil } +func (c *dumpConn) RemoteAddr() net.Addr { return nil } +func (c *dumpConn) SetDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetReadDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil } + +type neverEnding byte + +func (b neverEnding) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = byte(b) + } + return len(p), nil +} + +// DumpRequestOut is like DumpRequest but includes +// headers that the standard http.Transport adds, +// such as User-Agent. +func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { + save := req.Body + dummyBody := false + if !body || req.Body == nil { + req.Body = nil + if req.ContentLength != 0 { + req.Body = ioutil.NopCloser(io.LimitReader(neverEnding('x'), req.ContentLength)) + dummyBody = true + } + } else { + var err error + save, req.Body, err = drainBody(req.Body) + if err != nil { + return nil, err + } + } + + // Since we're using the actual Transport code to write the request, + // switch to http so the Transport doesn't try to do an SSL + // negotiation with our dumpConn and its bytes.Buffer & pipe. + // The wire format for https and http are the same, anyway. + reqSend := req + if req.URL.Scheme == "https" { + reqSend = new(http.Request) + *reqSend = *req + reqSend.URL = new(url.URL) + *reqSend.URL = *req.URL + reqSend.URL.Scheme = "http" + } + + // Use the actual Transport code to record what we would send + // on the wire, but not using TCP. Use a Transport with a + // custom dialer that returns a fake net.Conn that waits + // for the full input (and recording it), and then responds + // with a dummy response. + var buf bytes.Buffer // records the output + pr, pw := io.Pipe() + dr := &delegateReader{c: make(chan io.Reader)} + // Wait for the request before replying with a dummy response: + go func() { + http.ReadRequest(bufio.NewReader(pr)) + dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\n\r\n") + }() + + t := &http.Transport{ + Dial: func(net, addr string) (net.Conn, error) { + return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil + }, + } + defer t.CloseIdleConnections() + + _, err := t.RoundTrip(reqSend) + + req.Body = save + if err != nil { + return nil, err + } + dump := buf.Bytes() + + // If we used a dummy body above, remove it now. + // TODO: if the req.ContentLength is large, we allocate memory + // unnecessarily just to slice it off here. But this is just + // a debug function, so this is acceptable for now. We could + // discard the body earlier if this matters. + if dummyBody { + if i := bytes.Index(dump, []byte("\r\n\r\n")); i >= 0 { + dump = dump[:i+4] + } + } + return dump, nil +} + +// delegateReader is a reader that delegates to another reader, +// once it arrives on a channel. +type delegateReader struct { + c chan io.Reader + r io.Reader // nil until received from c +} + +func (r *delegateReader) Read(p []byte) (int, error) { + if r.r == nil { + r.r = <-r.c + } + return r.r.Read(p) +} + +// Return value if nonempty, def otherwise. +func valueOrDefault(value, def string) string { + if value != "" { + return value + } + return def +} + +var reqWriteExcludeHeaderDump = map[string]bool{ + "Host": true, // not in Header map anyway + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// dumpAsReceived writes req to w in the form as it was received, or +// at least as accurately as possible from the information retained in +// the request. +func dumpAsReceived(req *http.Request, w io.Writer) error { + return nil +} + +// DumpRequest returns the as-received wire representation of req, +// optionally including the request body, for debugging. +// DumpRequest is semantically a no-op, but in order to +// dump the body, it reads the body data into memory and +// changes req.Body to refer to the in-memory copy. +// The documentation for http.Request.Write details which fields +// of req are used. +func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { + save := req.Body + if !body || req.Body == nil { + req.Body = nil + } else { + save, req.Body, err = drainBody(req.Body) + if err != nil { + return + } + } + + var b bytes.Buffer + + fmt.Fprintf(&b, "%s %s HTTP/%d.%d\r\n", valueOrDefault(req.Method, "GET"), + req.URL.RequestURI(), req.ProtoMajor, req.ProtoMinor) + + host := req.Host + if host == "" && req.URL != nil { + host = req.URL.Host + } + if host != "" { + fmt.Fprintf(&b, "Host: %s\r\n", host) + } + + chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" + if len(req.TransferEncoding) > 0 { + fmt.Fprintf(&b, "Transfer-Encoding: %s\r\n", strings.Join(req.TransferEncoding, ",")) + } + if req.Close { + fmt.Fprintf(&b, "Connection: close\r\n") + } + + err = req.Header.WriteSubset(&b, reqWriteExcludeHeaderDump) + if err != nil { + return + } + + io.WriteString(&b, "\r\n") + + if req.Body != nil { + var dest io.Writer = &b + if chunked { + dest = NewChunkedWriter(dest) + } + _, err = io.Copy(dest, req.Body) + if chunked { + dest.(io.Closer).Close() + io.WriteString(&b, "\r\n") + } + } + + req.Body = save + if err != nil { + return + } + dump = b.Bytes() + return +} + +// errNoBody is a sentinel error value used by failureToReadBody so we can detect +// that the lack of body was intentional. +var errNoBody = errors.New("sentinel error value") + +// failureToReadBody is a io.ReadCloser that just returns errNoBody on +// Read. It's swapped in when we don't actually want to consume the +// body, but need a non-nil one, and want to distinguish the error +// from reading the dummy body. +type failureToReadBody struct{} + +func (failureToReadBody) Read([]byte) (int, error) { return 0, errNoBody } +func (failureToReadBody) Close() error { return nil } + +var emptyBody = ioutil.NopCloser(strings.NewReader("")) + +// DumpResponse is like DumpRequest but dumps a response. +func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) { + var b bytes.Buffer + save := resp.Body + savecl := resp.ContentLength + + if !body { + resp.Body = failureToReadBody{} + } else if resp.Body == nil { + resp.Body = emptyBody + } else { + save, resp.Body, err = drainBody(resp.Body) + if err != nil { + return + } + } + err = resp.Write(&b) + if err == errNoBody { + err = nil + } + resp.Body = save + resp.ContentLength = savecl + if err != nil { + return nil, err + } + return b.Bytes(), nil +} diff --git a/src/net/http/httputil/dump_test.go b/src/net/http/httputil/dump_test.go new file mode 100644 index 000000000..e1ffb3935 --- /dev/null +++ b/src/net/http/httputil/dump_test.go @@ -0,0 +1,263 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httputil + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "runtime" + "strings" + "testing" +) + +type dumpTest struct { + Req http.Request + Body interface{} // optional []byte or func() io.ReadCloser to populate Req.Body + + WantDump string + WantDumpOut string + NoBody bool // if true, set DumpRequest{,Out} body to false +} + +var dumpTests = []dumpTest{ + + // HTTP/1.1 => chunked coding; body; empty trailer + { + Req: http.Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/search", + }, + ProtoMajor: 1, + ProtoMinor: 1, + TransferEncoding: []string{"chunked"}, + }, + + Body: []byte("abcdef"), + + WantDump: "GET /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("abcdef") + chunk(""), + }, + + // Verify that DumpRequest preserves the HTTP version number, doesn't add a Host, + // and doesn't add a User-Agent. + { + Req: http.Request{ + Method: "GET", + URL: mustParseURL("/foo"), + ProtoMajor: 1, + ProtoMinor: 0, + Header: http.Header{ + "X-Foo": []string{"X-Bar"}, + }, + }, + + WantDump: "GET /foo HTTP/1.0\r\n" + + "X-Foo: X-Bar\r\n\r\n", + }, + + { + Req: *mustNewRequest("GET", "http://example.com/foo", nil), + + WantDumpOut: "GET /foo HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Accept-Encoding: gzip\r\n\r\n", + }, + + // Test that an https URL doesn't try to do an SSL negotiation + // with a bytes.Buffer and hang with all goroutines not + // runnable. + { + Req: *mustNewRequest("GET", "https://example.com/foo", nil), + + WantDumpOut: "GET /foo HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Accept-Encoding: gzip\r\n\r\n", + }, + + // Request with Body, but Dump requested without it. + { + Req: http.Request{ + Method: "POST", + URL: &url.URL{ + Scheme: "http", + Host: "post.tld", + Path: "/", + }, + ContentLength: 6, + ProtoMajor: 1, + ProtoMinor: 1, + }, + + Body: []byte("abcdef"), + + WantDumpOut: "POST / HTTP/1.1\r\n" + + "Host: post.tld\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Content-Length: 6\r\n" + + "Accept-Encoding: gzip\r\n\r\n", + + NoBody: true, + }, +} + +func TestDumpRequest(t *testing.T) { + numg0 := runtime.NumGoroutine() + for i, tt := range dumpTests { + setBody := func() { + if tt.Body == nil { + return + } + switch b := tt.Body.(type) { + case []byte: + tt.Req.Body = ioutil.NopCloser(bytes.NewReader(b)) + case func() io.ReadCloser: + tt.Req.Body = b() + } + } + setBody() + if tt.Req.Header == nil { + tt.Req.Header = make(http.Header) + } + + if tt.WantDump != "" { + setBody() + dump, err := DumpRequest(&tt.Req, !tt.NoBody) + if err != nil { + t.Errorf("DumpRequest #%d: %s", i, err) + continue + } + if string(dump) != tt.WantDump { + t.Errorf("DumpRequest %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDump, string(dump)) + continue + } + } + + if tt.WantDumpOut != "" { + setBody() + dump, err := DumpRequestOut(&tt.Req, !tt.NoBody) + if err != nil { + t.Errorf("DumpRequestOut #%d: %s", i, err) + continue + } + if string(dump) != tt.WantDumpOut { + t.Errorf("DumpRequestOut %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantDumpOut, string(dump)) + continue + } + } + } + if dg := runtime.NumGoroutine() - numg0; dg > 4 { + t.Errorf("Unexpectedly large number of new goroutines: %d new", dg) + } +} + +func chunk(s string) string { + return fmt.Sprintf("%x\r\n%s\r\n", len(s), s) +} + +func mustParseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(fmt.Sprintf("Error parsing URL %q: %v", s, err)) + } + return u +} + +func mustNewRequest(method, url string, body io.Reader) *http.Request { + req, err := http.NewRequest(method, url, body) + if err != nil { + panic(fmt.Sprintf("NewRequest(%q, %q, %p) err = %v", method, url, body, err)) + } + return req +} + +var dumpResTests = []struct { + res *http.Response + body bool + want string +}{ + { + res: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 50, + Header: http.Header{ + "Foo": []string{"Bar"}, + }, + Body: ioutil.NopCloser(strings.NewReader("foo")), // shouldn't be used + }, + body: false, // to verify we see 50, not empty or 3. + want: `HTTP/1.1 200 OK +Content-Length: 50 +Foo: Bar`, + }, + + { + res: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 3, + Body: ioutil.NopCloser(strings.NewReader("foo")), + }, + body: true, + want: `HTTP/1.1 200 OK +Content-Length: 3 + +foo`, + }, + + { + res: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: -1, + Body: ioutil.NopCloser(strings.NewReader("foo")), + TransferEncoding: []string{"chunked"}, + }, + body: true, + want: `HTTP/1.1 200 OK +Transfer-Encoding: chunked + +3 +foo +0`, + }, +} + +func TestDumpResponse(t *testing.T) { + for i, tt := range dumpResTests { + gotb, err := DumpResponse(tt.res, tt.body) + if err != nil { + t.Errorf("%d. DumpResponse = %v", i, err) + continue + } + got := string(gotb) + got = strings.TrimSpace(got) + got = strings.Replace(got, "\r", "", -1) + + if got != tt.want { + t.Errorf("%d.\nDumpResponse got:\n%s\n\nWant:\n%s\n", i, got, tt.want) + } + } +} diff --git a/src/net/http/httputil/httputil.go b/src/net/http/httputil/httputil.go new file mode 100644 index 000000000..2e523e9e2 --- /dev/null +++ b/src/net/http/httputil/httputil.go @@ -0,0 +1,39 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httputil provides HTTP utility functions, complementing the +// more common ones in the net/http package. +package httputil + +import ( + "io" + "net/http/internal" +) + +// NewChunkedReader returns a new chunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The chunkedReader returns io.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func NewChunkedReader(r io.Reader) io.Reader { + return internal.NewChunkedReader(r) +} + +// NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP +// "chunked" format before writing them to w. Closing the returned chunkedWriter +// sends the final 0-length chunk that marks the end of the stream. +// +// NewChunkedWriter is not needed by normal applications. The http +// package adds chunking automatically if handlers don't set a +// Content-Length header. Using NewChunkedWriter inside a handler +// would result in double chunking or chunking with a Content-Length +// length, both of which are wrong. +func NewChunkedWriter(w io.Writer) io.WriteCloser { + return internal.NewChunkedWriter(w) +} + +// ErrLineTooLong is returned when reading malformed chunked data +// with lines that are too long. +var ErrLineTooLong = internal.ErrLineTooLong diff --git a/src/net/http/httputil/persist.go b/src/net/http/httputil/persist.go new file mode 100644 index 000000000..987bcc96b --- /dev/null +++ b/src/net/http/httputil/persist.go @@ -0,0 +1,429 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httputil + +import ( + "bufio" + "errors" + "io" + "net" + "net/http" + "net/textproto" + "sync" +) + +var ( + ErrPersistEOF = &http.ProtocolError{ErrorString: "persistent connection closed"} + ErrClosed = &http.ProtocolError{ErrorString: "connection closed by user"} + ErrPipeline = &http.ProtocolError{ErrorString: "pipeline error"} +) + +// This is an API usage error - the local side is closed. +// ErrPersistEOF (above) reports that the remote side is closed. +var errClosed = errors.New("i/o operation on closed connection") + +// A ServerConn reads requests and sends responses over an underlying +// connection, until the HTTP keepalive logic commands an end. ServerConn +// also allows hijacking the underlying connection by calling Hijack +// to regain control over the connection. ServerConn supports pipe-lining, +// i.e. requests can be read out of sync (but in the same order) while the +// respective responses are sent. +// +// ServerConn is low-level and old. Applications should instead use Server +// in the net/http package. +type ServerConn struct { + lk sync.Mutex // read-write protects the following fields + c net.Conn + r *bufio.Reader + re, we error // read/write errors + lastbody io.ReadCloser + nread, nwritten int + pipereq map[*http.Request]uint + + pipe textproto.Pipeline +} + +// NewServerConn returns a new ServerConn reading and writing c. If r is not +// nil, it is the buffer to use when reading c. +// +// ServerConn is low-level and old. Applications should instead use Server +// in the net/http package. +func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { + if r == nil { + r = bufio.NewReader(c) + } + return &ServerConn{c: c, r: r, pipereq: make(map[*http.Request]uint)} +} + +// Hijack detaches the ServerConn and returns the underlying connection as well +// as the read-side bufio which may have some left over data. Hijack may be +// called before Read has signaled the end of the keep-alive logic. The user +// should not call Hijack while Read or Write is in progress. +func (sc *ServerConn) Hijack() (c net.Conn, r *bufio.Reader) { + sc.lk.Lock() + defer sc.lk.Unlock() + c = sc.c + r = sc.r + sc.c = nil + sc.r = nil + return +} + +// Close calls Hijack and then also closes the underlying connection +func (sc *ServerConn) Close() error { + c, _ := sc.Hijack() + if c != nil { + return c.Close() + } + return nil +} + +// Read returns the next request on the wire. An ErrPersistEOF is returned if +// it is gracefully determined that there are no more requests (e.g. after the +// first request on an HTTP/1.0 connection, or after a Connection:close on a +// HTTP/1.1 connection). +func (sc *ServerConn) Read() (req *http.Request, err error) { + + // Ensure ordered execution of Reads and Writes + id := sc.pipe.Next() + sc.pipe.StartRequest(id) + defer func() { + sc.pipe.EndRequest(id) + if req == nil { + sc.pipe.StartResponse(id) + sc.pipe.EndResponse(id) + } else { + // Remember the pipeline id of this request + sc.lk.Lock() + sc.pipereq[req] = id + sc.lk.Unlock() + } + }() + + sc.lk.Lock() + if sc.we != nil { // no point receiving if write-side broken or closed + defer sc.lk.Unlock() + return nil, sc.we + } + if sc.re != nil { + defer sc.lk.Unlock() + return nil, sc.re + } + if sc.r == nil { // connection closed by user in the meantime + defer sc.lk.Unlock() + return nil, errClosed + } + r := sc.r + lastbody := sc.lastbody + sc.lastbody = nil + sc.lk.Unlock() + + // Make sure body is fully consumed, even if user does not call body.Close + if lastbody != nil { + // body.Close is assumed to be idempotent and multiple calls to + // it should return the error that its first invocation + // returned. + err = lastbody.Close() + if err != nil { + sc.lk.Lock() + defer sc.lk.Unlock() + sc.re = err + return nil, err + } + } + + req, err = http.ReadRequest(r) + sc.lk.Lock() + defer sc.lk.Unlock() + if err != nil { + if err == io.ErrUnexpectedEOF { + // A close from the opposing client is treated as a + // graceful close, even if there was some unparse-able + // data before the close. + sc.re = ErrPersistEOF + return nil, sc.re + } else { + sc.re = err + return req, err + } + } + sc.lastbody = req.Body + sc.nread++ + if req.Close { + sc.re = ErrPersistEOF + return req, sc.re + } + return req, err +} + +// Pending returns the number of unanswered requests +// that have been received on the connection. +func (sc *ServerConn) Pending() int { + sc.lk.Lock() + defer sc.lk.Unlock() + return sc.nread - sc.nwritten +} + +// Write writes resp in response to req. To close the connection gracefully, set the +// Response.Close field to true. Write should be considered operational until +// it returns an error, regardless of any errors returned on the Read side. +func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { + + // Retrieve the pipeline ID of this request/response pair + sc.lk.Lock() + id, ok := sc.pipereq[req] + delete(sc.pipereq, req) + if !ok { + sc.lk.Unlock() + return ErrPipeline + } + sc.lk.Unlock() + + // Ensure pipeline order + sc.pipe.StartResponse(id) + defer sc.pipe.EndResponse(id) + + sc.lk.Lock() + if sc.we != nil { + defer sc.lk.Unlock() + return sc.we + } + if sc.c == nil { // connection closed by user in the meantime + defer sc.lk.Unlock() + return ErrClosed + } + c := sc.c + if sc.nread <= sc.nwritten { + defer sc.lk.Unlock() + return errors.New("persist server pipe count") + } + if resp.Close { + // After signaling a keep-alive close, any pipelined unread + // requests will be lost. It is up to the user to drain them + // before signaling. + sc.re = ErrPersistEOF + } + sc.lk.Unlock() + + err := resp.Write(c) + sc.lk.Lock() + defer sc.lk.Unlock() + if err != nil { + sc.we = err + return err + } + sc.nwritten++ + + return nil +} + +// A ClientConn sends request and receives headers over an underlying +// connection, while respecting the HTTP keepalive logic. ClientConn +// supports hijacking the connection calling Hijack to +// regain control of the underlying net.Conn and deal with it as desired. +// +// ClientConn is low-level and old. Applications should instead use +// Client or Transport in the net/http package. +type ClientConn struct { + lk sync.Mutex // read-write protects the following fields + c net.Conn + r *bufio.Reader + re, we error // read/write errors + lastbody io.ReadCloser + nread, nwritten int + pipereq map[*http.Request]uint + + pipe textproto.Pipeline + writeReq func(*http.Request, io.Writer) error +} + +// NewClientConn returns a new ClientConn reading and writing c. If r is not +// nil, it is the buffer to use when reading c. +// +// ClientConn is low-level and old. Applications should use Client or +// Transport in the net/http package. +func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { + if r == nil { + r = bufio.NewReader(c) + } + return &ClientConn{ + c: c, + r: r, + pipereq: make(map[*http.Request]uint), + writeReq: (*http.Request).Write, + } +} + +// NewProxyClientConn works like NewClientConn but writes Requests +// using Request's WriteProxy method. +// +// New code should not use NewProxyClientConn. See Client or +// Transport in the net/http package instead. +func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn { + cc := NewClientConn(c, r) + cc.writeReq = (*http.Request).WriteProxy + return cc +} + +// Hijack detaches the ClientConn and returns the underlying connection as well +// as the read-side bufio which may have some left over data. Hijack may be +// called before the user or Read have signaled the end of the keep-alive +// logic. The user should not call Hijack while Read or Write is in progress. +func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { + cc.lk.Lock() + defer cc.lk.Unlock() + c = cc.c + r = cc.r + cc.c = nil + cc.r = nil + return +} + +// Close calls Hijack and then also closes the underlying connection +func (cc *ClientConn) Close() error { + c, _ := cc.Hijack() + if c != nil { + return c.Close() + } + return nil +} + +// Write writes a request. An ErrPersistEOF error is returned if the connection +// has been closed in an HTTP keepalive sense. If req.Close equals true, the +// keepalive connection is logically closed after this request and the opposing +// server is informed. An ErrUnexpectedEOF indicates the remote closed the +// underlying TCP connection, which is usually considered as graceful close. +func (cc *ClientConn) Write(req *http.Request) (err error) { + + // Ensure ordered execution of Writes + id := cc.pipe.Next() + cc.pipe.StartRequest(id) + defer func() { + cc.pipe.EndRequest(id) + if err != nil { + cc.pipe.StartResponse(id) + cc.pipe.EndResponse(id) + } else { + // Remember the pipeline id of this request + cc.lk.Lock() + cc.pipereq[req] = id + cc.lk.Unlock() + } + }() + + cc.lk.Lock() + if cc.re != nil { // no point sending if read-side closed or broken + defer cc.lk.Unlock() + return cc.re + } + if cc.we != nil { + defer cc.lk.Unlock() + return cc.we + } + if cc.c == nil { // connection closed by user in the meantime + defer cc.lk.Unlock() + return errClosed + } + c := cc.c + if req.Close { + // We write the EOF to the write-side error, because there + // still might be some pipelined reads + cc.we = ErrPersistEOF + } + cc.lk.Unlock() + + err = cc.writeReq(req, c) + cc.lk.Lock() + defer cc.lk.Unlock() + if err != nil { + cc.we = err + return err + } + cc.nwritten++ + + return nil +} + +// Pending returns the number of unanswered requests +// that have been sent on the connection. +func (cc *ClientConn) Pending() int { + cc.lk.Lock() + defer cc.lk.Unlock() + return cc.nwritten - cc.nread +} + +// Read reads the next response from the wire. A valid response might be +// returned together with an ErrPersistEOF, which means that the remote +// requested that this be the last request serviced. Read can be called +// concurrently with Write, but not with another Read. +func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { + // Retrieve the pipeline ID of this request/response pair + cc.lk.Lock() + id, ok := cc.pipereq[req] + delete(cc.pipereq, req) + if !ok { + cc.lk.Unlock() + return nil, ErrPipeline + } + cc.lk.Unlock() + + // Ensure pipeline order + cc.pipe.StartResponse(id) + defer cc.pipe.EndResponse(id) + + cc.lk.Lock() + if cc.re != nil { + defer cc.lk.Unlock() + return nil, cc.re + } + if cc.r == nil { // connection closed by user in the meantime + defer cc.lk.Unlock() + return nil, errClosed + } + r := cc.r + lastbody := cc.lastbody + cc.lastbody = nil + cc.lk.Unlock() + + // Make sure body is fully consumed, even if user does not call body.Close + if lastbody != nil { + // body.Close is assumed to be idempotent and multiple calls to + // it should return the error that its first invocation + // returned. + err = lastbody.Close() + if err != nil { + cc.lk.Lock() + defer cc.lk.Unlock() + cc.re = err + return nil, err + } + } + + resp, err = http.ReadResponse(r, req) + cc.lk.Lock() + defer cc.lk.Unlock() + if err != nil { + cc.re = err + return resp, err + } + cc.lastbody = resp.Body + + cc.nread++ + + if resp.Close { + cc.re = ErrPersistEOF // don't send any more requests + return resp, cc.re + } + return resp, err +} + +// Do is convenience method that writes a request and reads a response. +func (cc *ClientConn) Do(req *http.Request) (resp *http.Response, err error) { + err = cc.Write(req) + if err != nil { + return + } + return cc.Read(req) +} diff --git a/src/net/http/httputil/reverseproxy.go b/src/net/http/httputil/reverseproxy.go new file mode 100644 index 000000000..ab4637018 --- /dev/null +++ b/src/net/http/httputil/reverseproxy.go @@ -0,0 +1,225 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP reverse proxy handler + +package httputil + +import ( + "io" + "log" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// onExitFlushLoop is a callback set by tests to detect the state of the +// flushLoop() goroutine. +var onExitFlushLoop func() + +// ReverseProxy is an HTTP Handler that takes an incoming request and +// sends it to another server, proxying the response back to the +// client. +type ReverseProxy struct { + // Director must be a function which modifies + // the request into a new request to be sent + // using Transport. Its response is then copied + // back to the original client unmodified. + Director func(*http.Request) + + // The transport used to perform proxy requests. + // If nil, http.DefaultTransport is used. + Transport http.RoundTripper + + // FlushInterval specifies the flush interval + // to flush to the client while copying the + // response body. + // If zero, no periodic flushing is done. + FlushInterval time.Duration + + // ErrorLog specifies an optional logger for errors + // that occur when attempting to proxy the request. + // If nil, logging goes to os.Stderr via the log package's + // standard logger. + ErrorLog *log.Logger +} + +func singleJoiningSlash(a, b string) string { + aslash := strings.HasSuffix(a, "/") + bslash := strings.HasPrefix(b, "/") + switch { + case aslash && bslash: + return a + b[1:] + case !aslash && !bslash: + return a + "/" + b + } + return a + b +} + +// NewSingleHostReverseProxy returns a new ReverseProxy that rewrites +// URLs to the scheme, host, and base path provided in target. If the +// target's path is "/base" and the incoming request was for "/dir", +// the target request will be for /base/dir. +func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy { + targetQuery := target.RawQuery + director := func(req *http.Request) { + req.URL.Scheme = target.Scheme + req.URL.Host = target.Host + req.URL.Path = singleJoiningSlash(target.Path, req.URL.Path) + if targetQuery == "" || req.URL.RawQuery == "" { + req.URL.RawQuery = targetQuery + req.URL.RawQuery + } else { + req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery + } + } + return &ReverseProxy{Director: director} +} + +func copyHeader(dst, src http.Header) { + for k, vv := range src { + for _, v := range vv { + dst.Add(k, v) + } + } +} + +// Hop-by-hop headers. These are removed when sent to the backend. +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html +var hopHeaders = []string{ + "Connection", + "Keep-Alive", + "Proxy-Authenticate", + "Proxy-Authorization", + "Te", // canonicalized version of "TE" + "Trailers", + "Transfer-Encoding", + "Upgrade", +} + +func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + transport := p.Transport + if transport == nil { + transport = http.DefaultTransport + } + + outreq := new(http.Request) + *outreq = *req // includes shallow copies of maps, but okay + + p.Director(outreq) + outreq.Proto = "HTTP/1.1" + outreq.ProtoMajor = 1 + outreq.ProtoMinor = 1 + outreq.Close = false + + // Remove hop-by-hop headers to the backend. Especially + // important is "Connection" because we want a persistent + // connection, regardless of what the client sent to us. This + // is modifying the same underlying map from req (shallow + // copied above) so we only copy it if necessary. + copiedHeaders := false + for _, h := range hopHeaders { + if outreq.Header.Get(h) != "" { + if !copiedHeaders { + outreq.Header = make(http.Header) + copyHeader(outreq.Header, req.Header) + copiedHeaders = true + } + outreq.Header.Del(h) + } + } + + if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil { + // If we aren't the first proxy retain prior + // X-Forwarded-For information as a comma+space + // separated list and fold multiple headers into one. + if prior, ok := outreq.Header["X-Forwarded-For"]; ok { + clientIP = strings.Join(prior, ", ") + ", " + clientIP + } + outreq.Header.Set("X-Forwarded-For", clientIP) + } + + res, err := transport.RoundTrip(outreq) + if err != nil { + p.logf("http: proxy error: %v", err) + rw.WriteHeader(http.StatusInternalServerError) + return + } + defer res.Body.Close() + + for _, h := range hopHeaders { + res.Header.Del(h) + } + + copyHeader(rw.Header(), res.Header) + + rw.WriteHeader(res.StatusCode) + p.copyResponse(rw, res.Body) +} + +func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader) { + if p.FlushInterval != 0 { + if wf, ok := dst.(writeFlusher); ok { + mlw := &maxLatencyWriter{ + dst: wf, + latency: p.FlushInterval, + done: make(chan bool), + } + go mlw.flushLoop() + defer mlw.stop() + dst = mlw + } + } + + io.Copy(dst, src) +} + +func (p *ReverseProxy) logf(format string, args ...interface{}) { + if p.ErrorLog != nil { + p.ErrorLog.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +type writeFlusher interface { + io.Writer + http.Flusher +} + +type maxLatencyWriter struct { + dst writeFlusher + latency time.Duration + + lk sync.Mutex // protects Write + Flush + done chan bool +} + +func (m *maxLatencyWriter) Write(p []byte) (int, error) { + m.lk.Lock() + defer m.lk.Unlock() + return m.dst.Write(p) +} + +func (m *maxLatencyWriter) flushLoop() { + t := time.NewTicker(m.latency) + defer t.Stop() + for { + select { + case <-m.done: + if onExitFlushLoop != nil { + onExitFlushLoop() + } + return + case <-t.C: + m.lk.Lock() + m.dst.Flush() + m.lk.Unlock() + } + } +} + +func (m *maxLatencyWriter) stop() { m.done <- true } diff --git a/src/net/http/httputil/reverseproxy_test.go b/src/net/http/httputil/reverseproxy_test.go new file mode 100644 index 000000000..e9539b44b --- /dev/null +++ b/src/net/http/httputil/reverseproxy_test.go @@ -0,0 +1,213 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Reverse proxy tests. + +package httputil + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +const fakeHopHeader = "X-Fake-Hop-Header-For-Test" + +func init() { + hopHeaders = append(hopHeaders, fakeHopHeader) +} + +func TestReverseProxy(t *testing.T) { + const backendResponse = "I am the backend" + const backendStatus = 404 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.TransferEncoding) > 0 { + t.Errorf("backend got unexpected TransferEncoding: %v", r.TransferEncoding) + } + if r.Header.Get("X-Forwarded-For") == "" { + t.Errorf("didn't get X-Forwarded-For header") + } + if c := r.Header.Get("Connection"); c != "" { + t.Errorf("handler got Connection header value %q", c) + } + if c := r.Header.Get("Upgrade"); c != "" { + t.Errorf("handler got Upgrade header value %q", c) + } + if g, e := r.Host, "some-name"; g != e { + t.Errorf("backend got Host header %q, want %q", g, e) + } + w.Header().Set("X-Foo", "bar") + w.Header().Set("Upgrade", "foo") + w.Header().Set(fakeHopHeader, "foo") + w.Header().Add("X-Multi-Value", "foo") + w.Header().Add("X-Multi-Value", "bar") + http.SetCookie(w, &http.Cookie{Name: "flavor", Value: "chocolateChip"}) + w.WriteHeader(backendStatus) + w.Write([]byte(backendResponse)) + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := NewSingleHostReverseProxy(backendURL) + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + getReq, _ := http.NewRequest("GET", frontend.URL, nil) + getReq.Host = "some-name" + getReq.Header.Set("Connection", "close") + getReq.Header.Set("Upgrade", "foo") + getReq.Close = true + res, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + if g, e := res.StatusCode, backendStatus; g != e { + t.Errorf("got res.StatusCode %d; expected %d", g, e) + } + if g, e := res.Header.Get("X-Foo"), "bar"; g != e { + t.Errorf("got X-Foo %q; expected %q", g, e) + } + if c := res.Header.Get(fakeHopHeader); c != "" { + t.Errorf("got %s header value %q", fakeHopHeader, c) + } + if g, e := len(res.Header["X-Multi-Value"]), 2; g != e { + t.Errorf("got %d X-Multi-Value header values; expected %d", g, e) + } + if g, e := len(res.Header["Set-Cookie"]), 1; g != e { + t.Fatalf("got %d SetCookies, want %d", g, e) + } + if cookie := res.Cookies()[0]; cookie.Name != "flavor" { + t.Errorf("unexpected cookie %q", cookie.Name) + } + bodyBytes, _ := ioutil.ReadAll(res.Body) + if g, e := string(bodyBytes), backendResponse; g != e { + t.Errorf("got body %q; expected %q", g, e) + } +} + +func TestXForwardedFor(t *testing.T) { + const prevForwardedFor = "client ip" + const backendResponse = "I am the backend" + const backendStatus = 404 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("X-Forwarded-For") == "" { + t.Errorf("didn't get X-Forwarded-For header") + } + if !strings.Contains(r.Header.Get("X-Forwarded-For"), prevForwardedFor) { + t.Errorf("X-Forwarded-For didn't contain prior data") + } + w.WriteHeader(backendStatus) + w.Write([]byte(backendResponse)) + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := NewSingleHostReverseProxy(backendURL) + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + getReq, _ := http.NewRequest("GET", frontend.URL, nil) + getReq.Host = "some-name" + getReq.Header.Set("Connection", "close") + getReq.Header.Set("X-Forwarded-For", prevForwardedFor) + getReq.Close = true + res, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + if g, e := res.StatusCode, backendStatus; g != e { + t.Errorf("got res.StatusCode %d; expected %d", g, e) + } + bodyBytes, _ := ioutil.ReadAll(res.Body) + if g, e := string(bodyBytes), backendResponse; g != e { + t.Errorf("got body %q; expected %q", g, e) + } +} + +var proxyQueryTests = []struct { + baseSuffix string // suffix to add to backend URL + reqSuffix string // suffix to add to frontend's request URL + want string // what backend should see for final request URL (without ?) +}{ + {"", "", ""}, + {"?sta=tic", "?us=er", "sta=tic&us=er"}, + {"", "?us=er", "us=er"}, + {"?sta=tic", "", "sta=tic"}, +} + +func TestReverseProxyQuery(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Got-Query", r.URL.RawQuery) + w.Write([]byte("hi")) + })) + defer backend.Close() + + for i, tt := range proxyQueryTests { + backendURL, err := url.Parse(backend.URL + tt.baseSuffix) + if err != nil { + t.Fatal(err) + } + frontend := httptest.NewServer(NewSingleHostReverseProxy(backendURL)) + req, _ := http.NewRequest("GET", frontend.URL+tt.reqSuffix, nil) + req.Close = true + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%d. Get: %v", i, err) + } + if g, e := res.Header.Get("X-Got-Query"), tt.want; g != e { + t.Errorf("%d. got query %q; expected %q", i, g, e) + } + res.Body.Close() + frontend.Close() + } +} + +func TestReverseProxyFlushInterval(t *testing.T) { + const expected = "hi" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(expected)) + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.FlushInterval = time.Microsecond + + done := make(chan bool) + onExitFlushLoop = func() { done <- true } + defer func() { onExitFlushLoop = nil }() + + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + req, _ := http.NewRequest("GET", frontend.URL, nil) + req.Close = true + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Get: %v", err) + } + defer res.Body.Close() + if bodyBytes, _ := ioutil.ReadAll(res.Body); string(bodyBytes) != expected { + t.Errorf("got body %q; expected %q", bodyBytes, expected) + } + + select { + case <-done: + // OK + case <-time.After(5 * time.Second): + t.Error("maxLatencyWriter flushLoop() never exited") + } +} diff --git a/src/net/http/internal/chunked.go b/src/net/http/internal/chunked.go new file mode 100644 index 000000000..9294deb3e --- /dev/null +++ b/src/net/http/internal/chunked.go @@ -0,0 +1,202 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The wire protocol for HTTP's "chunked" Transfer-Encoding. + +// Package internal contains HTTP internals shared by net/http and +// net/http/httputil. +package internal + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" +) + +const maxLineLength = 4096 // assumed <= bufio.defaultBufSize + +var ErrLineTooLong = errors.New("header line too long") + +// NewChunkedReader returns a new chunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The chunkedReader returns io.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func NewChunkedReader(r io.Reader) io.Reader { + br, ok := r.(*bufio.Reader) + if !ok { + br = bufio.NewReader(r) + } + return &chunkedReader{r: br} +} + +type chunkedReader struct { + r *bufio.Reader + n uint64 // unread bytes in chunk + err error + buf [2]byte +} + +func (cr *chunkedReader) beginChunk() { + // chunk-size CRLF + var line []byte + line, cr.err = readLine(cr.r) + if cr.err != nil { + return + } + cr.n, cr.err = parseHexUint(line) + if cr.err != nil { + return + } + if cr.n == 0 { + cr.err = io.EOF + } +} + +func (cr *chunkedReader) chunkHeaderAvailable() bool { + n := cr.r.Buffered() + if n > 0 { + peek, _ := cr.r.Peek(n) + return bytes.IndexByte(peek, '\n') >= 0 + } + return false +} + +func (cr *chunkedReader) Read(b []uint8) (n int, err error) { + for cr.err == nil { + if cr.n == 0 { + if n > 0 && !cr.chunkHeaderAvailable() { + // We've read enough. Don't potentially block + // reading a new chunk header. + break + } + cr.beginChunk() + continue + } + if len(b) == 0 { + break + } + rbuf := b + if uint64(len(rbuf)) > cr.n { + rbuf = rbuf[:cr.n] + } + var n0 int + n0, cr.err = cr.r.Read(rbuf) + n += n0 + b = b[n0:] + cr.n -= uint64(n0) + // If we're at the end of a chunk, read the next two + // bytes to verify they are "\r\n". + if cr.n == 0 && cr.err == nil { + if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { + if cr.buf[0] != '\r' || cr.buf[1] != '\n' { + cr.err = errors.New("malformed chunked encoding") + } + } + } + } + return n, cr.err +} + +// Read a line of bytes (up to \n) from b. +// Give up if the line exceeds maxLineLength. +// The returned bytes are a pointer into storage in +// the bufio, so they are only valid until the next bufio read. +func readLine(b *bufio.Reader) (p []byte, err error) { + if p, err = b.ReadSlice('\n'); err != nil { + // We always know when EOF is coming. + // If the caller asked for a line, there should be a line. + if err == io.EOF { + err = io.ErrUnexpectedEOF + } else if err == bufio.ErrBufferFull { + err = ErrLineTooLong + } + return nil, err + } + if len(p) >= maxLineLength { + return nil, ErrLineTooLong + } + return trimTrailingWhitespace(p), nil +} + +func trimTrailingWhitespace(b []byte) []byte { + for len(b) > 0 && isASCIISpace(b[len(b)-1]) { + b = b[:len(b)-1] + } + return b +} + +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +// NewChunkedWriter returns a new chunkedWriter that translates writes into HTTP +// "chunked" format before writing them to w. Closing the returned chunkedWriter +// sends the final 0-length chunk that marks the end of the stream. +// +// NewChunkedWriter is not needed by normal applications. The http +// package adds chunking automatically if handlers don't set a +// Content-Length header. Using newChunkedWriter inside a handler +// would result in double chunking or chunking with a Content-Length +// length, both of which are wrong. +func NewChunkedWriter(w io.Writer) io.WriteCloser { + return &chunkedWriter{w} +} + +// Writing to chunkedWriter translates to writing in HTTP chunked Transfer +// Encoding wire format to the underlying Wire chunkedWriter. +type chunkedWriter struct { + Wire io.Writer +} + +// Write the contents of data as one chunk to Wire. +// NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has +// a bug since it does not check for success of io.WriteString +func (cw *chunkedWriter) Write(data []byte) (n int, err error) { + + // Don't send 0-length data. It looks like EOF for chunked encoding. + if len(data) == 0 { + return 0, nil + } + + if _, err = fmt.Fprintf(cw.Wire, "%x\r\n", len(data)); err != nil { + return 0, err + } + if n, err = cw.Wire.Write(data); err != nil { + return + } + if n != len(data) { + err = io.ErrShortWrite + return + } + _, err = io.WriteString(cw.Wire, "\r\n") + + return +} + +func (cw *chunkedWriter) Close() error { + _, err := io.WriteString(cw.Wire, "0\r\n") + return err +} + +func parseHexUint(v []byte) (n uint64, err error) { + for _, b := range v { + n <<= 4 + switch { + case '0' <= b && b <= '9': + b = b - '0' + case 'a' <= b && b <= 'f': + b = b - 'a' + 10 + case 'A' <= b && b <= 'F': + b = b - 'A' + 10 + default: + return 0, errors.New("invalid byte in chunk length") + } + n |= uint64(b) + } + return +} diff --git a/src/net/http/internal/chunked_test.go b/src/net/http/internal/chunked_test.go new file mode 100644 index 000000000..ebc626ea9 --- /dev/null +++ b/src/net/http/internal/chunked_test.go @@ -0,0 +1,156 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "strings" + "testing" +) + +func TestChunk(t *testing.T) { + var b bytes.Buffer + + w := NewChunkedWriter(&b) + const chunk1 = "hello, " + const chunk2 = "world! 0123456789abcdef" + w.Write([]byte(chunk1)) + w.Write([]byte(chunk2)) + w.Close() + + if g, e := b.String(), "7\r\nhello, \r\n17\r\nworld! 0123456789abcdef\r\n0\r\n"; g != e { + t.Fatalf("chunk writer wrote %q; want %q", g, e) + } + + r := NewChunkedReader(&b) + data, err := ioutil.ReadAll(r) + if err != nil { + t.Logf(`data: "%s"`, data) + t.Fatalf("ReadAll from reader: %v", err) + } + if g, e := string(data), chunk1+chunk2; g != e { + t.Errorf("chunk reader read %q; want %q", g, e) + } +} + +func TestChunkReadMultiple(t *testing.T) { + // Bunch of small chunks, all read together. + { + var b bytes.Buffer + w := NewChunkedWriter(&b) + w.Write([]byte("foo")) + w.Write([]byte("bar")) + w.Close() + + r := NewChunkedReader(&b) + buf := make([]byte, 10) + n, err := r.Read(buf) + if n != 6 || err != io.EOF { + t.Errorf("Read = %d, %v; want 6, EOF", n, err) + } + buf = buf[:n] + if string(buf) != "foobar" { + t.Errorf("Read = %q; want %q", buf, "foobar") + } + } + + // One big chunk followed by a little chunk, but the small bufio.Reader size + // should prevent the second chunk header from being read. + { + var b bytes.Buffer + w := NewChunkedWriter(&b) + // fillBufChunk is 11 bytes + 3 bytes header + 2 bytes footer = 16 bytes, + // the same as the bufio ReaderSize below (the minimum), so even + // though we're going to try to Read with a buffer larger enough to also + // receive "foo", the second chunk header won't be read yet. + const fillBufChunk = "0123456789a" + const shortChunk = "foo" + w.Write([]byte(fillBufChunk)) + w.Write([]byte(shortChunk)) + w.Close() + + r := NewChunkedReader(bufio.NewReaderSize(&b, 16)) + buf := make([]byte, len(fillBufChunk)+len(shortChunk)) + n, err := r.Read(buf) + if n != len(fillBufChunk) || err != nil { + t.Errorf("Read = %d, %v; want %d, nil", n, err, len(fillBufChunk)) + } + buf = buf[:n] + if string(buf) != fillBufChunk { + t.Errorf("Read = %q; want %q", buf, fillBufChunk) + } + + n, err = r.Read(buf) + if n != len(shortChunk) || err != io.EOF { + t.Errorf("Read = %d, %v; want %d, EOF", n, err, len(shortChunk)) + } + } + + // And test that we see an EOF chunk, even though our buffer is already full: + { + r := NewChunkedReader(bufio.NewReader(strings.NewReader("3\r\nfoo\r\n0\r\n"))) + buf := make([]byte, 3) + n, err := r.Read(buf) + if n != 3 || err != io.EOF { + t.Errorf("Read = %d, %v; want 3, EOF", n, err) + } + if string(buf) != "foo" { + t.Errorf("buf = %q; want foo", buf) + } + } +} + +func TestChunkReaderAllocs(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + var buf bytes.Buffer + w := NewChunkedWriter(&buf) + a, b, c := []byte("aaaaaa"), []byte("bbbbbbbbbbbb"), []byte("cccccccccccccccccccccccc") + w.Write(a) + w.Write(b) + w.Write(c) + w.Close() + + readBuf := make([]byte, len(a)+len(b)+len(c)+1) + byter := bytes.NewReader(buf.Bytes()) + bufr := bufio.NewReader(byter) + mallocs := testing.AllocsPerRun(100, func() { + byter.Seek(0, 0) + bufr.Reset(byter) + r := NewChunkedReader(bufr) + n, err := io.ReadFull(r, readBuf) + if n != len(readBuf)-1 { + t.Fatalf("read %d bytes; want %d", n, len(readBuf)-1) + } + if err != io.ErrUnexpectedEOF { + t.Fatalf("read error = %v; want ErrUnexpectedEOF", err) + } + }) + if mallocs > 1.5 { + t.Errorf("mallocs = %v; want 1", mallocs) + } +} + +func TestParseHexUint(t *testing.T) { + for i := uint64(0); i <= 1234; i++ { + line := []byte(fmt.Sprintf("%x", i)) + got, err := parseHexUint(line) + if err != nil { + t.Fatalf("on %d: %v", i, err) + } + if got != i { + t.Errorf("for input %q = %d; want %d", line, got, i) + } + } + _, err := parseHexUint([]byte("bogus")) + if err == nil { + t.Error("expected error on bogus input") + } +} diff --git a/src/net/http/jar.go b/src/net/http/jar.go new file mode 100644 index 000000000..5c3de0dad --- /dev/null +++ b/src/net/http/jar.go @@ -0,0 +1,27 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "net/url" +) + +// A CookieJar manages storage and use of cookies in HTTP requests. +// +// Implementations of CookieJar must be safe for concurrent use by multiple +// goroutines. +// +// The net/http/cookiejar package provides a CookieJar implementation. +type CookieJar interface { + // SetCookies handles the receipt of the cookies in a reply for the + // given URL. It may or may not choose to save the cookies, depending + // on the jar's policy and implementation. + SetCookies(u *url.URL, cookies []*Cookie) + + // Cookies returns the cookies to send in a request for the given URL. + // It is up to the implementation to honor the standard cookie use + // restrictions such as in RFC 6265. + Cookies(u *url.URL) []*Cookie +} diff --git a/src/net/http/lex.go b/src/net/http/lex.go new file mode 100644 index 000000000..cb33318f4 --- /dev/null +++ b/src/net/http/lex.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +// This file deals with lexical matters of HTTP + +var isTokenTable = [127]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} + +func isToken(r rune) bool { + i := int(r) + return i < len(isTokenTable) && isTokenTable[i] +} + +func isNotToken(r rune) bool { + return !isToken(r) +} diff --git a/src/net/http/lex_test.go b/src/net/http/lex_test.go new file mode 100644 index 000000000..6d9d294f7 --- /dev/null +++ b/src/net/http/lex_test.go @@ -0,0 +1,31 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "testing" +) + +func isChar(c rune) bool { return c <= 127 } + +func isCtl(c rune) bool { return c <= 31 || c == 127 } + +func isSeparator(c rune) bool { + switch c { + case '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t': + return true + } + return false +} + +func TestIsToken(t *testing.T) { + for i := 0; i <= 130; i++ { + r := rune(i) + expected := isChar(r) && !isCtl(r) && !isSeparator(r) + if isToken(r) != expected { + t.Errorf("isToken(0x%x) = %v", r, !expected) + } + } +} diff --git a/src/net/http/npn_test.go b/src/net/http/npn_test.go new file mode 100644 index 000000000..98b8930d0 --- /dev/null +++ b/src/net/http/npn_test.go @@ -0,0 +1,118 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "bufio" + "crypto/tls" + "fmt" + "io" + "io/ioutil" + . "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestNextProtoUpgrade(t *testing.T) { + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "path=%s,proto=", r.URL.Path) + if r.TLS != nil { + w.Write([]byte(r.TLS.NegotiatedProtocol)) + } + if r.RemoteAddr == "" { + t.Error("request with no RemoteAddr") + } + if r.Body == nil { + t.Errorf("request with nil Body") + } + })) + ts.TLS = &tls.Config{ + NextProtos: []string{"unhandled-proto", "tls-0.9"}, + } + ts.Config.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){ + "tls-0.9": handleTLSProtocol09, + } + ts.StartTLS() + defer ts.Close() + + tr := newTLSTransport(t, ts) + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + // Normal request, without NPN. + { + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if want := "path=/,proto="; string(body) != want { + t.Errorf("plain request = %q; want %q", body, want) + } + } + + // Request to an advertised but unhandled NPN protocol. + // Server will hang up. + { + tr.CloseIdleConnections() + tr.TLSClientConfig.NextProtos = []string{"unhandled-proto"} + _, err := c.Get(ts.URL) + if err == nil { + t.Errorf("expected error on unhandled-proto request") + } + } + + // Request using the "tls-0.9" protocol, which we register here. + // It is HTTP/0.9 over TLS. + { + tlsConfig := newTLSTransport(t, ts).TLSClientConfig + tlsConfig.NextProtos = []string{"tls-0.9"} + conn, err := tls.Dial("tcp", ts.Listener.Addr().String(), tlsConfig) + if err != nil { + t.Fatal(err) + } + conn.Write([]byte("GET /foo\n")) + body, err := ioutil.ReadAll(conn) + if err != nil { + t.Fatal(err) + } + if want := "path=/foo,proto=tls-0.9"; string(body) != want { + t.Errorf("plain request = %q; want %q", body, want) + } + } +} + +// handleTLSProtocol09 implements the HTTP/0.9 protocol over TLS, for the +// TestNextProtoUpgrade test. +func handleTLSProtocol09(srv *Server, conn *tls.Conn, h Handler) { + br := bufio.NewReader(conn) + line, err := br.ReadString('\n') + if err != nil { + return + } + line = strings.TrimSpace(line) + path := strings.TrimPrefix(line, "GET ") + if path == line { + return + } + req, _ := NewRequest("GET", path, nil) + req.Proto = "HTTP/0.9" + req.ProtoMajor = 0 + req.ProtoMinor = 9 + rw := &http09Writer{conn, make(Header)} + h.ServeHTTP(rw, req) +} + +type http09Writer struct { + io.Writer + h Header +} + +func (w http09Writer) Header() Header { return w.h } +func (w http09Writer) WriteHeader(int) {} // no headers diff --git a/src/net/http/pprof/pprof.go b/src/net/http/pprof/pprof.go new file mode 100644 index 000000000..0c7548e3e --- /dev/null +++ b/src/net/http/pprof/pprof.go @@ -0,0 +1,205 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pprof serves via its HTTP server runtime profiling data +// in the format expected by the pprof visualization tool. +// For more information about pprof, see +// http://code.google.com/p/google-perftools/. +// +// The package is typically only imported for the side effect of +// registering its HTTP handlers. +// The handled paths all begin with /debug/pprof/. +// +// To use pprof, link this package into your program: +// import _ "net/http/pprof" +// +// If your application is not already running an http server, you +// need to start one. Add "net/http" and "log" to your imports and +// the following code to your main function: +// +// go func() { +// log.Println(http.ListenAndServe("localhost:6060", nil)) +// }() +// +// Then use the pprof tool to look at the heap profile: +// +// go tool pprof http://localhost:6060/debug/pprof/heap +// +// Or to look at a 30-second CPU profile: +// +// go tool pprof http://localhost:6060/debug/pprof/profile +// +// Or to look at the goroutine blocking profile: +// +// go tool pprof http://localhost:6060/debug/pprof/block +// +// To view all available profiles, open http://localhost:6060/debug/pprof/ +// in your browser. +// +// For a study of the facility in action, visit +// +// http://blog.golang.org/2011/06/profiling-go-programs.html +// +package pprof + +import ( + "bufio" + "bytes" + "fmt" + "html/template" + "io" + "log" + "net/http" + "os" + "runtime" + "runtime/pprof" + "strconv" + "strings" + "time" +) + +func init() { + http.Handle("/debug/pprof/", http.HandlerFunc(Index)) + http.Handle("/debug/pprof/cmdline", http.HandlerFunc(Cmdline)) + http.Handle("/debug/pprof/profile", http.HandlerFunc(Profile)) + http.Handle("/debug/pprof/symbol", http.HandlerFunc(Symbol)) +} + +// Cmdline responds with the running program's +// command line, with arguments separated by NUL bytes. +// The package initialization registers it as /debug/pprof/cmdline. +func Cmdline(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprintf(w, strings.Join(os.Args, "\x00")) +} + +// Profile responds with the pprof-formatted cpu profile. +// The package initialization registers it as /debug/pprof/profile. +func Profile(w http.ResponseWriter, r *http.Request) { + sec, _ := strconv.ParseInt(r.FormValue("seconds"), 10, 64) + if sec == 0 { + sec = 30 + } + + // Set Content Type assuming StartCPUProfile will work, + // because if it does it starts writing. + w.Header().Set("Content-Type", "application/octet-stream") + if err := pprof.StartCPUProfile(w); err != nil { + // StartCPUProfile failed, so no writes yet. + // Can change header back to text content + // and send error code. + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusInternalServerError) + fmt.Fprintf(w, "Could not enable CPU profiling: %s\n", err) + return + } + time.Sleep(time.Duration(sec) * time.Second) + pprof.StopCPUProfile() +} + +// Symbol looks up the program counters listed in the request, +// responding with a table mapping program counters to function names. +// The package initialization registers it as /debug/pprof/symbol. +func Symbol(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + // We have to read the whole POST body before + // writing any output. Buffer the output here. + var buf bytes.Buffer + + // We don't know how many symbols we have, but we + // do have symbol information. Pprof only cares whether + // this number is 0 (no symbols available) or > 0. + fmt.Fprintf(&buf, "num_symbols: 1\n") + + var b *bufio.Reader + if r.Method == "POST" { + b = bufio.NewReader(r.Body) + } else { + b = bufio.NewReader(strings.NewReader(r.URL.RawQuery)) + } + + for { + word, err := b.ReadSlice('+') + if err == nil { + word = word[0 : len(word)-1] // trim + + } + pc, _ := strconv.ParseUint(string(word), 0, 64) + if pc != 0 { + f := runtime.FuncForPC(uintptr(pc)) + if f != nil { + fmt.Fprintf(&buf, "%#x %s\n", pc, f.Name()) + } + } + + // Wait until here to check for err; the last + // symbol will have an err because it doesn't end in +. + if err != nil { + if err != io.EOF { + fmt.Fprintf(&buf, "reading request: %v\n", err) + } + break + } + } + + w.Write(buf.Bytes()) +} + +// Handler returns an HTTP handler that serves the named profile. +func Handler(name string) http.Handler { + return handler(name) +} + +type handler string + +func (name handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + debug, _ := strconv.Atoi(r.FormValue("debug")) + p := pprof.Lookup(string(name)) + if p == nil { + w.WriteHeader(404) + fmt.Fprintf(w, "Unknown profile: %s\n", name) + return + } + p.WriteTo(w, debug) + return +} + +// Index responds with the pprof-formatted profile named by the request. +// For example, "/debug/pprof/heap" serves the "heap" profile. +// Index responds to a request for "/debug/pprof/" with an HTML page +// listing the available profiles. +func Index(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/debug/pprof/") { + name := strings.TrimPrefix(r.URL.Path, "/debug/pprof/") + if name != "" { + handler(name).ServeHTTP(w, r) + return + } + } + + profiles := pprof.Profiles() + if err := indexTmpl.Execute(w, profiles); err != nil { + log.Print(err) + } +} + +var indexTmpl = template.Must(template.New("index").Parse(`<html> +<head> +<title>/debug/pprof/</title> +</head> +/debug/pprof/<br> +<br> +<body> +profiles:<br> +<table> +{{range .}} +<tr><td align=right>{{.Count}}<td><a href="/debug/pprof/{{.Name}}?debug=1">{{.Name}}</a> +{{end}} +</table> +<br> +<a href="/debug/pprof/goroutine?debug=2">full goroutine stack dump</a><br> +</body> +</html> +`)) diff --git a/src/net/http/proxy_test.go b/src/net/http/proxy_test.go new file mode 100644 index 000000000..b6aed3792 --- /dev/null +++ b/src/net/http/proxy_test.go @@ -0,0 +1,81 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "net/url" + "os" + "testing" +) + +// TODO(mattn): +// test ProxyAuth + +var UseProxyTests = []struct { + host string + match bool +}{ + // Never proxy localhost: + {"localhost:80", false}, + {"127.0.0.1", false}, + {"127.0.0.2", false}, + {"[::1]", false}, + {"[::2]", true}, // not a loopback address + + {"barbaz.net", false}, // match as .barbaz.net + {"foobar.com", false}, // have a port but match + {"foofoobar.com", true}, // not match as a part of foobar.com + {"baz.com", true}, // not match as a part of barbaz.com + {"localhost.net", true}, // not match as suffix of address + {"local.localhost", true}, // not match as prefix as address + {"barbarbaz.net", true}, // not match because NO_PROXY have a '.' + {"www.foobar.com", false}, // match because NO_PROXY includes "foobar.com" +} + +func TestUseProxy(t *testing.T) { + ResetProxyEnv() + os.Setenv("NO_PROXY", "foobar.com, .barbaz.net") + for _, test := range UseProxyTests { + if useProxy(test.host+":80") != test.match { + t.Errorf("useProxy(%v) = %v, want %v", test.host, !test.match, test.match) + } + } +} + +var cacheKeysTests = []struct { + proxy string + scheme string + addr string + key string +}{ + {"", "http", "foo.com", "|http|foo.com"}, + {"", "https", "foo.com", "|https|foo.com"}, + {"http://foo.com", "http", "foo.com", "http://foo.com|http|"}, + {"http://foo.com", "https", "foo.com", "http://foo.com|https|foo.com"}, +} + +func TestCacheKeys(t *testing.T) { + for _, tt := range cacheKeysTests { + var proxy *url.URL + if tt.proxy != "" { + u, err := url.Parse(tt.proxy) + if err != nil { + t.Fatal(err) + } + proxy = u + } + cm := connectMethod{proxy, tt.scheme, tt.addr} + if got := cm.key().String(); got != tt.key { + t.Fatalf("{%q, %q, %q} cache key = %q; want %q", tt.proxy, tt.scheme, tt.addr, got, tt.key) + } + } +} + +func ResetProxyEnv() { + for _, v := range []string{"HTTP_PROXY", "http_proxy", "NO_PROXY", "no_proxy"} { + os.Setenv(v, "") + } + ResetCachedEnvironment() +} diff --git a/src/net/http/race.go b/src/net/http/race.go new file mode 100644 index 000000000..766503967 --- /dev/null +++ b/src/net/http/race.go @@ -0,0 +1,11 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build race + +package http + +func init() { + raceEnabled = true +} diff --git a/src/net/http/range_test.go b/src/net/http/range_test.go new file mode 100644 index 000000000..ef911af7b --- /dev/null +++ b/src/net/http/range_test.go @@ -0,0 +1,79 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "testing" +) + +var ParseRangeTests = []struct { + s string + length int64 + r []httpRange +}{ + {"", 0, nil}, + {"", 1000, nil}, + {"foo", 0, nil}, + {"bytes=", 0, nil}, + {"bytes=7", 10, nil}, + {"bytes= 7 ", 10, nil}, + {"bytes=1-", 0, nil}, + {"bytes=5-4", 10, nil}, + {"bytes=0-2,5-4", 10, nil}, + {"bytes=2-5,4-3", 10, nil}, + {"bytes=--5,4--3", 10, nil}, + {"bytes=A-", 10, nil}, + {"bytes=A- ", 10, nil}, + {"bytes=A-Z", 10, nil}, + {"bytes= -Z", 10, nil}, + {"bytes=5-Z", 10, nil}, + {"bytes=Ran-dom, garbage", 10, nil}, + {"bytes=0x01-0x02", 10, nil}, + {"bytes= ", 10, nil}, + {"bytes= , , , ", 10, nil}, + + {"bytes=0-9", 10, []httpRange{{0, 10}}}, + {"bytes=0-", 10, []httpRange{{0, 10}}}, + {"bytes=5-", 10, []httpRange{{5, 5}}}, + {"bytes=0-20", 10, []httpRange{{0, 10}}}, + {"bytes=15-,0-5", 10, nil}, + {"bytes=1-2,5-", 10, []httpRange{{1, 2}, {5, 5}}}, + {"bytes=-2 , 7-", 11, []httpRange{{9, 2}, {7, 4}}}, + {"bytes=0-0 ,2-2, 7-", 11, []httpRange{{0, 1}, {2, 1}, {7, 4}}}, + {"bytes=-5", 10, []httpRange{{5, 5}}}, + {"bytes=-15", 10, []httpRange{{0, 10}}}, + {"bytes=0-499", 10000, []httpRange{{0, 500}}}, + {"bytes=500-999", 10000, []httpRange{{500, 500}}}, + {"bytes=-500", 10000, []httpRange{{9500, 500}}}, + {"bytes=9500-", 10000, []httpRange{{9500, 500}}}, + {"bytes=0-0,-1", 10000, []httpRange{{0, 1}, {9999, 1}}}, + {"bytes=500-600,601-999", 10000, []httpRange{{500, 101}, {601, 399}}}, + {"bytes=500-700,601-999", 10000, []httpRange{{500, 201}, {601, 399}}}, + + // Match Apache laxity: + {"bytes= 1 -2 , 4- 5, 7 - 8 , ,,", 11, []httpRange{{1, 2}, {4, 2}, {7, 2}}}, +} + +func TestParseRange(t *testing.T) { + for _, test := range ParseRangeTests { + r := test.r + ranges, err := parseRange(test.s, test.length) + if err != nil && r != nil { + t.Errorf("parseRange(%q) returned error %q", test.s, err) + } + if len(ranges) != len(r) { + t.Errorf("len(parseRange(%q)) = %d, want %d", test.s, len(ranges), len(r)) + continue + } + for i := range r { + if ranges[i].start != r[i].start { + t.Errorf("parseRange(%q)[%d].start = %d, want %d", test.s, i, ranges[i].start, r[i].start) + } + if ranges[i].length != r[i].length { + t.Errorf("parseRange(%q)[%d].length = %d, want %d", test.s, i, ranges[i].length, r[i].length) + } + } + } +} diff --git a/src/net/http/readrequest_test.go b/src/net/http/readrequest_test.go new file mode 100644 index 000000000..e930d99af --- /dev/null +++ b/src/net/http/readrequest_test.go @@ -0,0 +1,358 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/url" + "reflect" + "strings" + "testing" +) + +type reqTest struct { + Raw string + Req *Request + Body string + Trailer Header + Error string +} + +var noError = "" +var noBody = "" +var noTrailer Header = nil + +var reqTests = []reqTest{ + // Baseline test; All Request fields included for template use + { + "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + + "Host: www.techcrunch.com\r\n" + + "User-Agent: Fake\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + + "Accept-Language: en-us,en;q=0.5\r\n" + + "Accept-Encoding: gzip,deflate\r\n" + + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + + "Keep-Alive: 300\r\n" + + "Content-Length: 7\r\n" + + "Proxy-Connection: keep-alive\r\n\r\n" + + "abcdef\n???", + + &Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "www.techcrunch.com", + Path: "/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "Accept-Language": {"en-us,en;q=0.5"}, + "Accept-Encoding": {"gzip,deflate"}, + "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, + "Keep-Alive": {"300"}, + "Proxy-Connection": {"keep-alive"}, + "Content-Length": {"7"}, + "User-Agent": {"Fake"}, + }, + Close: false, + ContentLength: 7, + Host: "www.techcrunch.com", + RequestURI: "http://www.techcrunch.com/", + }, + + "abcdef\n", + + noTrailer, + noError, + }, + + // GET request with no body (the normal case) + { + "GET / HTTP/1.1\r\n" + + "Host: foo.com\r\n\r\n", + + &Request{ + Method: "GET", + URL: &url.URL{ + Path: "/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: false, + ContentLength: 0, + Host: "foo.com", + RequestURI: "/", + }, + + noBody, + noTrailer, + noError, + }, + + // Tests that we don't parse a path that looks like a + // scheme-relative URI as a scheme-relative URI. + { + "GET //user@host/is/actually/a/path/ HTTP/1.1\r\n" + + "Host: test\r\n\r\n", + + &Request{ + Method: "GET", + URL: &url.URL{ + Path: "//user@host/is/actually/a/path/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: false, + ContentLength: 0, + Host: "test", + RequestURI: "//user@host/is/actually/a/path/", + }, + + noBody, + noTrailer, + noError, + }, + + // Tests a bogus abs_path on the Request-Line (RFC 2616 section 5.1.2) + { + "GET ../../../../etc/passwd HTTP/1.1\r\n" + + "Host: test\r\n\r\n", + nil, + noBody, + noTrailer, + "parse ../../../../etc/passwd: invalid URI for request", + }, + + // Tests missing URL: + { + "GET HTTP/1.1\r\n" + + "Host: test\r\n\r\n", + nil, + noBody, + noTrailer, + "parse : empty url", + }, + + // Tests chunked body with trailer: + { + "POST / HTTP/1.1\r\n" + + "Host: foo.com\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + "3\r\nfoo\r\n" + + "3\r\nbar\r\n" + + "0\r\n" + + "Trailer-Key: Trailer-Value\r\n" + + "\r\n", + &Request{ + Method: "POST", + URL: &url.URL{ + Path: "/", + }, + TransferEncoding: []string{"chunked"}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + ContentLength: -1, + Host: "foo.com", + RequestURI: "/", + }, + + "foobar", + Header{ + "Trailer-Key": {"Trailer-Value"}, + }, + noError, + }, + + // CONNECT request with domain name: + { + "CONNECT www.google.com:443 HTTP/1.1\r\n\r\n", + + &Request{ + Method: "CONNECT", + URL: &url.URL{ + Host: "www.google.com:443", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: false, + ContentLength: 0, + Host: "www.google.com:443", + RequestURI: "www.google.com:443", + }, + + noBody, + noTrailer, + noError, + }, + + // CONNECT request with IP address: + { + "CONNECT 127.0.0.1:6060 HTTP/1.1\r\n\r\n", + + &Request{ + Method: "CONNECT", + URL: &url.URL{ + Host: "127.0.0.1:6060", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: false, + ContentLength: 0, + Host: "127.0.0.1:6060", + RequestURI: "127.0.0.1:6060", + }, + + noBody, + noTrailer, + noError, + }, + + // CONNECT request for RPC: + { + "CONNECT /_goRPC_ HTTP/1.1\r\n\r\n", + + &Request{ + Method: "CONNECT", + URL: &url.URL{ + Path: "/_goRPC_", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: false, + ContentLength: 0, + Host: "", + RequestURI: "/_goRPC_", + }, + + noBody, + noTrailer, + noError, + }, + + // SSDP Notify request. golang.org/issue/3692 + { + "NOTIFY * HTTP/1.1\r\nServer: foo\r\n\r\n", + &Request{ + Method: "NOTIFY", + URL: &url.URL{ + Path: "*", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{ + "Server": []string{"foo"}, + }, + Close: false, + ContentLength: 0, + RequestURI: "*", + }, + + noBody, + noTrailer, + noError, + }, + + // OPTIONS request. Similar to golang.org/issue/3692 + { + "OPTIONS * HTTP/1.1\r\nServer: foo\r\n\r\n", + &Request{ + Method: "OPTIONS", + URL: &url.URL{ + Path: "*", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{ + "Server": []string{"foo"}, + }, + Close: false, + ContentLength: 0, + RequestURI: "*", + }, + + noBody, + noTrailer, + noError, + }, + + // Connection: close. golang.org/issue/8261 + { + "GET / HTTP/1.1\r\nHost: issue8261.com\r\nConnection: close\r\n\r\n", + &Request{ + Method: "GET", + URL: &url.URL{ + Path: "/", + }, + Header: Header{ + // This wasn't removed from Go 1.0 to + // Go 1.3, so locking it in that we + // keep this: + "Connection": []string{"close"}, + }, + Host: "issue8261.com", + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Close: true, + RequestURI: "/", + }, + + noBody, + noTrailer, + noError, + }, +} + +func TestReadRequest(t *testing.T) { + for i := range reqTests { + tt := &reqTests[i] + req, err := ReadRequest(bufio.NewReader(strings.NewReader(tt.Raw))) + if err != nil { + if err.Error() != tt.Error { + t.Errorf("#%d: error %q, want error %q", i, err.Error(), tt.Error) + } + continue + } + rbody := req.Body + req.Body = nil + testName := fmt.Sprintf("Test %d (%q)", i, tt.Raw) + diff(t, testName, req, tt.Req) + var bout bytes.Buffer + if rbody != nil { + _, err := io.Copy(&bout, rbody) + if err != nil { + t.Fatalf("%s: copying body: %v", testName, err) + } + rbody.Close() + } + body := bout.String() + if body != tt.Body { + t.Errorf("%s: Body = %q want %q", testName, body, tt.Body) + } + if !reflect.DeepEqual(tt.Trailer, req.Trailer) { + t.Errorf("%s: Trailers differ.\n got: %v\nwant: %v", testName, req.Trailer, tt.Trailer) + } + } +} diff --git a/src/net/http/request.go b/src/net/http/request.go new file mode 100644 index 000000000..263c26c9b --- /dev/null +++ b/src/net/http/request.go @@ -0,0 +1,920 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP Request reading and parsing. + +package http + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "mime" + "mime/multipart" + "net/textproto" + "net/url" + "strconv" + "strings" + "sync" +) + +const ( + maxValueLength = 4096 + maxHeaderLines = 1024 + chunkSize = 4 << 10 // 4 KB chunks + defaultMaxMemory = 32 << 20 // 32 MB +) + +// ErrMissingFile is returned by FormFile when the provided file field name +// is either not present in the request or not a file field. +var ErrMissingFile = errors.New("http: no such file") + +// HTTP request parsing errors. +type ProtocolError struct { + ErrorString string +} + +func (err *ProtocolError) Error() string { return err.ErrorString } + +var ( + ErrHeaderTooLong = &ProtocolError{"header too long"} + ErrShortBody = &ProtocolError{"entity body too short"} + ErrNotSupported = &ProtocolError{"feature not supported"} + ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} + ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} + ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} +) + +type badStringError struct { + what string + str string +} + +func (e *badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } + +// Headers that Request.Write handles itself and should be skipped. +var reqWriteExcludeHeader = map[string]bool{ + "Host": true, // not in Header map anyway + "User-Agent": true, + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// A Request represents an HTTP request received by a server +// or to be sent by a client. +// +// The field semantics differ slightly between client and server +// usage. In addition to the notes on the fields below, see the +// documentation for Request.Write and RoundTripper. +type Request struct { + // Method specifies the HTTP method (GET, POST, PUT, etc.). + // For client requests an empty string means GET. + Method string + + // URL specifies either the URI being requested (for server + // requests) or the URL to access (for client requests). + // + // For server requests the URL is parsed from the URI + // supplied on the Request-Line as stored in RequestURI. For + // most requests, fields other than Path and RawQuery will be + // empty. (See RFC 2616, Section 5.1.2) + // + // For client requests, the URL's Host specifies the server to + // connect to, while the Request's Host field optionally + // specifies the Host header value to send in the HTTP + // request. + URL *url.URL + + // The protocol version for incoming requests. + // Client requests always use HTTP/1.1. + Proto string // "HTTP/1.0" + ProtoMajor int // 1 + ProtoMinor int // 0 + + // A header maps request lines to their values. + // If the header says + // + // accept-encoding: gzip, deflate + // Accept-Language: en-us + // Connection: keep-alive + // + // then + // + // Header = map[string][]string{ + // "Accept-Encoding": {"gzip, deflate"}, + // "Accept-Language": {"en-us"}, + // "Connection": {"keep-alive"}, + // } + // + // HTTP defines that header names are case-insensitive. + // The request parser implements this by canonicalizing the + // name, making the first character and any characters + // following a hyphen uppercase and the rest lowercase. + // + // For client requests certain headers are automatically + // added and may override values in Header. + // + // See the documentation for the Request.Write method. + Header Header + + // Body is the request's body. + // + // For client requests a nil body means the request has no + // body, such as a GET request. The HTTP Client's Transport + // is responsible for calling the Close method. + // + // For server requests the Request Body is always non-nil + // but will return EOF immediately when no body is present. + // The Server will close the request body. The ServeHTTP + // Handler does not need to. + Body io.ReadCloser + + // ContentLength records the length of the associated content. + // The value -1 indicates that the length is unknown. + // Values >= 0 indicate that the given number of bytes may + // be read from Body. + // For client requests, a value of 0 means unknown if Body is not nil. + ContentLength int64 + + // TransferEncoding lists the transfer encodings from outermost to + // innermost. An empty list denotes the "identity" encoding. + // TransferEncoding can usually be ignored; chunked encoding is + // automatically added and removed as necessary when sending and + // receiving requests. + TransferEncoding []string + + // Close indicates whether to close the connection after + // replying to this request (for servers) or after sending + // the request (for clients). + Close bool + + // For server requests Host specifies the host on which the + // URL is sought. Per RFC 2616, this is either the value of + // the "Host" header or the host name given in the URL itself. + // It may be of the form "host:port". + // + // For client requests Host optionally overrides the Host + // header to send. If empty, the Request.Write method uses + // the value of URL.Host. + Host string + + // Form contains the parsed form data, including both the URL + // field's query parameters and the POST or PUT form data. + // This field is only available after ParseForm is called. + // The HTTP client ignores Form and uses Body instead. + Form url.Values + + // PostForm contains the parsed form data from POST or PUT + // body parameters. + // This field is only available after ParseForm is called. + // The HTTP client ignores PostForm and uses Body instead. + PostForm url.Values + + // MultipartForm is the parsed multipart form, including file uploads. + // This field is only available after ParseMultipartForm is called. + // The HTTP client ignores MultipartForm and uses Body instead. + MultipartForm *multipart.Form + + // Trailer specifies additional headers that are sent after the request + // body. + // + // For server requests the Trailer map initially contains only the + // trailer keys, with nil values. (The client declares which trailers it + // will later send.) While the handler is reading from Body, it must + // not reference Trailer. After reading from Body returns EOF, Trailer + // can be read again and will contain non-nil values, if they were sent + // by the client. + // + // For client requests Trailer must be initialized to a map containing + // the trailer keys to later send. The values may be nil or their final + // values. The ContentLength must be 0 or -1, to send a chunked request. + // After the HTTP request is sent the map values can be updated while + // the request body is read. Once the body returns EOF, the caller must + // not mutate Trailer. + // + // Few HTTP clients, servers, or proxies support HTTP trailers. + Trailer Header + + // RemoteAddr allows HTTP servers and other software to record + // the network address that sent the request, usually for + // logging. This field is not filled in by ReadRequest and + // has no defined format. The HTTP server in this package + // sets RemoteAddr to an "IP:port" address before invoking a + // handler. + // This field is ignored by the HTTP client. + RemoteAddr string + + // RequestURI is the unmodified Request-URI of the + // Request-Line (RFC 2616, Section 5.1) as sent by the client + // to a server. Usually the URL field should be used instead. + // It is an error to set this field in an HTTP client request. + RequestURI string + + // TLS allows HTTP servers and other software to record + // information about the TLS connection on which the request + // was received. This field is not filled in by ReadRequest. + // The HTTP server in this package sets the field for + // TLS-enabled connections before invoking a handler; + // otherwise it leaves the field nil. + // This field is ignored by the HTTP client. + TLS *tls.ConnectionState +} + +// ProtoAtLeast reports whether the HTTP protocol used +// in the request is at least major.minor. +func (r *Request) ProtoAtLeast(major, minor int) bool { + return r.ProtoMajor > major || + r.ProtoMajor == major && r.ProtoMinor >= minor +} + +// UserAgent returns the client's User-Agent, if sent in the request. +func (r *Request) UserAgent() string { + return r.Header.Get("User-Agent") +} + +// Cookies parses and returns the HTTP cookies sent with the request. +func (r *Request) Cookies() []*Cookie { + return readCookies(r.Header, "") +} + +var ErrNoCookie = errors.New("http: named cookie not present") + +// Cookie returns the named cookie provided in the request or +// ErrNoCookie if not found. +func (r *Request) Cookie(name string) (*Cookie, error) { + for _, c := range readCookies(r.Header, name) { + return c, nil + } + return nil, ErrNoCookie +} + +// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, +// AddCookie does not attach more than one Cookie header field. That +// means all cookies, if any, are written into the same line, +// separated by semicolon. +func (r *Request) AddCookie(c *Cookie) { + s := fmt.Sprintf("%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) + if c := r.Header.Get("Cookie"); c != "" { + r.Header.Set("Cookie", c+"; "+s) + } else { + r.Header.Set("Cookie", s) + } +} + +// Referer returns the referring URL, if sent in the request. +// +// Referer is misspelled as in the request itself, a mistake from the +// earliest days of HTTP. This value can also be fetched from the +// Header map as Header["Referer"]; the benefit of making it available +// as a method is that the compiler can diagnose programs that use the +// alternate (correct English) spelling req.Referrer() but cannot +// diagnose programs that use Header["Referrer"]. +func (r *Request) Referer() string { + return r.Header.Get("Referer") +} + +// multipartByReader is a sentinel value. +// Its presence in Request.MultipartForm indicates that parsing of the request +// body has been handed off to a MultipartReader instead of ParseMultipartFrom. +var multipartByReader = &multipart.Form{ + Value: make(map[string][]string), + File: make(map[string][]*multipart.FileHeader), +} + +// MultipartReader returns a MIME multipart reader if this is a +// multipart/form-data POST request, else returns nil and an error. +// Use this function instead of ParseMultipartForm to +// process the request body as a stream. +func (r *Request) MultipartReader() (*multipart.Reader, error) { + if r.MultipartForm == multipartByReader { + return nil, errors.New("http: MultipartReader called twice") + } + if r.MultipartForm != nil { + return nil, errors.New("http: multipart handled by ParseMultipartForm") + } + r.MultipartForm = multipartByReader + return r.multipartReader() +} + +func (r *Request) multipartReader() (*multipart.Reader, error) { + v := r.Header.Get("Content-Type") + if v == "" { + return nil, ErrNotMultipart + } + d, params, err := mime.ParseMediaType(v) + if err != nil || d != "multipart/form-data" { + return nil, ErrNotMultipart + } + boundary, ok := params["boundary"] + if !ok { + return nil, ErrMissingBoundary + } + return multipart.NewReader(r.Body, boundary), nil +} + +// Return value if nonempty, def otherwise. +func valueOrDefault(value, def string) string { + if value != "" { + return value + } + return def +} + +// NOTE: This is not intended to reflect the actual Go version being used. +// It was changed from "Go http package" to "Go 1.1 package http" at the +// time of the Go 1.1 release because the former User-Agent had ended up +// on a blacklist for some intrusion detection systems. +// See https://codereview.appspot.com/7532043. +const defaultUserAgent = "Go 1.1 package http" + +// Write writes an HTTP/1.1 request -- header and body -- in wire format. +// This method consults the following fields of the request: +// Host +// URL +// Method (defaults to "GET") +// Header +// ContentLength +// TransferEncoding +// Body +// +// If Body is present, Content-Length is <= 0 and TransferEncoding +// hasn't been set to "identity", Write adds "Transfer-Encoding: +// chunked" to the header. Body is closed after it is sent. +func (r *Request) Write(w io.Writer) error { + return r.write(w, false, nil) +} + +// WriteProxy is like Write but writes the request in the form +// expected by an HTTP proxy. In particular, WriteProxy writes the +// initial Request-URI line of the request with an absolute URI, per +// section 5.1.2 of RFC 2616, including the scheme and host. +// In either case, WriteProxy also writes a Host header, using +// either r.Host or r.URL.Host. +func (r *Request) WriteProxy(w io.Writer) error { + return r.write(w, true, nil) +} + +// extraHeaders may be nil +func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header) error { + host := req.Host + if host == "" { + if req.URL == nil { + return errors.New("http: Request.Write on Request with no Host or URL set") + } + host = req.URL.Host + } + + ruri := req.URL.RequestURI() + if usingProxy && req.URL.Scheme != "" && req.URL.Opaque == "" { + ruri = req.URL.Scheme + "://" + host + ruri + } else if req.Method == "CONNECT" && req.URL.Path == "" { + // CONNECT requests normally give just the host and port, not a full URL. + ruri = host + } + // TODO(bradfitz): escape at least newlines in ruri? + + // Wrap the writer in a bufio Writer if it's not already buffered. + // Don't always call NewWriter, as that forces a bytes.Buffer + // and other small bufio Writers to have a minimum 4k buffer + // size. + var bw *bufio.Writer + if _, ok := w.(io.ByteWriter); !ok { + bw = bufio.NewWriter(w) + w = bw + } + + _, err := fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) + if err != nil { + return err + } + + // Header lines + _, err = fmt.Fprintf(w, "Host: %s\r\n", host) + if err != nil { + return err + } + + // Use the defaultUserAgent unless the Header contains one, which + // may be blank to not send the header. + userAgent := defaultUserAgent + if req.Header != nil { + if ua := req.Header["User-Agent"]; len(ua) > 0 { + userAgent = ua[0] + } + } + if userAgent != "" { + _, err = fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent) + if err != nil { + return err + } + } + + // Process Body,ContentLength,Close,Trailer + tw, err := newTransferWriter(req) + if err != nil { + return err + } + err = tw.WriteHeader(w) + if err != nil { + return err + } + + err = req.Header.WriteSubset(w, reqWriteExcludeHeader) + if err != nil { + return err + } + + if extraHeaders != nil { + err = extraHeaders.Write(w) + if err != nil { + return err + } + } + + _, err = io.WriteString(w, "\r\n") + if err != nil { + return err + } + + // Write body and trailer + err = tw.WriteBody(w) + if err != nil { + return err + } + + if bw != nil { + return bw.Flush() + } + return nil +} + +// ParseHTTPVersion parses a HTTP version string. +// "HTTP/1.0" returns (1, 0, true). +func ParseHTTPVersion(vers string) (major, minor int, ok bool) { + const Big = 1000000 // arbitrary upper bound + switch vers { + case "HTTP/1.1": + return 1, 1, true + case "HTTP/1.0": + return 1, 0, true + } + if !strings.HasPrefix(vers, "HTTP/") { + return 0, 0, false + } + dot := strings.Index(vers, ".") + if dot < 0 { + return 0, 0, false + } + major, err := strconv.Atoi(vers[5:dot]) + if err != nil || major < 0 || major > Big { + return 0, 0, false + } + minor, err = strconv.Atoi(vers[dot+1:]) + if err != nil || minor < 0 || minor > Big { + return 0, 0, false + } + return major, minor, true +} + +// NewRequest returns a new Request given a method, URL, and optional body. +// +// If the provided body is also an io.Closer, the returned +// Request.Body is set to body and will be closed by the Client +// methods Do, Post, and PostForm, and Transport.RoundTrip. +func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { + u, err := url.Parse(urlStr) + if err != nil { + return nil, err + } + rc, ok := body.(io.ReadCloser) + if !ok && body != nil { + rc = ioutil.NopCloser(body) + } + req := &Request{ + Method: method, + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(Header), + Body: rc, + Host: u.Host, + } + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + case *bytes.Reader: + req.ContentLength = int64(v.Len()) + case *strings.Reader: + req.ContentLength = int64(v.Len()) + } + } + + return req, nil +} + +// BasicAuth returns the username and password provided in the request's +// Authorization header, if the request uses HTTP Basic Authentication. +// See RFC 2617, Section 2. +func (r *Request) BasicAuth() (username, password string, ok bool) { + auth := r.Header.Get("Authorization") + if auth == "" { + return + } + return parseBasicAuth(auth) +} + +// parseBasicAuth parses an HTTP Basic Authentication string. +// "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true). +func parseBasicAuth(auth string) (username, password string, ok bool) { + if !strings.HasPrefix(auth, "Basic ") { + return + } + c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) + if err != nil { + return + } + cs := string(c) + s := strings.IndexByte(cs, ':') + if s < 0 { + return + } + return cs[:s], cs[s+1:], true +} + +// SetBasicAuth sets the request's Authorization header to use HTTP +// Basic Authentication with the provided username and password. +// +// With HTTP Basic Authentication the provided username and password +// are not encrypted. +func (r *Request) SetBasicAuth(username, password string) { + r.Header.Set("Authorization", "Basic "+basicAuth(username, password)) +} + +// parseRequestLine parses "GET /foo HTTP/1.1" into its three parts. +func parseRequestLine(line string) (method, requestURI, proto string, ok bool) { + s1 := strings.Index(line, " ") + s2 := strings.Index(line[s1+1:], " ") + if s1 < 0 || s2 < 0 { + return + } + s2 += s1 + 1 + return line[:s1], line[s1+1 : s2], line[s2+1:], true +} + +var textprotoReaderPool sync.Pool + +func newTextprotoReader(br *bufio.Reader) *textproto.Reader { + if v := textprotoReaderPool.Get(); v != nil { + tr := v.(*textproto.Reader) + tr.R = br + return tr + } + return textproto.NewReader(br) +} + +func putTextprotoReader(r *textproto.Reader) { + r.R = nil + textprotoReaderPool.Put(r) +} + +// ReadRequest reads and parses a request from b. +func ReadRequest(b *bufio.Reader) (req *Request, err error) { + + tp := newTextprotoReader(b) + req = new(Request) + + // First line: GET /index.html HTTP/1.0 + var s string + if s, err = tp.ReadLine(); err != nil { + return nil, err + } + defer func() { + putTextprotoReader(tp) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + var ok bool + req.Method, req.RequestURI, req.Proto, ok = parseRequestLine(s) + if !ok { + return nil, &badStringError{"malformed HTTP request", s} + } + rawurl := req.RequestURI + if req.ProtoMajor, req.ProtoMinor, ok = ParseHTTPVersion(req.Proto); !ok { + return nil, &badStringError{"malformed HTTP version", req.Proto} + } + + // CONNECT requests are used two different ways, and neither uses a full URL: + // The standard use is to tunnel HTTPS through an HTTP proxy. + // It looks like "CONNECT www.google.com:443 HTTP/1.1", and the parameter is + // just the authority section of a URL. This information should go in req.URL.Host. + // + // The net/rpc package also uses CONNECT, but there the parameter is a path + // that starts with a slash. It can be parsed with the regular URL parser, + // and the path will end up in req.URL.Path, where it needs to be in order for + // RPC to work. + justAuthority := req.Method == "CONNECT" && !strings.HasPrefix(rawurl, "/") + if justAuthority { + rawurl = "http://" + rawurl + } + + if req.URL, err = url.ParseRequestURI(rawurl); err != nil { + return nil, err + } + + if justAuthority { + // Strip the bogus "http://" back off. + req.URL.Scheme = "" + } + + // Subsequent lines: Key: value. + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + req.Header = Header(mimeHeader) + + // RFC2616: Must treat + // GET /index.html HTTP/1.1 + // Host: www.google.com + // and + // GET http://www.google.com/index.html HTTP/1.1 + // Host: doesntmatter + // the same. In the second case, any Host line is ignored. + req.Host = req.URL.Host + if req.Host == "" { + req.Host = req.Header.get("Host") + } + delete(req.Header, "Host") + + fixPragmaCacheControl(req.Header) + + err = readTransfer(req, b) + if err != nil { + return nil, err + } + + req.Close = shouldClose(req.ProtoMajor, req.ProtoMinor, req.Header, false) + return req, nil +} + +// MaxBytesReader is similar to io.LimitReader but is intended for +// limiting the size of incoming request bodies. In contrast to +// io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a +// non-EOF error for a Read beyond the limit, and Closes the +// underlying reader when its Close method is called. +// +// MaxBytesReader prevents clients from accidentally or maliciously +// sending a large request and wasting server resources. +func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { + return &maxBytesReader{w: w, r: r, n: n} +} + +type maxBytesReader struct { + w ResponseWriter + r io.ReadCloser // underlying reader + n int64 // max bytes remaining + stopped bool +} + +func (l *maxBytesReader) Read(p []byte) (n int, err error) { + if l.n <= 0 { + if !l.stopped { + l.stopped = true + if res, ok := l.w.(*response); ok { + res.requestTooLarge() + } + } + return 0, errors.New("http: request body too large") + } + if int64(len(p)) > l.n { + p = p[:l.n] + } + n, err = l.r.Read(p) + l.n -= int64(n) + return +} + +func (l *maxBytesReader) Close() error { + return l.r.Close() +} + +func copyValues(dst, src url.Values) { + for k, vs := range src { + for _, value := range vs { + dst.Add(k, value) + } + } +} + +func parsePostForm(r *Request) (vs url.Values, err error) { + if r.Body == nil { + err = errors.New("missing form body") + return + } + ct := r.Header.Get("Content-Type") + // RFC 2616, section 7.2.1 - empty type + // SHOULD be treated as application/octet-stream + if ct == "" { + ct = "application/octet-stream" + } + ct, _, err = mime.ParseMediaType(ct) + switch { + case ct == "application/x-www-form-urlencoded": + var reader io.Reader = r.Body + maxFormSize := int64(1<<63 - 1) + if _, ok := r.Body.(*maxBytesReader); !ok { + maxFormSize = int64(10 << 20) // 10 MB is a lot of text. + reader = io.LimitReader(r.Body, maxFormSize+1) + } + b, e := ioutil.ReadAll(reader) + if e != nil { + if err == nil { + err = e + } + break + } + if int64(len(b)) > maxFormSize { + err = errors.New("http: POST too large") + return + } + vs, e = url.ParseQuery(string(b)) + if err == nil { + err = e + } + case ct == "multipart/form-data": + // handled by ParseMultipartForm (which is calling us, or should be) + // TODO(bradfitz): there are too many possible + // orders to call too many functions here. + // Clean this up and write more tests. + // request_test.go contains the start of this, + // in TestParseMultipartFormOrder and others. + } + return +} + +// ParseForm parses the raw query from the URL and updates r.Form. +// +// For POST or PUT requests, it also parses the request body as a form and +// put the results into both r.PostForm and r.Form. +// POST and PUT body parameters take precedence over URL query string values +// in r.Form. +// +// If the request Body's size has not already been limited by MaxBytesReader, +// the size is capped at 10MB. +// +// ParseMultipartForm calls ParseForm automatically. +// It is idempotent. +func (r *Request) ParseForm() error { + var err error + if r.PostForm == nil { + if r.Method == "POST" || r.Method == "PUT" || r.Method == "PATCH" { + r.PostForm, err = parsePostForm(r) + } + if r.PostForm == nil { + r.PostForm = make(url.Values) + } + } + if r.Form == nil { + if len(r.PostForm) > 0 { + r.Form = make(url.Values) + copyValues(r.Form, r.PostForm) + } + var newValues url.Values + if r.URL != nil { + var e error + newValues, e = url.ParseQuery(r.URL.RawQuery) + if err == nil { + err = e + } + } + if newValues == nil { + newValues = make(url.Values) + } + if r.Form == nil { + r.Form = newValues + } else { + copyValues(r.Form, newValues) + } + } + return err +} + +// ParseMultipartForm parses a request body as multipart/form-data. +// The whole request body is parsed and up to a total of maxMemory bytes of +// its file parts are stored in memory, with the remainder stored on +// disk in temporary files. +// ParseMultipartForm calls ParseForm if necessary. +// After one call to ParseMultipartForm, subsequent calls have no effect. +func (r *Request) ParseMultipartForm(maxMemory int64) error { + if r.MultipartForm == multipartByReader { + return errors.New("http: multipart handled by MultipartReader") + } + if r.Form == nil { + err := r.ParseForm() + if err != nil { + return err + } + } + if r.MultipartForm != nil { + return nil + } + + mr, err := r.multipartReader() + if err != nil { + return err + } + + f, err := mr.ReadForm(maxMemory) + if err != nil { + return err + } + for k, v := range f.Value { + r.Form[k] = append(r.Form[k], v...) + } + r.MultipartForm = f + + return nil +} + +// FormValue returns the first value for the named component of the query. +// POST and PUT body parameters take precedence over URL query string values. +// FormValue calls ParseMultipartForm and ParseForm if necessary and ignores +// any errors returned by these functions. +// To access multiple values of the same key use ParseForm. +func (r *Request) FormValue(key string) string { + if r.Form == nil { + r.ParseMultipartForm(defaultMaxMemory) + } + if vs := r.Form[key]; len(vs) > 0 { + return vs[0] + } + return "" +} + +// PostFormValue returns the first value for the named component of the POST +// or PUT request body. URL query parameters are ignored. +// PostFormValue calls ParseMultipartForm and ParseForm if necessary and ignores +// any errors returned by these functions. +func (r *Request) PostFormValue(key string) string { + if r.PostForm == nil { + r.ParseMultipartForm(defaultMaxMemory) + } + if vs := r.PostForm[key]; len(vs) > 0 { + return vs[0] + } + return "" +} + +// FormFile returns the first file for the provided form key. +// FormFile calls ParseMultipartForm and ParseForm if necessary. +func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) { + if r.MultipartForm == multipartByReader { + return nil, nil, errors.New("http: multipart handled by MultipartReader") + } + if r.MultipartForm == nil { + err := r.ParseMultipartForm(defaultMaxMemory) + if err != nil { + return nil, nil, err + } + } + if r.MultipartForm != nil && r.MultipartForm.File != nil { + if fhs := r.MultipartForm.File[key]; len(fhs) > 0 { + f, err := fhs[0].Open() + return f, fhs[0], err + } + } + return nil, nil, ErrMissingFile +} + +func (r *Request) expectsContinue() bool { + return hasToken(r.Header.get("Expect"), "100-continue") +} + +func (r *Request) wantsHttp10KeepAlive() bool { + if r.ProtoMajor != 1 || r.ProtoMinor != 0 { + return false + } + return hasToken(r.Header.get("Connection"), "keep-alive") +} + +func (r *Request) wantsClose() bool { + return hasToken(r.Header.get("Connection"), "close") +} + +func (r *Request) closeBody() { + if r.Body != nil { + r.Body.Close() + } +} diff --git a/src/net/http/request_test.go b/src/net/http/request_test.go new file mode 100644 index 000000000..759ea4e8b --- /dev/null +++ b/src/net/http/request_test.go @@ -0,0 +1,680 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "bufio" + "bytes" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + . "net/http" + "net/http/httptest" + "net/url" + "os" + "reflect" + "regexp" + "strings" + "testing" +) + +func TestQuery(t *testing.T) { + req := &Request{Method: "GET"} + req.URL, _ = url.Parse("http://www.google.com/search?q=foo&q=bar") + if q := req.FormValue("q"); q != "foo" { + t.Errorf(`req.FormValue("q") = %q, want "foo"`, q) + } +} + +func TestPostQuery(t *testing.T) { + req, _ := NewRequest("POST", "http://www.google.com/search?q=foo&q=bar&both=x&prio=1&empty=not", + strings.NewReader("z=post&both=y&prio=2&empty=")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") + + if q := req.FormValue("q"); q != "foo" { + t.Errorf(`req.FormValue("q") = %q, want "foo"`, q) + } + if z := req.FormValue("z"); z != "post" { + t.Errorf(`req.FormValue("z") = %q, want "post"`, z) + } + if bq, found := req.PostForm["q"]; found { + t.Errorf(`req.PostForm["q"] = %q, want no entry in map`, bq) + } + if bz := req.PostFormValue("z"); bz != "post" { + t.Errorf(`req.PostFormValue("z") = %q, want "post"`, bz) + } + if qs := req.Form["q"]; !reflect.DeepEqual(qs, []string{"foo", "bar"}) { + t.Errorf(`req.Form["q"] = %q, want ["foo", "bar"]`, qs) + } + if both := req.Form["both"]; !reflect.DeepEqual(both, []string{"y", "x"}) { + t.Errorf(`req.Form["both"] = %q, want ["y", "x"]`, both) + } + if prio := req.FormValue("prio"); prio != "2" { + t.Errorf(`req.FormValue("prio") = %q, want "2" (from body)`, prio) + } + if empty := req.FormValue("empty"); empty != "" { + t.Errorf(`req.FormValue("empty") = %q, want "" (from body)`, empty) + } +} + +func TestPatchQuery(t *testing.T) { + req, _ := NewRequest("PATCH", "http://www.google.com/search?q=foo&q=bar&both=x&prio=1&empty=not", + strings.NewReader("z=post&both=y&prio=2&empty=")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") + + if q := req.FormValue("q"); q != "foo" { + t.Errorf(`req.FormValue("q") = %q, want "foo"`, q) + } + if z := req.FormValue("z"); z != "post" { + t.Errorf(`req.FormValue("z") = %q, want "post"`, z) + } + if bq, found := req.PostForm["q"]; found { + t.Errorf(`req.PostForm["q"] = %q, want no entry in map`, bq) + } + if bz := req.PostFormValue("z"); bz != "post" { + t.Errorf(`req.PostFormValue("z") = %q, want "post"`, bz) + } + if qs := req.Form["q"]; !reflect.DeepEqual(qs, []string{"foo", "bar"}) { + t.Errorf(`req.Form["q"] = %q, want ["foo", "bar"]`, qs) + } + if both := req.Form["both"]; !reflect.DeepEqual(both, []string{"y", "x"}) { + t.Errorf(`req.Form["both"] = %q, want ["y", "x"]`, both) + } + if prio := req.FormValue("prio"); prio != "2" { + t.Errorf(`req.FormValue("prio") = %q, want "2" (from body)`, prio) + } + if empty := req.FormValue("empty"); empty != "" { + t.Errorf(`req.FormValue("empty") = %q, want "" (from body)`, empty) + } +} + +type stringMap map[string][]string +type parseContentTypeTest struct { + shouldError bool + contentType stringMap +} + +var parseContentTypeTests = []parseContentTypeTest{ + {false, stringMap{"Content-Type": {"text/plain"}}}, + // Empty content type is legal - shoult be treated as + // application/octet-stream (RFC 2616, section 7.2.1) + {false, stringMap{}}, + {true, stringMap{"Content-Type": {"text/plain; boundary="}}}, + {false, stringMap{"Content-Type": {"application/unknown"}}}, +} + +func TestParseFormUnknownContentType(t *testing.T) { + for i, test := range parseContentTypeTests { + req := &Request{ + Method: "POST", + Header: Header(test.contentType), + Body: ioutil.NopCloser(strings.NewReader("body")), + } + err := req.ParseForm() + switch { + case err == nil && test.shouldError: + t.Errorf("test %d should have returned error", i) + case err != nil && !test.shouldError: + t.Errorf("test %d should not have returned error, got %v", i, err) + } + } +} + +func TestParseFormInitializeOnError(t *testing.T) { + nilBody, _ := NewRequest("POST", "http://www.google.com/search?q=foo", nil) + tests := []*Request{ + nilBody, + {Method: "GET", URL: nil}, + } + for i, req := range tests { + err := req.ParseForm() + if req.Form == nil { + t.Errorf("%d. Form not initialized, error %v", i, err) + } + if req.PostForm == nil { + t.Errorf("%d. PostForm not initialized, error %v", i, err) + } + } +} + +func TestMultipartReader(t *testing.T) { + req := &Request{ + Method: "POST", + Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}}, + Body: ioutil.NopCloser(new(bytes.Buffer)), + } + multipart, err := req.MultipartReader() + if multipart == nil { + t.Errorf("expected multipart; error: %v", err) + } + + req.Header = Header{"Content-Type": {"text/plain"}} + multipart, err = req.MultipartReader() + if multipart != nil { + t.Error("unexpected multipart for text/plain") + } +} + +func TestParseMultipartForm(t *testing.T) { + req := &Request{ + Method: "POST", + Header: Header{"Content-Type": {`multipart/form-data; boundary="foo123"`}}, + Body: ioutil.NopCloser(new(bytes.Buffer)), + } + err := req.ParseMultipartForm(25) + if err == nil { + t.Error("expected multipart EOF, got nil") + } + + req.Header = Header{"Content-Type": {"text/plain"}} + err = req.ParseMultipartForm(25) + if err != ErrNotMultipart { + t.Error("expected ErrNotMultipart for text/plain") + } +} + +func TestRedirect(t *testing.T) { + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + switch r.URL.Path { + case "/": + w.Header().Set("Location", "/foo/") + w.WriteHeader(StatusSeeOther) + case "/foo/": + fmt.Fprintf(w, "foo") + default: + w.WriteHeader(StatusBadRequest) + } + })) + defer ts.Close() + + var end = regexp.MustCompile("/foo/$") + r, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + r.Body.Close() + url := r.Request.URL.String() + if r.StatusCode != 200 || !end.MatchString(url) { + t.Fatalf("Get got status %d at %q, want 200 matching /foo/$", r.StatusCode, url) + } +} + +func TestSetBasicAuth(t *testing.T) { + r, _ := NewRequest("GET", "http://example.com/", nil) + r.SetBasicAuth("Aladdin", "open sesame") + if g, e := r.Header.Get("Authorization"), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="; g != e { + t.Errorf("got header %q, want %q", g, e) + } +} + +func TestMultipartRequest(t *testing.T) { + // Test that we can read the values and files of a + // multipart request with FormValue and FormFile, + // and that ParseMultipartForm can be called multiple times. + req := newTestMultipartRequest(t) + if err := req.ParseMultipartForm(25); err != nil { + t.Fatal("ParseMultipartForm first call:", err) + } + defer req.MultipartForm.RemoveAll() + validateTestMultipartContents(t, req, false) + if err := req.ParseMultipartForm(25); err != nil { + t.Fatal("ParseMultipartForm second call:", err) + } + validateTestMultipartContents(t, req, false) +} + +func TestMultipartRequestAuto(t *testing.T) { + // Test that FormValue and FormFile automatically invoke + // ParseMultipartForm and return the right values. + req := newTestMultipartRequest(t) + defer func() { + if req.MultipartForm != nil { + req.MultipartForm.RemoveAll() + } + }() + validateTestMultipartContents(t, req, true) +} + +func TestMissingFileMultipartRequest(t *testing.T) { + // Test that FormFile returns an error if + // the named file is missing. + req := newTestMultipartRequest(t) + testMissingFile(t, req) +} + +// Test that FormValue invokes ParseMultipartForm. +func TestFormValueCallsParseMultipartForm(t *testing.T) { + req, _ := NewRequest("POST", "http://www.google.com/", strings.NewReader("z=post")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") + if req.Form != nil { + t.Fatal("Unexpected request Form, want nil") + } + req.FormValue("z") + if req.Form == nil { + t.Fatal("ParseMultipartForm not called by FormValue") + } +} + +// Test that FormFile invokes ParseMultipartForm. +func TestFormFileCallsParseMultipartForm(t *testing.T) { + req := newTestMultipartRequest(t) + if req.Form != nil { + t.Fatal("Unexpected request Form, want nil") + } + req.FormFile("") + if req.Form == nil { + t.Fatal("ParseMultipartForm not called by FormFile") + } +} + +// Test that ParseMultipartForm errors if called +// after MultipartReader on the same request. +func TestParseMultipartFormOrder(t *testing.T) { + req := newTestMultipartRequest(t) + if _, err := req.MultipartReader(); err != nil { + t.Fatalf("MultipartReader: %v", err) + } + if err := req.ParseMultipartForm(1024); err == nil { + t.Fatal("expected an error from ParseMultipartForm after call to MultipartReader") + } +} + +// Test that MultipartReader errors if called +// after ParseMultipartForm on the same request. +func TestMultipartReaderOrder(t *testing.T) { + req := newTestMultipartRequest(t) + if err := req.ParseMultipartForm(25); err != nil { + t.Fatalf("ParseMultipartForm: %v", err) + } + defer req.MultipartForm.RemoveAll() + if _, err := req.MultipartReader(); err == nil { + t.Fatal("expected an error from MultipartReader after call to ParseMultipartForm") + } +} + +// Test that FormFile errors if called after +// MultipartReader on the same request. +func TestFormFileOrder(t *testing.T) { + req := newTestMultipartRequest(t) + if _, err := req.MultipartReader(); err != nil { + t.Fatalf("MultipartReader: %v", err) + } + if _, _, err := req.FormFile(""); err == nil { + t.Fatal("expected an error from FormFile after call to MultipartReader") + } +} + +var readRequestErrorTests = []struct { + in string + err error +}{ + {"GET / HTTP/1.1\r\nheader:foo\r\n\r\n", nil}, + {"GET / HTTP/1.1\r\nheader:foo\r\n", io.ErrUnexpectedEOF}, + {"", io.EOF}, +} + +func TestReadRequestErrors(t *testing.T) { + for i, tt := range readRequestErrorTests { + _, err := ReadRequest(bufio.NewReader(strings.NewReader(tt.in))) + if err != tt.err { + t.Errorf("%d. got error = %v; want %v", i, err, tt.err) + } + } +} + +func TestNewRequestHost(t *testing.T) { + req, err := NewRequest("GET", "http://localhost:1234/", nil) + if err != nil { + t.Fatal(err) + } + if req.Host != "localhost:1234" { + t.Errorf("Host = %q; want localhost:1234", req.Host) + } +} + +func TestNewRequestContentLength(t *testing.T) { + readByte := func(r io.Reader) io.Reader { + var b [1]byte + r.Read(b[:]) + return r + } + tests := []struct { + r io.Reader + want int64 + }{ + {bytes.NewReader([]byte("123")), 3}, + {bytes.NewBuffer([]byte("1234")), 4}, + {strings.NewReader("12345"), 5}, + // Not detected: + {struct{ io.Reader }{strings.NewReader("xyz")}, 0}, + {io.NewSectionReader(strings.NewReader("x"), 0, 6), 0}, + {readByte(io.NewSectionReader(strings.NewReader("xy"), 0, 6)), 0}, + } + for _, tt := range tests { + req, err := NewRequest("POST", "http://localhost/", tt.r) + if err != nil { + t.Fatal(err) + } + if req.ContentLength != tt.want { + t.Errorf("ContentLength(%T) = %d; want %d", tt.r, req.ContentLength, tt.want) + } + } +} + +var parseHTTPVersionTests = []struct { + vers string + major, minor int + ok bool +}{ + {"HTTP/0.9", 0, 9, true}, + {"HTTP/1.0", 1, 0, true}, + {"HTTP/1.1", 1, 1, true}, + {"HTTP/3.14", 3, 14, true}, + + {"HTTP", 0, 0, false}, + {"HTTP/one.one", 0, 0, false}, + {"HTTP/1.1/", 0, 0, false}, + {"HTTP/-1,0", 0, 0, false}, + {"HTTP/0,-1", 0, 0, false}, + {"HTTP/", 0, 0, false}, + {"HTTP/1,1", 0, 0, false}, +} + +func TestParseHTTPVersion(t *testing.T) { + for _, tt := range parseHTTPVersionTests { + major, minor, ok := ParseHTTPVersion(tt.vers) + if ok != tt.ok || major != tt.major || minor != tt.minor { + type version struct { + major, minor int + ok bool + } + t.Errorf("failed to parse %q, expected: %#v, got %#v", tt.vers, version{tt.major, tt.minor, tt.ok}, version{major, minor, ok}) + } + } +} + +type getBasicAuthTest struct { + username, password string + ok bool +} + +type parseBasicAuthTest getBasicAuthTest + +type basicAuthCredentialsTest struct { + username, password string +} + +var getBasicAuthTests = []struct { + username, password string + ok bool +}{ + {"Aladdin", "open sesame", true}, + {"Aladdin", "open:sesame", true}, + {"", "", true}, +} + +func TestGetBasicAuth(t *testing.T) { + for _, tt := range getBasicAuthTests { + r, _ := NewRequest("GET", "http://example.com/", nil) + r.SetBasicAuth(tt.username, tt.password) + username, password, ok := r.BasicAuth() + if ok != tt.ok || username != tt.username || password != tt.password { + t.Errorf("BasicAuth() = %#v, want %#v", getBasicAuthTest{username, password, ok}, + getBasicAuthTest{tt.username, tt.password, tt.ok}) + } + } + // Unauthenticated request. + r, _ := NewRequest("GET", "http://example.com/", nil) + username, password, ok := r.BasicAuth() + if ok { + t.Errorf("expected false from BasicAuth when the request is unauthenticated") + } + want := basicAuthCredentialsTest{"", ""} + if username != want.username || password != want.password { + t.Errorf("expected credentials: %#v when the request is unauthenticated, got %#v", + want, basicAuthCredentialsTest{username, password}) + } +} + +var parseBasicAuthTests = []struct { + header, username, password string + ok bool +}{ + {"Basic " + base64.StdEncoding.EncodeToString([]byte("Aladdin:open sesame")), "Aladdin", "open sesame", true}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte("Aladdin:open:sesame")), "Aladdin", "open:sesame", true}, + {"Basic " + base64.StdEncoding.EncodeToString([]byte(":")), "", "", true}, + {"Basic" + base64.StdEncoding.EncodeToString([]byte("Aladdin:open sesame")), "", "", false}, + {base64.StdEncoding.EncodeToString([]byte("Aladdin:open sesame")), "", "", false}, + {"Basic ", "", "", false}, + {"Basic Aladdin:open sesame", "", "", false}, + {`Digest username="Aladdin"`, "", "", false}, +} + +func TestParseBasicAuth(t *testing.T) { + for _, tt := range parseBasicAuthTests { + r, _ := NewRequest("GET", "http://example.com/", nil) + r.Header.Set("Authorization", tt.header) + username, password, ok := r.BasicAuth() + if ok != tt.ok || username != tt.username || password != tt.password { + t.Errorf("BasicAuth() = %#v, want %#v", getBasicAuthTest{username, password, ok}, + getBasicAuthTest{tt.username, tt.password, tt.ok}) + } + } +} + +type logWrites struct { + t *testing.T + dst *[]string +} + +func (l logWrites) WriteByte(c byte) error { + l.t.Fatalf("unexpected WriteByte call") + return nil +} + +func (l logWrites) Write(p []byte) (n int, err error) { + *l.dst = append(*l.dst, string(p)) + return len(p), nil +} + +func TestRequestWriteBufferedWriter(t *testing.T) { + got := []string{} + req, _ := NewRequest("GET", "http://foo.com/", nil) + req.Write(logWrites{t, &got}) + want := []string{ + "GET / HTTP/1.1\r\n", + "Host: foo.com\r\n", + "User-Agent: " + DefaultUserAgent + "\r\n", + "\r\n", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Writes = %q\n Want = %q", got, want) + } +} + +func testMissingFile(t *testing.T, req *Request) { + f, fh, err := req.FormFile("missing") + if f != nil { + t.Errorf("FormFile file = %v, want nil", f) + } + if fh != nil { + t.Errorf("FormFile file header = %q, want nil", fh) + } + if err != ErrMissingFile { + t.Errorf("FormFile err = %q, want ErrMissingFile", err) + } +} + +func newTestMultipartRequest(t *testing.T) *Request { + b := strings.NewReader(strings.Replace(message, "\n", "\r\n", -1)) + req, err := NewRequest("POST", "/", b) + if err != nil { + t.Fatal("NewRequest:", err) + } + ctype := fmt.Sprintf(`multipart/form-data; boundary="%s"`, boundary) + req.Header.Set("Content-type", ctype) + return req +} + +func validateTestMultipartContents(t *testing.T, req *Request, allMem bool) { + if g, e := req.FormValue("texta"), textaValue; g != e { + t.Errorf("texta value = %q, want %q", g, e) + } + if g, e := req.FormValue("textb"), textbValue; g != e { + t.Errorf("textb value = %q, want %q", g, e) + } + if g := req.FormValue("missing"); g != "" { + t.Errorf("missing value = %q, want empty string", g) + } + + assertMem := func(n string, fd multipart.File) { + if _, ok := fd.(*os.File); ok { + t.Error(n, " is *os.File, should not be") + } + } + fda := testMultipartFile(t, req, "filea", "filea.txt", fileaContents) + defer fda.Close() + assertMem("filea", fda) + fdb := testMultipartFile(t, req, "fileb", "fileb.txt", filebContents) + defer fdb.Close() + if allMem { + assertMem("fileb", fdb) + } else { + if _, ok := fdb.(*os.File); !ok { + t.Errorf("fileb has unexpected underlying type %T", fdb) + } + } + + testMissingFile(t, req) +} + +func testMultipartFile(t *testing.T, req *Request, key, expectFilename, expectContent string) multipart.File { + f, fh, err := req.FormFile(key) + if err != nil { + t.Fatalf("FormFile(%q): %q", key, err) + } + if fh.Filename != expectFilename { + t.Errorf("filename = %q, want %q", fh.Filename, expectFilename) + } + var b bytes.Buffer + _, err = io.Copy(&b, f) + if err != nil { + t.Fatal("copying contents:", err) + } + if g := b.String(); g != expectContent { + t.Errorf("contents = %q, want %q", g, expectContent) + } + return f +} + +const ( + fileaContents = "This is a test file." + filebContents = "Another test file." + textaValue = "foo" + textbValue = "bar" + boundary = `MyBoundary` +) + +const message = ` +--MyBoundary +Content-Disposition: form-data; name="filea"; filename="filea.txt" +Content-Type: text/plain + +` + fileaContents + ` +--MyBoundary +Content-Disposition: form-data; name="fileb"; filename="fileb.txt" +Content-Type: text/plain + +` + filebContents + ` +--MyBoundary +Content-Disposition: form-data; name="texta" + +` + textaValue + ` +--MyBoundary +Content-Disposition: form-data; name="textb" + +` + textbValue + ` +--MyBoundary-- +` + +func benchmarkReadRequest(b *testing.B, request string) { + request = request + "\n" // final \n + request = strings.Replace(request, "\n", "\r\n", -1) // expand \n to \r\n + b.SetBytes(int64(len(request))) + r := bufio.NewReader(&infiniteReader{buf: []byte(request)}) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ReadRequest(r) + if err != nil { + b.Fatalf("failed to read request: %v", err) + } + } +} + +// infiniteReader satisfies Read requests as if the contents of buf +// loop indefinitely. +type infiniteReader struct { + buf []byte + offset int +} + +func (r *infiniteReader) Read(b []byte) (int, error) { + n := copy(b, r.buf[r.offset:]) + r.offset = (r.offset + n) % len(r.buf) + return n, nil +} + +func BenchmarkReadRequestChrome(b *testing.B) { + // https://github.com/felixge/node-http-perf/blob/master/fixtures/get.http + benchmarkReadRequest(b, `GET / HTTP/1.1 +Host: localhost:8080 +Connection: keep-alive +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.52 Safari/537.17 +Accept-Encoding: gzip,deflate,sdch +Accept-Language: en-US,en;q=0.8 +Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3 +Cookie: __utma=1.1978842379.1323102373.1323102373.1323102373.1; EPi:NumberOfVisits=1,2012-02-28T13:42:18; CrmSession=5b707226b9563e1bc69084d07a107c98; plushContainerWidth=100%25; plushNoTopMenu=0; hudson_auto_refresh=false +`) +} + +func BenchmarkReadRequestCurl(b *testing.B) { + // curl http://localhost:8080/ + benchmarkReadRequest(b, `GET / HTTP/1.1 +User-Agent: curl/7.27.0 +Host: localhost:8080 +Accept: */* +`) +} + +func BenchmarkReadRequestApachebench(b *testing.B) { + // ab -n 1 -c 1 http://localhost:8080/ + benchmarkReadRequest(b, `GET / HTTP/1.0 +Host: localhost:8080 +User-Agent: ApacheBench/2.3 +Accept: */* +`) +} + +func BenchmarkReadRequestSiege(b *testing.B) { + // siege -r 1 -c 1 http://localhost:8080/ + benchmarkReadRequest(b, `GET / HTTP/1.1 +Host: localhost:8080 +Accept: */* +Accept-Encoding: gzip +User-Agent: JoeDog/1.00 [en] (X11; I; Siege 2.70) +Connection: keep-alive +`) +} + +func BenchmarkReadRequestWrk(b *testing.B) { + // wrk -t 1 -r 1 -c 1 http://localhost:8080/ + benchmarkReadRequest(b, `GET / HTTP/1.1 +Host: localhost:8080 +`) +} diff --git a/src/net/http/requestwrite_test.go b/src/net/http/requestwrite_test.go new file mode 100644 index 000000000..7a6bd5878 --- /dev/null +++ b/src/net/http/requestwrite_test.go @@ -0,0 +1,623 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "net/url" + "strings" + "testing" +) + +type reqWriteTest struct { + Req Request + Body interface{} // optional []byte or func() io.ReadCloser to populate Req.Body + + // Any of these three may be empty to skip that test. + WantWrite string // Request.Write + WantProxy string // Request.WriteProxy + + WantError error // wanted error from Request.Write +} + +var reqWriteTests = []reqWriteTest{ + // HTTP/1.1 => chunked coding; no body; no trailer + { + Req: Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "www.techcrunch.com", + Path: "/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}, + "Accept-Charset": {"ISO-8859-1,utf-8;q=0.7,*;q=0.7"}, + "Accept-Encoding": {"gzip,deflate"}, + "Accept-Language": {"en-us,en;q=0.5"}, + "Keep-Alive": {"300"}, + "Proxy-Connection": {"keep-alive"}, + "User-Agent": {"Fake"}, + }, + Body: nil, + Close: false, + Host: "www.techcrunch.com", + Form: map[string][]string{}, + }, + + WantWrite: "GET / HTTP/1.1\r\n" + + "Host: www.techcrunch.com\r\n" + + "User-Agent: Fake\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + + "Accept-Encoding: gzip,deflate\r\n" + + "Accept-Language: en-us,en;q=0.5\r\n" + + "Keep-Alive: 300\r\n" + + "Proxy-Connection: keep-alive\r\n\r\n", + + WantProxy: "GET http://www.techcrunch.com/ HTTP/1.1\r\n" + + "Host: www.techcrunch.com\r\n" + + "User-Agent: Fake\r\n" + + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n" + + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + + "Accept-Encoding: gzip,deflate\r\n" + + "Accept-Language: en-us,en;q=0.5\r\n" + + "Keep-Alive: 300\r\n" + + "Proxy-Connection: keep-alive\r\n\r\n", + }, + // HTTP/1.1 => chunked coding; body; empty trailer + { + Req: Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/search", + }, + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + TransferEncoding: []string{"chunked"}, + }, + + Body: []byte("abcdef"), + + WantWrite: "GET /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("abcdef") + chunk(""), + + WantProxy: "GET http://www.google.com/search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("abcdef") + chunk(""), + }, + // HTTP/1.1 POST => chunked coding; body; empty trailer + { + Req: Request{ + Method: "POST", + URL: &url.URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/search", + }, + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: true, + TransferEncoding: []string{"chunked"}, + }, + + Body: []byte("abcdef"), + + WantWrite: "POST /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Connection: close\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("abcdef") + chunk(""), + + WantProxy: "POST http://www.google.com/search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Connection: close\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("abcdef") + chunk(""), + }, + + // HTTP/1.1 POST with Content-Length, no chunking + { + Req: Request{ + Method: "POST", + URL: &url.URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/search", + }, + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Close: true, + ContentLength: 6, + }, + + Body: []byte("abcdef"), + + WantWrite: "POST /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "abcdef", + + WantProxy: "POST http://www.google.com/search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Connection: close\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "abcdef", + }, + + // HTTP/1.1 POST with Content-Length in headers + { + Req: Request{ + Method: "POST", + URL: mustParseURL("http://example.com/"), + Host: "example.com", + Header: Header{ + "Content-Length": []string{"10"}, // ignored + }, + ContentLength: 6, + }, + + Body: []byte("abcdef"), + + WantWrite: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "abcdef", + + WantProxy: "POST http://example.com/ HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Content-Length: 6\r\n" + + "\r\n" + + "abcdef", + }, + + // default to HTTP/1.1 + { + Req: Request{ + Method: "GET", + URL: mustParseURL("/search"), + Host: "www.google.com", + }, + + WantWrite: "GET /search HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "\r\n", + }, + + // Request with a 0 ContentLength and a 0 byte body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + Body: func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 0)) }, + + // RFC 2616 Section 14.13 says Content-Length should be specified + // unless body is prohibited by the request method. + // Also, nginx expects it for POST and PUT. + WantWrite: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + + WantProxy: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + }, + + // Request with a 0 ContentLength and a 1 byte body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + Body: func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 1)) }, + + WantWrite: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("x") + chunk(""), + + WantProxy: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + chunk("x") + chunk(""), + }, + + // Request with a ContentLength of 10 but a 5 byte body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 10, // but we're going to send only 5 bytes + }, + Body: []byte("12345"), + WantError: errors.New("http: ContentLength=10 with Body length 5"), + }, + + // Request with a ContentLength of 4 but an 8 byte body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 4, // but we're going to try to send 8 bytes + }, + Body: []byte("12345678"), + WantError: errors.New("http: ContentLength=4 with Body length 8"), + }, + + // Request with a 5 ContentLength and nil body. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 5, // but we'll omit the body + }, + WantError: errors.New("http: Request.ContentLength=5 with nil Body"), + }, + + // Request with a 0 ContentLength and a body with 1 byte content and an error. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + Body: func() io.ReadCloser { + err := errors.New("Custom reader error") + errReader := &errorReader{err} + return ioutil.NopCloser(io.MultiReader(strings.NewReader("x"), errReader)) + }, + + WantError: errors.New("Custom reader error"), + }, + + // Request with a 0 ContentLength and a body without content and an error. + { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + Body: func() io.ReadCloser { + err := errors.New("Custom reader error") + errReader := &errorReader{err} + return ioutil.NopCloser(errReader) + }, + + WantError: errors.New("Custom reader error"), + }, + + // Verify that DumpRequest preserves the HTTP version number, doesn't add a Host, + // and doesn't add a User-Agent. + { + Req: Request{ + Method: "GET", + URL: mustParseURL("/foo"), + ProtoMajor: 1, + ProtoMinor: 0, + Header: Header{ + "X-Foo": []string{"X-Bar"}, + }, + }, + + WantWrite: "GET /foo HTTP/1.1\r\n" + + "Host: \r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "X-Foo: X-Bar\r\n\r\n", + }, + + // If no Request.Host and no Request.URL.Host, we send + // an empty Host header, and don't use + // Request.Header["Host"]. This is just testing that + // we don't change Go 1.0 behavior. + { + Req: Request{ + Method: "GET", + Host: "", + URL: &url.URL{ + Scheme: "http", + Host: "", + Path: "/search", + }, + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{ + "Host": []string{"bad.example.com"}, + }, + }, + + WantWrite: "GET /search HTTP/1.1\r\n" + + "Host: \r\n" + + "User-Agent: Go 1.1 package http\r\n\r\n", + }, + + // Opaque test #1 from golang.org/issue/4860 + { + Req: Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "www.google.com", + Opaque: "/%2F/%2F/", + }, + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + }, + + WantWrite: "GET /%2F/%2F/ HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n\r\n", + }, + + // Opaque test #2 from golang.org/issue/4860 + { + Req: Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "x.google.com", + Opaque: "//y.google.com/%2F/%2F/", + }, + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + }, + + WantWrite: "GET http://y.google.com/%2F/%2F/ HTTP/1.1\r\n" + + "Host: x.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n\r\n", + }, + + // Testing custom case in header keys. Issue 5022. + { + Req: Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{ + "ALL-CAPS": {"x"}, + }, + }, + + WantWrite: "GET / HTTP/1.1\r\n" + + "Host: www.google.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "ALL-CAPS: x\r\n" + + "\r\n", + }, +} + +func TestRequestWrite(t *testing.T) { + for i := range reqWriteTests { + tt := &reqWriteTests[i] + + setBody := func() { + if tt.Body == nil { + return + } + switch b := tt.Body.(type) { + case []byte: + tt.Req.Body = ioutil.NopCloser(bytes.NewReader(b)) + case func() io.ReadCloser: + tt.Req.Body = b() + } + } + setBody() + if tt.Req.Header == nil { + tt.Req.Header = make(Header) + } + + var braw bytes.Buffer + err := tt.Req.Write(&braw) + if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.WantError); g != e { + t.Errorf("writing #%d, err = %q, want %q", i, g, e) + continue + } + if err != nil { + continue + } + + if tt.WantWrite != "" { + sraw := braw.String() + if sraw != tt.WantWrite { + t.Errorf("Test %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantWrite, sraw) + continue + } + } + + if tt.WantProxy != "" { + setBody() + var praw bytes.Buffer + err = tt.Req.WriteProxy(&praw) + if err != nil { + t.Errorf("WriteProxy #%d: %s", i, err) + continue + } + sraw := praw.String() + if sraw != tt.WantProxy { + t.Errorf("Test Proxy %d, expecting:\n%s\nGot:\n%s\n", i, tt.WantProxy, sraw) + continue + } + } + } +} + +type closeChecker struct { + io.Reader + closed bool +} + +func (rc *closeChecker) Close() error { + rc.closed = true + return nil +} + +// TestRequestWriteClosesBody tests that Request.Write does close its request.Body. +// It also indirectly tests NewRequest and that it doesn't wrap an existing Closer +// inside a NopCloser, and that it serializes it correctly. +func TestRequestWriteClosesBody(t *testing.T) { + rc := &closeChecker{Reader: strings.NewReader("my body")} + req, _ := NewRequest("POST", "http://foo.com/", rc) + if req.ContentLength != 0 { + t.Errorf("got req.ContentLength %d, want 0", req.ContentLength) + } + buf := new(bytes.Buffer) + req.Write(buf) + if !rc.closed { + t.Error("body not closed after write") + } + expected := "POST / HTTP/1.1\r\n" + + "Host: foo.com\r\n" + + "User-Agent: Go 1.1 package http\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + // TODO: currently we don't buffer before chunking, so we get a + // single "m" chunk before the other chunks, as this was the 1-byte + // read from our MultiReader where we stiched the Body back together + // after sniffing whether the Body was 0 bytes or not. + chunk("m") + + chunk("y body") + + chunk("") + if buf.String() != expected { + t.Errorf("write:\n got: %s\nwant: %s", buf.String(), expected) + } +} + +func chunk(s string) string { + return fmt.Sprintf("%x\r\n%s\r\n", len(s), s) +} + +func mustParseURL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(fmt.Sprintf("Error parsing URL %q: %v", s, err)) + } + return u +} + +type writerFunc func([]byte) (int, error) + +func (f writerFunc) Write(p []byte) (int, error) { return f(p) } + +// TestRequestWriteError tests the Write err != nil checks in (*Request).write. +func TestRequestWriteError(t *testing.T) { + failAfter, writeCount := 0, 0 + errFail := errors.New("fake write failure") + + // w is the buffered io.Writer to write the request to. It + // fails exactly once on its Nth Write call, as controlled by + // failAfter. It also tracks the number of calls in + // writeCount. + w := struct { + io.ByteWriter // to avoid being wrapped by a bufio.Writer + io.Writer + }{ + nil, + writerFunc(func(p []byte) (n int, err error) { + writeCount++ + if failAfter == 0 { + err = errFail + } + failAfter-- + return len(p), err + }), + } + + req, _ := NewRequest("GET", "http://example.com/", nil) + const writeCalls = 4 // number of Write calls in current implementation + sawGood := false + for n := 0; n <= writeCalls+2; n++ { + failAfter = n + writeCount = 0 + err := req.Write(w) + var wantErr error + if n < writeCalls { + wantErr = errFail + } + if err != wantErr { + t.Errorf("for fail-after %d Writes, err = %v; want %v", n, err, wantErr) + continue + } + if err == nil { + sawGood = true + if writeCount != writeCalls { + t.Fatalf("writeCalls constant is outdated in test") + } + } + if writeCount > writeCalls || writeCount > n+1 { + t.Errorf("for fail-after %d, saw unexpectedly high (%d) write calls", n, writeCount) + } + } + if !sawGood { + t.Fatalf("writeCalls constant is outdated in test") + } +} diff --git a/src/net/http/response.go b/src/net/http/response.go new file mode 100644 index 000000000..5d2c39080 --- /dev/null +++ b/src/net/http/response.go @@ -0,0 +1,291 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP Response reading and parsing. + +package http + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "io" + "net/textproto" + "net/url" + "strconv" + "strings" +) + +var respExcludeHeader = map[string]bool{ + "Content-Length": true, + "Transfer-Encoding": true, + "Trailer": true, +} + +// Response represents the response from an HTTP request. +// +type Response struct { + Status string // e.g. "200 OK" + StatusCode int // e.g. 200 + Proto string // e.g. "HTTP/1.0" + ProtoMajor int // e.g. 1 + ProtoMinor int // e.g. 0 + + // Header maps header keys to values. If the response had multiple + // headers with the same key, they may be concatenated, with comma + // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers + // be semantically equivalent to a comma-delimited sequence.) Values + // duplicated by other fields in this struct (e.g., ContentLength) are + // omitted from Header. + // + // Keys in the map are canonicalized (see CanonicalHeaderKey). + Header Header + + // Body represents the response body. + // + // The http Client and Transport guarantee that Body is always + // non-nil, even on responses without a body or responses with + // a zero-length body. It is the caller's responsibility to + // close Body. + // + // The Body is automatically dechunked if the server replied + // with a "chunked" Transfer-Encoding. + Body io.ReadCloser + + // ContentLength records the length of the associated content. The + // value -1 indicates that the length is unknown. Unless Request.Method + // is "HEAD", values >= 0 indicate that the given number of bytes may + // be read from Body. + ContentLength int64 + + // Contains transfer encodings from outer-most to inner-most. Value is + // nil, means that "identity" encoding is used. + TransferEncoding []string + + // Close records whether the header directed that the connection be + // closed after reading Body. The value is advice for clients: neither + // ReadResponse nor Response.Write ever closes a connection. + Close bool + + // Trailer maps trailer keys to values, in the same + // format as the header. + Trailer Header + + // The Request that was sent to obtain this Response. + // Request's Body is nil (having already been consumed). + // This is only populated for Client requests. + Request *Request + + // TLS contains information about the TLS connection on which the + // response was received. It is nil for unencrypted responses. + // The pointer is shared between responses and should not be + // modified. + TLS *tls.ConnectionState +} + +// Cookies parses and returns the cookies set in the Set-Cookie headers. +func (r *Response) Cookies() []*Cookie { + return readSetCookies(r.Header) +} + +var ErrNoLocation = errors.New("http: no Location header in response") + +// Location returns the URL of the response's "Location" header, +// if present. Relative redirects are resolved relative to +// the Response's Request. ErrNoLocation is returned if no +// Location header is present. +func (r *Response) Location() (*url.URL, error) { + lv := r.Header.Get("Location") + if lv == "" { + return nil, ErrNoLocation + } + if r.Request != nil && r.Request.URL != nil { + return r.Request.URL.Parse(lv) + } + return url.Parse(lv) +} + +// ReadResponse reads and returns an HTTP response from r. +// The req parameter optionally specifies the Request that corresponds +// to this Response. If nil, a GET request is assumed. +// Clients must call resp.Body.Close when finished reading resp.Body. +// After that call, clients can inspect resp.Trailer to find key/value +// pairs included in the response trailer. +func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) { + tp := textproto.NewReader(r) + resp := &Response{ + Request: req, + } + + // Parse the first line of the response. + line, err := tp.ReadLine() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + f := strings.SplitN(line, " ", 3) + if len(f) < 2 { + return nil, &badStringError{"malformed HTTP response", line} + } + reasonPhrase := "" + if len(f) > 2 { + reasonPhrase = f[2] + } + resp.Status = f[1] + " " + reasonPhrase + resp.StatusCode, err = strconv.Atoi(f[1]) + if err != nil { + return nil, &badStringError{"malformed HTTP status code", f[1]} + } + + resp.Proto = f[0] + var ok bool + if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok { + return nil, &badStringError{"malformed HTTP version", resp.Proto} + } + + // Parse the response headers. + mimeHeader, err := tp.ReadMIMEHeader() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return nil, err + } + resp.Header = Header(mimeHeader) + + fixPragmaCacheControl(resp.Header) + + err = readTransfer(resp, r) + if err != nil { + return nil, err + } + + return resp, nil +} + +// RFC2616: Should treat +// Pragma: no-cache +// like +// Cache-Control: no-cache +func fixPragmaCacheControl(header Header) { + if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" { + if _, presentcc := header["Cache-Control"]; !presentcc { + header["Cache-Control"] = []string{"no-cache"} + } + } +} + +// ProtoAtLeast reports whether the HTTP protocol used +// in the response is at least major.minor. +func (r *Response) ProtoAtLeast(major, minor int) bool { + return r.ProtoMajor > major || + r.ProtoMajor == major && r.ProtoMinor >= minor +} + +// Writes the response (header, body and trailer) in wire format. This method +// consults the following fields of the response: +// +// StatusCode +// ProtoMajor +// ProtoMinor +// Request.Method +// TransferEncoding +// Trailer +// Body +// ContentLength +// Header, values for non-canonical keys will have unpredictable behavior +// +// Body is closed after it is sent. +func (r *Response) Write(w io.Writer) error { + // Status line + text := r.Status + if text == "" { + var ok bool + text, ok = statusText[r.StatusCode] + if !ok { + text = "status code " + strconv.Itoa(r.StatusCode) + } + } + protoMajor, protoMinor := strconv.Itoa(r.ProtoMajor), strconv.Itoa(r.ProtoMinor) + statusCode := strconv.Itoa(r.StatusCode) + " " + text = strings.TrimPrefix(text, statusCode) + if _, err := io.WriteString(w, "HTTP/"+protoMajor+"."+protoMinor+" "+statusCode+text+"\r\n"); err != nil { + return err + } + + // Clone it, so we can modify r1 as needed. + r1 := new(Response) + *r1 = *r + if r1.ContentLength == 0 && r1.Body != nil { + // Is it actually 0 length? Or just unknown? + var buf [1]byte + n, err := r1.Body.Read(buf[:]) + if err != nil && err != io.EOF { + return err + } + if n == 0 { + // Reset it to a known zero reader, in case underlying one + // is unhappy being read repeatedly. + r1.Body = eofReader + } else { + r1.ContentLength = -1 + r1.Body = struct { + io.Reader + io.Closer + }{ + io.MultiReader(bytes.NewReader(buf[:1]), r.Body), + r.Body, + } + } + } + // If we're sending a non-chunked HTTP/1.1 response without a + // content-length, the only way to do that is the old HTTP/1.0 + // way, by noting the EOF with a connection close, so we need + // to set Close. + if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) { + r1.Close = true + } + + // Process Body,ContentLength,Close,Trailer + tw, err := newTransferWriter(r1) + if err != nil { + return err + } + err = tw.WriteHeader(w) + if err != nil { + return err + } + + // Rest of header + err = r.Header.WriteSubset(w, respExcludeHeader) + if err != nil { + return err + } + + // contentLengthAlreadySent may have been already sent for + // POST/PUT requests, even if zero length. See Issue 8180. + contentLengthAlreadySent := tw.shouldSendContentLength() + if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent { + if _, err := io.WriteString(w, "Content-Length: 0\r\n"); err != nil { + return err + } + } + + // End-of-header + if _, err := io.WriteString(w, "\r\n"); err != nil { + return err + } + + // Write body and trailer + err = tw.WriteBody(w) + if err != nil { + return err + } + + // Success + return nil +} diff --git a/src/net/http/response_test.go b/src/net/http/response_test.go new file mode 100644 index 000000000..2dd0fad11 --- /dev/null +++ b/src/net/http/response_test.go @@ -0,0 +1,646 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "bytes" + "compress/gzip" + "crypto/rand" + "fmt" + "io" + "io/ioutil" + "net/http/internal" + "net/url" + "reflect" + "regexp" + "strings" + "testing" +) + +type respTest struct { + Raw string + Resp Response + Body string +} + +func dummyReq(method string) *Request { + return &Request{Method: method} +} + +func dummyReq11(method string) *Request { + return &Request{Method: method, Proto: "HTTP/1.1", ProtoMajor: 1, ProtoMinor: 1} +} + +var respTests = []respTest{ + // Unchunked response without Content-Length. + { + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{ + "Connection": {"close"}, // TODO(rsc): Delete? + }, + Close: true, + ContentLength: -1, + }, + + "Body here\n", + }, + + // Unchunked HTTP/1.1 response without Content-Length or + // Connection headers. + { + "HTTP/1.1 200 OK\r\n" + + "\r\n" + + "Body here\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Request: dummyReq("GET"), + Close: true, + ContentLength: -1, + }, + + "Body here\n", + }, + + // Unchunked HTTP/1.1 204 response without Content-Length. + { + "HTTP/1.1 204 No Content\r\n" + + "\r\n" + + "Body should not be read!\n", + + Response{ + Status: "204 No Content", + StatusCode: 204, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: Header{}, + Request: dummyReq("GET"), + Close: false, + ContentLength: 0, + }, + + "", + }, + + // Unchunked response with Content-Length. + { + "HTTP/1.0 200 OK\r\n" + + "Content-Length: 10\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{ + "Connection": {"close"}, + "Content-Length": {"10"}, + }, + Close: true, + ContentLength: 10, + }, + + "Body here\n", + }, + + // Chunked response without Content-Length. + { + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n" + + "0a\r\n" + + "Body here\n\r\n" + + "09\r\n" + + "continued\r\n" + + "0\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{}, + Close: false, + ContentLength: -1, + TransferEncoding: []string{"chunked"}, + }, + + "Body here\ncontinued", + }, + + // Chunked response with Content-Length. + { + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "Content-Length: 10\r\n" + + "\r\n" + + "0a\r\n" + + "Body here\n\r\n" + + "0\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{}, + Close: false, + ContentLength: -1, + TransferEncoding: []string{"chunked"}, + }, + + "Body here\n", + }, + + // Chunked response in response to a HEAD request + { + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("HEAD"), + Header: Header{}, + TransferEncoding: []string{"chunked"}, + Close: false, + ContentLength: -1, + }, + + "", + }, + + // Content-Length in response to a HEAD request + { + "HTTP/1.0 200 OK\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("HEAD"), + Header: Header{"Content-Length": {"256"}}, + TransferEncoding: nil, + Close: true, + ContentLength: 256, + }, + + "", + }, + + // Content-Length in response to a HEAD request with HTTP/1.1 + { + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 256\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("HEAD"), + Header: Header{"Content-Length": {"256"}}, + TransferEncoding: nil, + Close: false, + ContentLength: 256, + }, + + "", + }, + + // No Content-Length or Chunked in response to a HEAD request + { + "HTTP/1.0 200 OK\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("HEAD"), + Header: Header{}, + TransferEncoding: nil, + Close: true, + ContentLength: -1, + }, + + "", + }, + + // explicit Content-Length of 0. + { + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{ + "Content-Length": {"0"}, + }, + Close: false, + ContentLength: 0, + }, + + "", + }, + + // Status line without a Reason-Phrase, but trailing space. + // (permitted by RFC 2616) + { + "HTTP/1.0 303 \r\n\r\n", + Response{ + Status: "303 ", + StatusCode: 303, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Close: true, + ContentLength: -1, + }, + + "", + }, + + // Status line without a Reason-Phrase, and no trailing space. + // (not permitted by RFC 2616, but we'll accept it anyway) + { + "HTTP/1.0 303\r\n\r\n", + Response{ + Status: "303 ", + StatusCode: 303, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Close: true, + ContentLength: -1, + }, + + "", + }, + + // golang.org/issue/4767: don't special-case multipart/byteranges responses + { + `HTTP/1.1 206 Partial Content +Connection: close +Content-Type: multipart/byteranges; boundary=18a75608c8f47cef + +some body`, + Response{ + Status: "206 Partial Content", + StatusCode: 206, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{ + "Content-Type": []string{"multipart/byteranges; boundary=18a75608c8f47cef"}, + }, + Close: true, + ContentLength: -1, + }, + + "some body", + }, + + // Unchunked response without Content-Length, Request is nil + { + "HTTP/1.0 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "Body here\n", + + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.0", + ProtoMajor: 1, + ProtoMinor: 0, + Header: Header{ + "Connection": {"close"}, // TODO(rsc): Delete? + }, + Close: true, + ContentLength: -1, + }, + + "Body here\n", + }, +} + +func TestReadResponse(t *testing.T) { + for i, tt := range respTests { + resp, err := ReadResponse(bufio.NewReader(strings.NewReader(tt.Raw)), tt.Resp.Request) + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + rbody := resp.Body + resp.Body = nil + diff(t, fmt.Sprintf("#%d Response", i), resp, &tt.Resp) + var bout bytes.Buffer + if rbody != nil { + _, err = io.Copy(&bout, rbody) + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + rbody.Close() + } + body := bout.String() + if body != tt.Body { + t.Errorf("#%d: Body = %q want %q", i, body, tt.Body) + } + } +} + +func TestWriteResponse(t *testing.T) { + for i, tt := range respTests { + resp, err := ReadResponse(bufio.NewReader(strings.NewReader(tt.Raw)), tt.Resp.Request) + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + err = resp.Write(ioutil.Discard) + if err != nil { + t.Errorf("#%d: %v", i, err) + continue + } + } +} + +var readResponseCloseInMiddleTests = []struct { + chunked, compressed bool +}{ + {false, false}, + {true, false}, + {true, true}, +} + +// TestReadResponseCloseInMiddle tests that closing a body after +// reading only part of its contents advances the read to the end of +// the request, right up until the next request. +func TestReadResponseCloseInMiddle(t *testing.T) { + for _, test := range readResponseCloseInMiddleTests { + fatalf := func(format string, args ...interface{}) { + args = append([]interface{}{test.chunked, test.compressed}, args...) + t.Fatalf("on test chunked=%v, compressed=%v: "+format, args...) + } + checkErr := func(err error, msg string) { + if err == nil { + return + } + fatalf(msg+": %v", err) + } + var buf bytes.Buffer + buf.WriteString("HTTP/1.1 200 OK\r\n") + if test.chunked { + buf.WriteString("Transfer-Encoding: chunked\r\n") + } else { + buf.WriteString("Content-Length: 1000000\r\n") + } + var wr io.Writer = &buf + if test.chunked { + wr = internal.NewChunkedWriter(wr) + } + if test.compressed { + buf.WriteString("Content-Encoding: gzip\r\n") + wr = gzip.NewWriter(wr) + } + buf.WriteString("\r\n") + + chunk := bytes.Repeat([]byte{'x'}, 1000) + for i := 0; i < 1000; i++ { + if test.compressed { + // Otherwise this compresses too well. + _, err := io.ReadFull(rand.Reader, chunk) + checkErr(err, "rand.Reader ReadFull") + } + wr.Write(chunk) + } + if test.compressed { + err := wr.(*gzip.Writer).Close() + checkErr(err, "compressor close") + } + if test.chunked { + buf.WriteString("0\r\n\r\n") + } + buf.WriteString("Next Request Here") + + bufr := bufio.NewReader(&buf) + resp, err := ReadResponse(bufr, dummyReq("GET")) + checkErr(err, "ReadResponse") + expectedLength := int64(-1) + if !test.chunked { + expectedLength = 1000000 + } + if resp.ContentLength != expectedLength { + fatalf("expected response length %d, got %d", expectedLength, resp.ContentLength) + } + if resp.Body == nil { + fatalf("nil body") + } + if test.compressed { + gzReader, err := gzip.NewReader(resp.Body) + checkErr(err, "gzip.NewReader") + resp.Body = &readerAndCloser{gzReader, resp.Body} + } + + rbuf := make([]byte, 2500) + n, err := io.ReadFull(resp.Body, rbuf) + checkErr(err, "2500 byte ReadFull") + if n != 2500 { + fatalf("ReadFull only read %d bytes", n) + } + if test.compressed == false && !bytes.Equal(bytes.Repeat([]byte{'x'}, 2500), rbuf) { + fatalf("ReadFull didn't read 2500 'x'; got %q", string(rbuf)) + } + resp.Body.Close() + + rest, err := ioutil.ReadAll(bufr) + checkErr(err, "ReadAll on remainder") + if e, g := "Next Request Here", string(rest); e != g { + g = regexp.MustCompile(`(xx+)`).ReplaceAllStringFunc(g, func(match string) string { + return fmt.Sprintf("x(repeated x%d)", len(match)) + }) + fatalf("remainder = %q, expected %q", g, e) + } + } +} + +func diff(t *testing.T, prefix string, have, want interface{}) { + hv := reflect.ValueOf(have).Elem() + wv := reflect.ValueOf(want).Elem() + if hv.Type() != wv.Type() { + t.Errorf("%s: type mismatch %v want %v", prefix, hv.Type(), wv.Type()) + } + for i := 0; i < hv.NumField(); i++ { + hf := hv.Field(i).Interface() + wf := wv.Field(i).Interface() + if !reflect.DeepEqual(hf, wf) { + t.Errorf("%s: %s = %v want %v", prefix, hv.Type().Field(i).Name, hf, wf) + } + } +} + +type responseLocationTest struct { + location string // Response's Location header or "" + requrl string // Response.Request.URL or "" + want string + wantErr error +} + +var responseLocationTests = []responseLocationTest{ + {"/foo", "http://bar.com/baz", "http://bar.com/foo", nil}, + {"http://foo.com/", "http://bar.com/baz", "http://foo.com/", nil}, + {"", "http://bar.com/baz", "", ErrNoLocation}, +} + +func TestLocationResponse(t *testing.T) { + for i, tt := range responseLocationTests { + res := new(Response) + res.Header = make(Header) + res.Header.Set("Location", tt.location) + if tt.requrl != "" { + res.Request = &Request{} + var err error + res.Request.URL, err = url.Parse(tt.requrl) + if err != nil { + t.Fatalf("bad test URL %q: %v", tt.requrl, err) + } + } + + got, err := res.Location() + if tt.wantErr != nil { + if err == nil { + t.Errorf("%d. err=nil; want %q", i, tt.wantErr) + continue + } + if g, e := err.Error(), tt.wantErr.Error(); g != e { + t.Errorf("%d. err=%q; want %q", i, g, e) + continue + } + continue + } + if err != nil { + t.Errorf("%d. err=%q", i, err) + continue + } + if g, e := got.String(), tt.want; g != e { + t.Errorf("%d. Location=%q; want %q", i, g, e) + } + } +} + +func TestResponseStatusStutter(t *testing.T) { + r := &Response{ + Status: "123 some status", + StatusCode: 123, + ProtoMajor: 1, + ProtoMinor: 3, + } + var buf bytes.Buffer + r.Write(&buf) + if strings.Contains(buf.String(), "123 123") { + t.Errorf("stutter in status: %s", buf.String()) + } +} + +func TestResponseContentLengthShortBody(t *testing.T) { + const shortBody = "Short body, not 123 bytes." + br := bufio.NewReader(strings.NewReader("HTTP/1.1 200 OK\r\n" + + "Content-Length: 123\r\n" + + "\r\n" + + shortBody)) + res, err := ReadResponse(br, &Request{Method: "GET"}) + if err != nil { + t.Fatal(err) + } + if res.ContentLength != 123 { + t.Fatalf("Content-Length = %d; want 123", res.ContentLength) + } + var buf bytes.Buffer + n, err := io.Copy(&buf, res.Body) + if n != int64(len(shortBody)) { + t.Errorf("Copied %d bytes; want %d, len(%q)", n, len(shortBody), shortBody) + } + if buf.String() != shortBody { + t.Errorf("Read body %q; want %q", buf.String(), shortBody) + } + if err != io.ErrUnexpectedEOF { + t.Errorf("io.Copy error = %#v; want io.ErrUnexpectedEOF", err) + } +} + +func TestReadResponseUnexpectedEOF(t *testing.T) { + br := bufio.NewReader(strings.NewReader("HTTP/1.1 301 Moved Permanently\r\n" + + "Location: http://example.com")) + _, err := ReadResponse(br, nil) + if err != io.ErrUnexpectedEOF { + t.Errorf("ReadResponse = %v; want io.ErrUnexpectedEOF", err) + } +} + +func TestNeedsSniff(t *testing.T) { + // needsSniff returns true with an empty response. + r := &response{} + if got, want := r.needsSniff(), true; got != want { + t.Errorf("needsSniff = %t; want %t", got, want) + } + // needsSniff returns false when Content-Type = nil. + r.handlerHeader = Header{"Content-Type": nil} + if got, want := r.needsSniff(), false; got != want { + t.Errorf("needsSniff empty Content-Type = %t; want %t", got, want) + } +} diff --git a/src/net/http/responsewrite_test.go b/src/net/http/responsewrite_test.go new file mode 100644 index 000000000..585b13b85 --- /dev/null +++ b/src/net/http/responsewrite_test.go @@ -0,0 +1,226 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "io/ioutil" + "strings" + "testing" +) + +type respWriteTest struct { + Resp Response + Raw string +} + +func TestResponseWrite(t *testing.T) { + respWriteTests := []respWriteTest{ + // HTTP/1.0, identity coding; no trailer + { + Response{ + StatusCode: 503, + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("abcdef")), + ContentLength: 6, + }, + + "HTTP/1.0 503 Service Unavailable\r\n" + + "Content-Length: 6\r\n\r\n" + + "abcdef", + }, + // Unchunked response without Content-Length. + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("abcdef")), + ContentLength: -1, + }, + "HTTP/1.0 200 OK\r\n" + + "\r\n" + + "abcdef", + }, + // HTTP/1.1 response with unknown length and Connection: close + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("abcdef")), + ContentLength: -1, + Close: true, + }, + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "abcdef", + }, + // HTTP/1.1 response with unknown length and not setting connection: close + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq11("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("abcdef")), + ContentLength: -1, + Close: false, + }, + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\n" + + "abcdef", + }, + // HTTP/1.1 response with unknown length and not setting connection: close, but + // setting chunked. + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq11("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("abcdef")), + ContentLength: -1, + TransferEncoding: []string{"chunked"}, + Close: false, + }, + "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + "6\r\nabcdef\r\n0\r\n\r\n", + }, + // HTTP/1.1 response 0 content-length, and nil body + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq11("GET"), + Header: Header{}, + Body: nil, + ContentLength: 0, + Close: false, + }, + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + }, + // HTTP/1.1 response 0 content-length, and non-nil empty body + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq11("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("")), + ContentLength: 0, + Close: false, + }, + "HTTP/1.1 200 OK\r\n" + + "Content-Length: 0\r\n" + + "\r\n", + }, + // HTTP/1.1 response 0 content-length, and non-nil non-empty body + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq11("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("foo")), + ContentLength: 0, + Close: false, + }, + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "\r\nfoo", + }, + // HTTP/1.1, chunked coding; empty trailer; close + { + Response{ + StatusCode: 200, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{}, + Body: ioutil.NopCloser(strings.NewReader("abcdef")), + ContentLength: 6, + TransferEncoding: []string{"chunked"}, + Close: true, + }, + + "HTTP/1.1 200 OK\r\n" + + "Connection: close\r\n" + + "Transfer-Encoding: chunked\r\n\r\n" + + "6\r\nabcdef\r\n0\r\n\r\n", + }, + + // Header value with a newline character (Issue 914). + // Also tests removal of leading and trailing whitespace. + { + Response{ + StatusCode: 204, + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{ + "Foo": []string{" Bar\nBaz "}, + }, + Body: nil, + ContentLength: 0, + TransferEncoding: []string{"chunked"}, + Close: true, + }, + + "HTTP/1.1 204 No Content\r\n" + + "Connection: close\r\n" + + "Foo: Bar Baz\r\n" + + "\r\n", + }, + + // Want a single Content-Length header. Fixing issue 8180 where + // there were two. + { + Response{ + StatusCode: StatusOK, + ProtoMajor: 1, + ProtoMinor: 1, + Request: &Request{Method: "POST"}, + Header: Header{}, + ContentLength: 0, + TransferEncoding: nil, + Body: nil, + }, + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", + }, + } + + for i := range respWriteTests { + tt := &respWriteTests[i] + var braw bytes.Buffer + err := tt.Resp.Write(&braw) + if err != nil { + t.Errorf("error writing #%d: %s", i, err) + continue + } + sraw := braw.String() + if sraw != tt.Raw { + t.Errorf("Test %d, expecting:\n%q\nGot:\n%q\n", i, tt.Raw, sraw) + continue + } + } +} diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go new file mode 100644 index 000000000..ee4f20499 --- /dev/null +++ b/src/net/http/serve_test.go @@ -0,0 +1,2944 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// End-to-end serving tests + +package http_test + +import ( + "bufio" + "bytes" + "crypto/tls" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "math/rand" + "net" + . "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "reflect" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" +) + +type dummyAddr string +type oneConnListener struct { + conn net.Conn +} + +func (l *oneConnListener) Accept() (c net.Conn, err error) { + c = l.conn + if c == nil { + err = io.EOF + return + } + err = nil + l.conn = nil + return +} + +func (l *oneConnListener) Close() error { + return nil +} + +func (l *oneConnListener) Addr() net.Addr { + return dummyAddr("test-address") +} + +func (a dummyAddr) Network() string { + return string(a) +} + +func (a dummyAddr) String() string { + return string(a) +} + +type noopConn struct{} + +func (noopConn) LocalAddr() net.Addr { return dummyAddr("local-addr") } +func (noopConn) RemoteAddr() net.Addr { return dummyAddr("remote-addr") } +func (noopConn) SetDeadline(t time.Time) error { return nil } +func (noopConn) SetReadDeadline(t time.Time) error { return nil } +func (noopConn) SetWriteDeadline(t time.Time) error { return nil } + +type rwTestConn struct { + io.Reader + io.Writer + noopConn + + closeFunc func() error // called if non-nil + closec chan bool // else, if non-nil, send value to it on close +} + +func (c *rwTestConn) Close() error { + if c.closeFunc != nil { + return c.closeFunc() + } + select { + case c.closec <- true: + default: + } + return nil +} + +type testConn struct { + readBuf bytes.Buffer + writeBuf bytes.Buffer + closec chan bool // if non-nil, send value to it on close + noopConn +} + +func (c *testConn) Read(b []byte) (int, error) { + return c.readBuf.Read(b) +} + +func (c *testConn) Write(b []byte) (int, error) { + return c.writeBuf.Write(b) +} + +func (c *testConn) Close() error { + select { + case c.closec <- true: + default: + } + return nil +} + +// reqBytes treats req as a request (with \n delimiters) and returns it with \r\n delimiters, +// ending in \r\n\r\n +func reqBytes(req string) []byte { + return []byte(strings.Replace(strings.TrimSpace(req), "\n", "\r\n", -1) + "\r\n\r\n") +} + +type handlerTest struct { + handler Handler +} + +func newHandlerTest(h Handler) handlerTest { + return handlerTest{h} +} + +func (ht handlerTest) rawResponse(req string) string { + reqb := reqBytes(req) + var output bytes.Buffer + conn := &rwTestConn{ + Reader: bytes.NewReader(reqb), + Writer: &output, + closec: make(chan bool, 1), + } + ln := &oneConnListener{conn: conn} + go Serve(ln, ht.handler) + <-conn.closec + return output.String() +} + +func TestConsumingBodyOnNextConn(t *testing.T) { + conn := new(testConn) + for i := 0; i < 2; i++ { + conn.readBuf.Write([]byte( + "POST / HTTP/1.1\r\n" + + "Host: test\r\n" + + "Content-Length: 11\r\n" + + "\r\n" + + "foo=1&bar=1")) + } + + reqNum := 0 + ch := make(chan *Request) + servech := make(chan error) + listener := &oneConnListener{conn} + handler := func(res ResponseWriter, req *Request) { + reqNum++ + ch <- req + } + + go func() { + servech <- Serve(listener, HandlerFunc(handler)) + }() + + var req *Request + req = <-ch + if req == nil { + t.Fatal("Got nil first request.") + } + if req.Method != "POST" { + t.Errorf("For request #1's method, got %q; expected %q", + req.Method, "POST") + } + + req = <-ch + if req == nil { + t.Fatal("Got nil first request.") + } + if req.Method != "POST" { + t.Errorf("For request #2's method, got %q; expected %q", + req.Method, "POST") + } + + if serveerr := <-servech; serveerr != io.EOF { + t.Errorf("Serve returned %q; expected EOF", serveerr) + } +} + +type stringHandler string + +func (s stringHandler) ServeHTTP(w ResponseWriter, r *Request) { + w.Header().Set("Result", string(s)) +} + +var handlers = []struct { + pattern string + msg string +}{ + {"/", "Default"}, + {"/someDir/", "someDir"}, + {"someHost.com/someDir/", "someHost.com/someDir"}, +} + +var vtests = []struct { + url string + expected string +}{ + {"http://localhost/someDir/apage", "someDir"}, + {"http://localhost/otherDir/apage", "Default"}, + {"http://someHost.com/someDir/apage", "someHost.com/someDir"}, + {"http://otherHost.com/someDir/apage", "someDir"}, + {"http://otherHost.com/aDir/apage", "Default"}, + // redirections for trees + {"http://localhost/someDir", "/someDir/"}, + {"http://someHost.com/someDir", "/someDir/"}, +} + +func TestHostHandlers(t *testing.T) { + defer afterTest(t) + mux := NewServeMux() + for _, h := range handlers { + mux.Handle(h.pattern, stringHandler(h.msg)) + } + ts := httptest.NewServer(mux) + defer ts.Close() + + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + cc := httputil.NewClientConn(conn, nil) + for _, vt := range vtests { + var r *Response + var req Request + if req.URL, err = url.Parse(vt.url); err != nil { + t.Errorf("cannot parse url: %v", err) + continue + } + if err := cc.Write(&req); err != nil { + t.Errorf("writing request: %v", err) + continue + } + r, err := cc.Read(&req) + if err != nil { + t.Errorf("reading response: %v", err) + continue + } + switch r.StatusCode { + case StatusOK: + s := r.Header.Get("Result") + if s != vt.expected { + t.Errorf("Get(%q) = %q, want %q", vt.url, s, vt.expected) + } + case StatusMovedPermanently: + s := r.Header.Get("Location") + if s != vt.expected { + t.Errorf("Get(%q) = %q, want %q", vt.url, s, vt.expected) + } + default: + t.Errorf("Get(%q) unhandled status code %d", vt.url, r.StatusCode) + } + } +} + +var serveMuxRegister = []struct { + pattern string + h Handler +}{ + {"/dir/", serve(200)}, + {"/search", serve(201)}, + {"codesearch.google.com/search", serve(202)}, + {"codesearch.google.com/", serve(203)}, + {"example.com/", HandlerFunc(checkQueryStringHandler)}, +} + +// serve returns a handler that sends a response with the given code. +func serve(code int) HandlerFunc { + return func(w ResponseWriter, r *Request) { + w.WriteHeader(code) + } +} + +// checkQueryStringHandler checks if r.URL.RawQuery has the same value +// as the URL excluding the scheme and the query string and sends 200 +// response code if it is, 500 otherwise. +func checkQueryStringHandler(w ResponseWriter, r *Request) { + u := *r.URL + u.Scheme = "http" + u.Host = r.Host + u.RawQuery = "" + if "http://"+r.URL.RawQuery == u.String() { + w.WriteHeader(200) + } else { + w.WriteHeader(500) + } +} + +var serveMuxTests = []struct { + method string + host string + path string + code int + pattern string +}{ + {"GET", "google.com", "/", 404, ""}, + {"GET", "google.com", "/dir", 301, "/dir/"}, + {"GET", "google.com", "/dir/", 200, "/dir/"}, + {"GET", "google.com", "/dir/file", 200, "/dir/"}, + {"GET", "google.com", "/search", 201, "/search"}, + {"GET", "google.com", "/search/", 404, ""}, + {"GET", "google.com", "/search/foo", 404, ""}, + {"GET", "codesearch.google.com", "/search", 202, "codesearch.google.com/search"}, + {"GET", "codesearch.google.com", "/search/", 203, "codesearch.google.com/"}, + {"GET", "codesearch.google.com", "/search/foo", 203, "codesearch.google.com/"}, + {"GET", "codesearch.google.com", "/", 203, "codesearch.google.com/"}, + {"GET", "images.google.com", "/search", 201, "/search"}, + {"GET", "images.google.com", "/search/", 404, ""}, + {"GET", "images.google.com", "/search/foo", 404, ""}, + {"GET", "google.com", "/../search", 301, "/search"}, + {"GET", "google.com", "/dir/..", 301, ""}, + {"GET", "google.com", "/dir/..", 301, ""}, + {"GET", "google.com", "/dir/./file", 301, "/dir/"}, + + // The /foo -> /foo/ redirect applies to CONNECT requests + // but the path canonicalization does not. + {"CONNECT", "google.com", "/dir", 301, "/dir/"}, + {"CONNECT", "google.com", "/../search", 404, ""}, + {"CONNECT", "google.com", "/dir/..", 200, "/dir/"}, + {"CONNECT", "google.com", "/dir/..", 200, "/dir/"}, + {"CONNECT", "google.com", "/dir/./file", 200, "/dir/"}, +} + +func TestServeMuxHandler(t *testing.T) { + mux := NewServeMux() + for _, e := range serveMuxRegister { + mux.Handle(e.pattern, e.h) + } + + for _, tt := range serveMuxTests { + r := &Request{ + Method: tt.method, + Host: tt.host, + URL: &url.URL{ + Path: tt.path, + }, + } + h, pattern := mux.Handler(r) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, r) + if pattern != tt.pattern || rr.Code != tt.code { + t.Errorf("%s %s %s = %d, %q, want %d, %q", tt.method, tt.host, tt.path, rr.Code, pattern, tt.code, tt.pattern) + } + } +} + +var serveMuxTests2 = []struct { + method string + host string + url string + code int + redirOk bool +}{ + {"GET", "google.com", "/", 404, false}, + {"GET", "example.com", "/test/?example.com/test/", 200, false}, + {"GET", "example.com", "test/?example.com/test/", 200, true}, +} + +// TestServeMuxHandlerRedirects tests that automatic redirects generated by +// mux.Handler() shouldn't clear the request's query string. +func TestServeMuxHandlerRedirects(t *testing.T) { + mux := NewServeMux() + for _, e := range serveMuxRegister { + mux.Handle(e.pattern, e.h) + } + + for _, tt := range serveMuxTests2 { + tries := 1 + turl := tt.url + for tries > 0 { + u, e := url.Parse(turl) + if e != nil { + t.Fatal(e) + } + r := &Request{ + Method: tt.method, + Host: tt.host, + URL: u, + } + h, _ := mux.Handler(r) + rr := httptest.NewRecorder() + h.ServeHTTP(rr, r) + if rr.Code != 301 { + if rr.Code != tt.code { + t.Errorf("%s %s %s = %d, want %d", tt.method, tt.host, tt.url, rr.Code, tt.code) + } + break + } + if !tt.redirOk { + t.Errorf("%s %s %s, unexpected redirect", tt.method, tt.host, tt.url) + break + } + turl = rr.HeaderMap.Get("Location") + tries-- + } + if tries < 0 { + t.Errorf("%s %s %s, too many redirects", tt.method, tt.host, tt.url) + } + } +} + +// Tests for http://code.google.com/p/go/issues/detail?id=900 +func TestMuxRedirectLeadingSlashes(t *testing.T) { + paths := []string{"//foo.txt", "///foo.txt", "/../../foo.txt"} + for _, path := range paths { + req, err := ReadRequest(bufio.NewReader(strings.NewReader("GET " + path + " HTTP/1.1\r\nHost: test\r\n\r\n"))) + if err != nil { + t.Errorf("%s", err) + } + mux := NewServeMux() + resp := httptest.NewRecorder() + + mux.ServeHTTP(resp, req) + + if loc, expected := resp.Header().Get("Location"), "/foo.txt"; loc != expected { + t.Errorf("Expected Location header set to %q; got %q", expected, loc) + return + } + + if code, expected := resp.Code, StatusMovedPermanently; code != expected { + t.Errorf("Expected response code of StatusMovedPermanently; got %d", code) + return + } + } +} + +func TestServerTimeouts(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + reqNum := 0 + ts := httptest.NewUnstartedServer(HandlerFunc(func(res ResponseWriter, req *Request) { + reqNum++ + fmt.Fprintf(res, "req=%d", reqNum) + })) + ts.Config.ReadTimeout = 250 * time.Millisecond + ts.Config.WriteTimeout = 250 * time.Millisecond + ts.Start() + defer ts.Close() + + // Hit the HTTP server successfully. + tr := &Transport{DisableKeepAlives: true} // they interfere with this test + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + r, err := c.Get(ts.URL) + if err != nil { + t.Fatalf("http Get #1: %v", err) + } + got, _ := ioutil.ReadAll(r.Body) + expected := "req=1" + if string(got) != expected { + t.Errorf("Unexpected response for request #1; got %q; expected %q", + string(got), expected) + } + + // Slow client that should timeout. + t1 := time.Now() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + buf := make([]byte, 1) + n, err := conn.Read(buf) + latency := time.Since(t1) + if n != 0 || err != io.EOF { + t.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, io.EOF) + } + if latency < 200*time.Millisecond /* fudge from 250 ms above */ { + t.Errorf("got EOF after %s, want >= %s", latency, 200*time.Millisecond) + } + + // Hit the HTTP server successfully again, verifying that the + // previous slow connection didn't run our handler. (that we + // get "req=2", not "req=3") + r, err = Get(ts.URL) + if err != nil { + t.Fatalf("http Get #2: %v", err) + } + got, _ = ioutil.ReadAll(r.Body) + expected = "req=2" + if string(got) != expected { + t.Errorf("Get #2 got %q, want %q", string(got), expected) + } + + if !testing.Short() { + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer conn.Close() + go io.Copy(ioutil.Discard, conn) + for i := 0; i < 5; i++ { + _, err := conn.Write([]byte("GET / HTTP/1.1\r\nHost: foo\r\n\r\n")) + if err != nil { + t.Fatalf("on write %d: %v", i, err) + } + time.Sleep(ts.Config.ReadTimeout / 2) + } + } +} + +// golang.org/issue/4741 -- setting only a write timeout that triggers +// shouldn't cause a handler to block forever on reads (next HTTP +// request) that will never happen. +func TestOnlyWriteTimeout(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + var conn net.Conn + var afterTimeoutErrc = make(chan error, 1) + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, req *Request) { + buf := make([]byte, 512<<10) + _, err := w.Write(buf) + if err != nil { + t.Errorf("handler Write error: %v", err) + return + } + conn.SetWriteDeadline(time.Now().Add(-30 * time.Second)) + _, err = w.Write(buf) + afterTimeoutErrc <- err + })) + ts.Listener = trackLastConnListener{ts.Listener, &conn} + ts.Start() + defer ts.Close() + + tr := &Transport{DisableKeepAlives: false} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + errc := make(chan error) + go func() { + res, err := c.Get(ts.URL) + if err != nil { + errc <- err + return + } + _, err = io.Copy(ioutil.Discard, res.Body) + errc <- err + }() + select { + case err := <-errc: + if err == nil { + t.Errorf("expected an error from Get request") + } + case <-time.After(5 * time.Second): + t.Fatal("timeout waiting for Get error") + } + if err := <-afterTimeoutErrc; err == nil { + t.Error("expected write error after timeout") + } +} + +// trackLastConnListener tracks the last net.Conn that was accepted. +type trackLastConnListener struct { + net.Listener + last *net.Conn // destination +} + +func (l trackLastConnListener) Accept() (c net.Conn, err error) { + c, err = l.Listener.Accept() + *l.last = c + return +} + +// TestIdentityResponse verifies that a handler can unset +func TestIdentityResponse(t *testing.T) { + defer afterTest(t) + handler := HandlerFunc(func(rw ResponseWriter, req *Request) { + rw.Header().Set("Content-Length", "3") + rw.Header().Set("Transfer-Encoding", req.FormValue("te")) + switch { + case req.FormValue("overwrite") == "1": + _, err := rw.Write([]byte("foo TOO LONG")) + if err != ErrContentLength { + t.Errorf("expected ErrContentLength; got %v", err) + } + case req.FormValue("underwrite") == "1": + rw.Header().Set("Content-Length", "500") + rw.Write([]byte("too short")) + default: + rw.Write([]byte("foo")) + } + }) + + ts := httptest.NewServer(handler) + defer ts.Close() + + // Note: this relies on the assumption (which is true) that + // Get sends HTTP/1.1 or greater requests. Otherwise the + // server wouldn't have the choice to send back chunked + // responses. + for _, te := range []string{"", "identity"} { + url := ts.URL + "/?te=" + te + res, err := Get(url) + if err != nil { + t.Fatalf("error with Get of %s: %v", url, err) + } + if cl, expected := res.ContentLength, int64(3); cl != expected { + t.Errorf("for %s expected res.ContentLength of %d; got %d", url, expected, cl) + } + if cl, expected := res.Header.Get("Content-Length"), "3"; cl != expected { + t.Errorf("for %s expected Content-Length header of %q; got %q", url, expected, cl) + } + if tl, expected := len(res.TransferEncoding), 0; tl != expected { + t.Errorf("for %s expected len(res.TransferEncoding) of %d; got %d (%v)", + url, expected, tl, res.TransferEncoding) + } + res.Body.Close() + } + + // Verify that ErrContentLength is returned + url := ts.URL + "/?overwrite=1" + res, err := Get(url) + if err != nil { + t.Fatalf("error with Get of %s: %v", url, err) + } + res.Body.Close() + + // Verify that the connection is closed when the declared Content-Length + // is larger than what the handler wrote. + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + _, err = conn.Write([]byte("GET /?underwrite=1 HTTP/1.1\r\nHost: foo\r\n\r\n")) + if err != nil { + t.Fatalf("error writing: %v", err) + } + + // The ReadAll will hang for a failing test, so use a Timer to + // fail explicitly. + goTimeout(t, 2*time.Second, func() { + got, _ := ioutil.ReadAll(conn) + expectedSuffix := "\r\n\r\ntoo short" + if !strings.HasSuffix(string(got), expectedSuffix) { + t.Errorf("Expected output to end with %q; got response body %q", + expectedSuffix, string(got)) + } + }) +} + +func testTCPConnectionCloses(t *testing.T, req string, h Handler) { + defer afterTest(t) + s := httptest.NewServer(h) + defer s.Close() + + conn, err := net.Dial("tcp", s.Listener.Addr().String()) + if err != nil { + t.Fatal("dial error:", err) + } + defer conn.Close() + + _, err = fmt.Fprint(conn, req) + if err != nil { + t.Fatal("print error:", err) + } + + r := bufio.NewReader(conn) + res, err := ReadResponse(r, &Request{Method: "GET"}) + if err != nil { + t.Fatal("ReadResponse error:", err) + } + + didReadAll := make(chan bool, 1) + go func() { + select { + case <-time.After(5 * time.Second): + t.Error("body not closed after 5s") + return + case <-didReadAll: + } + }() + + _, err = ioutil.ReadAll(r) + if err != nil { + t.Fatal("read error:", err) + } + didReadAll <- true + + if !res.Close { + t.Errorf("Response.Close = false; want true") + } +} + +// TestServeHTTP10Close verifies that HTTP/1.0 requests won't be kept alive. +func TestServeHTTP10Close(t *testing.T) { + testTCPConnectionCloses(t, "GET / HTTP/1.0\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, "testdata/file") + })) +} + +// TestClientCanClose verifies that clients can also force a connection to close. +func TestClientCanClose(t *testing.T) { + testTCPConnectionCloses(t, "GET / HTTP/1.1\r\nConnection: close\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + // Nothing. + })) +} + +// TestHandlersCanSetConnectionClose verifies that handlers can force a connection to close, +// even for HTTP/1.1 requests. +func TestHandlersCanSetConnectionClose11(t *testing.T) { + testTCPConnectionCloses(t, "GET / HTTP/1.1\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + })) +} + +func TestHandlersCanSetConnectionClose10(t *testing.T) { + testTCPConnectionCloses(t, "GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + })) +} + +func TestSetsRemoteAddr(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "%s", r.RemoteAddr) + })) + defer ts.Close() + + res, err := Get(ts.URL) + if err != nil { + t.Fatalf("Get error: %v", err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("ReadAll error: %v", err) + } + ip := string(body) + if !strings.HasPrefix(ip, "127.0.0.1:") && !strings.HasPrefix(ip, "[::1]:") { + t.Fatalf("Expected local addr; got %q", ip) + } +} + +func TestChunkedResponseHeaders(t *testing.T) { + defer afterTest(t) + log.SetOutput(ioutil.Discard) // is noisy otherwise + defer log.SetOutput(os.Stderr) + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Length", "intentional gibberish") // we check that this is deleted + w.(Flusher).Flush() + fmt.Fprintf(w, "I am a chunked response.") + })) + defer ts.Close() + + res, err := Get(ts.URL) + if err != nil { + t.Fatalf("Get error: %v", err) + } + defer res.Body.Close() + if g, e := res.ContentLength, int64(-1); g != e { + t.Errorf("expected ContentLength of %d; got %d", e, g) + } + if g, e := res.TransferEncoding, []string{"chunked"}; !reflect.DeepEqual(g, e) { + t.Errorf("expected TransferEncoding of %v; got %v", e, g) + } + if _, haveCL := res.Header["Content-Length"]; haveCL { + t.Errorf("Unexpected Content-Length") + } +} + +// Test304Responses verifies that 304s don't declare that they're +// chunking in their response headers and aren't allowed to produce +// output. +func Test304Responses(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(StatusNotModified) + _, err := w.Write([]byte("illegal body")) + if err != ErrBodyNotAllowed { + t.Errorf("on Write, expected ErrBodyNotAllowed, got %v", err) + } + })) + defer ts.Close() + res, err := Get(ts.URL) + if err != nil { + t.Error(err) + } + if len(res.TransferEncoding) > 0 { + t.Errorf("expected no TransferEncoding; got %v", res.TransferEncoding) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Error(err) + } + if len(body) > 0 { + t.Errorf("got unexpected body %q", string(body)) + } +} + +// TestHeadResponses verifies that all MIME type sniffing and Content-Length +// counting of GET requests also happens on HEAD requests. +func TestHeadResponses(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + _, err := w.Write([]byte("<html>")) + if err != nil { + t.Errorf("ResponseWriter.Write: %v", err) + } + + // Also exercise the ReaderFrom path + _, err = io.Copy(w, strings.NewReader("789a")) + if err != nil { + t.Errorf("Copy(ResponseWriter, ...): %v", err) + } + })) + defer ts.Close() + res, err := Head(ts.URL) + if err != nil { + t.Error(err) + } + if len(res.TransferEncoding) > 0 { + t.Errorf("expected no TransferEncoding; got %v", res.TransferEncoding) + } + if ct := res.Header.Get("Content-Type"); ct != "text/html; charset=utf-8" { + t.Errorf("Content-Type: %q; want text/html; charset=utf-8", ct) + } + if v := res.ContentLength; v != 10 { + t.Errorf("Content-Length: %d; want 10", v) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Error(err) + } + if len(body) > 0 { + t.Errorf("got unexpected body %q", string(body)) + } +} + +func TestTLSHandshakeTimeout(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + errc := make(chanWriter, 10) // but only expecting 1 + ts.Config.ReadTimeout = 250 * time.Millisecond + ts.Config.ErrorLog = log.New(errc, "", 0) + ts.StartTLS() + defer ts.Close() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer conn.Close() + goTimeout(t, 10*time.Second, func() { + var buf [1]byte + n, err := conn.Read(buf[:]) + if err == nil || n != 0 { + t.Errorf("Read = %d, %v; want an error and no bytes", n, err) + } + }) + select { + case v := <-errc: + if !strings.Contains(v, "timeout") && !strings.Contains(v, "TLS handshake") { + t.Errorf("expected a TLS handshake timeout error; got %q", v) + } + case <-time.After(5 * time.Second): + t.Errorf("timeout waiting for logged error") + } +} + +func TestTLSServer(t *testing.T) { + defer afterTest(t) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.TLS != nil { + w.Header().Set("X-TLS-Set", "true") + if r.TLS.HandshakeComplete { + w.Header().Set("X-TLS-HandshakeComplete", "true") + } + } + })) + ts.Config.ErrorLog = log.New(ioutil.Discard, "", 0) + defer ts.Close() + + // Connect an idle TCP connection to this server before we run + // our real tests. This idle connection used to block forever + // in the TLS handshake, preventing future connections from + // being accepted. It may prevent future accidental blocking + // in newConn. + idleConn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer idleConn.Close() + goTimeout(t, 10*time.Second, func() { + if !strings.HasPrefix(ts.URL, "https://") { + t.Errorf("expected test TLS server to start with https://, got %q", ts.URL) + return + } + noVerifyTransport := &Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + client := &Client{Transport: noVerifyTransport} + res, err := client.Get(ts.URL) + if err != nil { + t.Error(err) + return + } + if res == nil { + t.Errorf("got nil Response") + return + } + defer res.Body.Close() + if res.Header.Get("X-TLS-Set") != "true" { + t.Errorf("expected X-TLS-Set response header") + return + } + if res.Header.Get("X-TLS-HandshakeComplete") != "true" { + t.Errorf("expected X-TLS-HandshakeComplete header") + } + }) +} + +type serverExpectTest struct { + contentLength int // of request body + chunked bool + expectation string // e.g. "100-continue" + readBody bool // whether handler should read the body (if false, sends StatusUnauthorized) + expectedResponse string // expected substring in first line of http response +} + +func expectTest(contentLength int, expectation string, readBody bool, expectedResponse string) serverExpectTest { + return serverExpectTest{ + contentLength: contentLength, + expectation: expectation, + readBody: readBody, + expectedResponse: expectedResponse, + } +} + +var serverExpectTests = []serverExpectTest{ + // Normal 100-continues, case-insensitive. + expectTest(100, "100-continue", true, "100 Continue"), + expectTest(100, "100-cOntInUE", true, "100 Continue"), + + // No 100-continue. + expectTest(100, "", true, "200 OK"), + + // 100-continue but requesting client to deny us, + // so it never reads the body. + expectTest(100, "100-continue", false, "401 Unauthorized"), + // Likewise without 100-continue: + expectTest(100, "", false, "401 Unauthorized"), + + // Non-standard expectations are failures + expectTest(0, "a-pony", false, "417 Expectation Failed"), + + // Expect-100 requested but no body (is apparently okay: Issue 7625) + expectTest(0, "100-continue", true, "200 OK"), + // Expect-100 requested but handler doesn't read the body + expectTest(0, "100-continue", false, "401 Unauthorized"), + // Expect-100 continue with no body, but a chunked body. + { + expectation: "100-continue", + readBody: true, + chunked: true, + expectedResponse: "100 Continue", + }, +} + +// Tests that the server responds to the "Expect" request header +// correctly. +func TestServerExpect(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + // Note using r.FormValue("readbody") because for POST + // requests that would read from r.Body, which we only + // conditionally want to do. + if strings.Contains(r.URL.RawQuery, "readbody=true") { + ioutil.ReadAll(r.Body) + w.Write([]byte("Hi")) + } else { + w.WriteHeader(StatusUnauthorized) + } + })) + defer ts.Close() + + runTest := func(test serverExpectTest) { + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer conn.Close() + + // Only send the body immediately if we're acting like an HTTP client + // that doesn't send 100-continue expectations. + writeBody := test.contentLength != 0 && strings.ToLower(test.expectation) != "100-continue" + + go func() { + contentLen := fmt.Sprintf("Content-Length: %d", test.contentLength) + if test.chunked { + contentLen = "Transfer-Encoding: chunked" + } + _, err := fmt.Fprintf(conn, "POST /?readbody=%v HTTP/1.1\r\n"+ + "Connection: close\r\n"+ + "%s\r\n"+ + "Expect: %s\r\nHost: foo\r\n\r\n", + test.readBody, contentLen, test.expectation) + if err != nil { + t.Errorf("On test %#v, error writing request headers: %v", test, err) + return + } + if writeBody { + var targ io.WriteCloser = struct { + io.Writer + io.Closer + }{ + conn, + ioutil.NopCloser(nil), + } + if test.chunked { + targ = httputil.NewChunkedWriter(conn) + } + body := strings.Repeat("A", test.contentLength) + _, err = fmt.Fprint(targ, body) + if err == nil { + err = targ.Close() + } + if err != nil { + if !test.readBody { + // Server likely already hung up on us. + // See larger comment below. + t.Logf("On test %#v, acceptable error writing request body: %v", test, err) + return + } + t.Errorf("On test %#v, error writing request body: %v", test, err) + } + } + }() + bufr := bufio.NewReader(conn) + line, err := bufr.ReadString('\n') + if err != nil { + if writeBody && !test.readBody { + // This is an acceptable failure due to a possible TCP race: + // We were still writing data and the server hung up on us. A TCP + // implementation may send a RST if our request body data was known + // to be lost, which may trigger our reads to fail. + // See RFC 1122 page 88. + t.Logf("On test %#v, acceptable error from ReadString: %v", test, err) + return + } + t.Fatalf("On test %#v, ReadString: %v", test, err) + } + if !strings.Contains(line, test.expectedResponse) { + t.Errorf("On test %#v, got first line = %q; want %q", test, line, test.expectedResponse) + } + } + + for _, test := range serverExpectTests { + runTest(test) + } +} + +// Under a ~256KB (maxPostHandlerReadBytes) threshold, the server +// should consume client request bodies that a handler didn't read. +func TestServerUnreadRequestBodyLittle(t *testing.T) { + conn := new(testConn) + body := strings.Repeat("x", 100<<10) + conn.readBuf.Write([]byte(fmt.Sprintf( + "POST / HTTP/1.1\r\n"+ + "Host: test\r\n"+ + "Content-Length: %d\r\n"+ + "\r\n", len(body)))) + conn.readBuf.Write([]byte(body)) + + done := make(chan bool) + + ls := &oneConnListener{conn} + go Serve(ls, HandlerFunc(func(rw ResponseWriter, req *Request) { + defer close(done) + if conn.readBuf.Len() < len(body)/2 { + t.Errorf("on request, read buffer length is %d; expected about 100 KB", conn.readBuf.Len()) + } + rw.WriteHeader(200) + rw.(Flusher).Flush() + if g, e := conn.readBuf.Len(), 0; g != e { + t.Errorf("after WriteHeader, read buffer length is %d; want %d", g, e) + } + if c := rw.Header().Get("Connection"); c != "" { + t.Errorf(`Connection header = %q; want ""`, c) + } + })) + <-done +} + +// Over a ~256KB (maxPostHandlerReadBytes) threshold, the server +// should ignore client request bodies that a handler didn't read +// and close the connection. +func TestServerUnreadRequestBodyLarge(t *testing.T) { + conn := new(testConn) + body := strings.Repeat("x", 1<<20) + conn.readBuf.Write([]byte(fmt.Sprintf( + "POST / HTTP/1.1\r\n"+ + "Host: test\r\n"+ + "Content-Length: %d\r\n"+ + "\r\n", len(body)))) + conn.readBuf.Write([]byte(body)) + conn.closec = make(chan bool, 1) + + ls := &oneConnListener{conn} + go Serve(ls, HandlerFunc(func(rw ResponseWriter, req *Request) { + if conn.readBuf.Len() < len(body)/2 { + t.Errorf("on request, read buffer length is %d; expected about 1MB", conn.readBuf.Len()) + } + rw.WriteHeader(200) + rw.(Flusher).Flush() + if conn.readBuf.Len() < len(body)/2 { + t.Errorf("post-WriteHeader, read buffer length is %d; expected about 1MB", conn.readBuf.Len()) + } + })) + <-conn.closec + + if res := conn.writeBuf.String(); !strings.Contains(res, "Connection: close") { + t.Errorf("Expected a Connection: close header; got response: %s", res) + } +} + +func TestTimeoutHandler(t *testing.T) { + defer afterTest(t) + sendHi := make(chan bool, 1) + writeErrors := make(chan error, 1) + sayHi := HandlerFunc(func(w ResponseWriter, r *Request) { + <-sendHi + _, werr := w.Write([]byte("hi")) + writeErrors <- werr + }) + timeout := make(chan time.Time, 1) // write to this to force timeouts + ts := httptest.NewServer(NewTestTimeoutHandler(sayHi, timeout)) + defer ts.Close() + + // Succeed without timing out: + sendHi <- true + res, err := Get(ts.URL) + if err != nil { + t.Error(err) + } + if g, e := res.StatusCode, StatusOK; g != e { + t.Errorf("got res.StatusCode %d; expected %d", g, e) + } + body, _ := ioutil.ReadAll(res.Body) + if g, e := string(body), "hi"; g != e { + t.Errorf("got body %q; expected %q", g, e) + } + if g := <-writeErrors; g != nil { + t.Errorf("got unexpected Write error on first request: %v", g) + } + + // Times out: + timeout <- time.Time{} + res, err = Get(ts.URL) + if err != nil { + t.Error(err) + } + if g, e := res.StatusCode, StatusServiceUnavailable; g != e { + t.Errorf("got res.StatusCode %d; expected %d", g, e) + } + body, _ = ioutil.ReadAll(res.Body) + if !strings.Contains(string(body), "<title>Timeout</title>") { + t.Errorf("expected timeout body; got %q", string(body)) + } + + // Now make the previously-timed out handler speak again, + // which verifies the panic is handled: + sendHi <- true + if g, e := <-writeErrors, ErrHandlerTimeout; g != e { + t.Errorf("expected Write error of %v; got %v", e, g) + } +} + +// See issues 8209 and 8414. +func TestTimeoutHandlerRace(t *testing.T) { + defer afterTest(t) + + delayHi := HandlerFunc(func(w ResponseWriter, r *Request) { + ms, _ := strconv.Atoi(r.URL.Path[1:]) + if ms == 0 { + ms = 1 + } + for i := 0; i < ms; i++ { + w.Write([]byte("hi")) + time.Sleep(time.Millisecond) + } + }) + + ts := httptest.NewServer(TimeoutHandler(delayHi, 20*time.Millisecond, "")) + defer ts.Close() + + var wg sync.WaitGroup + gate := make(chan bool, 10) + n := 50 + if testing.Short() { + n = 10 + gate = make(chan bool, 3) + } + for i := 0; i < n; i++ { + gate <- true + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-gate }() + res, err := Get(fmt.Sprintf("%s/%d", ts.URL, rand.Intn(50))) + if err == nil { + io.Copy(ioutil.Discard, res.Body) + res.Body.Close() + } + }() + } + wg.Wait() +} + +// See issues 8209 and 8414. +func TestTimeoutHandlerRaceHeader(t *testing.T) { + defer afterTest(t) + + delay204 := HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(204) + }) + + ts := httptest.NewServer(TimeoutHandler(delay204, time.Nanosecond, "")) + defer ts.Close() + + var wg sync.WaitGroup + gate := make(chan bool, 50) + n := 500 + if testing.Short() { + n = 10 + } + for i := 0; i < n; i++ { + gate <- true + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-gate }() + res, err := Get(ts.URL) + if err != nil { + t.Error(err) + return + } + defer res.Body.Close() + io.Copy(ioutil.Discard, res.Body) + }() + } + wg.Wait() +} + +// Verifies we don't path.Clean() on the wrong parts in redirects. +func TestRedirectMunging(t *testing.T) { + req, _ := NewRequest("GET", "http://example.com/", nil) + + resp := httptest.NewRecorder() + Redirect(resp, req, "/foo?next=http://bar.com/", 302) + if g, e := resp.Header().Get("Location"), "/foo?next=http://bar.com/"; g != e { + t.Errorf("Location header was %q; want %q", g, e) + } + + resp = httptest.NewRecorder() + Redirect(resp, req, "http://localhost:8080/_ah/login?continue=http://localhost:8080/", 302) + if g, e := resp.Header().Get("Location"), "http://localhost:8080/_ah/login?continue=http://localhost:8080/"; g != e { + t.Errorf("Location header was %q; want %q", g, e) + } +} + +func TestRedirectBadPath(t *testing.T) { + // This used to crash. It's not valid input (bad path), but it + // shouldn't crash. + rr := httptest.NewRecorder() + req := &Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "http", + Path: "not-empty-but-no-leading-slash", // bogus + }, + } + Redirect(rr, req, "", 304) + if rr.Code != 304 { + t.Errorf("Code = %d; want 304", rr.Code) + } +} + +// TestZeroLengthPostAndResponse exercises an optimization done by the Transport: +// when there is no body (either because the method doesn't permit a body, or an +// explicit Content-Length of zero is present), then the transport can re-use the +// connection immediately. But when it re-uses the connection, it typically closes +// the previous request's body, which is not optimal for zero-lengthed bodies, +// as the client would then see http.ErrBodyReadAfterClose and not 0, io.EOF. +func TestZeroLengthPostAndResponse(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + all, err := ioutil.ReadAll(r.Body) + if err != nil { + t.Fatalf("handler ReadAll: %v", err) + } + if len(all) != 0 { + t.Errorf("handler got %d bytes; expected 0", len(all)) + } + rw.Header().Set("Content-Length", "0") + })) + defer ts.Close() + + req, err := NewRequest("POST", ts.URL, strings.NewReader("")) + if err != nil { + t.Fatal(err) + } + req.ContentLength = 0 + + var resp [5]*Response + for i := range resp { + resp[i], err = DefaultClient.Do(req) + if err != nil { + t.Fatalf("client post #%d: %v", i, err) + } + } + + for i := range resp { + all, err := ioutil.ReadAll(resp[i].Body) + if err != nil { + t.Fatalf("req #%d: client ReadAll: %v", i, err) + } + if len(all) != 0 { + t.Errorf("req #%d: client got %d bytes; expected 0", i, len(all)) + } + } +} + +func TestHandlerPanicNil(t *testing.T) { + testHandlerPanic(t, false, nil) +} + +func TestHandlerPanic(t *testing.T) { + testHandlerPanic(t, false, "intentional death for testing") +} + +func TestHandlerPanicWithHijack(t *testing.T) { + testHandlerPanic(t, true, "intentional death for testing") +} + +func testHandlerPanic(t *testing.T, withHijack bool, panicValue interface{}) { + defer afterTest(t) + // Unlike the other tests that set the log output to ioutil.Discard + // to quiet the output, this test uses a pipe. The pipe serves three + // purposes: + // + // 1) The log.Print from the http server (generated by the caught + // panic) will go to the pipe instead of stderr, making the + // output quiet. + // + // 2) We read from the pipe to verify that the handler + // actually caught the panic and logged something. + // + // 3) The blocking Read call prevents this TestHandlerPanic + // function from exiting before the HTTP server handler + // finishes crashing. If this text function exited too + // early (and its defer log.SetOutput(os.Stderr) ran), + // then the crash output could spill into the next test. + pr, pw := io.Pipe() + log.SetOutput(pw) + defer log.SetOutput(os.Stderr) + defer pw.Close() + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if withHijack { + rwc, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Logf("unexpected error: %v", err) + } + defer rwc.Close() + } + panic(panicValue) + })) + defer ts.Close() + + // Do a blocking read on the log output pipe so its logging + // doesn't bleed into the next test. But wait only 5 seconds + // for it. + done := make(chan bool, 1) + go func() { + buf := make([]byte, 4<<10) + _, err := pr.Read(buf) + pr.Close() + if err != nil && err != io.EOF { + t.Error(err) + } + done <- true + }() + + _, err := Get(ts.URL) + if err == nil { + t.Logf("expected an error") + } + + if panicValue == nil { + return + } + + select { + case <-done: + return + case <-time.After(5 * time.Second): + t.Fatal("expected server handler to log an error") + } +} + +func TestNoDate(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header()["Date"] = nil + })) + defer ts.Close() + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + _, present := res.Header["Date"] + if present { + t.Fatalf("Expected no Date header; got %v", res.Header["Date"]) + } +} + +func TestStripPrefix(t *testing.T) { + defer afterTest(t) + h := HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Path", r.URL.Path) + }) + ts := httptest.NewServer(StripPrefix("/foo", h)) + defer ts.Close() + + res, err := Get(ts.URL + "/foo/bar") + if err != nil { + t.Fatal(err) + } + if g, e := res.Header.Get("X-Path"), "/bar"; g != e { + t.Errorf("test 1: got %s, want %s", g, e) + } + res.Body.Close() + + res, err = Get(ts.URL + "/bar") + if err != nil { + t.Fatal(err) + } + if g, e := res.StatusCode, 404; g != e { + t.Errorf("test 2: got status %v, want %v", g, e) + } + res.Body.Close() +} + +func TestRequestLimit(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + t.Fatalf("didn't expect to get request in Handler") + })) + defer ts.Close() + req, _ := NewRequest("GET", ts.URL, nil) + var bytesPerHeader = len("header12345: val12345\r\n") + for i := 0; i < ((DefaultMaxHeaderBytes+4096)/bytesPerHeader)+1; i++ { + req.Header.Set(fmt.Sprintf("header%05d", i), fmt.Sprintf("val%05d", i)) + } + res, err := DefaultClient.Do(req) + if err != nil { + // Some HTTP clients may fail on this undefined behavior (server replying and + // closing the connection while the request is still being written), but + // we do support it (at least currently), so we expect a response below. + t.Fatalf("Do: %v", err) + } + defer res.Body.Close() + if res.StatusCode != 413 { + t.Fatalf("expected 413 response status; got: %d %s", res.StatusCode, res.Status) + } +} + +type neverEnding byte + +func (b neverEnding) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = byte(b) + } + return len(p), nil +} + +type countReader struct { + r io.Reader + n *int64 +} + +func (cr countReader) Read(p []byte) (n int, err error) { + n, err = cr.r.Read(p) + atomic.AddInt64(cr.n, int64(n)) + return +} + +func TestRequestBodyLimit(t *testing.T) { + defer afterTest(t) + const limit = 1 << 20 + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + r.Body = MaxBytesReader(w, r.Body, limit) + n, err := io.Copy(ioutil.Discard, r.Body) + if err == nil { + t.Errorf("expected error from io.Copy") + } + if n != limit { + t.Errorf("io.Copy = %d, want %d", n, limit) + } + })) + defer ts.Close() + + nWritten := new(int64) + req, _ := NewRequest("POST", ts.URL, io.LimitReader(countReader{neverEnding('a'), nWritten}, limit*200)) + + // Send the POST, but don't care it succeeds or not. The + // remote side is going to reply and then close the TCP + // connection, and HTTP doesn't really define if that's + // allowed or not. Some HTTP clients will get the response + // and some (like ours, currently) will complain that the + // request write failed, without reading the response. + // + // But that's okay, since what we're really testing is that + // the remote side hung up on us before we wrote too much. + _, _ = DefaultClient.Do(req) + + if atomic.LoadInt64(nWritten) > limit*100 { + t.Errorf("handler restricted the request body to %d bytes, but client managed to write %d", + limit, nWritten) + } +} + +// TestClientWriteShutdown tests that if the client shuts down the write +// side of their TCP connection, the server doesn't send a 400 Bad Request. +func TestClientWriteShutdown(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + defer ts.Close() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + err = conn.(*net.TCPConn).CloseWrite() + if err != nil { + t.Fatalf("Dial: %v", err) + } + donec := make(chan bool) + go func() { + defer close(donec) + bs, err := ioutil.ReadAll(conn) + if err != nil { + t.Fatalf("ReadAll: %v", err) + } + got := string(bs) + if got != "" { + t.Errorf("read %q from server; want nothing", got) + } + }() + select { + case <-donec: + case <-time.After(10 * time.Second): + t.Fatalf("timeout") + } +} + +// Tests that chunked server responses that write 1 byte at a time are +// buffered before chunk headers are added, not after chunk headers. +func TestServerBufferedChunking(t *testing.T) { + conn := new(testConn) + conn.readBuf.Write([]byte("GET / HTTP/1.1\r\n\r\n")) + conn.closec = make(chan bool, 1) + ls := &oneConnListener{conn} + go Serve(ls, HandlerFunc(func(rw ResponseWriter, req *Request) { + rw.(Flusher).Flush() // force the Header to be sent, in chunking mode, not counting the length + rw.Write([]byte{'x'}) + rw.Write([]byte{'y'}) + rw.Write([]byte{'z'}) + })) + <-conn.closec + if !bytes.HasSuffix(conn.writeBuf.Bytes(), []byte("\r\n\r\n3\r\nxyz\r\n0\r\n\r\n")) { + t.Errorf("response didn't end with a single 3 byte 'xyz' chunk; got:\n%q", + conn.writeBuf.Bytes()) + } +} + +// Tests that the server flushes its response headers out when it's +// ignoring the response body and waits a bit before forcefully +// closing the TCP connection, causing the client to get a RST. +// See http://golang.org/issue/3595 +func TestServerGracefulClose(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + Error(w, "bye", StatusUnauthorized) + })) + defer ts.Close() + + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + const bodySize = 5 << 20 + req := []byte(fmt.Sprintf("POST / HTTP/1.1\r\nHost: foo.com\r\nContent-Length: %d\r\n\r\n", bodySize)) + for i := 0; i < bodySize; i++ { + req = append(req, 'x') + } + writeErr := make(chan error) + go func() { + _, err := conn.Write(req) + writeErr <- err + }() + br := bufio.NewReader(conn) + lineNum := 0 + for { + line, err := br.ReadString('\n') + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("ReadLine: %v", err) + } + lineNum++ + if lineNum == 1 && !strings.Contains(line, "401 Unauthorized") { + t.Errorf("Response line = %q; want a 401", line) + } + } + // Wait for write to finish. This is a broken pipe on both + // Darwin and Linux, but checking this isn't the point of + // the test. + <-writeErr +} + +func TestCaseSensitiveMethod(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Method != "get" { + t.Errorf(`Got method %q; want "get"`, r.Method) + } + })) + defer ts.Close() + req, _ := NewRequest("get", ts.URL, nil) + res, err := DefaultClient.Do(req) + if err != nil { + t.Error(err) + return + } + res.Body.Close() +} + +// TestContentLengthZero tests that for both an HTTP/1.0 and HTTP/1.1 +// request (both keep-alive), when a Handler never writes any +// response, the net/http package adds a "Content-Length: 0" response +// header. +func TestContentLengthZero(t *testing.T) { + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) {})) + defer ts.Close() + + for _, version := range []string{"HTTP/1.0", "HTTP/1.1"} { + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + _, err = fmt.Fprintf(conn, "GET / %v\r\nConnection: keep-alive\r\nHost: foo\r\n\r\n", version) + if err != nil { + t.Fatalf("error writing: %v", err) + } + req, _ := NewRequest("GET", "/", nil) + res, err := ReadResponse(bufio.NewReader(conn), req) + if err != nil { + t.Fatalf("error reading response: %v", err) + } + if te := res.TransferEncoding; len(te) > 0 { + t.Errorf("For version %q, Transfer-Encoding = %q; want none", version, te) + } + if cl := res.ContentLength; cl != 0 { + t.Errorf("For version %q, Content-Length = %v; want 0", version, cl) + } + conn.Close() + } +} + +func TestCloseNotifier(t *testing.T) { + defer afterTest(t) + gotReq := make(chan bool, 1) + sawClose := make(chan bool, 1) + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + gotReq <- true + cc := rw.(CloseNotifier).CloseNotify() + <-cc + sawClose <- true + })) + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("error dialing: %v", err) + } + diec := make(chan bool) + go func() { + _, err = fmt.Fprintf(conn, "GET / HTTP/1.1\r\nConnection: keep-alive\r\nHost: foo\r\n\r\n") + if err != nil { + t.Fatal(err) + } + <-diec + conn.Close() + }() +For: + for { + select { + case <-gotReq: + diec <- true + case <-sawClose: + break For + case <-time.After(5 * time.Second): + t.Fatal("timeout") + } + } + ts.Close() +} + +func TestCloseNotifierChanLeak(t *testing.T) { + defer afterTest(t) + req := reqBytes("GET / HTTP/1.0\nHost: golang.org") + for i := 0; i < 20; i++ { + var output bytes.Buffer + conn := &rwTestConn{ + Reader: bytes.NewReader(req), + Writer: &output, + closec: make(chan bool, 1), + } + ln := &oneConnListener{conn: conn} + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + // Ignore the return value and never read from + // it, testing that we don't leak goroutines + // on the sending side: + _ = rw.(CloseNotifier).CloseNotify() + }) + go Serve(ln, handler) + <-conn.closec + } +} + +func TestOptions(t *testing.T) { + uric := make(chan string, 2) // only expect 1, but leave space for 2 + mux := NewServeMux() + mux.HandleFunc("/", func(w ResponseWriter, r *Request) { + uric <- r.RequestURI + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + // An OPTIONS * request should succeed. + _, err = conn.Write([]byte("OPTIONS * HTTP/1.1\r\nHost: foo.com\r\n\r\n")) + if err != nil { + t.Fatal(err) + } + br := bufio.NewReader(conn) + res, err := ReadResponse(br, &Request{Method: "OPTIONS"}) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Errorf("Got non-200 response to OPTIONS *: %#v", res) + } + + // A GET * request on a ServeMux should fail. + _, err = conn.Write([]byte("GET * HTTP/1.1\r\nHost: foo.com\r\n\r\n")) + if err != nil { + t.Fatal(err) + } + res, err = ReadResponse(br, &Request{Method: "GET"}) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 400 { + t.Errorf("Got non-400 response to GET *: %#v", res) + } + + res, err = Get(ts.URL + "/second") + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if got := <-uric; got != "/second" { + t.Errorf("Handler saw request for %q; want /second", got) + } +} + +// Tests regarding the ordering of Write, WriteHeader, Header, and +// Flush calls. In Go 1.0, rw.WriteHeader immediately flushed the +// (*response).header to the wire. In Go 1.1, the actual wire flush is +// delayed, so we could maybe tack on a Content-Length and better +// Content-Type after we see more (or all) of the output. To preserve +// compatibility with Go 1, we need to be careful to track which +// headers were live at the time of WriteHeader, so we write the same +// ones, even if the handler modifies them (~erroneously) after the +// first Write. +func TestHeaderToWire(t *testing.T) { + tests := []struct { + name string + handler func(ResponseWriter, *Request) + check func(output string) error + }{ + { + name: "write without Header", + handler: func(rw ResponseWriter, r *Request) { + rw.Write([]byte("hello world")) + }, + check: func(got string) error { + if !strings.Contains(got, "Content-Length:") { + return errors.New("no content-length") + } + if !strings.Contains(got, "Content-Type: text/plain") { + return errors.New("no content-length") + } + return nil + }, + }, + { + name: "Header mutation before write", + handler: func(rw ResponseWriter, r *Request) { + h := rw.Header() + h.Set("Content-Type", "some/type") + rw.Write([]byte("hello world")) + h.Set("Too-Late", "bogus") + }, + check: func(got string) error { + if !strings.Contains(got, "Content-Length:") { + return errors.New("no content-length") + } + if !strings.Contains(got, "Content-Type: some/type") { + return errors.New("wrong content-type") + } + if strings.Contains(got, "Too-Late") { + return errors.New("don't want too-late header") + } + return nil + }, + }, + { + name: "write then useless Header mutation", + handler: func(rw ResponseWriter, r *Request) { + rw.Write([]byte("hello world")) + rw.Header().Set("Too-Late", "Write already wrote headers") + }, + check: func(got string) error { + if strings.Contains(got, "Too-Late") { + return errors.New("header appeared from after WriteHeader") + } + return nil + }, + }, + { + name: "flush then write", + handler: func(rw ResponseWriter, r *Request) { + rw.(Flusher).Flush() + rw.Write([]byte("post-flush")) + rw.Header().Set("Too-Late", "Write already wrote headers") + }, + check: func(got string) error { + if !strings.Contains(got, "Transfer-Encoding: chunked") { + return errors.New("not chunked") + } + if strings.Contains(got, "Too-Late") { + return errors.New("header appeared from after WriteHeader") + } + return nil + }, + }, + { + name: "header then flush", + handler: func(rw ResponseWriter, r *Request) { + rw.Header().Set("Content-Type", "some/type") + rw.(Flusher).Flush() + rw.Write([]byte("post-flush")) + rw.Header().Set("Too-Late", "Write already wrote headers") + }, + check: func(got string) error { + if !strings.Contains(got, "Transfer-Encoding: chunked") { + return errors.New("not chunked") + } + if strings.Contains(got, "Too-Late") { + return errors.New("header appeared from after WriteHeader") + } + if !strings.Contains(got, "Content-Type: some/type") { + return errors.New("wrong content-length") + } + return nil + }, + }, + { + name: "sniff-on-first-write content-type", + handler: func(rw ResponseWriter, r *Request) { + rw.Write([]byte("<html><head></head><body>some html</body></html>")) + rw.Header().Set("Content-Type", "x/wrong") + }, + check: func(got string) error { + if !strings.Contains(got, "Content-Type: text/html") { + return errors.New("wrong content-length; want html") + } + return nil + }, + }, + { + name: "explicit content-type wins", + handler: func(rw ResponseWriter, r *Request) { + rw.Header().Set("Content-Type", "some/type") + rw.Write([]byte("<html><head></head><body>some html</body></html>")) + }, + check: func(got string) error { + if !strings.Contains(got, "Content-Type: some/type") { + return errors.New("wrong content-length; want html") + } + return nil + }, + }, + { + name: "empty handler", + handler: func(rw ResponseWriter, r *Request) { + }, + check: func(got string) error { + if !strings.Contains(got, "Content-Type: text/plain") { + return errors.New("wrong content-length; want text/plain") + } + if !strings.Contains(got, "Content-Length: 0") { + return errors.New("want 0 content-length") + } + return nil + }, + }, + { + name: "only Header, no write", + handler: func(rw ResponseWriter, r *Request) { + rw.Header().Set("Some-Header", "some-value") + }, + check: func(got string) error { + if !strings.Contains(got, "Some-Header") { + return errors.New("didn't get header") + } + return nil + }, + }, + { + name: "WriteHeader call", + handler: func(rw ResponseWriter, r *Request) { + rw.WriteHeader(404) + rw.Header().Set("Too-Late", "some-value") + }, + check: func(got string) error { + if !strings.Contains(got, "404") { + return errors.New("wrong status") + } + if strings.Contains(got, "Some-Header") { + return errors.New("shouldn't have seen Too-Late") + } + return nil + }, + }, + } + for _, tc := range tests { + ht := newHandlerTest(HandlerFunc(tc.handler)) + got := ht.rawResponse("GET / HTTP/1.1\nHost: golang.org") + if err := tc.check(got); err != nil { + t.Errorf("%s: %v\nGot response:\n%s", tc.name, err, got) + } + } +} + +// goTimeout runs f, failing t if f takes more than ns to complete. +func goTimeout(t *testing.T, d time.Duration, f func()) { + ch := make(chan bool, 2) + timer := time.AfterFunc(d, func() { + t.Errorf("Timeout expired after %v", d) + ch <- true + }) + defer timer.Stop() + go func() { + defer func() { ch <- true }() + f() + }() + <-ch +} + +type errorListener struct { + errs []error +} + +func (l *errorListener) Accept() (c net.Conn, err error) { + if len(l.errs) == 0 { + return nil, io.EOF + } + err = l.errs[0] + l.errs = l.errs[1:] + return +} + +func (l *errorListener) Close() error { + return nil +} + +func (l *errorListener) Addr() net.Addr { + return dummyAddr("test-address") +} + +func TestAcceptMaxFds(t *testing.T) { + log.SetOutput(ioutil.Discard) // is noisy otherwise + defer log.SetOutput(os.Stderr) + + ln := &errorListener{[]error{ + &net.OpError{ + Op: "accept", + Err: syscall.EMFILE, + }}} + err := Serve(ln, HandlerFunc(HandlerFunc(func(ResponseWriter, *Request) {}))) + if err != io.EOF { + t.Errorf("got error %v, want EOF", err) + } +} + +func TestWriteAfterHijack(t *testing.T) { + req := reqBytes("GET / HTTP/1.1\nHost: golang.org") + var buf bytes.Buffer + wrotec := make(chan bool, 1) + conn := &rwTestConn{ + Reader: bytes.NewReader(req), + Writer: &buf, + closec: make(chan bool, 1), + } + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + conn, bufrw, err := rw.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + go func() { + bufrw.Write([]byte("[hijack-to-bufw]")) + bufrw.Flush() + conn.Write([]byte("[hijack-to-conn]")) + conn.Close() + wrotec <- true + }() + }) + ln := &oneConnListener{conn: conn} + go Serve(ln, handler) + <-conn.closec + <-wrotec + if g, w := buf.String(), "[hijack-to-bufw][hijack-to-conn]"; g != w { + t.Errorf("wrote %q; want %q", g, w) + } +} + +func TestDoubleHijack(t *testing.T) { + req := reqBytes("GET / HTTP/1.1\nHost: golang.org") + var buf bytes.Buffer + conn := &rwTestConn{ + Reader: bytes.NewReader(req), + Writer: &buf, + closec: make(chan bool, 1), + } + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + conn, _, err := rw.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + _, _, err = rw.(Hijacker).Hijack() + if err == nil { + t.Errorf("got err = nil; want err != nil") + } + conn.Close() + }) + ln := &oneConnListener{conn: conn} + go Serve(ln, handler) + <-conn.closec +} + +// http://code.google.com/p/go/issues/detail?id=5955 +// Note that this does not test the "request too large" +// exit path from the http server. This is intentional; +// not sending Connection: close is just a minor wire +// optimization and is pointless if dealing with a +// badly behaved client. +func TestHTTP10ConnectionHeader(t *testing.T) { + defer afterTest(t) + + mux := NewServeMux() + mux.Handle("/", HandlerFunc(func(resp ResponseWriter, req *Request) {})) + ts := httptest.NewServer(mux) + defer ts.Close() + + // net/http uses HTTP/1.1 for requests, so write requests manually + tests := []struct { + req string // raw http request + expect []string // expected Connection header(s) + }{ + { + req: "GET / HTTP/1.0\r\n\r\n", + expect: nil, + }, + { + req: "OPTIONS * HTTP/1.0\r\n\r\n", + expect: nil, + }, + { + req: "GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", + expect: []string{"keep-alive"}, + }, + } + + for _, tt := range tests { + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal("dial err:", err) + } + + _, err = fmt.Fprint(conn, tt.req) + if err != nil { + t.Fatal("conn write err:", err) + } + + resp, err := ReadResponse(bufio.NewReader(conn), &Request{Method: "GET"}) + if err != nil { + t.Fatal("ReadResponse err:", err) + } + conn.Close() + resp.Body.Close() + + got := resp.Header["Connection"] + if !reflect.DeepEqual(got, tt.expect) { + t.Errorf("wrong Connection headers for request %q. Got %q expect %q", tt.req, got, tt.expect) + } + } +} + +// See golang.org/issue/5660 +func TestServerReaderFromOrder(t *testing.T) { + defer afterTest(t) + pr, pw := io.Pipe() + const size = 3 << 20 + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + rw.Header().Set("Content-Type", "text/plain") // prevent sniffing path + done := make(chan bool) + go func() { + io.Copy(rw, pr) + close(done) + }() + time.Sleep(25 * time.Millisecond) // give Copy a chance to break things + n, err := io.Copy(ioutil.Discard, req.Body) + if err != nil { + t.Errorf("handler Copy: %v", err) + return + } + if n != size { + t.Errorf("handler Copy = %d; want %d", n, size) + } + pw.Write([]byte("hi")) + pw.Close() + <-done + })) + defer ts.Close() + + req, err := NewRequest("POST", ts.URL, io.LimitReader(neverEnding('a'), size)) + if err != nil { + t.Fatal(err) + } + res, err := DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if string(all) != "hi" { + t.Errorf("Body = %q; want hi", all) + } +} + +// Issue 6157, Issue 6685 +func TestCodesPreventingContentTypeAndBody(t *testing.T) { + for _, code := range []int{StatusNotModified, StatusNoContent, StatusContinue} { + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/header" { + w.Header().Set("Content-Length", "123") + } + w.WriteHeader(code) + if r.URL.Path == "/more" { + w.Write([]byte("stuff")) + } + })) + for _, req := range []string{ + "GET / HTTP/1.0", + "GET /header HTTP/1.0", + "GET /more HTTP/1.0", + "GET / HTTP/1.1", + "GET /header HTTP/1.1", + "GET /more HTTP/1.1", + } { + got := ht.rawResponse(req) + wantStatus := fmt.Sprintf("%d %s", code, StatusText(code)) + if !strings.Contains(got, wantStatus) { + t.Errorf("Code %d: Wanted %q Modified for %q: %s", code, wantStatus, req, got) + } else if strings.Contains(got, "Content-Length") { + t.Errorf("Code %d: Got a Content-Length from %q: %s", code, req, got) + } else if strings.Contains(got, "stuff") { + t.Errorf("Code %d: Response contains a body from %q: %s", code, req, got) + } + } + } +} + +func TestContentTypeOkayOn204(t *testing.T) { + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Length", "123") // suppressed + w.Header().Set("Content-Type", "foo/bar") + w.WriteHeader(204) + })) + got := ht.rawResponse("GET / HTTP/1.1") + if !strings.Contains(got, "Content-Type: foo/bar") { + t.Errorf("Response = %q; want Content-Type: foo/bar", got) + } + if strings.Contains(got, "Content-Length: 123") { + t.Errorf("Response = %q; don't want a Content-Length", got) + } +} + +// Issue 6995 +// A server Handler can receive a Request, and then turn around and +// give a copy of that Request.Body out to the Transport (e.g. any +// proxy). So then two people own that Request.Body (both the server +// and the http client), and both think they can close it on failure. +// Therefore, all incoming server requests Bodies need to be thread-safe. +func TestTransportAndServerSharedBodyRace(t *testing.T) { + defer afterTest(t) + + const bodySize = 1 << 20 + + unblockBackend := make(chan bool) + backend := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + io.CopyN(rw, req.Body, bodySize/2) + <-unblockBackend + })) + defer backend.Close() + + backendRespc := make(chan *Response, 1) + proxy := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + if req.RequestURI == "/foo" { + rw.Write([]byte("bar")) + return + } + req2, _ := NewRequest("POST", backend.URL, req.Body) + req2.ContentLength = bodySize + + bresp, err := DefaultClient.Do(req2) + if err != nil { + t.Errorf("Proxy outbound request: %v", err) + return + } + _, err = io.CopyN(ioutil.Discard, bresp.Body, bodySize/4) + if err != nil { + t.Errorf("Proxy copy error: %v", err) + return + } + backendRespc <- bresp // to close later + + // Try to cause a race: Both the DefaultTransport and the proxy handler's Server + // will try to read/close req.Body (aka req2.Body) + DefaultTransport.(*Transport).CancelRequest(req2) + rw.Write([]byte("OK")) + })) + defer proxy.Close() + + req, _ := NewRequest("POST", proxy.URL, io.LimitReader(neverEnding('a'), bodySize)) + res, err := DefaultClient.Do(req) + if err != nil { + t.Fatalf("Original request: %v", err) + } + + // Cleanup, so we don't leak goroutines. + res.Body.Close() + close(unblockBackend) + (<-backendRespc).Body.Close() +} + +// Test that a hanging Request.Body.Read from another goroutine can't +// cause the Handler goroutine's Request.Body.Close to block. +func TestRequestBodyCloseDoesntBlock(t *testing.T) { + t.Skipf("Skipping known issue; see golang.org/issue/7121") + if testing.Short() { + t.Skip("skipping in -short mode") + } + defer afterTest(t) + + readErrCh := make(chan error, 1) + errCh := make(chan error, 2) + + server := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + go func(body io.Reader) { + _, err := body.Read(make([]byte, 100)) + readErrCh <- err + }(req.Body) + time.Sleep(500 * time.Millisecond) + })) + defer server.Close() + + closeConn := make(chan bool) + defer close(closeConn) + go func() { + conn, err := net.Dial("tcp", server.Listener.Addr().String()) + if err != nil { + errCh <- err + return + } + defer conn.Close() + _, err = conn.Write([]byte("POST / HTTP/1.1\r\nConnection: close\r\nHost: foo\r\nContent-Length: 100000\r\n\r\n")) + if err != nil { + errCh <- err + return + } + // And now just block, making the server block on our + // 100000 bytes of body that will never arrive. + <-closeConn + }() + select { + case err := <-readErrCh: + if err == nil { + t.Error("Read was nil. Expected error.") + } + case err := <-errCh: + t.Error(err) + case <-time.After(5 * time.Second): + t.Error("timeout") + } +} + +func TestResponseWriterWriteStringAllocs(t *testing.T) { + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/s" { + io.WriteString(w, "Hello world") + } else { + w.Write([]byte("Hello world")) + } + })) + before := testing.AllocsPerRun(50, func() { ht.rawResponse("GET / HTTP/1.0") }) + after := testing.AllocsPerRun(50, func() { ht.rawResponse("GET /s HTTP/1.0") }) + if int(after) >= int(before) { + t.Errorf("WriteString allocs of %v >= Write allocs of %v", after, before) + } +} + +func TestAppendTime(t *testing.T) { + var b [len(TimeFormat)]byte + t1 := time.Date(2013, 9, 21, 15, 41, 0, 0, time.FixedZone("CEST", 2*60*60)) + res := ExportAppendTime(b[:0], t1) + t2, err := ParseTime(string(res)) + if err != nil { + t.Fatalf("Error parsing time: %s", err) + } + if !t1.Equal(t2) { + t.Fatalf("Times differ; expected: %v, got %v (%s)", t1, t2, string(res)) + } +} + +func TestServerConnState(t *testing.T) { + defer afterTest(t) + handler := map[string]func(w ResponseWriter, r *Request){ + "/": func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "Hello.") + }, + "/close": func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + fmt.Fprintf(w, "Hello.") + }, + "/hijack": func(w ResponseWriter, r *Request) { + c, _, _ := w.(Hijacker).Hijack() + c.Write([]byte("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nHello.")) + c.Close() + }, + "/hijack-panic": func(w ResponseWriter, r *Request) { + c, _, _ := w.(Hijacker).Hijack() + c.Write([]byte("HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nHello.")) + c.Close() + panic("intentional panic") + }, + } + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { + handler[r.URL.Path](w, r) + })) + defer ts.Close() + + var mu sync.Mutex // guard stateLog and connID + var stateLog = map[int][]ConnState{} + var connID = map[net.Conn]int{} + + ts.Config.ErrorLog = log.New(ioutil.Discard, "", 0) + ts.Config.ConnState = func(c net.Conn, state ConnState) { + if c == nil { + t.Errorf("nil conn seen in state %s", state) + return + } + mu.Lock() + defer mu.Unlock() + id, ok := connID[c] + if !ok { + id = len(connID) + 1 + connID[c] = id + } + stateLog[id] = append(stateLog[id], state) + } + ts.Start() + + mustGet(t, ts.URL+"/") + mustGet(t, ts.URL+"/close") + + mustGet(t, ts.URL+"/") + mustGet(t, ts.URL+"/", "Connection", "close") + + mustGet(t, ts.URL+"/hijack") + mustGet(t, ts.URL+"/hijack-panic") + + // New->Closed + { + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + c.Close() + } + + // New->Active->Closed + { + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(c, "BOGUS REQUEST\r\n\r\n"); err != nil { + t.Fatal(err) + } + c.Close() + } + + // New->Idle->Closed + { + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(c, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n"); err != nil { + t.Fatal(err) + } + res, err := ReadResponse(bufio.NewReader(c), nil) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(ioutil.Discard, res.Body); err != nil { + t.Fatal(err) + } + c.Close() + } + + want := map[int][]ConnState{ + 1: {StateNew, StateActive, StateIdle, StateActive, StateClosed}, + 2: {StateNew, StateActive, StateIdle, StateActive, StateClosed}, + 3: {StateNew, StateActive, StateHijacked}, + 4: {StateNew, StateActive, StateHijacked}, + 5: {StateNew, StateClosed}, + 6: {StateNew, StateActive, StateClosed}, + 7: {StateNew, StateActive, StateIdle, StateClosed}, + } + logString := func(m map[int][]ConnState) string { + var b bytes.Buffer + for id, l := range m { + fmt.Fprintf(&b, "Conn %d: ", id) + for _, s := range l { + fmt.Fprintf(&b, "%s ", s) + } + b.WriteString("\n") + } + return b.String() + } + + for i := 0; i < 5; i++ { + time.Sleep(time.Duration(i) * 50 * time.Millisecond) + mu.Lock() + match := reflect.DeepEqual(stateLog, want) + mu.Unlock() + if match { + return + } + } + + mu.Lock() + t.Errorf("Unexpected events.\nGot log: %s\n Want: %s\n", logString(stateLog), logString(want)) + mu.Unlock() +} + +func mustGet(t *testing.T, url string, headers ...string) { + req, err := NewRequest("GET", url, nil) + if err != nil { + t.Fatal(err) + } + for len(headers) > 0 { + req.Header.Add(headers[0], headers[1]) + headers = headers[2:] + } + res, err := DefaultClient.Do(req) + if err != nil { + t.Errorf("Error fetching %s: %v", url, err) + return + } + _, err = ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + t.Errorf("Error reading %s: %v", url, err) + } +} + +func TestServerKeepAlivesEnabled(t *testing.T) { + defer afterTest(t) + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + ts.Config.SetKeepAlivesEnabled(false) + ts.Start() + defer ts.Close() + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if !res.Close { + t.Errorf("Body.Close == false; want true") + } +} + +// golang.org/issue/7856 +func TestServerEmptyBodyRace(t *testing.T) { + defer afterTest(t) + var n int32 + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + atomic.AddInt32(&n, 1) + })) + defer ts.Close() + var wg sync.WaitGroup + const reqs = 20 + for i := 0; i < reqs; i++ { + wg.Add(1) + go func() { + defer wg.Done() + res, err := Get(ts.URL) + if err != nil { + t.Error(err) + return + } + defer res.Body.Close() + _, err = io.Copy(ioutil.Discard, res.Body) + if err != nil { + t.Error(err) + return + } + }() + } + wg.Wait() + if got := atomic.LoadInt32(&n); got != reqs { + t.Errorf("handler ran %d times; want %d", got, reqs) + } +} + +func TestServerConnStateNew(t *testing.T) { + sawNew := false // if the test is buggy, we'll race on this variable. + srv := &Server{ + ConnState: func(c net.Conn, state ConnState) { + if state == StateNew { + sawNew = true // testing that this write isn't racy + } + }, + Handler: HandlerFunc(func(w ResponseWriter, r *Request) {}), // irrelevant + } + srv.Serve(&oneConnListener{ + conn: &rwTestConn{ + Reader: strings.NewReader("GET / HTTP/1.1\r\nHost: foo\r\n\r\n"), + Writer: ioutil.Discard, + }, + }) + if !sawNew { // testing that this read isn't racy + t.Error("StateNew not seen") + } +} + +func BenchmarkClientServer(b *testing.B) { + b.ReportAllocs() + b.StopTimer() + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + fmt.Fprintf(rw, "Hello world.\n") + })) + defer ts.Close() + b.StartTimer() + + for i := 0; i < b.N; i++ { + res, err := Get(ts.URL) + if err != nil { + b.Fatal("Get:", err) + } + all, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + b.Fatal("ReadAll:", err) + } + body := string(all) + if body != "Hello world.\n" { + b.Fatal("Got body:", body) + } + } + + b.StopTimer() +} + +func BenchmarkClientServerParallel4(b *testing.B) { + benchmarkClientServerParallel(b, 4, false) +} + +func BenchmarkClientServerParallel64(b *testing.B) { + benchmarkClientServerParallel(b, 64, false) +} + +func BenchmarkClientServerParallelTLS4(b *testing.B) { + benchmarkClientServerParallel(b, 4, true) +} + +func BenchmarkClientServerParallelTLS64(b *testing.B) { + benchmarkClientServerParallel(b, 64, true) +} + +func benchmarkClientServerParallel(b *testing.B, parallelism int, useTLS bool) { + b.ReportAllocs() + ts := httptest.NewUnstartedServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + fmt.Fprintf(rw, "Hello world.\n") + })) + if useTLS { + ts.StartTLS() + } else { + ts.Start() + } + defer ts.Close() + b.ResetTimer() + b.SetParallelism(parallelism) + b.RunParallel(func(pb *testing.PB) { + noVerifyTransport := &Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + client := &Client{Transport: noVerifyTransport} + for pb.Next() { + res, err := client.Get(ts.URL) + if err != nil { + b.Logf("Get: %v", err) + continue + } + all, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + b.Logf("ReadAll: %v", err) + continue + } + body := string(all) + if body != "Hello world.\n" { + panic("Got body: " + body) + } + } + }) +} + +// A benchmark for profiling the server without the HTTP client code. +// The client code runs in a subprocess. +// +// For use like: +// $ go test -c +// $ ./http.test -test.run=XX -test.bench=BenchmarkServer -test.benchtime=15s -test.cpuprofile=http.prof +// $ go tool pprof http.test http.prof +// (pprof) web +func BenchmarkServer(b *testing.B) { + b.ReportAllocs() + // Child process mode; + if url := os.Getenv("TEST_BENCH_SERVER_URL"); url != "" { + n, err := strconv.Atoi(os.Getenv("TEST_BENCH_CLIENT_N")) + if err != nil { + panic(err) + } + for i := 0; i < n; i++ { + res, err := Get(url) + if err != nil { + log.Panicf("Get: %v", err) + } + all, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + log.Panicf("ReadAll: %v", err) + } + body := string(all) + if body != "Hello world.\n" { + log.Panicf("Got body: %q", body) + } + } + os.Exit(0) + return + } + + var res = []byte("Hello world.\n") + b.StopTimer() + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, r *Request) { + rw.Header().Set("Content-Type", "text/html; charset=utf-8") + rw.Write(res) + })) + defer ts.Close() + b.StartTimer() + + cmd := exec.Command(os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkServer") + cmd.Env = append([]string{ + fmt.Sprintf("TEST_BENCH_CLIENT_N=%d", b.N), + fmt.Sprintf("TEST_BENCH_SERVER_URL=%s", ts.URL), + }, os.Environ()...) + out, err := cmd.CombinedOutput() + if err != nil { + b.Errorf("Test failure: %v, with output: %s", err, out) + } +} + +func BenchmarkServerFakeConnNoKeepAlive(b *testing.B) { + b.ReportAllocs() + req := reqBytes(`GET / HTTP/1.0 +Host: golang.org +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.52 Safari/537.17 +Accept-Encoding: gzip,deflate,sdch +Accept-Language: en-US,en;q=0.8 +Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3 +`) + res := []byte("Hello world!\n") + + conn := &testConn{ + // testConn.Close will not push into the channel + // if it's full. + closec: make(chan bool, 1), + } + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + rw.Header().Set("Content-Type", "text/html; charset=utf-8") + rw.Write(res) + }) + ln := new(oneConnListener) + for i := 0; i < b.N; i++ { + conn.readBuf.Reset() + conn.writeBuf.Reset() + conn.readBuf.Write(req) + ln.conn = conn + Serve(ln, handler) + <-conn.closec + } +} + +// repeatReader reads content count times, then EOFs. +type repeatReader struct { + content []byte + count int + off int +} + +func (r *repeatReader) Read(p []byte) (n int, err error) { + if r.count <= 0 { + return 0, io.EOF + } + n = copy(p, r.content[r.off:]) + r.off += n + if r.off == len(r.content) { + r.count-- + r.off = 0 + } + return +} + +func BenchmarkServerFakeConnWithKeepAlive(b *testing.B) { + b.ReportAllocs() + + req := reqBytes(`GET / HTTP/1.1 +Host: golang.org +Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 +User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.52 Safari/537.17 +Accept-Encoding: gzip,deflate,sdch +Accept-Language: en-US,en;q=0.8 +Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3 +`) + res := []byte("Hello world!\n") + + conn := &rwTestConn{ + Reader: &repeatReader{content: req, count: b.N}, + Writer: ioutil.Discard, + closec: make(chan bool, 1), + } + handled := 0 + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + handled++ + rw.Header().Set("Content-Type", "text/html; charset=utf-8") + rw.Write(res) + }) + ln := &oneConnListener{conn: conn} + go Serve(ln, handler) + <-conn.closec + if b.N != handled { + b.Errorf("b.N=%d but handled %d", b.N, handled) + } +} + +// same as above, but representing the most simple possible request +// and handler. Notably: the handler does not call rw.Header(). +func BenchmarkServerFakeConnWithKeepAliveLite(b *testing.B) { + b.ReportAllocs() + + req := reqBytes(`GET / HTTP/1.1 +Host: golang.org +`) + res := []byte("Hello world!\n") + + conn := &rwTestConn{ + Reader: &repeatReader{content: req, count: b.N}, + Writer: ioutil.Discard, + closec: make(chan bool, 1), + } + handled := 0 + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + handled++ + rw.Write(res) + }) + ln := &oneConnListener{conn: conn} + go Serve(ln, handler) + <-conn.closec + if b.N != handled { + b.Errorf("b.N=%d but handled %d", b.N, handled) + } +} + +const someResponse = "<html>some response</html>" + +// A Response that's just no bigger than 2KB, the buffer-before-chunking threshold. +var response = bytes.Repeat([]byte(someResponse), 2<<10/len(someResponse)) + +// Both Content-Type and Content-Length set. Should be no buffering. +func BenchmarkServerHandlerTypeLen(b *testing.B) { + benchmarkHandler(b, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Type", "text/html") + w.Header().Set("Content-Length", strconv.Itoa(len(response))) + w.Write(response) + })) +} + +// A Content-Type is set, but no length. No sniffing, but will count the Content-Length. +func BenchmarkServerHandlerNoLen(b *testing.B) { + benchmarkHandler(b, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Type", "text/html") + w.Write(response) + })) +} + +// A Content-Length is set, but the Content-Type will be sniffed. +func BenchmarkServerHandlerNoType(b *testing.B) { + benchmarkHandler(b, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Length", strconv.Itoa(len(response))) + w.Write(response) + })) +} + +// Neither a Content-Type or Content-Length, so sniffed and counted. +func BenchmarkServerHandlerNoHeader(b *testing.B) { + benchmarkHandler(b, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Write(response) + })) +} + +func benchmarkHandler(b *testing.B, h Handler) { + b.ReportAllocs() + req := reqBytes(`GET / HTTP/1.1 +Host: golang.org +`) + conn := &rwTestConn{ + Reader: &repeatReader{content: req, count: b.N}, + Writer: ioutil.Discard, + closec: make(chan bool, 1), + } + handled := 0 + handler := HandlerFunc(func(rw ResponseWriter, r *Request) { + handled++ + h.ServeHTTP(rw, r) + }) + ln := &oneConnListener{conn: conn} + go Serve(ln, handler) + <-conn.closec + if b.N != handled { + b.Errorf("b.N=%d but handled %d", b.N, handled) + } +} + +func BenchmarkServerHijack(b *testing.B) { + b.ReportAllocs() + req := reqBytes(`GET / HTTP/1.1 +Host: golang.org +`) + h := HandlerFunc(func(w ResponseWriter, r *Request) { + conn, _, err := w.(Hijacker).Hijack() + if err != nil { + panic(err) + } + conn.Close() + }) + conn := &rwTestConn{ + Writer: ioutil.Discard, + closec: make(chan bool, 1), + } + ln := &oneConnListener{conn: conn} + for i := 0; i < b.N; i++ { + conn.Reader = bytes.NewReader(req) + ln.conn = conn + Serve(ln, h) + <-conn.closec + } +} diff --git a/src/net/http/server.go b/src/net/http/server.go new file mode 100644 index 000000000..203037e9f --- /dev/null +++ b/src/net/http/server.go @@ -0,0 +1,2051 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP server. See RFC 2616. + +package http + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/url" + "os" + "path" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// Errors introduced by the HTTP server. +var ( + ErrWriteAfterFlush = errors.New("Conn.Write called after Flush") + ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") + ErrHijacked = errors.New("Conn has been hijacked") + ErrContentLength = errors.New("Conn.Write wrote more than the declared Content-Length") +) + +// Objects implementing the Handler interface can be +// registered to serve a particular path or subtree +// in the HTTP server. +// +// ServeHTTP should write reply headers and data to the ResponseWriter +// and then return. Returning signals that the request is finished +// and that the HTTP server can move on to the next request on +// the connection. +type Handler interface { + ServeHTTP(ResponseWriter, *Request) +} + +// A ResponseWriter interface is used by an HTTP handler to +// construct an HTTP response. +type ResponseWriter interface { + // Header returns the header map that will be sent by WriteHeader. + // Changing the header after a call to WriteHeader (or Write) has + // no effect. + Header() Header + + // Write writes the data to the connection as part of an HTTP reply. + // If WriteHeader has not yet been called, Write calls WriteHeader(http.StatusOK) + // before writing the data. If the Header does not contain a + // Content-Type line, Write adds a Content-Type set to the result of passing + // the initial 512 bytes of written data to DetectContentType. + Write([]byte) (int, error) + + // WriteHeader sends an HTTP response header with status code. + // If WriteHeader is not called explicitly, the first call to Write + // will trigger an implicit WriteHeader(http.StatusOK). + // Thus explicit calls to WriteHeader are mainly used to + // send error codes. + WriteHeader(int) +} + +// The Flusher interface is implemented by ResponseWriters that allow +// an HTTP handler to flush buffered data to the client. +// +// Note that even for ResponseWriters that support Flush, +// if the client is connected through an HTTP proxy, +// the buffered data may not reach the client until the response +// completes. +type Flusher interface { + // Flush sends any buffered data to the client. + Flush() +} + +// The Hijacker interface is implemented by ResponseWriters that allow +// an HTTP handler to take over the connection. +type Hijacker interface { + // Hijack lets the caller take over the connection. + // After a call to Hijack(), the HTTP server library + // will not do anything else with the connection. + // It becomes the caller's responsibility to manage + // and close the connection. + Hijack() (net.Conn, *bufio.ReadWriter, error) +} + +// The CloseNotifier interface is implemented by ResponseWriters which +// allow detecting when the underlying connection has gone away. +// +// This mechanism can be used to cancel long operations on the server +// if the client has disconnected before the response is ready. +type CloseNotifier interface { + // CloseNotify returns a channel that receives a single value + // when the client connection has gone away. + CloseNotify() <-chan bool +} + +// A conn represents the server side of an HTTP connection. +type conn struct { + remoteAddr string // network address of remote side + server *Server // the Server on which the connection arrived + rwc net.Conn // i/o connection + sr liveSwitchReader // where the LimitReader reads from; usually the rwc + lr *io.LimitedReader // io.LimitReader(sr) + buf *bufio.ReadWriter // buffered(lr,rwc), reading from bufio->limitReader->sr->rwc + tlsState *tls.ConnectionState // or nil when not using TLS + + mu sync.Mutex // guards the following + clientGone bool // if client has disconnected mid-request + closeNotifyc chan bool // made lazily + hijackedv bool // connection has been hijacked by handler +} + +func (c *conn) hijacked() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.hijackedv +} + +func (c *conn) hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.hijackedv { + return nil, nil, ErrHijacked + } + if c.closeNotifyc != nil { + return nil, nil, errors.New("http: Hijack is incompatible with use of CloseNotifier") + } + c.hijackedv = true + rwc = c.rwc + buf = c.buf + c.rwc = nil + c.buf = nil + c.setState(rwc, StateHijacked) + return +} + +func (c *conn) closeNotify() <-chan bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.closeNotifyc == nil { + c.closeNotifyc = make(chan bool, 1) + if c.hijackedv { + // to obey the function signature, even though + // it'll never receive a value. + return c.closeNotifyc + } + pr, pw := io.Pipe() + + readSource := c.sr.r + c.sr.Lock() + c.sr.r = pr + c.sr.Unlock() + go func() { + _, err := io.Copy(pw, readSource) + if err == nil { + err = io.EOF + } + pw.CloseWithError(err) + c.noteClientGone() + }() + } + return c.closeNotifyc +} + +func (c *conn) noteClientGone() { + c.mu.Lock() + defer c.mu.Unlock() + if c.closeNotifyc != nil && !c.clientGone { + c.closeNotifyc <- true + } + c.clientGone = true +} + +// A switchReader can have its Reader changed at runtime. +// It's not safe for concurrent Reads and switches. +type switchReader struct { + io.Reader +} + +// A switchWriter can have its Writer changed at runtime. +// It's not safe for concurrent Writes and switches. +type switchWriter struct { + io.Writer +} + +// A liveSwitchReader is a switchReader that's safe for concurrent +// reads and switches, if its mutex is held. +type liveSwitchReader struct { + sync.Mutex + r io.Reader +} + +func (sr *liveSwitchReader) Read(p []byte) (n int, err error) { + sr.Lock() + r := sr.r + sr.Unlock() + return r.Read(p) +} + +// This should be >= 512 bytes for DetectContentType, +// but otherwise it's somewhat arbitrary. +const bufferBeforeChunkingSize = 2048 + +// chunkWriter writes to a response's conn buffer, and is the writer +// wrapped by the response.bufw buffered writer. +// +// chunkWriter also is responsible for finalizing the Header, including +// conditionally setting the Content-Type and setting a Content-Length +// in cases where the handler's final output is smaller than the buffer +// size. It also conditionally adds chunk headers, when in chunking mode. +// +// See the comment above (*response).Write for the entire write flow. +type chunkWriter struct { + res *response + + // header is either nil or a deep clone of res.handlerHeader + // at the time of res.WriteHeader, if res.WriteHeader is + // called and extra buffering is being done to calculate + // Content-Type and/or Content-Length. + header Header + + // wroteHeader tells whether the header's been written to "the + // wire" (or rather: w.conn.buf). this is unlike + // (*response).wroteHeader, which tells only whether it was + // logically written. + wroteHeader bool + + // set by the writeHeader method: + chunking bool // using chunked transfer encoding for reply body +} + +var ( + crlf = []byte("\r\n") + colonSpace = []byte(": ") +) + +func (cw *chunkWriter) Write(p []byte) (n int, err error) { + if !cw.wroteHeader { + cw.writeHeader(p) + } + if cw.res.req.Method == "HEAD" { + // Eat writes. + return len(p), nil + } + if cw.chunking { + _, err = fmt.Fprintf(cw.res.conn.buf, "%x\r\n", len(p)) + if err != nil { + cw.res.conn.rwc.Close() + return + } + } + n, err = cw.res.conn.buf.Write(p) + if cw.chunking && err == nil { + _, err = cw.res.conn.buf.Write(crlf) + } + if err != nil { + cw.res.conn.rwc.Close() + } + return +} + +func (cw *chunkWriter) flush() { + if !cw.wroteHeader { + cw.writeHeader(nil) + } + cw.res.conn.buf.Flush() +} + +func (cw *chunkWriter) close() { + if !cw.wroteHeader { + cw.writeHeader(nil) + } + if cw.chunking { + // zero EOF chunk, trailer key/value pairs (currently + // unsupported in Go's server), followed by a blank + // line. + cw.res.conn.buf.WriteString("0\r\n\r\n") + } +} + +// A response represents the server side of an HTTP response. +type response struct { + conn *conn + req *Request // request for this response + wroteHeader bool // reply header has been (logically) written + wroteContinue bool // 100 Continue response was written + + w *bufio.Writer // buffers output in chunks to chunkWriter + cw chunkWriter + sw *switchWriter // of the bufio.Writer, for return to putBufioWriter + + // handlerHeader is the Header that Handlers get access to, + // which may be retained and mutated even after WriteHeader. + // handlerHeader is copied into cw.header at WriteHeader + // time, and privately mutated thereafter. + handlerHeader Header + calledHeader bool // handler accessed handlerHeader via Header + + written int64 // number of bytes written in body + contentLength int64 // explicitly-declared Content-Length; or -1 + status int // status code passed to WriteHeader + + // close connection after this reply. set on request and + // updated after response from handler if there's a + // "Connection: keep-alive" response header and a + // Content-Length. + closeAfterReply bool + + // requestBodyLimitHit is set by requestTooLarge when + // maxBytesReader hits its max size. It is checked in + // WriteHeader, to make sure we don't consume the + // remaining request body to try to advance to the next HTTP + // request. Instead, when this is set, we stop reading + // subsequent requests on this connection and stop reading + // input from it. + requestBodyLimitHit bool + + handlerDone bool // set true when the handler exits + + // Buffers for Date and Content-Length + dateBuf [len(TimeFormat)]byte + clenBuf [10]byte +} + +// requestTooLarge is called by maxBytesReader when too much input has +// been read from the client. +func (w *response) requestTooLarge() { + w.closeAfterReply = true + w.requestBodyLimitHit = true + if !w.wroteHeader { + w.Header().Set("Connection", "close") + } +} + +// needsSniff reports whether a Content-Type still needs to be sniffed. +func (w *response) needsSniff() bool { + _, haveType := w.handlerHeader["Content-Type"] + return !w.cw.wroteHeader && !haveType && w.written < sniffLen +} + +// writerOnly hides an io.Writer value's optional ReadFrom method +// from io.Copy. +type writerOnly struct { + io.Writer +} + +func srcIsRegularFile(src io.Reader) (isRegular bool, err error) { + switch v := src.(type) { + case *os.File: + fi, err := v.Stat() + if err != nil { + return false, err + } + return fi.Mode().IsRegular(), nil + case *io.LimitedReader: + return srcIsRegularFile(v.R) + default: + return + } +} + +// ReadFrom is here to optimize copying from an *os.File regular file +// to a *net.TCPConn with sendfile. +func (w *response) ReadFrom(src io.Reader) (n int64, err error) { + // Our underlying w.conn.rwc is usually a *TCPConn (with its + // own ReadFrom method). If not, or if our src isn't a regular + // file, just fall back to the normal copy method. + rf, ok := w.conn.rwc.(io.ReaderFrom) + regFile, err := srcIsRegularFile(src) + if err != nil { + return 0, err + } + if !ok || !regFile { + return io.Copy(writerOnly{w}, src) + } + + // sendfile path: + + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + + if w.needsSniff() { + n0, err := io.Copy(writerOnly{w}, io.LimitReader(src, sniffLen)) + n += n0 + if err != nil { + return n, err + } + } + + w.w.Flush() // get rid of any previous writes + w.cw.flush() // make sure Header is written; flush data to rwc + + // Now that cw has been flushed, its chunking field is guaranteed initialized. + if !w.cw.chunking && w.bodyAllowed() { + n0, err := rf.ReadFrom(src) + n += n0 + w.written += n0 + return n, err + } + + n0, err := io.Copy(writerOnly{w}, src) + n += n0 + return n, err +} + +// noLimit is an effective infinite upper bound for io.LimitedReader +const noLimit int64 = (1 << 63) - 1 + +// debugServerConnections controls whether all server connections are wrapped +// with a verbose logging wrapper. +const debugServerConnections = false + +// Create new connection from rwc. +func (srv *Server) newConn(rwc net.Conn) (c *conn, err error) { + c = new(conn) + c.remoteAddr = rwc.RemoteAddr().String() + c.server = srv + c.rwc = rwc + if debugServerConnections { + c.rwc = newLoggingConn("server", c.rwc) + } + c.sr = liveSwitchReader{r: c.rwc} + c.lr = io.LimitReader(&c.sr, noLimit).(*io.LimitedReader) + br := newBufioReader(c.lr) + bw := newBufioWriterSize(c.rwc, 4<<10) + c.buf = bufio.NewReadWriter(br, bw) + return c, nil +} + +var ( + bufioReaderPool sync.Pool + bufioWriter2kPool sync.Pool + bufioWriter4kPool sync.Pool +) + +func bufioWriterPool(size int) *sync.Pool { + switch size { + case 2 << 10: + return &bufioWriter2kPool + case 4 << 10: + return &bufioWriter4kPool + } + return nil +} + +func newBufioReader(r io.Reader) *bufio.Reader { + if v := bufioReaderPool.Get(); v != nil { + br := v.(*bufio.Reader) + br.Reset(r) + return br + } + return bufio.NewReader(r) +} + +func putBufioReader(br *bufio.Reader) { + br.Reset(nil) + bufioReaderPool.Put(br) +} + +func newBufioWriterSize(w io.Writer, size int) *bufio.Writer { + pool := bufioWriterPool(size) + if pool != nil { + if v := pool.Get(); v != nil { + bw := v.(*bufio.Writer) + bw.Reset(w) + return bw + } + } + return bufio.NewWriterSize(w, size) +} + +func putBufioWriter(bw *bufio.Writer) { + bw.Reset(nil) + if pool := bufioWriterPool(bw.Available()); pool != nil { + pool.Put(bw) + } +} + +// DefaultMaxHeaderBytes is the maximum permitted size of the headers +// in an HTTP request. +// This can be overridden by setting Server.MaxHeaderBytes. +const DefaultMaxHeaderBytes = 1 << 20 // 1 MB + +func (srv *Server) maxHeaderBytes() int { + if srv.MaxHeaderBytes > 0 { + return srv.MaxHeaderBytes + } + return DefaultMaxHeaderBytes +} + +func (srv *Server) initialLimitedReaderSize() int64 { + return int64(srv.maxHeaderBytes()) + 4096 // bufio slop +} + +// wrapper around io.ReaderCloser which on first read, sends an +// HTTP/1.1 100 Continue header +type expectContinueReader struct { + resp *response + readCloser io.ReadCloser + closed bool +} + +func (ecr *expectContinueReader) Read(p []byte) (n int, err error) { + if ecr.closed { + return 0, ErrBodyReadAfterClose + } + if !ecr.resp.wroteContinue && !ecr.resp.conn.hijacked() { + ecr.resp.wroteContinue = true + ecr.resp.conn.buf.WriteString("HTTP/1.1 100 Continue\r\n\r\n") + ecr.resp.conn.buf.Flush() + } + return ecr.readCloser.Read(p) +} + +func (ecr *expectContinueReader) Close() error { + ecr.closed = true + return ecr.readCloser.Close() +} + +// TimeFormat is the time format to use with +// time.Parse and time.Time.Format when parsing +// or generating times in HTTP headers. +// It is like time.RFC1123 but hard codes GMT as the time zone. +const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" + +// appendTime is a non-allocating version of []byte(t.UTC().Format(TimeFormat)) +func appendTime(b []byte, t time.Time) []byte { + const days = "SunMonTueWedThuFriSat" + const months = "JanFebMarAprMayJunJulAugSepOctNovDec" + + t = t.UTC() + yy, mm, dd := t.Date() + hh, mn, ss := t.Clock() + day := days[3*t.Weekday():] + mon := months[3*(mm-1):] + + return append(b, + day[0], day[1], day[2], ',', ' ', + byte('0'+dd/10), byte('0'+dd%10), ' ', + mon[0], mon[1], mon[2], ' ', + byte('0'+yy/1000), byte('0'+(yy/100)%10), byte('0'+(yy/10)%10), byte('0'+yy%10), ' ', + byte('0'+hh/10), byte('0'+hh%10), ':', + byte('0'+mn/10), byte('0'+mn%10), ':', + byte('0'+ss/10), byte('0'+ss%10), ' ', + 'G', 'M', 'T') +} + +var errTooLarge = errors.New("http: request too large") + +// Read next request from connection. +func (c *conn) readRequest() (w *response, err error) { + if c.hijacked() { + return nil, ErrHijacked + } + + if d := c.server.ReadTimeout; d != 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + } + if d := c.server.WriteTimeout; d != 0 { + defer func() { + c.rwc.SetWriteDeadline(time.Now().Add(d)) + }() + } + + c.lr.N = c.server.initialLimitedReaderSize() + var req *Request + if req, err = ReadRequest(c.buf.Reader); err != nil { + if c.lr.N == 0 { + return nil, errTooLarge + } + return nil, err + } + c.lr.N = noLimit + + req.RemoteAddr = c.remoteAddr + req.TLS = c.tlsState + + w = &response{ + conn: c, + req: req, + handlerHeader: make(Header), + contentLength: -1, + } + w.cw.res = w + w.w = newBufioWriterSize(&w.cw, bufferBeforeChunkingSize) + return w, nil +} + +func (w *response) Header() Header { + if w.cw.header == nil && w.wroteHeader && !w.cw.wroteHeader { + // Accessing the header between logically writing it + // and physically writing it means we need to allocate + // a clone to snapshot the logically written state. + w.cw.header = w.handlerHeader.clone() + } + w.calledHeader = true + return w.handlerHeader +} + +// maxPostHandlerReadBytes is the max number of Request.Body bytes not +// consumed by a handler that the server will read from the client +// in order to keep a connection alive. If there are more bytes than +// this then the server to be paranoid instead sends a "Connection: +// close" response. +// +// This number is approximately what a typical machine's TCP buffer +// size is anyway. (if we have the bytes on the machine, we might as +// well read them) +const maxPostHandlerReadBytes = 256 << 10 + +func (w *response) WriteHeader(code int) { + if w.conn.hijacked() { + w.conn.server.logf("http: response.WriteHeader on hijacked connection") + return + } + if w.wroteHeader { + w.conn.server.logf("http: multiple response.WriteHeader calls") + return + } + w.wroteHeader = true + w.status = code + + if w.calledHeader && w.cw.header == nil { + w.cw.header = w.handlerHeader.clone() + } + + if cl := w.handlerHeader.get("Content-Length"); cl != "" { + v, err := strconv.ParseInt(cl, 10, 64) + if err == nil && v >= 0 { + w.contentLength = v + } else { + w.conn.server.logf("http: invalid Content-Length of %q", cl) + w.handlerHeader.Del("Content-Length") + } + } +} + +// extraHeader is the set of headers sometimes added by chunkWriter.writeHeader. +// This type is used to avoid extra allocations from cloning and/or populating +// the response Header map and all its 1-element slices. +type extraHeader struct { + contentType string + connection string + transferEncoding string + date []byte // written if not nil + contentLength []byte // written if not nil +} + +// Sorted the same as extraHeader.Write's loop. +var extraHeaderKeys = [][]byte{ + []byte("Content-Type"), + []byte("Connection"), + []byte("Transfer-Encoding"), +} + +var ( + headerContentLength = []byte("Content-Length: ") + headerDate = []byte("Date: ") +) + +// Write writes the headers described in h to w. +// +// This method has a value receiver, despite the somewhat large size +// of h, because it prevents an allocation. The escape analysis isn't +// smart enough to realize this function doesn't mutate h. +func (h extraHeader) Write(w *bufio.Writer) { + if h.date != nil { + w.Write(headerDate) + w.Write(h.date) + w.Write(crlf) + } + if h.contentLength != nil { + w.Write(headerContentLength) + w.Write(h.contentLength) + w.Write(crlf) + } + for i, v := range []string{h.contentType, h.connection, h.transferEncoding} { + if v != "" { + w.Write(extraHeaderKeys[i]) + w.Write(colonSpace) + w.WriteString(v) + w.Write(crlf) + } + } +} + +// writeHeader finalizes the header sent to the client and writes it +// to cw.res.conn.buf. +// +// p is not written by writeHeader, but is the first chunk of the body +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. It's also used to set the Content-Length, if the +// total body size was small and the handler has already finished +// running. +func (cw *chunkWriter) writeHeader(p []byte) { + if cw.wroteHeader { + return + } + cw.wroteHeader = true + + w := cw.res + keepAlivesEnabled := w.conn.server.doKeepAlives() + isHEAD := w.req.Method == "HEAD" + + // header is written out to w.conn.buf below. Depending on the + // state of the handler, we either own the map or not. If we + // don't own it, the exclude map is created lazily for + // WriteSubset to remove headers. The setHeader struct holds + // headers we need to add. + header := cw.header + owned := header != nil + if !owned { + header = w.handlerHeader + } + var excludeHeader map[string]bool + delHeader := func(key string) { + if owned { + header.Del(key) + return + } + if _, ok := header[key]; !ok { + return + } + if excludeHeader == nil { + excludeHeader = make(map[string]bool) + } + excludeHeader[key] = true + } + var setHeader extraHeader + + // If the handler is done but never sent a Content-Length + // response header and this is our first (and last) write, set + // it, even to zero. This helps HTTP/1.0 clients keep their + // "keep-alive" connections alive. + // Exceptions: 304/204/1xx responses never get Content-Length, and if + // it was a HEAD request, we don't know the difference between + // 0 actual bytes and 0 bytes because the handler noticed it + // was a HEAD request and chose not to write anything. So for + // HEAD, the handler should either write the Content-Length or + // write non-zero bytes. If it's actually 0 bytes and the + // handler never looked at the Request.Method, we just don't + // send a Content-Length header. + if w.handlerDone && bodyAllowedForStatus(w.status) && header.get("Content-Length") == "" && (!isHEAD || len(p) > 0) { + w.contentLength = int64(len(p)) + setHeader.contentLength = strconv.AppendInt(cw.res.clenBuf[:0], int64(len(p)), 10) + } + + // If this was an HTTP/1.0 request with keep-alive and we sent a + // Content-Length back, we can make this a keep-alive response ... + if w.req.wantsHttp10KeepAlive() && keepAlivesEnabled { + sentLength := header.get("Content-Length") != "" + if sentLength && header.get("Connection") == "keep-alive" { + w.closeAfterReply = false + } + } + + // Check for a explicit (and valid) Content-Length header. + hasCL := w.contentLength != -1 + + if w.req.wantsHttp10KeepAlive() && (isHEAD || hasCL) { + _, connectionHeaderSet := header["Connection"] + if !connectionHeaderSet { + setHeader.connection = "keep-alive" + } + } else if !w.req.ProtoAtLeast(1, 1) || w.req.wantsClose() { + w.closeAfterReply = true + } + + if header.get("Connection") == "close" || !keepAlivesEnabled { + w.closeAfterReply = true + } + + // Per RFC 2616, we should consume the request body before + // replying, if the handler hasn't already done so. But we + // don't want to do an unbounded amount of reading here for + // DoS reasons, so we only try up to a threshold. + if w.req.ContentLength != 0 && !w.closeAfterReply { + ecr, isExpecter := w.req.Body.(*expectContinueReader) + if !isExpecter || ecr.resp.wroteContinue { + n, _ := io.CopyN(ioutil.Discard, w.req.Body, maxPostHandlerReadBytes+1) + if n >= maxPostHandlerReadBytes { + w.requestTooLarge() + delHeader("Connection") + setHeader.connection = "close" + } else { + w.req.Body.Close() + } + } + } + + code := w.status + if bodyAllowedForStatus(code) { + // If no content type, apply sniffing algorithm to body. + _, haveType := header["Content-Type"] + if !haveType { + setHeader.contentType = DetectContentType(p) + } + } else { + for _, k := range suppressedHeaders(code) { + delHeader(k) + } + } + + if _, ok := header["Date"]; !ok { + setHeader.date = appendTime(cw.res.dateBuf[:0], time.Now()) + } + + te := header.get("Transfer-Encoding") + hasTE := te != "" + if hasCL && hasTE && te != "identity" { + // TODO: return an error if WriteHeader gets a return parameter + // For now just ignore the Content-Length. + w.conn.server.logf("http: WriteHeader called with both Transfer-Encoding of %q and a Content-Length of %d", + te, w.contentLength) + delHeader("Content-Length") + hasCL = false + } + + if w.req.Method == "HEAD" || !bodyAllowedForStatus(code) { + // do nothing + } else if code == StatusNoContent { + delHeader("Transfer-Encoding") + } else if hasCL { + delHeader("Transfer-Encoding") + } else if w.req.ProtoAtLeast(1, 1) { + // HTTP/1.1 or greater: use chunked transfer encoding + // to avoid closing the connection at EOF. + // TODO: this blows away any custom or stacked Transfer-Encoding they + // might have set. Deal with that as need arises once we have a valid + // use case. + cw.chunking = true + setHeader.transferEncoding = "chunked" + } else { + // HTTP version < 1.1: cannot do chunked transfer + // encoding and we don't know the Content-Length so + // signal EOF by closing connection. + w.closeAfterReply = true + delHeader("Transfer-Encoding") // in case already set + } + + // Cannot use Content-Length with non-identity Transfer-Encoding. + if cw.chunking { + delHeader("Content-Length") + } + if !w.req.ProtoAtLeast(1, 0) { + return + } + + if w.closeAfterReply && (!keepAlivesEnabled || !hasToken(cw.header.get("Connection"), "close")) { + delHeader("Connection") + if w.req.ProtoAtLeast(1, 1) { + setHeader.connection = "close" + } + } + + w.conn.buf.WriteString(statusLine(w.req, code)) + cw.header.WriteSubset(w.conn.buf, excludeHeader) + setHeader.Write(w.conn.buf.Writer) + w.conn.buf.Write(crlf) +} + +// statusLines is a cache of Status-Line strings, keyed by code (for +// HTTP/1.1) or negative code (for HTTP/1.0). This is faster than a +// map keyed by struct of two fields. This map's max size is bounded +// by 2*len(statusText), two protocol types for each known official +// status code in the statusText map. +var ( + statusMu sync.RWMutex + statusLines = make(map[int]string) +) + +// statusLine returns a response Status-Line (RFC 2616 Section 6.1) +// for the given request and response status code. +func statusLine(req *Request, code int) string { + // Fast path: + key := code + proto11 := req.ProtoAtLeast(1, 1) + if !proto11 { + key = -key + } + statusMu.RLock() + line, ok := statusLines[key] + statusMu.RUnlock() + if ok { + return line + } + + // Slow path: + proto := "HTTP/1.0" + if proto11 { + proto = "HTTP/1.1" + } + codestring := strconv.Itoa(code) + text, ok := statusText[code] + if !ok { + text = "status code " + codestring + } + line = proto + " " + codestring + " " + text + "\r\n" + if ok { + statusMu.Lock() + defer statusMu.Unlock() + statusLines[key] = line + } + return line +} + +// bodyAllowed returns true if a Write is allowed for this response type. +// It's illegal to call this before the header has been flushed. +func (w *response) bodyAllowed() bool { + if !w.wroteHeader { + panic("") + } + return bodyAllowedForStatus(w.status) +} + +// The Life Of A Write is like this: +// +// Handler starts. No header has been sent. The handler can either +// write a header, or just start writing. Writing before sending a header +// sends an implicitly empty 200 OK header. +// +// If the handler didn't declare a Content-Length up front, we either +// go into chunking mode or, if the handler finishes running before +// the chunking buffer size, we compute a Content-Length and send that +// in the header instead. +// +// Likewise, if the handler didn't set a Content-Type, we sniff that +// from the initial chunk of output. +// +// The Writers are wired together like: +// +// 1. *response (the ResponseWriter) -> +// 2. (*response).w, a *bufio.Writer of bufferBeforeChunkingSize bytes +// 3. chunkWriter.Writer (whose writeHeader finalizes Content-Length/Type) +// and which writes the chunk headers, if needed. +// 4. conn.buf, a bufio.Writer of default (4kB) bytes +// 5. the rwc, the net.Conn. +// +// TODO(bradfitz): short-circuit some of the buffering when the +// initial header contains both a Content-Type and Content-Length. +// Also short-circuit in (1) when the header's been sent and not in +// chunking mode, writing directly to (4) instead, if (2) has no +// buffered data. More generally, we could short-circuit from (1) to +// (3) even in chunking mode if the write size from (1) is over some +// threshold and nothing is in (2). The answer might be mostly making +// bufferBeforeChunkingSize smaller and having bufio's fast-paths deal +// with this instead. +func (w *response) Write(data []byte) (n int, err error) { + return w.write(len(data), data, "") +} + +func (w *response) WriteString(data string) (n int, err error) { + return w.write(len(data), nil, data) +} + +// either dataB or dataS is non-zero. +func (w *response) write(lenData int, dataB []byte, dataS string) (n int, err error) { + if w.conn.hijacked() { + w.conn.server.logf("http: response.Write on hijacked connection") + return 0, ErrHijacked + } + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + if lenData == 0 { + return 0, nil + } + if !w.bodyAllowed() { + return 0, ErrBodyNotAllowed + } + + w.written += int64(lenData) // ignoring errors, for errorKludge + if w.contentLength != -1 && w.written > w.contentLength { + return 0, ErrContentLength + } + if dataB != nil { + return w.w.Write(dataB) + } else { + return w.w.WriteString(dataS) + } +} + +func (w *response) finishRequest() { + w.handlerDone = true + + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + + w.w.Flush() + putBufioWriter(w.w) + w.cw.close() + w.conn.buf.Flush() + + // Close the body (regardless of w.closeAfterReply) so we can + // re-use its bufio.Reader later safely. + w.req.Body.Close() + + if w.req.MultipartForm != nil { + w.req.MultipartForm.RemoveAll() + } + + if w.req.Method != "HEAD" && w.contentLength != -1 && w.bodyAllowed() && w.contentLength != w.written { + // Did not write enough. Avoid getting out of sync. + w.closeAfterReply = true + } +} + +func (w *response) Flush() { + if !w.wroteHeader { + w.WriteHeader(StatusOK) + } + w.w.Flush() + w.cw.flush() +} + +func (c *conn) finalFlush() { + if c.buf != nil { + c.buf.Flush() + + // Steal the bufio.Reader (~4KB worth of memory) and its associated + // reader for a future connection. + putBufioReader(c.buf.Reader) + + // Steal the bufio.Writer (~4KB worth of memory) and its associated + // writer for a future connection. + putBufioWriter(c.buf.Writer) + + c.buf = nil + } +} + +// Close the connection. +func (c *conn) close() { + c.finalFlush() + if c.rwc != nil { + c.rwc.Close() + c.rwc = nil + } +} + +// rstAvoidanceDelay is the amount of time we sleep after closing the +// write side of a TCP connection before closing the entire socket. +// By sleeping, we increase the chances that the client sees our FIN +// and processes its final data before they process the subsequent RST +// from closing a connection with known unread data. +// This RST seems to occur mostly on BSD systems. (And Windows?) +// This timeout is somewhat arbitrary (~latency around the planet). +const rstAvoidanceDelay = 500 * time.Millisecond + +// closeWrite flushes any outstanding data and sends a FIN packet (if +// client is connected via TCP), signalling that we're done. We then +// pause for a bit, hoping the client processes it before `any +// subsequent RST. +// +// See http://golang.org/issue/3595 +func (c *conn) closeWriteAndWait() { + c.finalFlush() + if tcp, ok := c.rwc.(*net.TCPConn); ok { + tcp.CloseWrite() + } + time.Sleep(rstAvoidanceDelay) +} + +// validNPN reports whether the proto is not a blacklisted Next +// Protocol Negotiation protocol. Empty and built-in protocol types +// are blacklisted and can't be overridden with alternate +// implementations. +func validNPN(proto string) bool { + switch proto { + case "", "http/1.1", "http/1.0": + return false + } + return true +} + +func (c *conn) setState(nc net.Conn, state ConnState) { + if hook := c.server.ConnState; hook != nil { + hook(nc, state) + } +} + +// Serve a new connection. +func (c *conn) serve() { + origConn := c.rwc // copy it before it's set nil on Close or Hijack + defer func() { + if err := recover(); err != nil { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + c.server.logf("http: panic serving %v: %v\n%s", c.remoteAddr, err, buf) + } + if !c.hijacked() { + c.close() + c.setState(origConn, StateClosed) + } + }() + + if tlsConn, ok := c.rwc.(*tls.Conn); ok { + if d := c.server.ReadTimeout; d != 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + } + if d := c.server.WriteTimeout; d != 0 { + c.rwc.SetWriteDeadline(time.Now().Add(d)) + } + if err := tlsConn.Handshake(); err != nil { + c.server.logf("http: TLS handshake error from %s: %v", c.rwc.RemoteAddr(), err) + return + } + c.tlsState = new(tls.ConnectionState) + *c.tlsState = tlsConn.ConnectionState() + if proto := c.tlsState.NegotiatedProtocol; validNPN(proto) { + if fn := c.server.TLSNextProto[proto]; fn != nil { + h := initNPNRequest{tlsConn, serverHandler{c.server}} + fn(c.server, tlsConn, h) + } + return + } + } + + for { + w, err := c.readRequest() + if c.lr.N != c.server.initialLimitedReaderSize() { + // If we read any bytes off the wire, we're active. + c.setState(c.rwc, StateActive) + } + if err != nil { + if err == errTooLarge { + // Their HTTP client may or may not be + // able to read this if we're + // responding to them and hanging up + // while they're still writing their + // request. Undefined behavior. + io.WriteString(c.rwc, "HTTP/1.1 413 Request Entity Too Large\r\n\r\n") + c.closeWriteAndWait() + break + } else if err == io.EOF { + break // Don't reply + } else if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + break // Don't reply + } + io.WriteString(c.rwc, "HTTP/1.1 400 Bad Request\r\n\r\n") + break + } + + // Expect 100 Continue support + req := w.req + if req.expectsContinue() { + if req.ProtoAtLeast(1, 1) && req.ContentLength != 0 { + // Wrap the Body reader with one that replies on the connection + req.Body = &expectContinueReader{readCloser: req.Body, resp: w} + } + req.Header.Del("Expect") + } else if req.Header.get("Expect") != "" { + w.sendExpectationFailed() + break + } + + // HTTP cannot have multiple simultaneous active requests.[*] + // Until the server replies to this request, it can't read another, + // so we might as well run the handler in this goroutine. + // [*] Not strictly true: HTTP pipelining. We could let them all process + // in parallel even if their responses need to be serialized. + serverHandler{c.server}.ServeHTTP(w, w.req) + if c.hijacked() { + return + } + w.finishRequest() + if w.closeAfterReply { + if w.requestBodyLimitHit { + c.closeWriteAndWait() + } + break + } + c.setState(c.rwc, StateIdle) + } +} + +func (w *response) sendExpectationFailed() { + // TODO(bradfitz): let ServeHTTP handlers handle + // requests with non-standard expectation[s]? Seems + // theoretical at best, and doesn't fit into the + // current ServeHTTP model anyway. We'd need to + // make the ResponseWriter an optional + // "ExpectReplier" interface or something. + // + // For now we'll just obey RFC 2616 14.20 which says + // "If a server receives a request containing an + // Expect field that includes an expectation- + // extension that it does not support, it MUST + // respond with a 417 (Expectation Failed) status." + w.Header().Set("Connection", "close") + w.WriteHeader(StatusExpectationFailed) + w.finishRequest() +} + +// Hijack implements the Hijacker.Hijack method. Our response is both a ResponseWriter +// and a Hijacker. +func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { + if w.wroteHeader { + w.cw.flush() + } + // Release the bufioWriter that writes to the chunk writer, it is not + // used after a connection has been hijacked. + rwc, buf, err = w.conn.hijack() + if err == nil { + putBufioWriter(w.w) + w.w = nil + } + return rwc, buf, err +} + +func (w *response) CloseNotify() <-chan bool { + return w.conn.closeNotify() +} + +// The HandlerFunc type is an adapter to allow the use of +// ordinary functions as HTTP handlers. If f is a function +// with the appropriate signature, HandlerFunc(f) is a +// Handler object that calls f. +type HandlerFunc func(ResponseWriter, *Request) + +// ServeHTTP calls f(w, r). +func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { + f(w, r) +} + +// Helper handlers + +// Error replies to the request with the specified error message and HTTP code. +// The error message should be plain text. +func Error(w ResponseWriter, error string, code int) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(code) + fmt.Fprintln(w, error) +} + +// NotFound replies to the request with an HTTP 404 not found error. +func NotFound(w ResponseWriter, r *Request) { Error(w, "404 page not found", StatusNotFound) } + +// NotFoundHandler returns a simple request handler +// that replies to each request with a ``404 page not found'' reply. +func NotFoundHandler() Handler { return HandlerFunc(NotFound) } + +// StripPrefix returns a handler that serves HTTP requests +// by removing the given prefix from the request URL's Path +// and invoking the handler h. StripPrefix handles a +// request for a path that doesn't begin with prefix by +// replying with an HTTP 404 not found error. +func StripPrefix(prefix string, h Handler) Handler { + if prefix == "" { + return h + } + return HandlerFunc(func(w ResponseWriter, r *Request) { + if p := strings.TrimPrefix(r.URL.Path, prefix); len(p) < len(r.URL.Path) { + r.URL.Path = p + h.ServeHTTP(w, r) + } else { + NotFound(w, r) + } + }) +} + +// Redirect replies to the request with a redirect to url, +// which may be a path relative to the request path. +func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { + if u, err := url.Parse(urlStr); err == nil { + // If url was relative, make absolute by + // combining with request path. + // The browser would probably do this for us, + // but doing it ourselves is more reliable. + + // NOTE(rsc): RFC 2616 says that the Location + // line must be an absolute URI, like + // "http://www.google.com/redirect/", + // not a path like "/redirect/". + // Unfortunately, we don't know what to + // put in the host name section to get the + // client to connect to us again, so we can't + // know the right absolute URI to send back. + // Because of this problem, no one pays attention + // to the RFC; they all send back just a new path. + // So do we. + oldpath := r.URL.Path + if oldpath == "" { // should not happen, but avoid a crash if it does + oldpath = "/" + } + if u.Scheme == "" { + // no leading http://server + if urlStr == "" || urlStr[0] != '/' { + // make relative path absolute + olddir, _ := path.Split(oldpath) + urlStr = olddir + urlStr + } + + var query string + if i := strings.Index(urlStr, "?"); i != -1 { + urlStr, query = urlStr[:i], urlStr[i:] + } + + // clean up but preserve trailing slash + trailing := strings.HasSuffix(urlStr, "/") + urlStr = path.Clean(urlStr) + if trailing && !strings.HasSuffix(urlStr, "/") { + urlStr += "/" + } + urlStr += query + } + } + + w.Header().Set("Location", urlStr) + w.WriteHeader(code) + + // RFC2616 recommends that a short note "SHOULD" be included in the + // response because older user agents may not understand 301/307. + // Shouldn't send the response for POST or HEAD; that leaves GET. + if r.Method == "GET" { + note := "<a href=\"" + htmlEscape(urlStr) + "\">" + statusText[code] + "</a>.\n" + fmt.Fprintln(w, note) + } +} + +var htmlReplacer = strings.NewReplacer( + "&", "&", + "<", "<", + ">", ">", + // """ is shorter than """. + `"`, """, + // "'" is shorter than "'" and apos was not in HTML until HTML5. + "'", "'", +) + +func htmlEscape(s string) string { + return htmlReplacer.Replace(s) +} + +// Redirect to a fixed URL +type redirectHandler struct { + url string + code int +} + +func (rh *redirectHandler) ServeHTTP(w ResponseWriter, r *Request) { + Redirect(w, r, rh.url, rh.code) +} + +// RedirectHandler returns a request handler that redirects +// each request it receives to the given url using the given +// status code. +func RedirectHandler(url string, code int) Handler { + return &redirectHandler{url, code} +} + +// ServeMux is an HTTP request multiplexer. +// It matches the URL of each incoming request against a list of registered +// patterns and calls the handler for the pattern that +// most closely matches the URL. +// +// Patterns name fixed, rooted paths, like "/favicon.ico", +// or rooted subtrees, like "/images/" (note the trailing slash). +// Longer patterns take precedence over shorter ones, so that +// if there are handlers registered for both "/images/" +// and "/images/thumbnails/", the latter handler will be +// called for paths beginning "/images/thumbnails/" and the +// former will receive requests for any other paths in the +// "/images/" subtree. +// +// Note that since a pattern ending in a slash names a rooted subtree, +// the pattern "/" matches all paths not matched by other registered +// patterns, not just the URL with Path == "/". +// +// Patterns may optionally begin with a host name, restricting matches to +// URLs on that host only. Host-specific patterns take precedence over +// general patterns, so that a handler might register for the two patterns +// "/codesearch" and "codesearch.google.com/" without also taking over +// requests for "http://www.google.com/". +// +// ServeMux also takes care of sanitizing the URL request path, +// redirecting any request containing . or .. elements to an +// equivalent .- and ..-free URL. +type ServeMux struct { + mu sync.RWMutex + m map[string]muxEntry + hosts bool // whether any patterns contain hostnames +} + +type muxEntry struct { + explicit bool + h Handler + pattern string +} + +// NewServeMux allocates and returns a new ServeMux. +func NewServeMux() *ServeMux { return &ServeMux{m: make(map[string]muxEntry)} } + +// DefaultServeMux is the default ServeMux used by Serve. +var DefaultServeMux = NewServeMux() + +// Does path match pattern? +func pathMatch(pattern, path string) bool { + if len(pattern) == 0 { + // should not happen + return false + } + n := len(pattern) + if pattern[n-1] != '/' { + return pattern == path + } + return len(path) >= n && path[0:n] == pattern +} + +// Return the canonical path for p, eliminating . and .. elements. +func cleanPath(p string) string { + if p == "" { + return "/" + } + if p[0] != '/' { + p = "/" + p + } + np := path.Clean(p) + // path.Clean removes trailing slash except for root; + // put the trailing slash back if necessary. + if p[len(p)-1] == '/' && np != "/" { + np += "/" + } + return np +} + +// Find a handler on a handler map given a path string +// Most-specific (longest) pattern wins +func (mux *ServeMux) match(path string) (h Handler, pattern string) { + var n = 0 + for k, v := range mux.m { + if !pathMatch(k, path) { + continue + } + if h == nil || len(k) > n { + n = len(k) + h = v.h + pattern = v.pattern + } + } + return +} + +// Handler returns the handler to use for the given request, +// consulting r.Method, r.Host, and r.URL.Path. It always returns +// a non-nil handler. If the path is not in its canonical form, the +// handler will be an internally-generated handler that redirects +// to the canonical path. +// +// Handler also returns the registered pattern that matches the +// request or, in the case of internally-generated redirects, +// the pattern that will match after following the redirect. +// +// If there is no registered handler that applies to the request, +// Handler returns a ``page not found'' handler and an empty pattern. +func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { + if r.Method != "CONNECT" { + if p := cleanPath(r.URL.Path); p != r.URL.Path { + _, pattern = mux.handler(r.Host, p) + url := *r.URL + url.Path = p + return RedirectHandler(url.String(), StatusMovedPermanently), pattern + } + } + + return mux.handler(r.Host, r.URL.Path) +} + +// handler is the main implementation of Handler. +// The path is known to be in canonical form, except for CONNECT methods. +func (mux *ServeMux) handler(host, path string) (h Handler, pattern string) { + mux.mu.RLock() + defer mux.mu.RUnlock() + + // Host-specific pattern takes precedence over generic ones + if mux.hosts { + h, pattern = mux.match(host + path) + } + if h == nil { + h, pattern = mux.match(path) + } + if h == nil { + h, pattern = NotFoundHandler(), "" + } + return +} + +// ServeHTTP dispatches the request to the handler whose +// pattern most closely matches the request URL. +func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { + if r.RequestURI == "*" { + if r.ProtoAtLeast(1, 1) { + w.Header().Set("Connection", "close") + } + w.WriteHeader(StatusBadRequest) + return + } + h, _ := mux.Handler(r) + h.ServeHTTP(w, r) +} + +// Handle registers the handler for the given pattern. +// If a handler already exists for pattern, Handle panics. +func (mux *ServeMux) Handle(pattern string, handler Handler) { + mux.mu.Lock() + defer mux.mu.Unlock() + + if pattern == "" { + panic("http: invalid pattern " + pattern) + } + if handler == nil { + panic("http: nil handler") + } + if mux.m[pattern].explicit { + panic("http: multiple registrations for " + pattern) + } + + mux.m[pattern] = muxEntry{explicit: true, h: handler, pattern: pattern} + + if pattern[0] != '/' { + mux.hosts = true + } + + // Helpful behavior: + // If pattern is /tree/, insert an implicit permanent redirect for /tree. + // It can be overridden by an explicit registration. + n := len(pattern) + if n > 0 && pattern[n-1] == '/' && !mux.m[pattern[0:n-1]].explicit { + // If pattern contains a host name, strip it and use remaining + // path for redirect. + path := pattern + if pattern[0] != '/' { + // In pattern, at least the last character is a '/', so + // strings.Index can't be -1. + path = pattern[strings.Index(pattern, "/"):] + } + mux.m[pattern[0:n-1]] = muxEntry{h: RedirectHandler(path, StatusMovedPermanently), pattern: pattern} + } +} + +// HandleFunc registers the handler function for the given pattern. +func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { + mux.Handle(pattern, HandlerFunc(handler)) +} + +// Handle registers the handler for the given pattern +// in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. +func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } + +// HandleFunc registers the handler function for the given pattern +// in the DefaultServeMux. +// The documentation for ServeMux explains how patterns are matched. +func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { + DefaultServeMux.HandleFunc(pattern, handler) +} + +// Serve accepts incoming HTTP connections on the listener l, +// creating a new service goroutine for each. The service goroutines +// read requests and then call handler to reply to them. +// Handler is typically nil, in which case the DefaultServeMux is used. +func Serve(l net.Listener, handler Handler) error { + srv := &Server{Handler: handler} + return srv.Serve(l) +} + +// A Server defines parameters for running an HTTP server. +// The zero value for Server is a valid configuration. +type Server struct { + Addr string // TCP address to listen on, ":http" if empty + Handler Handler // handler to invoke, http.DefaultServeMux if nil + ReadTimeout time.Duration // maximum duration before timing out read of the request + WriteTimeout time.Duration // maximum duration before timing out write of the response + MaxHeaderBytes int // maximum size of request headers, DefaultMaxHeaderBytes if 0 + TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS + + // TLSNextProto optionally specifies a function to take over + // ownership of the provided TLS connection when an NPN + // protocol upgrade has occurred. The map key is the protocol + // name negotiated. The Handler argument should be used to + // handle HTTP requests and will initialize the Request's TLS + // and RemoteAddr if not already set. The connection is + // automatically closed when the function returns. + TLSNextProto map[string]func(*Server, *tls.Conn, Handler) + + // ConnState specifies an optional callback function that is + // called when a client connection changes state. See the + // ConnState type and associated constants for details. + ConnState func(net.Conn, ConnState) + + // ErrorLog specifies an optional logger for errors accepting + // connections and unexpected behavior from handlers. + // If nil, logging goes to os.Stderr via the log package's + // standard logger. + ErrorLog *log.Logger + + disableKeepAlives int32 // accessed atomically. +} + +// A ConnState represents the state of a client connection to a server. +// It's used by the optional Server.ConnState hook. +type ConnState int + +const ( + // StateNew represents a new connection that is expected to + // send a request immediately. Connections begin at this + // state and then transition to either StateActive or + // StateClosed. + StateNew ConnState = iota + + // StateActive represents a connection that has read 1 or more + // bytes of a request. The Server.ConnState hook for + // StateActive fires before the request has entered a handler + // and doesn't fire again until the request has been + // handled. After the request is handled, the state + // transitions to StateClosed, StateHijacked, or StateIdle. + StateActive + + // StateIdle represents a connection that has finished + // handling a request and is in the keep-alive state, waiting + // for a new request. Connections transition from StateIdle + // to either StateActive or StateClosed. + StateIdle + + // StateHijacked represents a hijacked connection. + // This is a terminal state. It does not transition to StateClosed. + StateHijacked + + // StateClosed represents a closed connection. + // This is a terminal state. Hijacked connections do not + // transition to StateClosed. + StateClosed +) + +var stateName = map[ConnState]string{ + StateNew: "new", + StateActive: "active", + StateIdle: "idle", + StateHijacked: "hijacked", + StateClosed: "closed", +} + +func (c ConnState) String() string { + return stateName[c] +} + +// serverHandler delegates to either the server's Handler or +// DefaultServeMux and also handles "OPTIONS *" requests. +type serverHandler struct { + srv *Server +} + +func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) { + handler := sh.srv.Handler + if handler == nil { + handler = DefaultServeMux + } + if req.RequestURI == "*" && req.Method == "OPTIONS" { + handler = globalOptionsHandler{} + } + handler.ServeHTTP(rw, req) +} + +// ListenAndServe listens on the TCP network address srv.Addr and then +// calls Serve to handle requests on incoming connections. If +// srv.Addr is blank, ":http" is used. +func (srv *Server) ListenAndServe() error { + addr := srv.Addr + if addr == "" { + addr = ":http" + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + return srv.Serve(tcpKeepAliveListener{ln.(*net.TCPListener)}) +} + +// Serve accepts incoming connections on the Listener l, creating a +// new service goroutine for each. The service goroutines read requests and +// then call srv.Handler to reply to them. +func (srv *Server) Serve(l net.Listener) error { + defer l.Close() + var tempDelay time.Duration // how long to sleep on accept failure + for { + rw, e := l.Accept() + if e != nil { + if ne, ok := e.(net.Error); ok && ne.Temporary() { + if tempDelay == 0 { + tempDelay = 5 * time.Millisecond + } else { + tempDelay *= 2 + } + if max := 1 * time.Second; tempDelay > max { + tempDelay = max + } + srv.logf("http: Accept error: %v; retrying in %v", e, tempDelay) + time.Sleep(tempDelay) + continue + } + return e + } + tempDelay = 0 + c, err := srv.newConn(rw) + if err != nil { + continue + } + c.setState(c.rwc, StateNew) // before Serve can return + go c.serve() + } +} + +func (s *Server) doKeepAlives() bool { + return atomic.LoadInt32(&s.disableKeepAlives) == 0 +} + +// SetKeepAlivesEnabled controls whether HTTP keep-alives are enabled. +// By default, keep-alives are always enabled. Only very +// resource-constrained environments or servers in the process of +// shutting down should disable them. +func (s *Server) SetKeepAlivesEnabled(v bool) { + if v { + atomic.StoreInt32(&s.disableKeepAlives, 0) + } else { + atomic.StoreInt32(&s.disableKeepAlives, 1) + } +} + +func (s *Server) logf(format string, args ...interface{}) { + if s.ErrorLog != nil { + s.ErrorLog.Printf(format, args...) + } else { + log.Printf(format, args...) + } +} + +// ListenAndServe listens on the TCP network address addr +// and then calls Serve with handler to handle requests +// on incoming connections. Handler is typically nil, +// in which case the DefaultServeMux is used. +// +// A trivial example server is: +// +// package main +// +// import ( +// "io" +// "net/http" +// "log" +// ) +// +// // hello world, the web server +// func HelloServer(w http.ResponseWriter, req *http.Request) { +// io.WriteString(w, "hello, world!\n") +// } +// +// func main() { +// http.HandleFunc("/hello", HelloServer) +// err := http.ListenAndServe(":12345", nil) +// if err != nil { +// log.Fatal("ListenAndServe: ", err) +// } +// } +func ListenAndServe(addr string, handler Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServe() +} + +// ListenAndServeTLS acts identically to ListenAndServe, except that it +// expects HTTPS connections. Additionally, files containing a certificate and +// matching private key for the server must be provided. If the certificate +// is signed by a certificate authority, the certFile should be the concatenation +// of the server's certificate followed by the CA's certificate. +// +// A trivial example server is: +// +// import ( +// "log" +// "net/http" +// ) +// +// func handler(w http.ResponseWriter, req *http.Request) { +// w.Header().Set("Content-Type", "text/plain") +// w.Write([]byte("This is an example server.\n")) +// } +// +// func main() { +// http.HandleFunc("/", handler) +// log.Printf("About to listen on 10443. Go to https://127.0.0.1:10443/") +// err := http.ListenAndServeTLS(":10443", "cert.pem", "key.pem", nil) +// if err != nil { +// log.Fatal(err) +// } +// } +// +// One can use generate_cert.go in crypto/tls to generate cert.pem and key.pem. +func ListenAndServeTLS(addr string, certFile string, keyFile string, handler Handler) error { + server := &Server{Addr: addr, Handler: handler} + return server.ListenAndServeTLS(certFile, keyFile) +} + +// ListenAndServeTLS listens on the TCP network address srv.Addr and +// then calls Serve to handle requests on incoming TLS connections. +// +// Filenames containing a certificate and matching private key for +// the server must be provided. If the certificate is signed by a +// certificate authority, the certFile should be the concatenation +// of the server's certificate followed by the CA's certificate. +// +// If srv.Addr is blank, ":https" is used. +func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { + addr := srv.Addr + if addr == "" { + addr = ":https" + } + config := &tls.Config{} + if srv.TLSConfig != nil { + *config = *srv.TLSConfig + } + if config.NextProtos == nil { + config.NextProtos = []string{"http/1.1"} + } + + var err error + config.Certificates = make([]tls.Certificate, 1) + config.Certificates[0], err = tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return err + } + + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + + tlsListener := tls.NewListener(tcpKeepAliveListener{ln.(*net.TCPListener)}, config) + return srv.Serve(tlsListener) +} + +// TimeoutHandler returns a Handler that runs h with the given time limit. +// +// The new Handler calls h.ServeHTTP to handle each request, but if a +// call runs for longer than its time limit, the handler responds with +// a 503 Service Unavailable error and the given message in its body. +// (If msg is empty, a suitable default message will be sent.) +// After such a timeout, writes by h to its ResponseWriter will return +// ErrHandlerTimeout. +func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { + f := func() <-chan time.Time { + return time.After(dt) + } + return &timeoutHandler{h, f, msg} +} + +// ErrHandlerTimeout is returned on ResponseWriter Write calls +// in handlers which have timed out. +var ErrHandlerTimeout = errors.New("http: Handler timeout") + +type timeoutHandler struct { + handler Handler + timeout func() <-chan time.Time // returns channel producing a timeout + body string +} + +func (h *timeoutHandler) errorBody() string { + if h.body != "" { + return h.body + } + return "<html><head><title>Timeout</title></head><body><h1>Timeout</h1></body></html>" +} + +func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { + done := make(chan bool, 1) + tw := &timeoutWriter{w: w} + go func() { + h.handler.ServeHTTP(tw, r) + done <- true + }() + select { + case <-done: + return + case <-h.timeout(): + tw.mu.Lock() + defer tw.mu.Unlock() + if !tw.wroteHeader { + tw.w.WriteHeader(StatusServiceUnavailable) + tw.w.Write([]byte(h.errorBody())) + } + tw.timedOut = true + } +} + +type timeoutWriter struct { + w ResponseWriter + + mu sync.Mutex + timedOut bool + wroteHeader bool +} + +func (tw *timeoutWriter) Header() Header { + return tw.w.Header() +} + +func (tw *timeoutWriter) Write(p []byte) (int, error) { + tw.mu.Lock() + defer tw.mu.Unlock() + tw.wroteHeader = true // implicitly at least + if tw.timedOut { + return 0, ErrHandlerTimeout + } + return tw.w.Write(p) +} + +func (tw *timeoutWriter) WriteHeader(code int) { + tw.mu.Lock() + defer tw.mu.Unlock() + if tw.timedOut || tw.wroteHeader { + return + } + tw.wroteHeader = true + tw.w.WriteHeader(code) +} + +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by ListenAndServe and ListenAndServeTLS so +// dead TCP connections (e.g. closing laptop mid-download) eventually +// go away. +type tcpKeepAliveListener struct { + *net.TCPListener +} + +func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) { + tc, err := ln.AcceptTCP() + if err != nil { + return + } + tc.SetKeepAlive(true) + tc.SetKeepAlivePeriod(3 * time.Minute) + return tc, nil +} + +// globalOptionsHandler responds to "OPTIONS *" requests. +type globalOptionsHandler struct{} + +func (globalOptionsHandler) ServeHTTP(w ResponseWriter, r *Request) { + w.Header().Set("Content-Length", "0") + if r.ContentLength != 0 { + // Read up to 4KB of OPTIONS body (as mentioned in the + // spec as being reserved for future use), but anything + // over that is considered a waste of server resources + // (or an attack) and we abort and close the connection, + // courtesy of MaxBytesReader's EOF behavior. + mb := MaxBytesReader(w, r.Body, 4<<10) + io.Copy(ioutil.Discard, mb) + } +} + +type eofReaderWithWriteTo struct{} + +func (eofReaderWithWriteTo) WriteTo(io.Writer) (int64, error) { return 0, nil } +func (eofReaderWithWriteTo) Read([]byte) (int, error) { return 0, io.EOF } + +// eofReader is a non-nil io.ReadCloser that always returns EOF. +// It has a WriteTo method so io.Copy won't need a buffer. +var eofReader = &struct { + eofReaderWithWriteTo + io.Closer +}{ + eofReaderWithWriteTo{}, + ioutil.NopCloser(nil), +} + +// Verify that an io.Copy from an eofReader won't require a buffer. +var _ io.WriterTo = eofReader + +// initNPNRequest is an HTTP handler that initializes certain +// uninitialized fields in its *Request. Such partially-initialized +// Requests come from NPN protocol handlers. +type initNPNRequest struct { + c *tls.Conn + h serverHandler +} + +func (h initNPNRequest) ServeHTTP(rw ResponseWriter, req *Request) { + if req.TLS == nil { + req.TLS = &tls.ConnectionState{} + *req.TLS = h.c.ConnectionState() + } + if req.Body == nil { + req.Body = eofReader + } + if req.RemoteAddr == "" { + req.RemoteAddr = h.c.RemoteAddr().String() + } + h.h.ServeHTTP(rw, req) +} + +// loggingConn is used for debugging. +type loggingConn struct { + name string + net.Conn +} + +var ( + uniqNameMu sync.Mutex + uniqNameNext = make(map[string]int) +) + +func newLoggingConn(baseName string, c net.Conn) net.Conn { + uniqNameMu.Lock() + defer uniqNameMu.Unlock() + uniqNameNext[baseName]++ + return &loggingConn{ + name: fmt.Sprintf("%s-%d", baseName, uniqNameNext[baseName]), + Conn: c, + } +} + +func (c *loggingConn) Write(p []byte) (n int, err error) { + log.Printf("%s.Write(%d) = ....", c.name, len(p)) + n, err = c.Conn.Write(p) + log.Printf("%s.Write(%d) = %d, %v", c.name, len(p), n, err) + return +} + +func (c *loggingConn) Read(p []byte) (n int, err error) { + log.Printf("%s.Read(%d) = ....", c.name, len(p)) + n, err = c.Conn.Read(p) + log.Printf("%s.Read(%d) = %d, %v", c.name, len(p), n, err) + return +} + +func (c *loggingConn) Close() (err error) { + log.Printf("%s.Close() = ...", c.name) + err = c.Conn.Close() + log.Printf("%s.Close() = %v", c.name, err) + return +} diff --git a/src/net/http/sniff.go b/src/net/http/sniff.go new file mode 100644 index 000000000..68f519b05 --- /dev/null +++ b/src/net/http/sniff.go @@ -0,0 +1,214 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "encoding/binary" +) + +// The algorithm uses at most sniffLen bytes to make its decision. +const sniffLen = 512 + +// DetectContentType implements the algorithm described +// at http://mimesniff.spec.whatwg.org/ to determine the +// Content-Type of the given data. It considers at most the +// first 512 bytes of data. DetectContentType always returns +// a valid MIME type: if it cannot determine a more specific one, it +// returns "application/octet-stream". +func DetectContentType(data []byte) string { + if len(data) > sniffLen { + data = data[:sniffLen] + } + + // Index of the first non-whitespace byte in data. + firstNonWS := 0 + for ; firstNonWS < len(data) && isWS(data[firstNonWS]); firstNonWS++ { + } + + for _, sig := range sniffSignatures { + if ct := sig.match(data, firstNonWS); ct != "" { + return ct + } + } + + return "application/octet-stream" // fallback +} + +func isWS(b byte) bool { + return bytes.IndexByte([]byte("\t\n\x0C\r "), b) != -1 +} + +type sniffSig interface { + // match returns the MIME type of the data, or "" if unknown. + match(data []byte, firstNonWS int) string +} + +// Data matching the table in section 6. +var sniffSignatures = []sniffSig{ + htmlSig("<!DOCTYPE HTML"), + htmlSig("<HTML"), + htmlSig("<HEAD"), + htmlSig("<SCRIPT"), + htmlSig("<IFRAME"), + htmlSig("<H1"), + htmlSig("<DIV"), + htmlSig("<FONT"), + htmlSig("<TABLE"), + htmlSig("<A"), + htmlSig("<STYLE"), + htmlSig("<TITLE"), + htmlSig("<B"), + htmlSig("<BODY"), + htmlSig("<BR"), + htmlSig("<P"), + htmlSig("<!--"), + + &maskedSig{mask: []byte("\xFF\xFF\xFF\xFF\xFF"), pat: []byte("<?xml"), skipWS: true, ct: "text/xml; charset=utf-8"}, + + &exactSig{[]byte("%PDF-"), "application/pdf"}, + &exactSig{[]byte("%!PS-Adobe-"), "application/postscript"}, + + // UTF BOMs. + &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFE\xFF\x00\x00"), ct: "text/plain; charset=utf-16be"}, + &maskedSig{mask: []byte("\xFF\xFF\x00\x00"), pat: []byte("\xFF\xFE\x00\x00"), ct: "text/plain; charset=utf-16le"}, + &maskedSig{mask: []byte("\xFF\xFF\xFF\x00"), pat: []byte("\xEF\xBB\xBF\x00"), ct: "text/plain; charset=utf-8"}, + + &exactSig{[]byte("GIF87a"), "image/gif"}, + &exactSig{[]byte("GIF89a"), "image/gif"}, + &exactSig{[]byte("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), "image/png"}, + &exactSig{[]byte("\xFF\xD8\xFF"), "image/jpeg"}, + &exactSig{[]byte("BM"), "image/bmp"}, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF"), + pat: []byte("RIFF\x00\x00\x00\x00WEBPVP"), + ct: "image/webp", + }, + &exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"}, + &exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"}, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), + pat: []byte("RIFF\x00\x00\x00\x00WAVE"), + ct: "audio/wave", + }, + &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"}, + &exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"}, + &exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"}, + &exactSig{[]byte("\x1F\x8B\x08"), "application/x-gzip"}, + + // TODO(dsymonds): Re-enable this when the spec is sorted w.r.t. MP4. + //mp4Sig(0), + + textSig(0), // should be last +} + +type exactSig struct { + sig []byte + ct string +} + +func (e *exactSig) match(data []byte, firstNonWS int) string { + if bytes.HasPrefix(data, e.sig) { + return e.ct + } + return "" +} + +type maskedSig struct { + mask, pat []byte + skipWS bool + ct string +} + +func (m *maskedSig) match(data []byte, firstNonWS int) string { + if m.skipWS { + data = data[firstNonWS:] + } + if len(data) < len(m.mask) { + return "" + } + for i, mask := range m.mask { + db := data[i] & mask + if db != m.pat[i] { + return "" + } + } + return m.ct +} + +type htmlSig []byte + +func (h htmlSig) match(data []byte, firstNonWS int) string { + data = data[firstNonWS:] + if len(data) < len(h)+1 { + return "" + } + for i, b := range h { + db := data[i] + if 'A' <= b && b <= 'Z' { + db &= 0xDF + } + if b != db { + return "" + } + } + // Next byte must be space or right angle bracket. + if db := data[len(h)]; db != ' ' && db != '>' { + return "" + } + return "text/html; charset=utf-8" +} + +type mp4Sig int + +func (mp4Sig) match(data []byte, firstNonWS int) string { + // c.f. section 6.1. + if len(data) < 8 { + return "" + } + boxSize := int(binary.BigEndian.Uint32(data[:4])) + if boxSize%4 != 0 || len(data) < boxSize { + return "" + } + if !bytes.Equal(data[4:8], []byte("ftyp")) { + return "" + } + for st := 8; st < boxSize; st += 4 { + if st == 12 { + // minor version number + continue + } + seg := string(data[st : st+3]) + switch seg { + case "mp4", "iso", "M4V", "M4P", "M4B": + return "video/mp4" + /* The remainder are not in the spec. + case "M4A": + return "audio/mp4" + case "3gp": + return "video/3gpp" + case "jp2": + return "image/jp2" // JPEG 2000 + */ + } + } + return "" +} + +type textSig int + +func (textSig) match(data []byte, firstNonWS int) string { + // c.f. section 5, step 4. + for _, b := range data[firstNonWS:] { + switch { + case 0x00 <= b && b <= 0x08, + b == 0x0B, + 0x0E <= b && b <= 0x1A, + 0x1C <= b && b <= 0x1F: + return "" + } + } + return "text/plain; charset=utf-8" +} diff --git a/src/net/http/sniff_test.go b/src/net/http/sniff_test.go new file mode 100644 index 000000000..24ca27afc --- /dev/null +++ b/src/net/http/sniff_test.go @@ -0,0 +1,171 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "log" + . "net/http" + "net/http/httptest" + "reflect" + "strconv" + "strings" + "testing" +) + +var sniffTests = []struct { + desc string + data []byte + contentType string +}{ + // Some nonsense. + {"Empty", []byte{}, "text/plain; charset=utf-8"}, + {"Binary", []byte{1, 2, 3}, "application/octet-stream"}, + + {"HTML document #1", []byte(`<HtMl><bOdY>blah blah blah</body></html>`), "text/html; charset=utf-8"}, + {"HTML document #2", []byte(`<HTML></HTML>`), "text/html; charset=utf-8"}, + {"HTML document #3 (leading whitespace)", []byte(` <!DOCTYPE HTML>...`), "text/html; charset=utf-8"}, + {"HTML document #4 (leading CRLF)", []byte("\r\n<html>..."), "text/html; charset=utf-8"}, + + {"Plain text", []byte(`This is not HTML. It has ☃ though.`), "text/plain; charset=utf-8"}, + + {"XML", []byte("\n<?xml!"), "text/xml; charset=utf-8"}, + + // Image types. + {"GIF 87a", []byte(`GIF87a`), "image/gif"}, + {"GIF 89a", []byte(`GIF89a...`), "image/gif"}, + + // TODO(dsymonds): Re-enable this when the spec is sorted w.r.t. MP4. + //{"MP4 video", []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom<\x06t\xbfmdat"), "video/mp4"}, + //{"MP4 audio", []byte("\x00\x00\x00\x20ftypM4A \x00\x00\x00\x00M4A mp42isom\x00\x00\x00\x00"), "audio/mp4"}, +} + +func TestDetectContentType(t *testing.T) { + for _, tt := range sniffTests { + ct := DetectContentType(tt.data) + if ct != tt.contentType { + t.Errorf("%v: DetectContentType = %q, want %q", tt.desc, ct, tt.contentType) + } + } +} + +func TestServerContentType(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + i, _ := strconv.Atoi(r.FormValue("i")) + tt := sniffTests[i] + n, err := w.Write(tt.data) + if n != len(tt.data) || err != nil { + log.Fatalf("%v: Write(%q) = %v, %v want %d, nil", tt.desc, tt.data, n, err, len(tt.data)) + } + })) + defer ts.Close() + + for i, tt := range sniffTests { + resp, err := Get(ts.URL + "/?i=" + strconv.Itoa(i)) + if err != nil { + t.Errorf("%v: %v", tt.desc, err) + continue + } + if ct := resp.Header.Get("Content-Type"); ct != tt.contentType { + t.Errorf("%v: Content-Type = %q, want %q", tt.desc, ct, tt.contentType) + } + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("%v: reading body: %v", tt.desc, err) + } else if !bytes.Equal(data, tt.data) { + t.Errorf("%v: data is %q, want %q", tt.desc, data, tt.data) + } + resp.Body.Close() + } +} + +// Issue 5953: shouldn't sniff if the handler set a Content-Type header, +// even if it's the empty string. +func TestServerIssue5953(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header()["Content-Type"] = []string{""} + fmt.Fprintf(w, "<html><head></head><body>hi</body></html>") + })) + defer ts.Close() + + resp, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + + got := resp.Header["Content-Type"] + want := []string{""} + if !reflect.DeepEqual(got, want) { + t.Errorf("Content-Type = %q; want %q", got, want) + } + resp.Body.Close() +} + +func TestContentTypeWithCopy(t *testing.T) { + defer afterTest(t) + + const ( + input = "\n<html>\n\t<head>\n" + expected = "text/html; charset=utf-8" + ) + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + // Use io.Copy from a bytes.Buffer to trigger ReadFrom. + buf := bytes.NewBuffer([]byte(input)) + n, err := io.Copy(w, buf) + if int(n) != len(input) || err != nil { + t.Errorf("io.Copy(w, %q) = %v, %v want %d, nil", input, n, err, len(input)) + } + })) + defer ts.Close() + + resp, err := Get(ts.URL) + if err != nil { + t.Fatalf("Get: %v", err) + } + if ct := resp.Header.Get("Content-Type"); ct != expected { + t.Errorf("Content-Type = %q, want %q", ct, expected) + } + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("reading body: %v", err) + } else if !bytes.Equal(data, []byte(input)) { + t.Errorf("data is %q, want %q", data, input) + } + resp.Body.Close() +} + +func TestSniffWriteSize(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + size, _ := strconv.Atoi(r.FormValue("size")) + written, err := io.WriteString(w, strings.Repeat("a", size)) + if err != nil { + t.Errorf("write of %d bytes: %v", size, err) + return + } + if written != size { + t.Errorf("write of %d bytes wrote %d bytes", size, written) + } + })) + defer ts.Close() + for _, size := range []int{0, 1, 200, 600, 999, 1000, 1023, 1024, 512 << 10, 1 << 20} { + res, err := Get(fmt.Sprintf("%s/?size=%d", ts.URL, size)) + if err != nil { + t.Fatalf("size %d: %v", size, err) + } + if _, err := io.Copy(ioutil.Discard, res.Body); err != nil { + t.Fatalf("size %d: io.Copy of body = %v", size, err) + } + if err := res.Body.Close(); err != nil { + t.Fatalf("size %d: body Close = %v", size, err) + } + } +} diff --git a/src/net/http/status.go b/src/net/http/status.go new file mode 100644 index 000000000..d253bd5cb --- /dev/null +++ b/src/net/http/status.go @@ -0,0 +1,120 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +// HTTP status codes, defined in RFC 2616. +const ( + StatusContinue = 100 + StatusSwitchingProtocols = 101 + + StatusOK = 200 + StatusCreated = 201 + StatusAccepted = 202 + StatusNonAuthoritativeInfo = 203 + StatusNoContent = 204 + StatusResetContent = 205 + StatusPartialContent = 206 + + StatusMultipleChoices = 300 + StatusMovedPermanently = 301 + StatusFound = 302 + StatusSeeOther = 303 + StatusNotModified = 304 + StatusUseProxy = 305 + StatusTemporaryRedirect = 307 + + StatusBadRequest = 400 + StatusUnauthorized = 401 + StatusPaymentRequired = 402 + StatusForbidden = 403 + StatusNotFound = 404 + StatusMethodNotAllowed = 405 + StatusNotAcceptable = 406 + StatusProxyAuthRequired = 407 + StatusRequestTimeout = 408 + StatusConflict = 409 + StatusGone = 410 + StatusLengthRequired = 411 + StatusPreconditionFailed = 412 + StatusRequestEntityTooLarge = 413 + StatusRequestURITooLong = 414 + StatusUnsupportedMediaType = 415 + StatusRequestedRangeNotSatisfiable = 416 + StatusExpectationFailed = 417 + StatusTeapot = 418 + + StatusInternalServerError = 500 + StatusNotImplemented = 501 + StatusBadGateway = 502 + StatusServiceUnavailable = 503 + StatusGatewayTimeout = 504 + StatusHTTPVersionNotSupported = 505 + + // New HTTP status codes from RFC 6585. Not exported yet in Go 1.1. + // See discussion at https://codereview.appspot.com/7678043/ + statusPreconditionRequired = 428 + statusTooManyRequests = 429 + statusRequestHeaderFieldsTooLarge = 431 + statusNetworkAuthenticationRequired = 511 +) + +var statusText = map[int]string{ + StatusContinue: "Continue", + StatusSwitchingProtocols: "Switching Protocols", + + StatusOK: "OK", + StatusCreated: "Created", + StatusAccepted: "Accepted", + StatusNonAuthoritativeInfo: "Non-Authoritative Information", + StatusNoContent: "No Content", + StatusResetContent: "Reset Content", + StatusPartialContent: "Partial Content", + + StatusMultipleChoices: "Multiple Choices", + StatusMovedPermanently: "Moved Permanently", + StatusFound: "Found", + StatusSeeOther: "See Other", + StatusNotModified: "Not Modified", + StatusUseProxy: "Use Proxy", + StatusTemporaryRedirect: "Temporary Redirect", + + StatusBadRequest: "Bad Request", + StatusUnauthorized: "Unauthorized", + StatusPaymentRequired: "Payment Required", + StatusForbidden: "Forbidden", + StatusNotFound: "Not Found", + StatusMethodNotAllowed: "Method Not Allowed", + StatusNotAcceptable: "Not Acceptable", + StatusProxyAuthRequired: "Proxy Authentication Required", + StatusRequestTimeout: "Request Timeout", + StatusConflict: "Conflict", + StatusGone: "Gone", + StatusLengthRequired: "Length Required", + StatusPreconditionFailed: "Precondition Failed", + StatusRequestEntityTooLarge: "Request Entity Too Large", + StatusRequestURITooLong: "Request URI Too Long", + StatusUnsupportedMediaType: "Unsupported Media Type", + StatusRequestedRangeNotSatisfiable: "Requested Range Not Satisfiable", + StatusExpectationFailed: "Expectation Failed", + StatusTeapot: "I'm a teapot", + + StatusInternalServerError: "Internal Server Error", + StatusNotImplemented: "Not Implemented", + StatusBadGateway: "Bad Gateway", + StatusServiceUnavailable: "Service Unavailable", + StatusGatewayTimeout: "Gateway Timeout", + StatusHTTPVersionNotSupported: "HTTP Version Not Supported", + + statusPreconditionRequired: "Precondition Required", + statusTooManyRequests: "Too Many Requests", + statusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", + statusNetworkAuthenticationRequired: "Network Authentication Required", +} + +// StatusText returns a text for the HTTP status code. It returns the empty +// string if the code is unknown. +func StatusText(code int) string { + return statusText[code] +} diff --git a/src/net/http/testdata/file b/src/net/http/testdata/file new file mode 100644 index 000000000..11f11f9be --- /dev/null +++ b/src/net/http/testdata/file @@ -0,0 +1 @@ +0123456789 diff --git a/src/net/http/testdata/index.html b/src/net/http/testdata/index.html new file mode 100644 index 000000000..da8e1e93d --- /dev/null +++ b/src/net/http/testdata/index.html @@ -0,0 +1 @@ +index.html says hello diff --git a/src/net/http/testdata/style.css b/src/net/http/testdata/style.css new file mode 100644 index 000000000..208d16d42 --- /dev/null +++ b/src/net/http/testdata/style.css @@ -0,0 +1 @@ +body {} diff --git a/src/net/http/transfer.go b/src/net/http/transfer.go new file mode 100644 index 000000000..520500330 --- /dev/null +++ b/src/net/http/transfer.go @@ -0,0 +1,737 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http/internal" + "net/textproto" + "sort" + "strconv" + "strings" + "sync" +) + +// ErrLineTooLong is returned when reading request or response bodies +// with malformed chunked encoding. +var ErrLineTooLong = internal.ErrLineTooLong + +type errorReader struct { + err error +} + +func (r *errorReader) Read(p []byte) (n int, err error) { + return 0, r.err +} + +// transferWriter inspects the fields of a user-supplied Request or Response, +// sanitizes them without changing the user object and provides methods for +// writing the respective header, body and trailer in wire format. +type transferWriter struct { + Method string + Body io.Reader + BodyCloser io.Closer + ResponseToHEAD bool + ContentLength int64 // -1 means unknown, 0 means exactly none + Close bool + TransferEncoding []string + Trailer Header +} + +func newTransferWriter(r interface{}) (t *transferWriter, err error) { + t = &transferWriter{} + + // Extract relevant fields + atLeastHTTP11 := false + switch rr := r.(type) { + case *Request: + if rr.ContentLength != 0 && rr.Body == nil { + return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) + } + t.Method = rr.Method + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = rr.ContentLength + t.Close = rr.Close + t.TransferEncoding = rr.TransferEncoding + t.Trailer = rr.Trailer + atLeastHTTP11 = rr.ProtoAtLeast(1, 1) + if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 { + if t.ContentLength == 0 { + // Test to see if it's actually zero or just unset. + var buf [1]byte + n, rerr := io.ReadFull(t.Body, buf[:]) + if rerr != nil && rerr != io.EOF { + t.ContentLength = -1 + t.Body = &errorReader{rerr} + } else if n == 1 { + // Oh, guess there is data in this Body Reader after all. + // The ContentLength field just wasn't set. + // Stich the Body back together again, re-attaching our + // consumed byte. + t.ContentLength = -1 + t.Body = io.MultiReader(bytes.NewReader(buf[:]), t.Body) + } else { + // Body is actually empty. + t.Body = nil + t.BodyCloser = nil + } + } + if t.ContentLength < 0 { + t.TransferEncoding = []string{"chunked"} + } + } + case *Response: + if rr.Request != nil { + t.Method = rr.Request.Method + } + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = rr.ContentLength + t.Close = rr.Close + t.TransferEncoding = rr.TransferEncoding + t.Trailer = rr.Trailer + atLeastHTTP11 = rr.ProtoAtLeast(1, 1) + t.ResponseToHEAD = noBodyExpected(t.Method) + } + + // Sanitize Body,ContentLength,TransferEncoding + if t.ResponseToHEAD { + t.Body = nil + if chunked(t.TransferEncoding) { + t.ContentLength = -1 + } + } else { + if !atLeastHTTP11 || t.Body == nil { + t.TransferEncoding = nil + } + if chunked(t.TransferEncoding) { + t.ContentLength = -1 + } else if t.Body == nil { // no chunking, no body + t.ContentLength = 0 + } + } + + // Sanitize Trailer + if !chunked(t.TransferEncoding) { + t.Trailer = nil + } + + return t, nil +} + +func noBodyExpected(requestMethod string) bool { + return requestMethod == "HEAD" +} + +func (t *transferWriter) shouldSendContentLength() bool { + if chunked(t.TransferEncoding) { + return false + } + if t.ContentLength > 0 { + return true + } + // Many servers expect a Content-Length for these methods + if t.Method == "POST" || t.Method == "PUT" { + return true + } + if t.ContentLength == 0 && isIdentity(t.TransferEncoding) { + return true + } + + return false +} + +func (t *transferWriter) WriteHeader(w io.Writer) error { + if t.Close { + if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil { + return err + } + } + + // Write Content-Length and/or Transfer-Encoding whose values are a + // function of the sanitized field triple (Body, ContentLength, + // TransferEncoding) + if t.shouldSendContentLength() { + if _, err := io.WriteString(w, "Content-Length: "); err != nil { + return err + } + if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil { + return err + } + } else if chunked(t.TransferEncoding) { + if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil { + return err + } + } + + // Write Trailer header + if t.Trailer != nil { + keys := make([]string, 0, len(t.Trailer)) + for k := range t.Trailer { + k = CanonicalHeaderKey(k) + switch k { + case "Transfer-Encoding", "Trailer", "Content-Length": + return &badStringError{"invalid Trailer key", k} + } + keys = append(keys, k) + } + if len(keys) > 0 { + sort.Strings(keys) + // TODO: could do better allocation-wise here, but trailers are rare, + // so being lazy for now. + if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil { + return err + } + } + } + + return nil +} + +func (t *transferWriter) WriteBody(w io.Writer) error { + var err error + var ncopy int64 + + // Write body + if t.Body != nil { + if chunked(t.TransferEncoding) { + cw := internal.NewChunkedWriter(w) + _, err = io.Copy(cw, t.Body) + if err == nil { + err = cw.Close() + } + } else if t.ContentLength == -1 { + ncopy, err = io.Copy(w, t.Body) + } else { + ncopy, err = io.Copy(w, io.LimitReader(t.Body, t.ContentLength)) + if err != nil { + return err + } + var nextra int64 + nextra, err = io.Copy(ioutil.Discard, t.Body) + ncopy += nextra + } + if err != nil { + return err + } + if err = t.BodyCloser.Close(); err != nil { + return err + } + } + + if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy { + return fmt.Errorf("http: ContentLength=%d with Body length %d", + t.ContentLength, ncopy) + } + + // TODO(petar): Place trailer writer code here. + if chunked(t.TransferEncoding) { + // Write Trailer header + if t.Trailer != nil { + if err := t.Trailer.Write(w); err != nil { + return err + } + } + // Last chunk, empty trailer + _, err = io.WriteString(w, "\r\n") + } + return err +} + +type transferReader struct { + // Input + Header Header + StatusCode int + RequestMethod string + ProtoMajor int + ProtoMinor int + // Output + Body io.ReadCloser + ContentLength int64 + TransferEncoding []string + Close bool + Trailer Header +} + +// bodyAllowedForStatus reports whether a given response status code +// permits a body. See RFC2616, section 4.4. +func bodyAllowedForStatus(status int) bool { + switch { + case status >= 100 && status <= 199: + return false + case status == 204: + return false + case status == 304: + return false + } + return true +} + +var ( + suppressedHeaders304 = []string{"Content-Type", "Content-Length", "Transfer-Encoding"} + suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"} +) + +func suppressedHeaders(status int) []string { + switch { + case status == 304: + // RFC 2616 section 10.3.5: "the response MUST NOT include other entity-headers" + return suppressedHeaders304 + case !bodyAllowedForStatus(status): + return suppressedHeadersNoBody + } + return nil +} + +// msg is *Request or *Response. +func readTransfer(msg interface{}, r *bufio.Reader) (err error) { + t := &transferReader{RequestMethod: "GET"} + + // Unify input + isResponse := false + switch rr := msg.(type) { + case *Response: + t.Header = rr.Header + t.StatusCode = rr.StatusCode + t.ProtoMajor = rr.ProtoMajor + t.ProtoMinor = rr.ProtoMinor + t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true) + isResponse = true + if rr.Request != nil { + t.RequestMethod = rr.Request.Method + } + case *Request: + t.Header = rr.Header + t.ProtoMajor = rr.ProtoMajor + t.ProtoMinor = rr.ProtoMinor + // Transfer semantics for Requests are exactly like those for + // Responses with status code 200, responding to a GET method + t.StatusCode = 200 + default: + panic("unexpected type") + } + + // Default to HTTP/1.1 + if t.ProtoMajor == 0 && t.ProtoMinor == 0 { + t.ProtoMajor, t.ProtoMinor = 1, 1 + } + + // Transfer encoding, content length + t.TransferEncoding, err = fixTransferEncoding(t.RequestMethod, t.Header) + if err != nil { + return err + } + + realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.TransferEncoding) + if err != nil { + return err + } + if isResponse && t.RequestMethod == "HEAD" { + if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { + return err + } else { + t.ContentLength = n + } + } else { + t.ContentLength = realLength + } + + // Trailer + t.Trailer, err = fixTrailer(t.Header, t.TransferEncoding) + if err != nil { + return err + } + + // If there is no Content-Length or chunked Transfer-Encoding on a *Response + // and the status is not 1xx, 204 or 304, then the body is unbounded. + // See RFC2616, section 4.4. + switch msg.(type) { + case *Response: + if realLength == -1 && + !chunked(t.TransferEncoding) && + bodyAllowedForStatus(t.StatusCode) { + // Unbounded body. + t.Close = true + } + } + + // Prepare body reader. ContentLength < 0 means chunked encoding + // or close connection when finished, since multipart is not supported yet + switch { + case chunked(t.TransferEncoding): + if noBodyExpected(t.RequestMethod) { + t.Body = eofReader + } else { + t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} + } + case realLength == 0: + t.Body = eofReader + case realLength > 0: + t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} + default: + // realLength < 0, i.e. "Content-Length" not mentioned in header + if t.Close { + // Close semantics (i.e. HTTP/1.0) + t.Body = &body{src: r, closing: t.Close} + } else { + // Persistent connection (i.e. HTTP/1.1) + t.Body = eofReader + } + } + + // Unify output + switch rr := msg.(type) { + case *Request: + rr.Body = t.Body + rr.ContentLength = t.ContentLength + rr.TransferEncoding = t.TransferEncoding + rr.Close = t.Close + rr.Trailer = t.Trailer + case *Response: + rr.Body = t.Body + rr.ContentLength = t.ContentLength + rr.TransferEncoding = t.TransferEncoding + rr.Close = t.Close + rr.Trailer = t.Trailer + } + + return nil +} + +// Checks whether chunked is part of the encodings stack +func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" } + +// Checks whether the encoding is explicitly "identity". +func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" } + +// Sanitize transfer encoding +func fixTransferEncoding(requestMethod string, header Header) ([]string, error) { + raw, present := header["Transfer-Encoding"] + if !present { + return nil, nil + } + + delete(header, "Transfer-Encoding") + + encodings := strings.Split(raw[0], ",") + te := make([]string, 0, len(encodings)) + // TODO: Even though we only support "identity" and "chunked" + // encodings, the loop below is designed with foresight. One + // invariant that must be maintained is that, if present, + // chunked encoding must always come first. + for _, encoding := range encodings { + encoding = strings.ToLower(strings.TrimSpace(encoding)) + // "identity" encoding is not recorded + if encoding == "identity" { + break + } + if encoding != "chunked" { + return nil, &badStringError{"unsupported transfer encoding", encoding} + } + te = te[0 : len(te)+1] + te[len(te)-1] = encoding + } + if len(te) > 1 { + return nil, &badStringError{"too many transfer encodings", strings.Join(te, ",")} + } + if len(te) > 0 { + // Chunked encoding trumps Content-Length. See RFC 2616 + // Section 4.4. Currently len(te) > 0 implies chunked + // encoding. + delete(header, "Content-Length") + return te, nil + } + + return nil, nil +} + +// Determine the expected body length, using RFC 2616 Section 4.4. This +// function is not a method, because ultimately it should be shared by +// ReadResponse and ReadRequest. +func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { + + // Logic based on response type or status + if noBodyExpected(requestMethod) { + return 0, nil + } + if status/100 == 1 { + return 0, nil + } + switch status { + case 204, 304: + return 0, nil + } + + // Logic based on Transfer-Encoding + if chunked(te) { + return -1, nil + } + + // Logic based on Content-Length + cl := strings.TrimSpace(header.get("Content-Length")) + if cl != "" { + n, err := parseContentLength(cl) + if err != nil { + return -1, err + } + return n, nil + } else { + header.Del("Content-Length") + } + + if !isResponse && requestMethod == "GET" { + // RFC 2616 doesn't explicitly permit nor forbid an + // entity-body on a GET request so we permit one if + // declared, but we default to 0 here (not -1 below) + // if there's no mention of a body. + return 0, nil + } + + // Body-EOF logic based on other methods (like closing, or chunked coding) + return -1, nil +} + +// Determine whether to hang up after sending a request and body, or +// receiving a response and body +// 'header' is the request headers +func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { + if major < 1 { + return true + } else if major == 1 && minor == 0 { + if !strings.Contains(strings.ToLower(header.get("Connection")), "keep-alive") { + return true + } + return false + } else { + // TODO: Should split on commas, toss surrounding white space, + // and check each field. + if strings.ToLower(header.get("Connection")) == "close" { + if removeCloseHeader { + header.Del("Connection") + } + return true + } + } + return false +} + +// Parse the trailer header +func fixTrailer(header Header, te []string) (Header, error) { + raw := header.get("Trailer") + if raw == "" { + return nil, nil + } + + header.Del("Trailer") + trailer := make(Header) + keys := strings.Split(raw, ",") + for _, key := range keys { + key = CanonicalHeaderKey(strings.TrimSpace(key)) + switch key { + case "Transfer-Encoding", "Trailer", "Content-Length": + return nil, &badStringError{"bad trailer key", key} + } + trailer[key] = nil + } + if len(trailer) == 0 { + return nil, nil + } + if !chunked(te) { + // Trailer and no chunking + return nil, ErrUnexpectedTrailer + } + return trailer, nil +} + +// body turns a Reader into a ReadCloser. +// Close ensures that the body has been fully read +// and then reads the trailer if necessary. +type body struct { + src io.Reader + hdr interface{} // non-nil (Response or Request) value means read trailer + r *bufio.Reader // underlying wire-format reader for the trailer + closing bool // is the connection to be closed after reading body? + + mu sync.Mutex // guards closed, and calls to Read and Close + closed bool +} + +// ErrBodyReadAfterClose is returned when reading a Request or Response +// Body after the body has been closed. This typically happens when the body is +// read after an HTTP Handler calls WriteHeader or Write on its +// ResponseWriter. +var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") + +func (b *body) Read(p []byte) (n int, err error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return 0, ErrBodyReadAfterClose + } + return b.readLocked(p) +} + +// Must hold b.mu. +func (b *body) readLocked(p []byte) (n int, err error) { + n, err = b.src.Read(p) + + if err == io.EOF { + // Chunked case. Read the trailer. + if b.hdr != nil { + if e := b.readTrailer(); e != nil { + err = e + } + b.hdr = nil + } else { + // If the server declared the Content-Length, our body is a LimitedReader + // and we need to check whether this EOF arrived early. + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 { + err = io.ErrUnexpectedEOF + } + } + } + + // If we can return an EOF here along with the read data, do + // so. This is optional per the io.Reader contract, but doing + // so helps the HTTP transport code recycle its connection + // earlier (since it will see this EOF itself), even if the + // client doesn't do future reads or Close. + if err == nil && n > 0 { + if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 { + err = io.EOF + } + } + + return n, err +} + +var ( + singleCRLF = []byte("\r\n") + doubleCRLF = []byte("\r\n\r\n") +) + +func seeUpcomingDoubleCRLF(r *bufio.Reader) bool { + for peekSize := 4; ; peekSize++ { + // This loop stops when Peek returns an error, + // which it does when r's buffer has been filled. + buf, err := r.Peek(peekSize) + if bytes.HasSuffix(buf, doubleCRLF) { + return true + } + if err != nil { + break + } + } + return false +} + +var errTrailerEOF = errors.New("http: unexpected EOF reading trailer") + +func (b *body) readTrailer() error { + // The common case, since nobody uses trailers. + buf, err := b.r.Peek(2) + if bytes.Equal(buf, singleCRLF) { + b.r.ReadByte() + b.r.ReadByte() + return nil + } + if len(buf) < 2 { + return errTrailerEOF + } + if err != nil { + return err + } + + // Make sure there's a header terminator coming up, to prevent + // a DoS with an unbounded size Trailer. It's not easy to + // slip in a LimitReader here, as textproto.NewReader requires + // a concrete *bufio.Reader. Also, we can't get all the way + // back up to our conn's LimitedReader that *might* be backing + // this bufio.Reader. Instead, a hack: we iteratively Peek up + // to the bufio.Reader's max size, looking for a double CRLF. + // This limits the trailer to the underlying buffer size, typically 4kB. + if !seeUpcomingDoubleCRLF(b.r) { + return errors.New("http: suspiciously long trailer after chunked body") + } + + hdr, err := textproto.NewReader(b.r).ReadMIMEHeader() + if err != nil { + if err == io.EOF { + return errTrailerEOF + } + return err + } + switch rr := b.hdr.(type) { + case *Request: + mergeSetHeader(&rr.Trailer, Header(hdr)) + case *Response: + mergeSetHeader(&rr.Trailer, Header(hdr)) + } + return nil +} + +func mergeSetHeader(dst *Header, src Header) { + if *dst == nil { + *dst = src + return + } + for k, vv := range src { + (*dst)[k] = vv + } +} + +func (b *body) Close() error { + b.mu.Lock() + defer b.mu.Unlock() + if b.closed { + return nil + } + var err error + switch { + case b.hdr == nil && b.closing: + // no trailer and closing the connection next. + // no point in reading to EOF. + default: + // Fully consume the body, which will also lead to us reading + // the trailer headers after the body, if present. + _, err = io.Copy(ioutil.Discard, bodyLocked{b}) + } + b.closed = true + return err +} + +// bodyLocked is a io.Reader reading from a *body when its mutex is +// already held. +type bodyLocked struct { + b *body +} + +func (bl bodyLocked) Read(p []byte) (n int, err error) { + if bl.b.closed { + return 0, ErrBodyReadAfterClose + } + return bl.b.readLocked(p) +} + +// parseContentLength trims whitespace from s and returns -1 if no value +// is set, or the value if it's >= 0. +func parseContentLength(cl string) (int64, error) { + cl = strings.TrimSpace(cl) + if cl == "" { + return -1, nil + } + n, err := strconv.ParseInt(cl, 10, 64) + if err != nil || n < 0 { + return 0, &badStringError{"bad Content-Length", cl} + } + return n, nil + +} diff --git a/src/net/http/transfer_test.go b/src/net/http/transfer_test.go new file mode 100644 index 000000000..48cd540b9 --- /dev/null +++ b/src/net/http/transfer_test.go @@ -0,0 +1,64 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bufio" + "io" + "strings" + "testing" +) + +func TestBodyReadBadTrailer(t *testing.T) { + b := &body{ + src: strings.NewReader("foobar"), + hdr: true, // force reading the trailer + r: bufio.NewReader(strings.NewReader("")), + } + buf := make([]byte, 7) + n, err := b.Read(buf[:3]) + got := string(buf[:n]) + if got != "foo" || err != nil { + t.Fatalf(`first Read = %d (%q), %v; want 3 ("foo")`, n, got, err) + } + + n, err = b.Read(buf[:]) + got = string(buf[:n]) + if got != "bar" || err != nil { + t.Fatalf(`second Read = %d (%q), %v; want 3 ("bar")`, n, got, err) + } + + n, err = b.Read(buf[:]) + got = string(buf[:n]) + if err == nil { + t.Errorf("final Read was successful (%q), expected error from trailer read", got) + } +} + +func TestFinalChunkedBodyReadEOF(t *testing.T) { + res, err := ReadResponse(bufio.NewReader(strings.NewReader( + "HTTP/1.1 200 OK\r\n"+ + "Transfer-Encoding: chunked\r\n"+ + "\r\n"+ + "0a\r\n"+ + "Body here\n\r\n"+ + "09\r\n"+ + "continued\r\n"+ + "0\r\n"+ + "\r\n")), nil) + if err != nil { + t.Fatal(err) + } + want := "Body here\ncontinued" + buf := make([]byte, len(want)) + n, err := res.Body.Read(buf) + if n != len(want) || err != io.EOF { + t.Logf("body = %#v", res.Body) + t.Errorf("Read = %v, %v; want %d, EOF", n, err, len(want)) + } + if string(buf) != want { + t.Errorf("buf = %q; want %q", buf, want) + } +} diff --git a/src/net/http/transport.go b/src/net/http/transport.go new file mode 100644 index 000000000..527ed8bdd --- /dev/null +++ b/src/net/http/transport.go @@ -0,0 +1,1242 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// HTTP client implementation. See RFC 2616. +// +// This is the low-level Transport implementation of RoundTripper. +// The high-level interface is in client.go. + +package http + +import ( + "bufio" + "compress/gzip" + "crypto/tls" + "errors" + "fmt" + "io" + "log" + "net" + "net/url" + "os" + "strings" + "sync" + "time" +) + +// DefaultTransport is the default implementation of Transport and is +// used by DefaultClient. It establishes network connections as needed +// and caches them for reuse by subsequent calls. It uses HTTP proxies +// as directed by the $HTTP_PROXY and $NO_PROXY (or $http_proxy and +// $no_proxy) environment variables. +var DefaultTransport RoundTripper = &Transport{ + Proxy: ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, +} + +// DefaultMaxIdleConnsPerHost is the default value of Transport's +// MaxIdleConnsPerHost. +const DefaultMaxIdleConnsPerHost = 2 + +// Transport is an implementation of RoundTripper that supports HTTP, +// HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). +// Transport can also cache connections for future re-use. +type Transport struct { + idleMu sync.Mutex + idleConn map[connectMethodKey][]*persistConn + idleConnCh map[connectMethodKey]chan *persistConn + reqMu sync.Mutex + reqCanceler map[*Request]func() + altMu sync.RWMutex + altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper + + // Proxy specifies a function to return a proxy for a given + // Request. If the function returns a non-nil error, the + // request is aborted with the provided error. + // If Proxy is nil or returns a nil *URL, no proxy is used. + Proxy func(*Request) (*url.URL, error) + + // Dial specifies the dial function for creating unencrypted + // TCP connections. + // If Dial is nil, net.Dial is used. + Dial func(network, addr string) (net.Conn, error) + + // DialTLS specifies an optional dial function for creating + // TLS connections for non-proxied HTTPS requests. + // + // If DialTLS is nil, Dial and TLSClientConfig are used. + // + // If DialTLS is set, the Dial hook is not used for HTTPS + // requests and the TLSClientConfig and TLSHandshakeTimeout + // are ignored. The returned net.Conn is assumed to already be + // past the TLS handshake. + DialTLS func(network, addr string) (net.Conn, error) + + // TLSClientConfig specifies the TLS configuration to use with + // tls.Client. If nil, the default configuration is used. + TLSClientConfig *tls.Config + + // TLSHandshakeTimeout specifies the maximum amount of time waiting to + // wait for a TLS handshake. Zero means no timeout. + TLSHandshakeTimeout time.Duration + + // DisableKeepAlives, if true, prevents re-use of TCP connections + // between different HTTP requests. + DisableKeepAlives bool + + // DisableCompression, if true, prevents the Transport from + // requesting compression with an "Accept-Encoding: gzip" + // request header when the Request contains no existing + // Accept-Encoding value. If the Transport requests gzip on + // its own and gets a gzipped response, it's transparently + // decoded in the Response.Body. However, if the user + // explicitly requested gzip it is not automatically + // uncompressed. + DisableCompression bool + + // MaxIdleConnsPerHost, if non-zero, controls the maximum idle + // (keep-alive) to keep per-host. If zero, + // DefaultMaxIdleConnsPerHost is used. + MaxIdleConnsPerHost int + + // ResponseHeaderTimeout, if non-zero, specifies the amount of + // time to wait for a server's response headers after fully + // writing the request (including its body, if any). This + // time does not include the time to read the response body. + ResponseHeaderTimeout time.Duration + + // TODO: tunable on global max cached connections + // TODO: tunable on timeout on cached connections +} + +// ProxyFromEnvironment returns the URL of the proxy to use for a +// given request, as indicated by the environment variables +// $HTTP_PROXY and $NO_PROXY (or $http_proxy and $no_proxy). +// An error is returned if the proxy environment is invalid. +// A nil URL and nil error are returned if no proxy is defined in the +// environment, or a proxy should not be used for the given request. +// +// As a special case, if req.URL.Host is "localhost" (with or without +// a port number), then a nil URL and nil error will be returned. +func ProxyFromEnvironment(req *Request) (*url.URL, error) { + proxy := httpProxyEnv.Get() + if proxy == "" { + return nil, nil + } + if !useProxy(canonicalAddr(req.URL)) { + return nil, nil + } + proxyURL, err := url.Parse(proxy) + if err != nil || !strings.HasPrefix(proxyURL.Scheme, "http") { + // proxy was bogus. Try prepending "http://" to it and + // see if that parses correctly. If not, we fall + // through and complain about the original one. + if proxyURL, err := url.Parse("http://" + proxy); err == nil { + return proxyURL, nil + } + } + if err != nil { + return nil, fmt.Errorf("invalid proxy address %q: %v", proxy, err) + } + return proxyURL, nil +} + +// ProxyURL returns a proxy function (for use in a Transport) +// that always returns the same URL. +func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { + return func(*Request) (*url.URL, error) { + return fixedURL, nil + } +} + +// transportRequest is a wrapper around a *Request that adds +// optional extra headers to write. +type transportRequest struct { + *Request // original request, not to be mutated + extra Header // extra headers to write, or nil +} + +func (tr *transportRequest) extraHeaders() Header { + if tr.extra == nil { + tr.extra = make(Header) + } + return tr.extra +} + +// RoundTrip implements the RoundTripper interface. +// +// For higher-level HTTP client support (such as handling of cookies +// and redirects), see Get, Post, and the Client type. +func (t *Transport) RoundTrip(req *Request) (resp *Response, err error) { + if req.URL == nil { + req.closeBody() + return nil, errors.New("http: nil Request.URL") + } + if req.Header == nil { + req.closeBody() + return nil, errors.New("http: nil Request.Header") + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + t.altMu.RLock() + var rt RoundTripper + if t.altProto != nil { + rt = t.altProto[req.URL.Scheme] + } + t.altMu.RUnlock() + if rt == nil { + req.closeBody() + return nil, &badStringError{"unsupported protocol scheme", req.URL.Scheme} + } + return rt.RoundTrip(req) + } + if req.URL.Host == "" { + req.closeBody() + return nil, errors.New("http: no Host in request URL") + } + treq := &transportRequest{Request: req} + cm, err := t.connectMethodForRequest(treq) + if err != nil { + req.closeBody() + return nil, err + } + + // Get the cached or newly-created connection to either the + // host (for http or https), the http proxy, or the http proxy + // pre-CONNECTed to https server. In any case, we'll be ready + // to send it requests. + pconn, err := t.getConn(req, cm) + if err != nil { + t.setReqCanceler(req, nil) + req.closeBody() + return nil, err + } + + return pconn.roundTrip(treq) +} + +// RegisterProtocol registers a new protocol with scheme. +// The Transport will pass requests using the given scheme to rt. +// It is rt's responsibility to simulate HTTP request semantics. +// +// RegisterProtocol can be used by other packages to provide +// implementations of protocol schemes like "ftp" or "file". +func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { + if scheme == "http" || scheme == "https" { + panic("protocol " + scheme + " already registered") + } + t.altMu.Lock() + defer t.altMu.Unlock() + if t.altProto == nil { + t.altProto = make(map[string]RoundTripper) + } + if _, exists := t.altProto[scheme]; exists { + panic("protocol " + scheme + " already registered") + } + t.altProto[scheme] = rt +} + +// CloseIdleConnections closes any connections which were previously +// connected from previous requests but are now sitting idle in +// a "keep-alive" state. It does not interrupt any connections currently +// in use. +func (t *Transport) CloseIdleConnections() { + t.idleMu.Lock() + m := t.idleConn + t.idleConn = nil + t.idleConnCh = nil + t.idleMu.Unlock() + for _, conns := range m { + for _, pconn := range conns { + pconn.close() + } + } +} + +// CancelRequest cancels an in-flight request by closing its +// connection. +func (t *Transport) CancelRequest(req *Request) { + t.reqMu.Lock() + cancel := t.reqCanceler[req] + t.reqMu.Unlock() + if cancel != nil { + cancel() + } +} + +// +// Private implementation past this point. +// + +var ( + httpProxyEnv = &envOnce{ + names: []string{"HTTP_PROXY", "http_proxy"}, + } + noProxyEnv = &envOnce{ + names: []string{"NO_PROXY", "no_proxy"}, + } +) + +// envOnce looks up an environment variable (optionally by multiple +// names) once. It mitigates expensive lookups on some platforms +// (e.g. Windows). +type envOnce struct { + names []string + once sync.Once + val string +} + +func (e *envOnce) Get() string { + e.once.Do(e.init) + return e.val +} + +func (e *envOnce) init() { + for _, n := range e.names { + e.val = os.Getenv(n) + if e.val != "" { + return + } + } +} + +// reset is used by tests +func (e *envOnce) reset() { + e.once = sync.Once{} + e.val = "" +} + +func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { + cm.targetScheme = treq.URL.Scheme + cm.targetAddr = canonicalAddr(treq.URL) + if t.Proxy != nil { + cm.proxyURL, err = t.Proxy(treq.Request) + } + return cm, nil +} + +// proxyAuth returns the Proxy-Authorization header to set +// on requests, if applicable. +func (cm *connectMethod) proxyAuth() string { + if cm.proxyURL == nil { + return "" + } + if u := cm.proxyURL.User; u != nil { + username := u.Username() + password, _ := u.Password() + return "Basic " + basicAuth(username, password) + } + return "" +} + +// putIdleConn adds pconn to the list of idle persistent connections awaiting +// a new request. +// If pconn is no longer needed or not in a good state, putIdleConn +// returns false. +func (t *Transport) putIdleConn(pconn *persistConn) bool { + if t.DisableKeepAlives || t.MaxIdleConnsPerHost < 0 { + pconn.close() + return false + } + if pconn.isBroken() { + return false + } + key := pconn.cacheKey + max := t.MaxIdleConnsPerHost + if max == 0 { + max = DefaultMaxIdleConnsPerHost + } + t.idleMu.Lock() + + waitingDialer := t.idleConnCh[key] + select { + case waitingDialer <- pconn: + // We're done with this pconn and somebody else is + // currently waiting for a conn of this type (they're + // actively dialing, but this conn is ready + // first). Chrome calls this socket late binding. See + // https://insouciant.org/tech/connection-management-in-chromium/ + t.idleMu.Unlock() + return true + default: + if waitingDialer != nil { + // They had populated this, but their dial won + // first, so we can clean up this map entry. + delete(t.idleConnCh, key) + } + } + if t.idleConn == nil { + t.idleConn = make(map[connectMethodKey][]*persistConn) + } + if len(t.idleConn[key]) >= max { + t.idleMu.Unlock() + pconn.close() + return false + } + for _, exist := range t.idleConn[key] { + if exist == pconn { + log.Fatalf("dup idle pconn %p in freelist", pconn) + } + } + t.idleConn[key] = append(t.idleConn[key], pconn) + t.idleMu.Unlock() + return true +} + +// getIdleConnCh returns a channel to receive and return idle +// persistent connection for the given connectMethod. +// It may return nil, if persistent connections are not being used. +func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { + if t.DisableKeepAlives { + return nil + } + key := cm.key() + t.idleMu.Lock() + defer t.idleMu.Unlock() + if t.idleConnCh == nil { + t.idleConnCh = make(map[connectMethodKey]chan *persistConn) + } + ch, ok := t.idleConnCh[key] + if !ok { + ch = make(chan *persistConn) + t.idleConnCh[key] = ch + } + return ch +} + +func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) { + key := cm.key() + t.idleMu.Lock() + defer t.idleMu.Unlock() + if t.idleConn == nil { + return nil + } + for { + pconns, ok := t.idleConn[key] + if !ok { + return nil + } + if len(pconns) == 1 { + pconn = pconns[0] + delete(t.idleConn, key) + } else { + // 2 or more cached connections; pop last + // TODO: queue? + pconn = pconns[len(pconns)-1] + t.idleConn[key] = pconns[:len(pconns)-1] + } + if !pconn.isBroken() { + return + } + } +} + +func (t *Transport) setReqCanceler(r *Request, fn func()) { + t.reqMu.Lock() + defer t.reqMu.Unlock() + if t.reqCanceler == nil { + t.reqCanceler = make(map[*Request]func()) + } + if fn != nil { + t.reqCanceler[r] = fn + } else { + delete(t.reqCanceler, r) + } +} + +func (t *Transport) dial(network, addr string) (c net.Conn, err error) { + if t.Dial != nil { + return t.Dial(network, addr) + } + return net.Dial(network, addr) +} + +// Testing hooks: +var prePendingDial, postPendingDial func() + +// getConn dials and creates a new persistConn to the target as +// specified in the connectMethod. This includes doing a proxy CONNECT +// and/or setting up TLS. If this doesn't return an error, the persistConn +// is ready to write requests to. +func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) { + if pc := t.getIdleConn(cm); pc != nil { + return pc, nil + } + + type dialRes struct { + pc *persistConn + err error + } + dialc := make(chan dialRes) + + handlePendingDial := func() { + if prePendingDial != nil { + prePendingDial() + } + go func() { + if v := <-dialc; v.err == nil { + t.putIdleConn(v.pc) + } + if postPendingDial != nil { + postPendingDial() + } + }() + } + + cancelc := make(chan struct{}) + t.setReqCanceler(req, func() { close(cancelc) }) + + go func() { + pc, err := t.dialConn(cm) + dialc <- dialRes{pc, err} + }() + + idleConnCh := t.getIdleConnCh(cm) + select { + case v := <-dialc: + // Our dial finished. + return v.pc, v.err + case pc := <-idleConnCh: + // Another request finished first and its net.Conn + // became available before our dial. Or somebody + // else's dial that they didn't use. + // But our dial is still going, so give it away + // when it finishes: + handlePendingDial() + return pc, nil + case <-cancelc: + handlePendingDial() + return nil, errors.New("net/http: request canceled while waiting for connection") + } +} + +func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { + pconn := &persistConn{ + t: t, + cacheKey: cm.key(), + reqch: make(chan requestAndChan, 1), + writech: make(chan writeRequest, 1), + closech: make(chan struct{}), + writeErrCh: make(chan error, 1), + } + tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil + if tlsDial { + var err error + pconn.conn, err = t.DialTLS("tcp", cm.addr()) + if err != nil { + return nil, err + } + if tc, ok := pconn.conn.(*tls.Conn); ok { + cs := tc.ConnectionState() + pconn.tlsState = &cs + } + } else { + conn, err := t.dial("tcp", cm.addr()) + if err != nil { + if cm.proxyURL != nil { + err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) + } + return nil, err + } + pconn.conn = conn + } + + // Proxy setup. + switch { + case cm.proxyURL == nil: + // Do nothing. Not using a proxy. + case cm.targetScheme == "http": + pconn.isProxy = true + if pa := cm.proxyAuth(); pa != "" { + pconn.mutateHeaderFunc = func(h Header) { + h.Set("Proxy-Authorization", pa) + } + } + case cm.targetScheme == "https": + conn := pconn.conn + connectReq := &Request{ + Method: "CONNECT", + URL: &url.URL{Opaque: cm.targetAddr}, + Host: cm.targetAddr, + Header: make(Header), + } + if pa := cm.proxyAuth(); pa != "" { + connectReq.Header.Set("Proxy-Authorization", pa) + } + connectReq.Write(conn) + + // Read response. + // Okay to use and discard buffered reader here, because + // TLS server will not speak until spoken to. + br := bufio.NewReader(conn) + resp, err := ReadResponse(br, connectReq) + if err != nil { + conn.Close() + return nil, err + } + if resp.StatusCode != 200 { + f := strings.SplitN(resp.Status, " ", 2) + conn.Close() + return nil, errors.New(f[1]) + } + } + + if cm.targetScheme == "https" && !tlsDial { + // Initiate TLS and check remote host name against certificate. + cfg := t.TLSClientConfig + if cfg == nil || cfg.ServerName == "" { + host := cm.tlsHost() + if cfg == nil { + cfg = &tls.Config{ServerName: host} + } else { + clone := *cfg // shallow clone + clone.ServerName = host + cfg = &clone + } + } + plainConn := pconn.conn + tlsConn := tls.Client(plainConn, cfg) + errc := make(chan error, 2) + var timer *time.Timer // for canceling TLS handshake + if d := t.TLSHandshakeTimeout; d != 0 { + timer = time.AfterFunc(d, func() { + errc <- tlsHandshakeTimeoutError{} + }) + } + go func() { + err := tlsConn.Handshake() + if timer != nil { + timer.Stop() + } + errc <- err + }() + if err := <-errc; err != nil { + plainConn.Close() + return nil, err + } + if !cfg.InsecureSkipVerify { + if err := tlsConn.VerifyHostname(cfg.ServerName); err != nil { + plainConn.Close() + return nil, err + } + } + cs := tlsConn.ConnectionState() + pconn.tlsState = &cs + pconn.conn = tlsConn + } + + pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF}) + pconn.bw = bufio.NewWriter(pconn.conn) + go pconn.readLoop() + go pconn.writeLoop() + return pconn, nil +} + +// useProxy returns true if requests to addr should use a proxy, +// according to the NO_PROXY or no_proxy environment variable. +// addr is always a canonicalAddr with a host and port. +func useProxy(addr string) bool { + if len(addr) == 0 { + return true + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return false + } + if host == "localhost" { + return false + } + if ip := net.ParseIP(host); ip != nil { + if ip.IsLoopback() { + return false + } + } + + no_proxy := noProxyEnv.Get() + if no_proxy == "*" { + return false + } + + addr = strings.ToLower(strings.TrimSpace(addr)) + if hasPort(addr) { + addr = addr[:strings.LastIndex(addr, ":")] + } + + for _, p := range strings.Split(no_proxy, ",") { + p = strings.ToLower(strings.TrimSpace(p)) + if len(p) == 0 { + continue + } + if hasPort(p) { + p = p[:strings.LastIndex(p, ":")] + } + if addr == p { + return false + } + if p[0] == '.' && (strings.HasSuffix(addr, p) || addr == p[1:]) { + // no_proxy ".foo.com" matches "bar.foo.com" or "foo.com" + return false + } + if p[0] != '.' && strings.HasSuffix(addr, p) && addr[len(addr)-len(p)-1] == '.' { + // no_proxy "foo.com" matches "bar.foo.com" + return false + } + } + return true +} + +// connectMethod is the map key (in its String form) for keeping persistent +// TCP connections alive for subsequent HTTP requests. +// +// A connect method may be of the following types: +// +// Cache key form Description +// ----------------- ------------------------- +// |http|foo.com http directly to server, no proxy +// |https|foo.com https directly to server, no proxy +// http://proxy.com|https|foo.com http to proxy, then CONNECT to foo.com +// http://proxy.com|http http to proxy, http to anywhere after that +// +// Note: no support to https to the proxy yet. +// +type connectMethod struct { + proxyURL *url.URL // nil for no proxy, else full proxy URL + targetScheme string // "http" or "https" + targetAddr string // Not used if proxy + http targetScheme (4th example in table) +} + +func (cm *connectMethod) key() connectMethodKey { + proxyStr := "" + targetAddr := cm.targetAddr + if cm.proxyURL != nil { + proxyStr = cm.proxyURL.String() + if cm.targetScheme == "http" { + targetAddr = "" + } + } + return connectMethodKey{ + proxy: proxyStr, + scheme: cm.targetScheme, + addr: targetAddr, + } +} + +// addr returns the first hop "host:port" to which we need to TCP connect. +func (cm *connectMethod) addr() string { + if cm.proxyURL != nil { + return canonicalAddr(cm.proxyURL) + } + return cm.targetAddr +} + +// tlsHost returns the host name to match against the peer's +// TLS certificate. +func (cm *connectMethod) tlsHost() string { + h := cm.targetAddr + if hasPort(h) { + h = h[:strings.LastIndex(h, ":")] + } + return h +} + +// connectMethodKey is the map key version of connectMethod, with a +// stringified proxy URL (or the empty string) instead of a pointer to +// a URL. +type connectMethodKey struct { + proxy, scheme, addr string +} + +func (k connectMethodKey) String() string { + // Only used by tests. + return fmt.Sprintf("%s|%s|%s", k.proxy, k.scheme, k.addr) +} + +// persistConn wraps a connection, usually a persistent one +// (but may be used for non-keep-alive requests as well) +type persistConn struct { + t *Transport + cacheKey connectMethodKey + conn net.Conn + tlsState *tls.ConnectionState + br *bufio.Reader // from conn + sawEOF bool // whether we've seen EOF from conn; owned by readLoop + bw *bufio.Writer // to conn + reqch chan requestAndChan // written by roundTrip; read by readLoop + writech chan writeRequest // written by roundTrip; read by writeLoop + closech chan struct{} // closed when conn closed + isProxy bool + // writeErrCh passes the request write error (usually nil) + // from the writeLoop goroutine to the readLoop which passes + // it off to the res.Body reader, which then uses it to decide + // whether or not a connection can be reused. Issue 7569. + writeErrCh chan error + + lk sync.Mutex // guards following fields + numExpectedResponses int + closed bool // whether conn has been closed + broken bool // an error has happened on this connection; marked broken so it's not reused. + // mutateHeaderFunc is an optional func to modify extra + // headers on each outbound request before it's written. (the + // original Request given to RoundTrip is not modified) + mutateHeaderFunc func(Header) +} + +// isBroken reports whether this connection is in a known broken state. +func (pc *persistConn) isBroken() bool { + pc.lk.Lock() + b := pc.broken + pc.lk.Unlock() + return b +} + +func (pc *persistConn) cancelRequest() { + pc.conn.Close() +} + +var remoteSideClosedFunc func(error) bool // or nil to use default + +func remoteSideClosed(err error) bool { + if err == io.EOF { + return true + } + if remoteSideClosedFunc != nil { + return remoteSideClosedFunc(err) + } + return false +} + +func (pc *persistConn) readLoop() { + alive := true + + for alive { + pb, err := pc.br.Peek(1) + + pc.lk.Lock() + if pc.numExpectedResponses == 0 { + if !pc.closed { + pc.closeLocked() + if len(pb) > 0 { + log.Printf("Unsolicited response received on idle HTTP channel starting with %q; err=%v", + string(pb), err) + } + } + pc.lk.Unlock() + return + } + pc.lk.Unlock() + + rc := <-pc.reqch + + var resp *Response + if err == nil { + resp, err = ReadResponse(pc.br, rc.req) + if err == nil && resp.StatusCode == 100 { + // Skip any 100-continue for now. + // TODO(bradfitz): if rc.req had "Expect: 100-continue", + // actually block the request body write and signal the + // writeLoop now to begin sending it. (Issue 2184) For now we + // eat it, since we're never expecting one. + resp, err = ReadResponse(pc.br, rc.req) + } + } + + if resp != nil { + resp.TLS = pc.tlsState + } + + hasBody := resp != nil && rc.req.Method != "HEAD" && resp.ContentLength != 0 + + if err != nil { + pc.close() + } else { + if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" { + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + resp.Body = &gzipReader{body: resp.Body} + } + resp.Body = &bodyEOFSignal{body: resp.Body} + } + + if err != nil || resp.Close || rc.req.Close || resp.StatusCode <= 199 { + // Don't do keep-alive on error if either party requested a close + // or we get an unexpected informational (1xx) response. + // StatusCode 100 is already handled above. + alive = false + } + + var waitForBodyRead chan bool + if hasBody { + waitForBodyRead = make(chan bool, 2) + resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error { + // Sending false here sets alive to + // false and closes the connection + // below. + waitForBodyRead <- false + return nil + } + resp.Body.(*bodyEOFSignal).fn = func(err error) { + waitForBodyRead <- alive && + err == nil && + !pc.sawEOF && + pc.wroteRequest() && + pc.t.putIdleConn(pc) + } + } + + if alive && !hasBody { + alive = !pc.sawEOF && + pc.wroteRequest() && + pc.t.putIdleConn(pc) + } + + rc.ch <- responseAndError{resp, err} + + // Wait for the just-returned response body to be fully consumed + // before we race and peek on the underlying bufio reader. + if waitForBodyRead != nil { + select { + case alive = <-waitForBodyRead: + case <-pc.closech: + alive = false + } + } + + pc.t.setReqCanceler(rc.req, nil) + + if !alive { + pc.close() + } + } +} + +func (pc *persistConn) writeLoop() { + for { + select { + case wr := <-pc.writech: + if pc.isBroken() { + wr.ch <- errors.New("http: can't write HTTP request on broken connection") + continue + } + err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra) + if err == nil { + err = pc.bw.Flush() + } + if err != nil { + pc.markBroken() + wr.req.Request.closeBody() + } + pc.writeErrCh <- err // to the body reader, which might recycle us + wr.ch <- err // to the roundTrip function + case <-pc.closech: + return + } + } +} + +// wroteRequest is a check before recycling a connection that the previous write +// (from writeLoop above) happened and was successful. +func (pc *persistConn) wroteRequest() bool { + select { + case err := <-pc.writeErrCh: + // Common case: the write happened well before the response, so + // avoid creating a timer. + return err == nil + default: + // Rare case: the request was written in writeLoop above but + // before it could send to pc.writeErrCh, the reader read it + // all, processed it, and called us here. In this case, give the + // write goroutine a bit of time to finish its send. + // + // Less rare case: We also get here in the legitimate case of + // Issue 7569, where the writer is still writing (or stalled), + // but the server has already replied. In this case, we don't + // want to wait too long, and we want to return false so this + // connection isn't re-used. + select { + case err := <-pc.writeErrCh: + return err == nil + case <-time.After(50 * time.Millisecond): + return false + } + } +} + +type responseAndError struct { + res *Response + err error +} + +type requestAndChan struct { + req *Request + ch chan responseAndError + + // did the Transport (as opposed to the client code) add an + // Accept-Encoding gzip header? only if it we set it do + // we transparently decode the gzip. + addedGzip bool +} + +// A writeRequest is sent by the readLoop's goroutine to the +// writeLoop's goroutine to write a request while the read loop +// concurrently waits on both the write response and the server's +// reply. +type writeRequest struct { + req *transportRequest + ch chan<- error +} + +type httpError struct { + err string + timeout bool +} + +func (e *httpError) Error() string { return e.err } +func (e *httpError) Timeout() bool { return e.timeout } +func (e *httpError) Temporary() bool { return true } + +var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} +var errClosed error = &httpError{err: "net/http: transport closed before response was received"} + +func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { + pc.t.setReqCanceler(req.Request, pc.cancelRequest) + pc.lk.Lock() + pc.numExpectedResponses++ + headerFn := pc.mutateHeaderFunc + pc.lk.Unlock() + + if headerFn != nil { + headerFn(req.extraHeaders()) + } + + // Ask for a compressed version if the caller didn't set their + // own value for Accept-Encoding. We only attempted to + // uncompress the gzip stream if we were the layer that + // requested it. + requestedGzip := false + if !pc.t.DisableCompression && req.Header.Get("Accept-Encoding") == "" && req.Method != "HEAD" { + // Request gzip only, not deflate. Deflate is ambiguous and + // not as universally supported anyway. + // See: http://www.gzip.org/zlib/zlib_faq.html#faq38 + // + // Note that we don't request this for HEAD requests, + // due to a bug in nginx: + // http://trac.nginx.org/nginx/ticket/358 + // http://golang.org/issue/5522 + requestedGzip = true + req.extraHeaders().Set("Accept-Encoding", "gzip") + } + + // Write the request concurrently with waiting for a response, + // in case the server decides to reply before reading our full + // request body. + writeErrCh := make(chan error, 1) + pc.writech <- writeRequest{req, writeErrCh} + + resc := make(chan responseAndError, 1) + pc.reqch <- requestAndChan{req.Request, resc, requestedGzip} + + var re responseAndError + var pconnDeadCh = pc.closech + var failTicker <-chan time.Time + var respHeaderTimer <-chan time.Time +WaitResponse: + for { + select { + case err := <-writeErrCh: + if err != nil { + re = responseAndError{nil, err} + pc.close() + break WaitResponse + } + if d := pc.t.ResponseHeaderTimeout; d > 0 { + respHeaderTimer = time.After(d) + } + case <-pconnDeadCh: + // The persist connection is dead. This shouldn't + // usually happen (only with Connection: close responses + // with no response bodies), but if it does happen it + // means either a) the remote server hung up on us + // prematurely, or b) the readLoop sent us a response & + // closed its closech at roughly the same time, and we + // selected this case first, in which case a response + // might still be coming soon. + // + // We can't avoid the select race in b) by using a unbuffered + // resc channel instead, because then goroutines can + // leak if we exit due to other errors. + pconnDeadCh = nil // avoid spinning + failTicker = time.After(100 * time.Millisecond) // arbitrary time to wait for resc + case <-failTicker: + re = responseAndError{err: errClosed} + break WaitResponse + case <-respHeaderTimer: + pc.close() + re = responseAndError{err: errTimeout} + break WaitResponse + case re = <-resc: + break WaitResponse + } + } + + pc.lk.Lock() + pc.numExpectedResponses-- + pc.lk.Unlock() + + if re.err != nil { + pc.t.setReqCanceler(req.Request, nil) + } + return re.res, re.err +} + +// markBroken marks a connection as broken (so it's not reused). +// It differs from close in that it doesn't close the underlying +// connection for use when it's still being read. +func (pc *persistConn) markBroken() { + pc.lk.Lock() + defer pc.lk.Unlock() + pc.broken = true +} + +func (pc *persistConn) close() { + pc.lk.Lock() + defer pc.lk.Unlock() + pc.closeLocked() +} + +func (pc *persistConn) closeLocked() { + pc.broken = true + if !pc.closed { + pc.conn.Close() + pc.closed = true + close(pc.closech) + } + pc.mutateHeaderFunc = nil +} + +var portMap = map[string]string{ + "http": "80", + "https": "443", +} + +// canonicalAddr returns url.Host but always with a ":port" suffix +func canonicalAddr(url *url.URL) string { + addr := url.Host + if !hasPort(addr) { + return addr + ":" + portMap[url.Scheme] + } + return addr +} + +// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most +// once, right before its final (error-producing) Read or Close call +// returns. If earlyCloseFn is non-nil and Close is called before +// io.EOF is seen, earlyCloseFn is called instead of fn, and its +// return value is the return value from Close. +type bodyEOFSignal struct { + body io.ReadCloser + mu sync.Mutex // guards following 4 fields + closed bool // whether Close has been called + rerr error // sticky Read error + fn func(error) // error will be nil on Read io.EOF + earlyCloseFn func() error // optional alt Close func used if io.EOF not seen +} + +func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { + es.mu.Lock() + closed, rerr := es.closed, es.rerr + es.mu.Unlock() + if closed { + return 0, errors.New("http: read on closed response body") + } + if rerr != nil { + return 0, rerr + } + + n, err = es.body.Read(p) + if err != nil { + es.mu.Lock() + defer es.mu.Unlock() + if es.rerr == nil { + es.rerr = err + } + es.condfn(err) + } + return +} + +func (es *bodyEOFSignal) Close() error { + es.mu.Lock() + defer es.mu.Unlock() + if es.closed { + return nil + } + es.closed = true + if es.earlyCloseFn != nil && es.rerr != io.EOF { + return es.earlyCloseFn() + } + err := es.body.Close() + es.condfn(err) + return err +} + +// caller must hold es.mu. +func (es *bodyEOFSignal) condfn(err error) { + if es.fn == nil { + return + } + if err == io.EOF { + err = nil + } + es.fn(err) + es.fn = nil +} + +// gzipReader wraps a response body so it can lazily +// call gzip.NewReader on the first call to Read +type gzipReader struct { + body io.ReadCloser // underlying Response.Body + zr io.Reader // lazily-initialized gzip reader +} + +func (gz *gzipReader) Read(p []byte) (n int, err error) { + if gz.zr == nil { + gz.zr, err = gzip.NewReader(gz.body) + if err != nil { + return 0, err + } + } + return gz.zr.Read(p) +} + +func (gz *gzipReader) Close() error { + return gz.body.Close() +} + +type readerAndCloser struct { + io.Reader + io.Closer +} + +type tlsHandshakeTimeoutError struct{} + +func (tlsHandshakeTimeoutError) Timeout() bool { return true } +func (tlsHandshakeTimeoutError) Temporary() bool { return true } +func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } + +type noteEOFReader struct { + r io.Reader + sawEOF *bool +} + +func (nr noteEOFReader) Read(p []byte) (n int, err error) { + n, err = nr.r.Read(p) + if err == io.EOF { + *nr.sawEOF = true + } + return +} diff --git a/src/net/http/transport_test.go b/src/net/http/transport_test.go new file mode 100644 index 000000000..3460d690e --- /dev/null +++ b/src/net/http/transport_test.go @@ -0,0 +1,2211 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Tests for transport.go + +package http_test + +import ( + "bufio" + "bytes" + "compress/gzip" + "crypto/rand" + "crypto/tls" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + . "net/http" + "net/http/httptest" + "net/url" + "os" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +// TODO: test 5 pipelined requests with responses: 1) OK, 2) OK, Connection: Close +// and then verify that the final 2 responses get errors back. + +// hostPortHandler writes back the client's "host:port". +var hostPortHandler = HandlerFunc(func(w ResponseWriter, r *Request) { + if r.FormValue("close") == "true" { + w.Header().Set("Connection", "close") + } + w.Write([]byte(r.RemoteAddr)) +}) + +// testCloseConn is a net.Conn tracked by a testConnSet. +type testCloseConn struct { + net.Conn + set *testConnSet +} + +func (c *testCloseConn) Close() error { + c.set.remove(c) + return c.Conn.Close() +} + +// testConnSet tracks a set of TCP connections and whether they've +// been closed. +type testConnSet struct { + t *testing.T + mu sync.Mutex // guards closed and list + closed map[net.Conn]bool + list []net.Conn // in order created +} + +func (tcs *testConnSet) insert(c net.Conn) { + tcs.mu.Lock() + defer tcs.mu.Unlock() + tcs.closed[c] = false + tcs.list = append(tcs.list, c) +} + +func (tcs *testConnSet) remove(c net.Conn) { + tcs.mu.Lock() + defer tcs.mu.Unlock() + tcs.closed[c] = true +} + +// some tests use this to manage raw tcp connections for later inspection +func makeTestDial(t *testing.T) (*testConnSet, func(n, addr string) (net.Conn, error)) { + connSet := &testConnSet{ + t: t, + closed: make(map[net.Conn]bool), + } + dial := func(n, addr string) (net.Conn, error) { + c, err := net.Dial(n, addr) + if err != nil { + return nil, err + } + tc := &testCloseConn{c, connSet} + connSet.insert(tc) + return tc, nil + } + return connSet, dial +} + +func (tcs *testConnSet) check(t *testing.T) { + tcs.mu.Lock() + defer tcs.mu.Unlock() + for i := 4; i >= 0; i-- { + for i, c := range tcs.list { + if tcs.closed[c] { + continue + } + if i != 0 { + tcs.mu.Unlock() + time.Sleep(50 * time.Millisecond) + tcs.mu.Lock() + continue + } + t.Errorf("TCP connection #%d, %p (of %d total) was not closed", i+1, c, len(tcs.list)) + } + } +} + +// Two subsequent requests and verify their response is the same. +// The response from the server is our own IP:port +func TestTransportKeepAlives(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(hostPortHandler) + defer ts.Close() + + for _, disableKeepAlive := range []bool{false, true} { + tr := &Transport{DisableKeepAlives: disableKeepAlive} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + fetch := func(n int) string { + res, err := c.Get(ts.URL) + if err != nil { + t.Fatalf("error in disableKeepAlive=%v, req #%d, GET: %v", disableKeepAlive, n, err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("error in disableKeepAlive=%v, req #%d, ReadAll: %v", disableKeepAlive, n, err) + } + return string(body) + } + + body1 := fetch(1) + body2 := fetch(2) + + bodiesDiffer := body1 != body2 + if bodiesDiffer != disableKeepAlive { + t.Errorf("error in disableKeepAlive=%v. unexpected bodiesDiffer=%v; body1=%q; body2=%q", + disableKeepAlive, bodiesDiffer, body1, body2) + } + } +} + +func TestTransportConnectionCloseOnResponse(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(hostPortHandler) + defer ts.Close() + + connSet, testDial := makeTestDial(t) + + for _, connectionClose := range []bool{false, true} { + tr := &Transport{ + Dial: testDial, + } + c := &Client{Transport: tr} + + fetch := func(n int) string { + req := new(Request) + var err error + req.URL, err = url.Parse(ts.URL + fmt.Sprintf("/?close=%v", connectionClose)) + if err != nil { + t.Fatalf("URL parse error: %v", err) + } + req.Method = "GET" + req.Proto = "HTTP/1.1" + req.ProtoMajor = 1 + req.ProtoMinor = 1 + + res, err := c.Do(req) + if err != nil { + t.Fatalf("error in connectionClose=%v, req #%d, Do: %v", connectionClose, n, err) + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("error in connectionClose=%v, req #%d, ReadAll: %v", connectionClose, n, err) + } + return string(body) + } + + body1 := fetch(1) + body2 := fetch(2) + bodiesDiffer := body1 != body2 + if bodiesDiffer != connectionClose { + t.Errorf("error in connectionClose=%v. unexpected bodiesDiffer=%v; body1=%q; body2=%q", + connectionClose, bodiesDiffer, body1, body2) + } + + tr.CloseIdleConnections() + } + + connSet.check(t) +} + +func TestTransportConnectionCloseOnRequest(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(hostPortHandler) + defer ts.Close() + + connSet, testDial := makeTestDial(t) + + for _, connectionClose := range []bool{false, true} { + tr := &Transport{ + Dial: testDial, + } + c := &Client{Transport: tr} + + fetch := func(n int) string { + req := new(Request) + var err error + req.URL, err = url.Parse(ts.URL) + if err != nil { + t.Fatalf("URL parse error: %v", err) + } + req.Method = "GET" + req.Proto = "HTTP/1.1" + req.ProtoMajor = 1 + req.ProtoMinor = 1 + req.Close = connectionClose + + res, err := c.Do(req) + if err != nil { + t.Fatalf("error in connectionClose=%v, req #%d, Do: %v", connectionClose, n, err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("error in connectionClose=%v, req #%d, ReadAll: %v", connectionClose, n, err) + } + return string(body) + } + + body1 := fetch(1) + body2 := fetch(2) + bodiesDiffer := body1 != body2 + if bodiesDiffer != connectionClose { + t.Errorf("error in connectionClose=%v. unexpected bodiesDiffer=%v; body1=%q; body2=%q", + connectionClose, bodiesDiffer, body1, body2) + } + + tr.CloseIdleConnections() + } + + connSet.check(t) +} + +func TestTransportIdleCacheKeys(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(hostPortHandler) + defer ts.Close() + + tr := &Transport{DisableKeepAlives: false} + c := &Client{Transport: tr} + + if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { + t.Errorf("After CloseIdleConnections expected %d idle conn cache keys; got %d", e, g) + } + + resp, err := c.Get(ts.URL) + if err != nil { + t.Error(err) + } + ioutil.ReadAll(resp.Body) + + keys := tr.IdleConnKeysForTesting() + if e, g := 1, len(keys); e != g { + t.Fatalf("After Get expected %d idle conn cache keys; got %d", e, g) + } + + if e := "|http|" + ts.Listener.Addr().String(); keys[0] != e { + t.Errorf("Expected idle cache key %q; got %q", e, keys[0]) + } + + tr.CloseIdleConnections() + if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { + t.Errorf("After CloseIdleConnections expected %d idle conn cache keys; got %d", e, g) + } +} + +// Tests that the HTTP transport re-uses connections when a client +// reads to the end of a response Body without closing it. +func TestTransportReadToEndReusesConn(t *testing.T) { + defer afterTest(t) + const msg = "foobar" + + var addrSeen map[string]int + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + addrSeen[r.RemoteAddr]++ + if r.URL.Path == "/chunked/" { + w.WriteHeader(200) + w.(http.Flusher).Flush() + } else { + w.Header().Set("Content-Type", strconv.Itoa(len(msg))) + w.WriteHeader(200) + } + w.Write([]byte(msg)) + })) + defer ts.Close() + + buf := make([]byte, len(msg)) + + for pi, path := range []string{"/content-length/", "/chunked/"} { + wantLen := []int{len(msg), -1}[pi] + addrSeen = make(map[string]int) + for i := 0; i < 3; i++ { + res, err := http.Get(ts.URL + path) + if err != nil { + t.Errorf("Get %s: %v", path, err) + continue + } + // We want to close this body eventually (before the + // defer afterTest at top runs), but not before the + // len(addrSeen) check at the bottom of this test, + // since Closing this early in the loop would risk + // making connections be re-used for the wrong reason. + defer res.Body.Close() + + if res.ContentLength != int64(wantLen) { + t.Errorf("%s res.ContentLength = %d; want %d", path, res.ContentLength, wantLen) + } + n, err := res.Body.Read(buf) + if n != len(msg) || err != io.EOF { + t.Errorf("%s Read = %v, %v; want %d, EOF", path, n, err, len(msg)) + } + } + if len(addrSeen) != 1 { + t.Errorf("for %s, server saw %d distinct client addresses; want 1", path, len(addrSeen)) + } + } +} + +func TestTransportMaxPerHostIdleConns(t *testing.T) { + defer afterTest(t) + resch := make(chan string) + gotReq := make(chan bool) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + gotReq <- true + msg := <-resch + _, err := w.Write([]byte(msg)) + if err != nil { + t.Fatalf("Write: %v", err) + } + })) + defer ts.Close() + maxIdleConns := 2 + tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConns} + c := &Client{Transport: tr} + + // Start 3 outstanding requests and wait for the server to get them. + // Their responses will hang until we write to resch, though. + donech := make(chan bool) + doReq := func() { + resp, err := c.Get(ts.URL) + if err != nil { + t.Error(err) + return + } + if _, err := ioutil.ReadAll(resp.Body); err != nil { + t.Errorf("ReadAll: %v", err) + return + } + donech <- true + } + go doReq() + <-gotReq + go doReq() + <-gotReq + go doReq() + <-gotReq + + if e, g := 0, len(tr.IdleConnKeysForTesting()); e != g { + t.Fatalf("Before writes, expected %d idle conn cache keys; got %d", e, g) + } + + resch <- "res1" + <-donech + keys := tr.IdleConnKeysForTesting() + if e, g := 1, len(keys); e != g { + t.Fatalf("after first response, expected %d idle conn cache keys; got %d", e, g) + } + cacheKey := "|http|" + ts.Listener.Addr().String() + if keys[0] != cacheKey { + t.Fatalf("Expected idle cache key %q; got %q", cacheKey, keys[0]) + } + if e, g := 1, tr.IdleConnCountForTesting(cacheKey); e != g { + t.Errorf("after first response, expected %d idle conns; got %d", e, g) + } + + resch <- "res2" + <-donech + if e, g := 2, tr.IdleConnCountForTesting(cacheKey); e != g { + t.Errorf("after second response, expected %d idle conns; got %d", e, g) + } + + resch <- "res3" + <-donech + if e, g := maxIdleConns, tr.IdleConnCountForTesting(cacheKey); e != g { + t.Errorf("after third response, still expected %d idle conns; got %d", e, g) + } +} + +func TestTransportServerClosingUnexpectedly(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(hostPortHandler) + defer ts.Close() + + tr := &Transport{} + c := &Client{Transport: tr} + + fetch := func(n, retries int) string { + condFatalf := func(format string, arg ...interface{}) { + if retries <= 0 { + t.Fatalf(format, arg...) + } + t.Logf("retrying shortly after expected error: "+format, arg...) + time.Sleep(time.Second / time.Duration(retries)) + } + for retries >= 0 { + retries-- + res, err := c.Get(ts.URL) + if err != nil { + condFatalf("error in req #%d, GET: %v", n, err) + continue + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + condFatalf("error in req #%d, ReadAll: %v", n, err) + continue + } + res.Body.Close() + return string(body) + } + panic("unreachable") + } + + body1 := fetch(1, 0) + body2 := fetch(2, 0) + + ts.CloseClientConnections() // surprise! + + // This test has an expected race. Sleeping for 25 ms prevents + // it on most fast machines, causing the next fetch() call to + // succeed quickly. But if we do get errors, fetch() will retry 5 + // times with some delays between. + time.Sleep(25 * time.Millisecond) + + body3 := fetch(3, 5) + + if body1 != body2 { + t.Errorf("expected body1 and body2 to be equal") + } + if body2 == body3 { + t.Errorf("expected body2 and body3 to be different") + } +} + +// Test for http://golang.org/issue/2616 (appropriate issue number) +// This fails pretty reliably with GOMAXPROCS=100 or something high. +func TestStressSurpriseServerCloses(t *testing.T) { + defer afterTest(t) + if testing.Short() { + t.Skip("skipping test in short mode") + } + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Length", "5") + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte("Hello")) + w.(Flusher).Flush() + conn, buf, _ := w.(Hijacker).Hijack() + buf.Flush() + conn.Close() + })) + defer ts.Close() + + tr := &Transport{DisableKeepAlives: false} + c := &Client{Transport: tr} + + // Do a bunch of traffic from different goroutines. Send to activityc + // after each request completes, regardless of whether it failed. + const ( + numClients = 50 + reqsPerClient = 250 + ) + activityc := make(chan bool) + for i := 0; i < numClients; i++ { + go func() { + for i := 0; i < reqsPerClient; i++ { + res, err := c.Get(ts.URL) + if err == nil { + // We expect errors since the server is + // hanging up on us after telling us to + // send more requests, so we don't + // actually care what the error is. + // But we want to close the body in cases + // where we won the race. + res.Body.Close() + } + activityc <- true + } + }() + } + + // Make sure all the request come back, one way or another. + for i := 0; i < numClients*reqsPerClient; i++ { + select { + case <-activityc: + case <-time.After(5 * time.Second): + t.Fatalf("presumed deadlock; no HTTP client activity seen in awhile") + } + } +} + +// TestTransportHeadResponses verifies that we deal with Content-Lengths +// with no bodies properly +func TestTransportHeadResponses(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Method != "HEAD" { + panic("expected HEAD; got " + r.Method) + } + w.Header().Set("Content-Length", "123") + w.WriteHeader(200) + })) + defer ts.Close() + + tr := &Transport{DisableKeepAlives: false} + c := &Client{Transport: tr} + for i := 0; i < 2; i++ { + res, err := c.Head(ts.URL) + if err != nil { + t.Errorf("error on loop %d: %v", i, err) + continue + } + if e, g := "123", res.Header.Get("Content-Length"); e != g { + t.Errorf("loop %d: expected Content-Length header of %q, got %q", i, e, g) + } + if e, g := int64(123), res.ContentLength; e != g { + t.Errorf("loop %d: expected res.ContentLength of %v, got %v", i, e, g) + } + if all, err := ioutil.ReadAll(res.Body); err != nil { + t.Errorf("loop %d: Body ReadAll: %v", i, err) + } else if len(all) != 0 { + t.Errorf("Bogus body %q", all) + } + } +} + +// TestTransportHeadChunkedResponse verifies that we ignore chunked transfer-encoding +// on responses to HEAD requests. +func TestTransportHeadChunkedResponse(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Method != "HEAD" { + panic("expected HEAD; got " + r.Method) + } + w.Header().Set("Transfer-Encoding", "chunked") // client should ignore + w.Header().Set("x-client-ipport", r.RemoteAddr) + w.WriteHeader(200) + })) + defer ts.Close() + + tr := &Transport{DisableKeepAlives: false} + c := &Client{Transport: tr} + + res1, err := c.Head(ts.URL) + if err != nil { + t.Fatalf("request 1 error: %v", err) + } + res2, err := c.Head(ts.URL) + if err != nil { + t.Fatalf("request 2 error: %v", err) + } + if v1, v2 := res1.Header.Get("x-client-ipport"), res2.Header.Get("x-client-ipport"); v1 != v2 { + t.Errorf("ip/ports differed between head requests: %q vs %q", v1, v2) + } +} + +var roundTripTests = []struct { + accept string + expectAccept string + compressed bool +}{ + // Requests with no accept-encoding header use transparent compression + {"", "gzip", false}, + // Requests with other accept-encoding should pass through unmodified + {"foo", "foo", false}, + // Requests with accept-encoding == gzip should be passed through + {"gzip", "gzip", true}, +} + +// Test that the modification made to the Request by the RoundTripper is cleaned up +func TestRoundTripGzip(t *testing.T) { + defer afterTest(t) + const responseBody = "test response body" + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + accept := req.Header.Get("Accept-Encoding") + if expect := req.FormValue("expect_accept"); accept != expect { + t.Errorf("in handler, test %v: Accept-Encoding = %q, want %q", + req.FormValue("testnum"), accept, expect) + } + if accept == "gzip" { + rw.Header().Set("Content-Encoding", "gzip") + gz := gzip.NewWriter(rw) + gz.Write([]byte(responseBody)) + gz.Close() + } else { + rw.Header().Set("Content-Encoding", accept) + rw.Write([]byte(responseBody)) + } + })) + defer ts.Close() + + for i, test := range roundTripTests { + // Test basic request (no accept-encoding) + req, _ := NewRequest("GET", fmt.Sprintf("%s/?testnum=%d&expect_accept=%s", ts.URL, i, test.expectAccept), nil) + if test.accept != "" { + req.Header.Set("Accept-Encoding", test.accept) + } + res, err := DefaultTransport.RoundTrip(req) + var body []byte + if test.compressed { + var r *gzip.Reader + r, err = gzip.NewReader(res.Body) + if err != nil { + t.Errorf("%d. gzip NewReader: %v", i, err) + continue + } + body, err = ioutil.ReadAll(r) + res.Body.Close() + } else { + body, err = ioutil.ReadAll(res.Body) + } + if err != nil { + t.Errorf("%d. Error: %q", i, err) + continue + } + if g, e := string(body), responseBody; g != e { + t.Errorf("%d. body = %q; want %q", i, g, e) + } + if g, e := req.Header.Get("Accept-Encoding"), test.accept; g != e { + t.Errorf("%d. Accept-Encoding = %q; want %q (it was mutated, in violation of RoundTrip contract)", i, g, e) + } + if g, e := res.Header.Get("Content-Encoding"), test.accept; g != e { + t.Errorf("%d. Content-Encoding = %q; want %q", i, g, e) + } + } + +} + +func TestTransportGzip(t *testing.T) { + defer afterTest(t) + const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const nRandBytes = 1024 * 1024 + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + if req.Method == "HEAD" { + if g := req.Header.Get("Accept-Encoding"); g != "" { + t.Errorf("HEAD request sent with Accept-Encoding of %q; want none", g) + } + return + } + if g, e := req.Header.Get("Accept-Encoding"), "gzip"; g != e { + t.Errorf("Accept-Encoding = %q, want %q", g, e) + } + rw.Header().Set("Content-Encoding", "gzip") + + var w io.Writer = rw + var buf bytes.Buffer + if req.FormValue("chunked") == "0" { + w = &buf + defer io.Copy(rw, &buf) + defer func() { + rw.Header().Set("Content-Length", strconv.Itoa(buf.Len())) + }() + } + gz := gzip.NewWriter(w) + gz.Write([]byte(testString)) + if req.FormValue("body") == "large" { + io.CopyN(gz, rand.Reader, nRandBytes) + } + gz.Close() + })) + defer ts.Close() + + for _, chunked := range []string{"1", "0"} { + c := &Client{Transport: &Transport{}} + + // First fetch something large, but only read some of it. + res, err := c.Get(ts.URL + "/?body=large&chunked=" + chunked) + if err != nil { + t.Fatalf("large get: %v", err) + } + buf := make([]byte, len(testString)) + n, err := io.ReadFull(res.Body, buf) + if err != nil { + t.Fatalf("partial read of large response: size=%d, %v", n, err) + } + if e, g := testString, string(buf); e != g { + t.Errorf("partial read got %q, expected %q", g, e) + } + res.Body.Close() + // Read on the body, even though it's closed + n, err = res.Body.Read(buf) + if n != 0 || err == nil { + t.Errorf("expected error post-closed large Read; got = %d, %v", n, err) + } + + // Then something small. + res, err = c.Get(ts.URL + "/?chunked=" + chunked) + if err != nil { + t.Fatal(err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if g, e := string(body), testString; g != e { + t.Fatalf("body = %q; want %q", g, e) + } + if g, e := res.Header.Get("Content-Encoding"), ""; g != e { + t.Fatalf("Content-Encoding = %q; want %q", g, e) + } + + // Read on the body after it's been fully read: + n, err = res.Body.Read(buf) + if n != 0 || err == nil { + t.Errorf("expected Read error after exhausted reads; got %d, %v", n, err) + } + res.Body.Close() + n, err = res.Body.Read(buf) + if n != 0 || err == nil { + t.Errorf("expected Read error after Close; got %d, %v", n, err) + } + } + + // And a HEAD request too, because they're always weird. + c := &Client{Transport: &Transport{}} + res, err := c.Head(ts.URL) + if err != nil { + t.Fatalf("Head: %v", err) + } + if res.StatusCode != 200 { + t.Errorf("Head status=%d; want=200", res.StatusCode) + } +} + +func TestTransportProxy(t *testing.T) { + defer afterTest(t) + ch := make(chan string, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ch <- "real server" + })) + defer ts.Close() + proxy := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ch <- "proxy for " + r.URL.String() + })) + defer proxy.Close() + + pu, err := url.Parse(proxy.URL) + if err != nil { + t.Fatal(err) + } + c := &Client{Transport: &Transport{Proxy: ProxyURL(pu)}} + c.Head(ts.URL) + got := <-ch + want := "proxy for " + ts.URL + "/" + if got != want { + t.Errorf("want %q, got %q", want, got) + } +} + +// TestTransportGzipRecursive sends a gzip quine and checks that the +// client gets the same value back. This is more cute than anything, +// but checks that we don't recurse forever, and checks that +// Content-Encoding is removed. +func TestTransportGzipRecursive(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", "gzip") + w.Write(rgz) + })) + defer ts.Close() + + c := &Client{Transport: &Transport{}} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(body, rgz) { + t.Fatalf("Incorrect result from recursive gz:\nhave=%x\nwant=%x", + body, rgz) + } + if g, e := res.Header.Get("Content-Encoding"), ""; g != e { + t.Fatalf("Content-Encoding = %q; want %q", g, e) + } +} + +// golang.org/issue/7750: request fails when server replies with +// a short gzip body +func TestTransportGzipShort(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", "gzip") + w.Write([]byte{0x1f, 0x8b}) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + _, err = ioutil.ReadAll(res.Body) + if err == nil { + t.Fatal("Expect an error from reading a body.") + } + if err != io.ErrUnexpectedEOF { + t.Errorf("ReadAll error = %v; want io.ErrUnexpectedEOF", err) + } +} + +// tests that persistent goroutine connections shut down when no longer desired. +func TestTransportPersistConnLeak(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + gotReqCh := make(chan bool) + unblockCh := make(chan bool) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + gotReqCh <- true + <-unblockCh + w.Header().Set("Content-Length", "0") + w.WriteHeader(204) + })) + defer ts.Close() + + tr := &Transport{} + c := &Client{Transport: tr} + + n0 := runtime.NumGoroutine() + + const numReq = 25 + didReqCh := make(chan bool) + for i := 0; i < numReq; i++ { + go func() { + res, err := c.Get(ts.URL) + didReqCh <- true + if err != nil { + t.Errorf("client fetch error: %v", err) + return + } + res.Body.Close() + }() + } + + // Wait for all goroutines to be stuck in the Handler. + for i := 0; i < numReq; i++ { + <-gotReqCh + } + + nhigh := runtime.NumGoroutine() + + // Tell all handlers to unblock and reply. + for i := 0; i < numReq; i++ { + unblockCh <- true + } + + // Wait for all HTTP clients to be done. + for i := 0; i < numReq; i++ { + <-didReqCh + } + + tr.CloseIdleConnections() + time.Sleep(100 * time.Millisecond) + runtime.GC() + runtime.GC() // even more. + nfinal := runtime.NumGoroutine() + + growth := nfinal - n0 + + // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. + // Previously we were leaking one per numReq. + if int(growth) > 5 { + t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) + t.Error("too many new goroutines") + } +} + +// golang.org/issue/4531: Transport leaks goroutines when +// request.ContentLength is explicitly short +func TestTransportPersistConnLeakShortBody(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + })) + defer ts.Close() + + tr := &Transport{} + c := &Client{Transport: tr} + + n0 := runtime.NumGoroutine() + body := []byte("Hello") + for i := 0; i < 20; i++ { + req, err := NewRequest("POST", ts.URL, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.ContentLength = int64(len(body) - 2) // explicitly short + _, err = c.Do(req) + if err == nil { + t.Fatal("Expect an error from writing too long of a body.") + } + } + nhigh := runtime.NumGoroutine() + tr.CloseIdleConnections() + time.Sleep(400 * time.Millisecond) + runtime.GC() + nfinal := runtime.NumGoroutine() + + growth := nfinal - n0 + + // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. + // Previously we were leaking one per numReq. + t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) + if int(growth) > 5 { + t.Error("too many new goroutines") + } +} + +// This used to crash; http://golang.org/issue/3266 +func TestTransportIdleConnCrash(t *testing.T) { + defer afterTest(t) + tr := &Transport{} + c := &Client{Transport: tr} + + unblockCh := make(chan bool, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + <-unblockCh + tr.CloseIdleConnections() + })) + defer ts.Close() + + didreq := make(chan bool) + go func() { + res, err := c.Get(ts.URL) + if err != nil { + t.Error(err) + } else { + res.Body.Close() // returns idle conn + } + didreq <- true + }() + unblockCh <- true + <-didreq +} + +// Test that the transport doesn't close the TCP connection early, +// before the response body has been read. This was a regression +// which sadly lacked a triggering test. The large response body made +// the old race easier to trigger. +func TestIssue3644(t *testing.T) { + defer afterTest(t) + const numFoos = 5000 + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "close") + for i := 0; i < numFoos; i++ { + w.Write([]byte("foo ")) + } + })) + defer ts.Close() + tr := &Transport{} + c := &Client{Transport: tr} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + bs, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if len(bs) != numFoos*len("foo ") { + t.Errorf("unexpected response length") + } +} + +// Test that a client receives a server's reply, even if the server doesn't read +// the entire request body. +func TestIssue3595(t *testing.T) { + defer afterTest(t) + const deniedMsg = "sorry, denied." + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + Error(w, deniedMsg, StatusUnauthorized) + })) + defer ts.Close() + tr := &Transport{} + c := &Client{Transport: tr} + res, err := c.Post(ts.URL, "application/octet-stream", neverEnding('a')) + if err != nil { + t.Errorf("Post: %v", err) + return + } + got, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("Body ReadAll: %v", err) + } + if !strings.Contains(string(got), deniedMsg) { + t.Errorf("Known bug: response %q does not contain %q", got, deniedMsg) + } +} + +// From http://golang.org/issue/4454 , +// "client fails to handle requests with no body and chunked encoding" +func TestChunkedNoContent(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(StatusNoContent) + })) + defer ts.Close() + + for _, closeBody := range []bool{true, false} { + c := &Client{Transport: &Transport{}} + const n = 4 + for i := 1; i <= n; i++ { + res, err := c.Get(ts.URL) + if err != nil { + t.Errorf("closingBody=%v, req %d/%d: %v", closeBody, i, n, err) + } else { + if closeBody { + res.Body.Close() + } + } + } + } +} + +func TestTransportConcurrency(t *testing.T) { + defer afterTest(t) + maxProcs, numReqs := 16, 500 + if testing.Short() { + maxProcs, numReqs = 4, 50 + } + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(maxProcs)) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "%v", r.FormValue("echo")) + })) + defer ts.Close() + + var wg sync.WaitGroup + wg.Add(numReqs) + + // Due to the Transport's "socket late binding" (see + // idleConnCh in transport.go), the numReqs HTTP requests + // below can finish with a dial still outstanding. To keep + // the leak checker happy, keep track of pending dials and + // wait for them to finish (and be closed or returned to the + // idle pool) before we close idle connections. + SetPendingDialHooks(func() { wg.Add(1) }, wg.Done) + defer SetPendingDialHooks(nil, nil) + + tr := &Transport{} + defer tr.CloseIdleConnections() + + c := &Client{Transport: tr} + reqs := make(chan string) + defer close(reqs) + + for i := 0; i < maxProcs*2; i++ { + go func() { + for req := range reqs { + res, err := c.Get(ts.URL + "/?echo=" + req) + if err != nil { + t.Errorf("error on req %s: %v", req, err) + wg.Done() + continue + } + all, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Errorf("read error on req %s: %v", req, err) + wg.Done() + continue + } + if string(all) != req { + t.Errorf("body of req %s = %q; want %q", req, all, req) + } + res.Body.Close() + wg.Done() + } + }() + } + for i := 0; i < numReqs; i++ { + reqs <- fmt.Sprintf("request-%d", i) + } + wg.Wait() +} + +func TestIssue4191_InfiniteGetTimeout(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + const debug = false + mux := NewServeMux() + mux.HandleFunc("/get", func(w ResponseWriter, r *Request) { + io.Copy(w, neverEnding('a')) + }) + ts := httptest.NewServer(mux) + timeout := 100 * time.Millisecond + + client := &Client{ + Transport: &Transport{ + Dial: func(n, addr string) (net.Conn, error) { + conn, err := net.Dial(n, addr) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(timeout)) + if debug { + conn = NewLoggingConn("client", conn) + } + return conn, nil + }, + DisableKeepAlives: true, + }, + } + + getFailed := false + nRuns := 5 + if testing.Short() { + nRuns = 1 + } + for i := 0; i < nRuns; i++ { + if debug { + println("run", i+1, "of", nRuns) + } + sres, err := client.Get(ts.URL + "/get") + if err != nil { + if !getFailed { + // Make the timeout longer, once. + getFailed = true + t.Logf("increasing timeout") + i-- + timeout *= 10 + continue + } + t.Errorf("Error issuing GET: %v", err) + break + } + _, err = io.Copy(ioutil.Discard, sres.Body) + if err == nil { + t.Errorf("Unexpected successful copy") + break + } + } + if debug { + println("tests complete; waiting for handlers to finish") + } + ts.Close() +} + +func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7237") + } + defer afterTest(t) + const debug = false + mux := NewServeMux() + mux.HandleFunc("/get", func(w ResponseWriter, r *Request) { + io.Copy(w, neverEnding('a')) + }) + mux.HandleFunc("/put", func(w ResponseWriter, r *Request) { + defer r.Body.Close() + io.Copy(ioutil.Discard, r.Body) + }) + ts := httptest.NewServer(mux) + timeout := 100 * time.Millisecond + + client := &Client{ + Transport: &Transport{ + Dial: func(n, addr string) (net.Conn, error) { + conn, err := net.Dial(n, addr) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(timeout)) + if debug { + conn = NewLoggingConn("client", conn) + } + return conn, nil + }, + DisableKeepAlives: true, + }, + } + + getFailed := false + nRuns := 5 + if testing.Short() { + nRuns = 1 + } + for i := 0; i < nRuns; i++ { + if debug { + println("run", i+1, "of", nRuns) + } + sres, err := client.Get(ts.URL + "/get") + if err != nil { + if !getFailed { + // Make the timeout longer, once. + getFailed = true + t.Logf("increasing timeout") + i-- + timeout *= 10 + continue + } + t.Errorf("Error issuing GET: %v", err) + break + } + req, _ := NewRequest("PUT", ts.URL+"/put", sres.Body) + _, err = client.Do(req) + if err == nil { + sres.Body.Close() + t.Errorf("Unexpected successful PUT") + break + } + sres.Body.Close() + } + if debug { + println("tests complete; waiting for handlers to finish") + } + ts.Close() +} + +func TestTransportResponseHeaderTimeout(t *testing.T) { + defer afterTest(t) + if testing.Short() { + t.Skip("skipping timeout test in -short mode") + } + inHandler := make(chan bool, 1) + mux := NewServeMux() + mux.HandleFunc("/fast", func(w ResponseWriter, r *Request) { + inHandler <- true + }) + mux.HandleFunc("/slow", func(w ResponseWriter, r *Request) { + inHandler <- true + time.Sleep(2 * time.Second) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + tr := &Transport{ + ResponseHeaderTimeout: 500 * time.Millisecond, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + tests := []struct { + path string + want int + wantErr string + }{ + {path: "/fast", want: 200}, + {path: "/slow", wantErr: "timeout awaiting response headers"}, + {path: "/fast", want: 200}, + } + for i, tt := range tests { + res, err := c.Get(ts.URL + tt.path) + select { + case <-inHandler: + case <-time.After(5 * time.Second): + t.Errorf("never entered handler for test index %d, %s", i, tt.path) + continue + } + if err != nil { + uerr, ok := err.(*url.Error) + if !ok { + t.Errorf("error is not an url.Error; got: %#v", err) + continue + } + nerr, ok := uerr.Err.(net.Error) + if !ok { + t.Errorf("error does not satisfy net.Error interface; got: %#v", err) + continue + } + if !nerr.Timeout() { + t.Errorf("want timeout error; got: %q", nerr) + continue + } + if strings.Contains(err.Error(), tt.wantErr) { + continue + } + t.Errorf("%d. unexpected error: %v", i, err) + continue + } + if tt.wantErr != "" { + t.Errorf("%d. no error. expected error: %v", i, tt.wantErr) + continue + } + if res.StatusCode != tt.want { + t.Errorf("%d for path %q status = %d; want %d", i, tt.path, res.StatusCode, tt.want) + } + } +} + +func TestTransportCancelRequest(t *testing.T) { + defer afterTest(t) + if testing.Short() { + t.Skip("skipping test in -short mode") + } + unblockc := make(chan bool) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "Hello") + w.(Flusher).Flush() // send headers and some body + <-unblockc + })) + defer ts.Close() + defer close(unblockc) + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + req, _ := NewRequest("GET", ts.URL, nil) + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + go func() { + time.Sleep(1 * time.Second) + tr.CancelRequest(req) + }() + t0 := time.Now() + body, err := ioutil.ReadAll(res.Body) + d := time.Since(t0) + + if err == nil { + t.Error("expected an error reading the body") + } + if string(body) != "Hello" { + t.Errorf("Body = %q; want Hello", body) + } + if d < 500*time.Millisecond { + t.Errorf("expected ~1 second delay; got %v", d) + } + // Verify no outstanding requests after readLoop/writeLoop + // goroutines shut down. + for tries := 3; tries > 0; tries-- { + n := tr.NumPendingRequestsForTesting() + if n == 0 { + break + } + time.Sleep(100 * time.Millisecond) + if tries == 1 { + t.Errorf("pending requests = %d; want 0", n) + } + } +} + +func TestTransportCancelRequestInDial(t *testing.T) { + defer afterTest(t) + if testing.Short() { + t.Skip("skipping test in -short mode") + } + var logbuf bytes.Buffer + eventLog := log.New(&logbuf, "", 0) + + unblockDial := make(chan bool) + defer close(unblockDial) + + inDial := make(chan bool) + tr := &Transport{ + Dial: func(network, addr string) (net.Conn, error) { + eventLog.Println("dial: blocking") + inDial <- true + <-unblockDial + return nil, errors.New("nope") + }, + } + cl := &Client{Transport: tr} + gotres := make(chan bool) + req, _ := NewRequest("GET", "http://something.no-network.tld/", nil) + go func() { + _, err := cl.Do(req) + eventLog.Printf("Get = %v", err) + gotres <- true + }() + + select { + case <-inDial: + case <-time.After(5 * time.Second): + t.Fatal("timeout; never saw blocking dial") + } + + eventLog.Printf("canceling") + tr.CancelRequest(req) + + select { + case <-gotres: + case <-time.After(5 * time.Second): + panic("hang. events are: " + logbuf.String()) + } + + got := logbuf.String() + want := `dial: blocking +canceling +Get = Get http://something.no-network.tld/: net/http: request canceled while waiting for connection +` + if got != want { + t.Errorf("Got events:\n%s\nWant:\n%s", got, want) + } +} + +// golang.org/issue/3672 -- Client can't close HTTP stream +// Calling Close on a Response.Body used to just read until EOF. +// Now it actually closes the TCP connection. +func TestTransportCloseResponseBody(t *testing.T) { + defer afterTest(t) + writeErr := make(chan error, 1) + msg := []byte("young\n") + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + for { + _, err := w.Write(msg) + if err != nil { + writeErr <- err + return + } + w.(Flusher).Flush() + } + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + req, _ := NewRequest("GET", ts.URL, nil) + defer tr.CancelRequest(req) + + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + + const repeats = 3 + buf := make([]byte, len(msg)*repeats) + want := bytes.Repeat(msg, repeats) + + _, err = io.ReadFull(res.Body, buf) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, want) { + t.Fatalf("read %q; want %q", buf, want) + } + didClose := make(chan error, 1) + go func() { + didClose <- res.Body.Close() + }() + select { + case err := <-didClose: + if err != nil { + t.Errorf("Close = %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("too long waiting for close") + } + select { + case err := <-writeErr: + if err == nil { + t.Errorf("expected non-nil write error") + } + case <-time.After(10 * time.Second): + t.Fatal("too long waiting for write error") + } +} + +type fooProto struct{} + +func (fooProto) RoundTrip(req *Request) (*Response, error) { + res := &Response{ + Status: "200 OK", + StatusCode: 200, + Header: make(Header), + Body: ioutil.NopCloser(strings.NewReader("You wanted " + req.URL.String())), + } + return res, nil +} + +func TestTransportAltProto(t *testing.T) { + defer afterTest(t) + tr := &Transport{} + c := &Client{Transport: tr} + tr.RegisterProtocol("foo", fooProto{}) + res, err := c.Get("foo://bar.com/path") + if err != nil { + t.Fatal(err) + } + bodyb, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + body := string(bodyb) + if e := "You wanted foo://bar.com/path"; body != e { + t.Errorf("got response %q, want %q", body, e) + } +} + +func TestTransportNoHost(t *testing.T) { + defer afterTest(t) + tr := &Transport{} + _, err := tr.RoundTrip(&Request{ + Header: make(Header), + URL: &url.URL{ + Scheme: "http", + }, + }) + want := "http: no Host in request URL" + if got := fmt.Sprint(err); got != want { + t.Errorf("error = %v; want %q", err, want) + } +} + +func TestTransportSocketLateBinding(t *testing.T) { + defer afterTest(t) + + mux := NewServeMux() + fooGate := make(chan bool, 1) + mux.HandleFunc("/foo", func(w ResponseWriter, r *Request) { + w.Header().Set("foo-ipport", r.RemoteAddr) + w.(Flusher).Flush() + <-fooGate + }) + mux.HandleFunc("/bar", func(w ResponseWriter, r *Request) { + w.Header().Set("bar-ipport", r.RemoteAddr) + }) + ts := httptest.NewServer(mux) + defer ts.Close() + + dialGate := make(chan bool, 1) + tr := &Transport{ + Dial: func(n, addr string) (net.Conn, error) { + if <-dialGate { + return net.Dial(n, addr) + } + return nil, errors.New("manually closed") + }, + DisableKeepAlives: false, + } + defer tr.CloseIdleConnections() + c := &Client{ + Transport: tr, + } + + dialGate <- true // only allow one dial + fooRes, err := c.Get(ts.URL + "/foo") + if err != nil { + t.Fatal(err) + } + fooAddr := fooRes.Header.Get("foo-ipport") + if fooAddr == "" { + t.Fatal("No addr on /foo request") + } + time.AfterFunc(200*time.Millisecond, func() { + // let the foo response finish so we can use its + // connection for /bar + fooGate <- true + io.Copy(ioutil.Discard, fooRes.Body) + fooRes.Body.Close() + }) + + barRes, err := c.Get(ts.URL + "/bar") + if err != nil { + t.Fatal(err) + } + barAddr := barRes.Header.Get("bar-ipport") + if barAddr != fooAddr { + t.Fatalf("/foo came from conn %q; /bar came from %q instead", fooAddr, barAddr) + } + barRes.Body.Close() + dialGate <- false +} + +// Issue 2184 +func TestTransportReading100Continue(t *testing.T) { + defer afterTest(t) + + const numReqs = 5 + reqBody := func(n int) string { return fmt.Sprintf("request body %d", n) } + reqID := func(n int) string { return fmt.Sprintf("REQ-ID-%d", n) } + + send100Response := func(w *io.PipeWriter, r *io.PipeReader) { + defer w.Close() + defer r.Close() + br := bufio.NewReader(r) + n := 0 + for { + n++ + req, err := ReadRequest(br) + if err == io.EOF { + return + } + if err != nil { + t.Error(err) + return + } + slurp, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("Server request body slurp: %v", err) + return + } + id := req.Header.Get("Request-Id") + resCode := req.Header.Get("X-Want-Response-Code") + if resCode == "" { + resCode = "100 Continue" + if string(slurp) != reqBody(n) { + t.Errorf("Server got %q, %v; want %q", slurp, err, reqBody(n)) + } + } + body := fmt.Sprintf("Response number %d", n) + v := []byte(strings.Replace(fmt.Sprintf(`HTTP/1.1 %s +Date: Thu, 28 Feb 2013 17:55:41 GMT + +HTTP/1.1 200 OK +Content-Type: text/html +Echo-Request-Id: %s +Content-Length: %d + +%s`, resCode, id, len(body), body), "\n", "\r\n", -1)) + w.Write(v) + if id == reqID(numReqs) { + return + } + } + + } + + tr := &Transport{ + Dial: func(n, addr string) (net.Conn, error) { + sr, sw := io.Pipe() // server read/write + cr, cw := io.Pipe() // client read/write + conn := &rwTestConn{ + Reader: cr, + Writer: sw, + closeFunc: func() error { + sw.Close() + cw.Close() + return nil + }, + } + go send100Response(cw, sr) + return conn, nil + }, + DisableKeepAlives: false, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + testResponse := func(req *Request, name string, wantCode int) { + res, err := c.Do(req) + if err != nil { + t.Fatalf("%s: Do: %v", name, err) + } + if res.StatusCode != wantCode { + t.Fatalf("%s: Response Statuscode=%d; want %d", name, res.StatusCode, wantCode) + } + if id, idBack := req.Header.Get("Request-Id"), res.Header.Get("Echo-Request-Id"); id != "" && id != idBack { + t.Errorf("%s: response id %q != request id %q", name, idBack, id) + } + _, err = ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("%s: Slurp error: %v", name, err) + } + } + + // Few 100 responses, making sure we're not off-by-one. + for i := 1; i <= numReqs; i++ { + req, _ := NewRequest("POST", "http://dummy.tld/", strings.NewReader(reqBody(i))) + req.Header.Set("Request-Id", reqID(i)) + testResponse(req, fmt.Sprintf("100, %d/%d", i, numReqs), 200) + } + + // And some other informational 1xx but non-100 responses, to test + // we return them but don't re-use the connection. + for i := 1; i <= numReqs; i++ { + req, _ := NewRequest("POST", "http://other.tld/", strings.NewReader(reqBody(i))) + req.Header.Set("X-Want-Response-Code", "123 Sesame Street") + testResponse(req, fmt.Sprintf("123, %d/%d", i, numReqs), 123) + } +} + +type proxyFromEnvTest struct { + req string // URL to fetch; blank means "http://example.com" + env string + noenv string + want string + wanterr error +} + +func (t proxyFromEnvTest) String() string { + var buf bytes.Buffer + if t.env != "" { + fmt.Fprintf(&buf, "http_proxy=%q", t.env) + } + if t.noenv != "" { + fmt.Fprintf(&buf, " no_proxy=%q", t.noenv) + } + req := "http://example.com" + if t.req != "" { + req = t.req + } + fmt.Fprintf(&buf, " req=%q", req) + return strings.TrimSpace(buf.String()) +} + +var proxyFromEnvTests = []proxyFromEnvTest{ + {env: "127.0.0.1:8080", want: "http://127.0.0.1:8080"}, + {env: "cache.corp.example.com:1234", want: "http://cache.corp.example.com:1234"}, + {env: "cache.corp.example.com", want: "http://cache.corp.example.com"}, + {env: "https://cache.corp.example.com", want: "https://cache.corp.example.com"}, + {env: "http://127.0.0.1:8080", want: "http://127.0.0.1:8080"}, + {env: "https://127.0.0.1:8080", want: "https://127.0.0.1:8080"}, + {want: "<nil>"}, + {noenv: "example.com", req: "http://example.com/", env: "proxy", want: "<nil>"}, + {noenv: ".example.com", req: "http://example.com/", env: "proxy", want: "<nil>"}, + {noenv: "ample.com", req: "http://example.com/", env: "proxy", want: "http://proxy"}, + {noenv: "example.com", req: "http://foo.example.com/", env: "proxy", want: "<nil>"}, + {noenv: ".foo.com", req: "http://example.com/", env: "proxy", want: "http://proxy"}, +} + +func TestProxyFromEnvironment(t *testing.T) { + ResetProxyEnv() + for _, tt := range proxyFromEnvTests { + os.Setenv("HTTP_PROXY", tt.env) + os.Setenv("NO_PROXY", tt.noenv) + ResetCachedEnvironment() + reqURL := tt.req + if reqURL == "" { + reqURL = "http://example.com" + } + req, _ := NewRequest("GET", reqURL, nil) + url, err := ProxyFromEnvironment(req) + if g, e := fmt.Sprintf("%v", err), fmt.Sprintf("%v", tt.wanterr); g != e { + t.Errorf("%v: got error = %q, want %q", tt, g, e) + continue + } + if got := fmt.Sprintf("%s", url); got != tt.want { + t.Errorf("%v: got URL = %q, want %q", tt, url, tt.want) + } + } +} + +func TestIdleConnChannelLeak(t *testing.T) { + var mu sync.Mutex + var n int + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + mu.Lock() + n++ + mu.Unlock() + })) + defer ts.Close() + + tr := &Transport{ + Dial: func(netw, addr string) (net.Conn, error) { + return net.Dial(netw, ts.Listener.Addr().String()) + }, + } + defer tr.CloseIdleConnections() + + c := &Client{Transport: tr} + + // First, without keep-alives. + for _, disableKeep := range []bool{true, false} { + tr.DisableKeepAlives = disableKeep + for i := 0; i < 5; i++ { + _, err := c.Get(fmt.Sprintf("http://foo-host-%d.tld/", i)) + if err != nil { + t.Fatal(err) + } + } + if got := tr.IdleConnChMapSizeForTesting(); got != 0 { + t.Fatalf("ForDisableKeepAlives = %v, map size = %d; want 0", disableKeep, got) + } + } +} + +// Verify the status quo: that the Client.Post function coerces its +// body into a ReadCloser if it's a Closer, and that the Transport +// then closes it. +func TestTransportClosesRequestBody(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(http.HandlerFunc(func(w ResponseWriter, r *Request) { + io.Copy(ioutil.Discard, r.Body) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + cl := &Client{Transport: tr} + + closes := 0 + + res, err := cl.Post(ts.URL, "text/plain", countCloseReader{&closes, strings.NewReader("hello")}) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if closes != 1 { + t.Errorf("closes = %d; want 1", closes) + } +} + +func TestTransportTLSHandshakeTimeout(t *testing.T) { + defer afterTest(t) + if testing.Short() { + t.Skip("skipping in short mode") + } + ln := newLocalListener(t) + defer ln.Close() + testdonec := make(chan struct{}) + defer close(testdonec) + + go func() { + c, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + <-testdonec + c.Close() + }() + + getdonec := make(chan struct{}) + go func() { + defer close(getdonec) + tr := &Transport{ + Dial: func(_, _ string) (net.Conn, error) { + return net.Dial("tcp", ln.Addr().String()) + }, + TLSHandshakeTimeout: 250 * time.Millisecond, + } + cl := &Client{Transport: tr} + _, err := cl.Get("https://dummy.tld/") + if err == nil { + t.Error("expected error") + return + } + ue, ok := err.(*url.Error) + if !ok { + t.Errorf("expected url.Error; got %#v", err) + return + } + ne, ok := ue.Err.(net.Error) + if !ok { + t.Errorf("expected net.Error; got %#v", err) + return + } + if !ne.Timeout() { + t.Errorf("expected timeout error; got %v", err) + } + if !strings.Contains(err.Error(), "handshake timeout") { + t.Errorf("expected 'handshake timeout' in error; got %v", err) + } + }() + select { + case <-getdonec: + case <-time.After(5 * time.Second): + t.Error("test timeout; TLS handshake hung?") + } +} + +// Trying to repro golang.org/issue/3514 +func TestTLSServerClosesConnection(t *testing.T) { + defer afterTest(t) + if runtime.GOOS == "windows" { + t.Skip("skipping flaky test on Windows; golang.org/issue/7634") + } + closedc := make(chan bool, 1) + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if strings.Contains(r.URL.Path, "/keep-alive-then-die") { + conn, _, _ := w.(Hijacker).Hijack() + conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nfoo")) + conn.Close() + closedc <- true + return + } + fmt.Fprintf(w, "hello") + })) + defer ts.Close() + tr := &Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + defer tr.CloseIdleConnections() + client := &Client{Transport: tr} + + var nSuccess = 0 + var errs []error + const trials = 20 + for i := 0; i < trials; i++ { + tr.CloseIdleConnections() + res, err := client.Get(ts.URL + "/keep-alive-then-die") + if err != nil { + t.Fatal(err) + } + <-closedc + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if string(slurp) != "foo" { + t.Errorf("Got %q, want foo", slurp) + } + + // Now try again and see if we successfully + // pick a new connection. + res, err = client.Get(ts.URL + "/") + if err != nil { + errs = append(errs, err) + continue + } + slurp, err = ioutil.ReadAll(res.Body) + if err != nil { + errs = append(errs, err) + continue + } + nSuccess++ + } + if nSuccess > 0 { + t.Logf("successes = %d of %d", nSuccess, trials) + } else { + t.Errorf("All runs failed:") + } + for _, err := range errs { + t.Logf(" err: %v", err) + } +} + +// byteFromChanReader is an io.Reader that reads a single byte at a +// time from the channel. When the channel is closed, the reader +// returns io.EOF. +type byteFromChanReader chan byte + +func (c byteFromChanReader) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + b, ok := <-c + if !ok { + return 0, io.EOF + } + p[0] = b + return 1, nil +} + +// Verifies that the Transport doesn't reuse a connection in the case +// where the server replies before the request has been fully +// written. We still honor that reply (see TestIssue3595), but don't +// send future requests on the connection because it's then in a +// questionable state. +// golang.org/issue/7569 +func TestTransportNoReuseAfterEarlyResponse(t *testing.T) { + defer afterTest(t) + var sconn struct { + sync.Mutex + c net.Conn + } + var getOkay bool + closeConn := func() { + sconn.Lock() + defer sconn.Unlock() + if sconn.c != nil { + sconn.c.Close() + sconn.c = nil + if !getOkay { + t.Logf("Closed server connection") + } + } + } + defer closeConn() + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Method == "GET" { + io.WriteString(w, "bar") + return + } + conn, _, _ := w.(Hijacker).Hijack() + sconn.Lock() + sconn.c = conn + sconn.Unlock() + conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nfoo")) // keep-alive + go io.Copy(ioutil.Discard, conn) + })) + defer ts.Close() + tr := &Transport{} + defer tr.CloseIdleConnections() + client := &Client{Transport: tr} + + const bodySize = 256 << 10 + finalBit := make(byteFromChanReader, 1) + req, _ := NewRequest("POST", ts.URL, io.MultiReader(io.LimitReader(neverEnding('x'), bodySize-1), finalBit)) + req.ContentLength = bodySize + res, err := client.Do(req) + if err := wantBody(res, err, "foo"); err != nil { + t.Errorf("POST response: %v", err) + } + donec := make(chan bool) + go func() { + defer close(donec) + res, err = client.Get(ts.URL) + if err := wantBody(res, err, "bar"); err != nil { + t.Errorf("GET response: %v", err) + return + } + getOkay = true // suppress test noise + }() + time.AfterFunc(5*time.Second, closeConn) + select { + case <-donec: + finalBit <- 'x' // unblock the writeloop of the first Post + close(finalBit) + case <-time.After(7 * time.Second): + t.Fatal("timeout waiting for GET request to finish") + } +} + +type errorReader struct { + err error +} + +func (e errorReader) Read(p []byte) (int, error) { return 0, e.err } + +type closerFunc func() error + +func (f closerFunc) Close() error { return f() } + +// Issue 6981 +func TestTransportClosesBodyOnError(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see http://golang.org/issue/7782") + } + defer afterTest(t) + readBody := make(chan error, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + _, err := ioutil.ReadAll(r.Body) + readBody <- err + })) + defer ts.Close() + fakeErr := errors.New("fake error") + didClose := make(chan bool, 1) + req, _ := NewRequest("POST", ts.URL, struct { + io.Reader + io.Closer + }{ + io.MultiReader(io.LimitReader(neverEnding('x'), 1<<20), errorReader{fakeErr}), + closerFunc(func() error { + select { + case didClose <- true: + default: + } + return nil + }), + }) + res, err := DefaultClient.Do(req) + if res != nil { + defer res.Body.Close() + } + if err == nil || !strings.Contains(err.Error(), fakeErr.Error()) { + t.Fatalf("Do error = %v; want something containing %q", err, fakeErr.Error()) + } + select { + case err := <-readBody: + if err == nil { + t.Errorf("Unexpected success reading request body from handler; want 'unexpected EOF reading trailer'") + } + case <-time.After(5 * time.Second): + t.Error("timeout waiting for server handler to complete") + } + select { + case <-didClose: + default: + t.Errorf("didn't see Body.Close") + } +} + +func TestTransportDialTLS(t *testing.T) { + var mu sync.Mutex // guards following + var gotReq, didDial bool + + ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { + mu.Lock() + gotReq = true + mu.Unlock() + })) + defer ts.Close() + tr := &Transport{ + DialTLS: func(netw, addr string) (net.Conn, error) { + mu.Lock() + didDial = true + mu.Unlock() + c, err := tls.Dial(netw, addr, &tls.Config{ + InsecureSkipVerify: true, + }) + if err != nil { + return nil, err + } + return c, c.Handshake() + }, + } + defer tr.CloseIdleConnections() + client := &Client{Transport: tr} + res, err := client.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + mu.Lock() + if !gotReq { + t.Error("didn't get request") + } + if !didDial { + t.Error("didn't use dial hook") + } +} + +func wantBody(res *http.Response, err error, want string) error { + if err != nil { + return err + } + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("error reading body: %v", err) + } + if string(slurp) != want { + return fmt.Errorf("body = %q; want %q", slurp, want) + } + if err := res.Body.Close(); err != nil { + return fmt.Errorf("body Close = %v", err) + } + return nil +} + +func newLocalListener(t *testing.T) net.Listener { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ln, err = net.Listen("tcp6", "[::1]:0") + } + if err != nil { + t.Fatal(err) + } + return ln +} + +type countCloseReader struct { + n *int + io.Reader +} + +func (cr countCloseReader) Close() error { + (*cr.n)++ + return nil +} + +// rgz is a gzip quine that uncompresses to itself. +var rgz = []byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, + 0x69, 0x76, 0x65, 0x00, 0x92, 0xef, 0xe6, 0xe0, + 0x60, 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, + 0xe2, 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, + 0x00, 0xe8, 0xff, 0x92, 0xef, 0xe6, 0xe0, 0x60, + 0x00, 0x83, 0xa2, 0xd4, 0xe4, 0xd2, 0xa2, 0xe2, + 0xcc, 0xb2, 0x54, 0x06, 0x00, 0x00, 0x17, 0x00, + 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, 0x06, 0x00, + 0x05, 0x00, 0xfa, 0xff, 0x42, 0x12, 0x46, 0x16, + 0x06, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, 0x05, + 0x00, 0xfa, 0xff, 0x00, 0x14, 0x00, 0xeb, 0xff, + 0x42, 0x12, 0x46, 0x16, 0x06, 0x00, 0x05, 0x00, + 0xfa, 0xff, 0x00, 0x05, 0x00, 0xfa, 0xff, 0x00, + 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, + 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, + 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, 0xeb, 0xff, + 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x14, 0x00, + 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, + 0x14, 0x00, 0xeb, 0xff, 0x42, 0x88, 0x21, 0xc4, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x00, 0x17, 0x00, 0xe8, 0xff, + 0x42, 0x88, 0x21, 0xc4, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, + 0x17, 0x00, 0xe8, 0xff, 0x42, 0x12, 0x46, 0x16, + 0x06, 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, + 0x00, 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, + 0x00, 0x00, 0x00, 0x42, 0x12, 0x46, 0x16, 0x06, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x01, 0x08, 0x00, + 0xf7, 0xff, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00, + 0x00, 0x00, 0x3d, 0xb1, 0x20, 0x85, 0xfa, 0x00, + 0x00, 0x00, +} diff --git a/src/net/http/triv.go b/src/net/http/triv.go new file mode 100644 index 000000000..232d65089 --- /dev/null +++ b/src/net/http/triv.go @@ -0,0 +1,141 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "expvar" + "flag" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "strconv" + "sync" +) + +// hello world, the web server +var helloRequests = expvar.NewInt("hello-requests") + +func HelloServer(w http.ResponseWriter, req *http.Request) { + helloRequests.Add(1) + io.WriteString(w, "hello, world!\n") +} + +// Simple counter server. POSTing to it will set the value. +type Counter struct { + mu sync.Mutex // protects n + n int +} + +// This makes Counter satisfy the expvar.Var interface, so we can export +// it directly. +func (ctr *Counter) String() string { + ctr.mu.Lock() + defer ctr.mu.Unlock() + return fmt.Sprintf("%d", ctr.n) +} + +func (ctr *Counter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + ctr.mu.Lock() + defer ctr.mu.Unlock() + switch req.Method { + case "GET": + ctr.n++ + case "POST": + buf := new(bytes.Buffer) + io.Copy(buf, req.Body) + body := buf.String() + if n, err := strconv.Atoi(body); err != nil { + fmt.Fprintf(w, "bad POST: %v\nbody: [%v]\n", err, body) + } else { + ctr.n = n + fmt.Fprint(w, "counter reset\n") + } + } + fmt.Fprintf(w, "counter = %d\n", ctr.n) +} + +// simple flag server +var booleanflag = flag.Bool("boolean", true, "another flag for testing") + +func FlagServer(w http.ResponseWriter, req *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + fmt.Fprint(w, "Flags:\n") + flag.VisitAll(func(f *flag.Flag) { + if f.Value.String() != f.DefValue { + fmt.Fprintf(w, "%s = %s [default = %s]\n", f.Name, f.Value.String(), f.DefValue) + } else { + fmt.Fprintf(w, "%s = %s\n", f.Name, f.Value.String()) + } + }) +} + +// simple argument server +func ArgServer(w http.ResponseWriter, req *http.Request) { + for _, s := range os.Args { + fmt.Fprint(w, s, " ") + } +} + +// a channel (just for the fun of it) +type Chan chan int + +func ChanCreate() Chan { + c := make(Chan) + go func(c Chan) { + for x := 0; ; x++ { + c <- x + } + }(c) + return c +} + +func (ch Chan) ServeHTTP(w http.ResponseWriter, req *http.Request) { + io.WriteString(w, fmt.Sprintf("channel send #%d\n", <-ch)) +} + +// exec a program, redirecting output +func DateServer(rw http.ResponseWriter, req *http.Request) { + rw.Header().Set("Content-Type", "text/plain; charset=utf-8") + + date, err := exec.Command("/bin/date").Output() + if err != nil { + http.Error(rw, err.Error(), 500) + return + } + rw.Write(date) +} + +func Logger(w http.ResponseWriter, req *http.Request) { + log.Print(req.URL) + http.Error(w, "oops", 404) +} + +var webroot = flag.String("root", os.Getenv("HOME"), "web root directory") + +func main() { + flag.Parse() + + // The counter is published as a variable directly. + ctr := new(Counter) + expvar.Publish("counter", ctr) + http.Handle("/counter", ctr) + http.Handle("/", http.HandlerFunc(Logger)) + http.Handle("/go/", http.StripPrefix("/go/", http.FileServer(http.Dir(*webroot)))) + http.Handle("/chan", ChanCreate()) + http.HandleFunc("/flags", FlagServer) + http.HandleFunc("/args", ArgServer) + http.HandleFunc("/go/hello", HelloServer) + http.HandleFunc("/date", DateServer) + err := http.ListenAndServe(":12345", nil) + if err != nil { + log.Panicln("ListenAndServe:", err) + } +} diff --git a/src/net/http/z_last_test.go b/src/net/http/z_last_test.go new file mode 100644 index 000000000..5a0cc1198 --- /dev/null +++ b/src/net/http/z_last_test.go @@ -0,0 +1,97 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http_test + +import ( + "net/http" + "runtime" + "sort" + "strings" + "testing" + "time" +) + +func interestingGoroutines() (gs []string) { + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + for _, g := range strings.Split(string(buf), "\n\n") { + sl := strings.SplitN(g, "\n", 2) + if len(sl) != 2 { + continue + } + stack := strings.TrimSpace(sl[1]) + if stack == "" || + strings.Contains(stack, "created by net.startServer") || + strings.Contains(stack, "created by testing.RunTests") || + strings.Contains(stack, "closeWriteAndWait") || + strings.Contains(stack, "testing.Main(") || + // These only show up with GOTRACEBACK=2; Issue 5005 (comment 28) + strings.Contains(stack, "runtime.goexit") || + strings.Contains(stack, "created by runtime.gc") || + strings.Contains(stack, "runtime.MHeap_Scavenger") { + continue + } + gs = append(gs, stack) + } + sort.Strings(gs) + return +} + +// Verify the other tests didn't leave any goroutines running. +// This is in a file named z_last_test.go so it sorts at the end. +func TestGoroutinesRunning(t *testing.T) { + if testing.Short() { + t.Skip("not counting goroutines for leakage in -short mode") + } + gs := interestingGoroutines() + + n := 0 + stackCount := make(map[string]int) + for _, g := range gs { + stackCount[g]++ + n++ + } + + t.Logf("num goroutines = %d", n) + if n > 0 { + t.Error("Too many goroutines.") + for stack, count := range stackCount { + t.Logf("%d instances of:\n%s", count, stack) + } + } +} + +func afterTest(t *testing.T) { + http.DefaultTransport.(*http.Transport).CloseIdleConnections() + if testing.Short() { + return + } + var bad string + badSubstring := map[string]string{ + ").readLoop(": "a Transport", + ").writeLoop(": "a Transport", + "created by net/http/httptest.(*Server).Start": "an httptest.Server", + "timeoutHandler": "a TimeoutHandler", + "net.(*netFD).connect(": "a timing out dial", + ").noteClientGone(": "a closenotifier sender", + } + var stacks string + for i := 0; i < 4; i++ { + bad = "" + stacks = strings.Join(interestingGoroutines(), "\n\n") + for substr, what := range badSubstring { + if strings.Contains(stacks, substr) { + bad = what + } + } + if bad == "" { + return + } + // Bad stuff found, but goroutines might just still be + // shutting down, so give it some time. + time.Sleep(250 * time.Millisecond) + } + t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks) +} diff --git a/src/net/interface.go b/src/net/interface.go new file mode 100644 index 000000000..2e9f1ebc6 --- /dev/null +++ b/src/net/interface.go @@ -0,0 +1,126 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "errors" + +var ( + errInvalidInterface = errors.New("invalid network interface") + errInvalidInterfaceIndex = errors.New("invalid network interface index") + errInvalidInterfaceName = errors.New("invalid network interface name") + errNoSuchInterface = errors.New("no such network interface") + errNoSuchMulticastInterface = errors.New("no such multicast network interface") +) + +// Interface represents a mapping between network interface name +// and index. It also represents network interface facility +// information. +type Interface struct { + Index int // positive integer that starts at one, zero is never used + MTU int // maximum transmission unit + Name string // e.g., "en0", "lo0", "eth0.100" + HardwareAddr HardwareAddr // IEEE MAC-48, EUI-48 and EUI-64 form + Flags Flags // e.g., FlagUp, FlagLoopback, FlagMulticast +} + +type Flags uint + +const ( + FlagUp Flags = 1 << iota // interface is up + FlagBroadcast // interface supports broadcast access capability + FlagLoopback // interface is a loopback interface + FlagPointToPoint // interface belongs to a point-to-point link + FlagMulticast // interface supports multicast access capability +) + +var flagNames = []string{ + "up", + "broadcast", + "loopback", + "pointtopoint", + "multicast", +} + +func (f Flags) String() string { + s := "" + for i, name := range flagNames { + if f&(1<<uint(i)) != 0 { + if s != "" { + s += "|" + } + s += name + } + } + if s == "" { + s = "0" + } + return s +} + +// Addrs returns interface addresses for a specific interface. +func (ifi *Interface) Addrs() ([]Addr, error) { + if ifi == nil { + return nil, errInvalidInterface + } + return interfaceAddrTable(ifi) +} + +// MulticastAddrs returns multicast, joined group addresses for +// a specific interface. +func (ifi *Interface) MulticastAddrs() ([]Addr, error) { + if ifi == nil { + return nil, errInvalidInterface + } + return interfaceMulticastAddrTable(ifi) +} + +// Interfaces returns a list of the system's network interfaces. +func Interfaces() ([]Interface, error) { + return interfaceTable(0) +} + +// InterfaceAddrs returns a list of the system's network interface +// addresses. +func InterfaceAddrs() ([]Addr, error) { + return interfaceAddrTable(nil) +} + +// InterfaceByIndex returns the interface specified by index. +func InterfaceByIndex(index int) (*Interface, error) { + if index <= 0 { + return nil, errInvalidInterfaceIndex + } + ift, err := interfaceTable(index) + if err != nil { + return nil, err + } + return interfaceByIndex(ift, index) +} + +func interfaceByIndex(ift []Interface, index int) (*Interface, error) { + for _, ifi := range ift { + if index == ifi.Index { + return &ifi, nil + } + } + return nil, errNoSuchInterface +} + +// InterfaceByName returns the interface specified by name. +func InterfaceByName(name string) (*Interface, error) { + if name == "" { + return nil, errInvalidInterfaceName + } + ift, err := interfaceTable(0) + if err != nil { + return nil, err + } + for _, ifi := range ift { + if name == ifi.Name { + return &ifi, nil + } + } + return nil, errNoSuchInterface +} diff --git a/src/net/interface_bsd.go b/src/net/interface_bsd.go new file mode 100644 index 000000000..16775579d --- /dev/null +++ b/src/net/interface_bsd.go @@ -0,0 +1,182 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package net + +import ( + "os" + "syscall" + "unsafe" +) + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otherwise it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, error) { + tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex) + if err != nil { + return nil, os.NewSyscallError("route rib", err) + } + msgs, err := syscall.ParseRoutingMessage(tab) + if err != nil { + return nil, os.NewSyscallError("route message", err) + } + return parseInterfaceTable(ifindex, msgs) +} + +func parseInterfaceTable(ifindex int, msgs []syscall.RoutingMessage) ([]Interface, error) { + var ift []Interface +loop: + for _, m := range msgs { + switch m := m.(type) { + case *syscall.InterfaceMessage: + if ifindex == 0 || ifindex == int(m.Header.Index) { + ifi, err := newLink(m) + if err != nil { + return nil, err + } + ift = append(ift, *ifi) + if ifindex == int(m.Header.Index) { + break loop + } + } + } + } + return ift, nil +} + +func newLink(m *syscall.InterfaceMessage) (*Interface, error) { + sas, err := syscall.ParseRoutingSockaddr(m) + if err != nil { + return nil, os.NewSyscallError("route sockaddr", err) + } + ifi := &Interface{Index: int(m.Header.Index), Flags: linkFlags(m.Header.Flags)} + for _, sa := range sas { + switch sa := sa.(type) { + case *syscall.SockaddrDatalink: + // NOTE: SockaddrDatalink.Data is minimum work area, + // can be larger. + m.Data = m.Data[unsafe.Offsetof(sa.Data):] + var name [syscall.IFNAMSIZ]byte + for i := 0; i < int(sa.Nlen); i++ { + name[i] = byte(m.Data[i]) + } + ifi.Name = string(name[:sa.Nlen]) + ifi.MTU = int(m.Header.Data.Mtu) + addr := make([]byte, sa.Alen) + for i := 0; i < int(sa.Alen); i++ { + addr[i] = byte(m.Data[int(sa.Nlen)+i]) + } + ifi.HardwareAddr = addr[:sa.Alen] + } + } + return ifi, nil +} + +func linkFlags(rawFlags int32) Flags { + var f Flags + if rawFlags&syscall.IFF_UP != 0 { + f |= FlagUp + } + if rawFlags&syscall.IFF_BROADCAST != 0 { + f |= FlagBroadcast + } + if rawFlags&syscall.IFF_LOOPBACK != 0 { + f |= FlagLoopback + } + if rawFlags&syscall.IFF_POINTOPOINT != 0 { + f |= FlagPointToPoint + } + if rawFlags&syscall.IFF_MULTICAST != 0 { + f |= FlagMulticast + } + return f +} + +// If the ifi is nil, interfaceAddrTable returns addresses for all +// network interfaces. Otherwise it returns addresses for a specific +// interface. +func interfaceAddrTable(ifi *Interface) ([]Addr, error) { + index := 0 + if ifi != nil { + index = ifi.Index + } + tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST, index) + if err != nil { + return nil, os.NewSyscallError("route rib", err) + } + msgs, err := syscall.ParseRoutingMessage(tab) + if err != nil { + return nil, os.NewSyscallError("route message", err) + } + var ift []Interface + if index == 0 { + ift, err = parseInterfaceTable(index, msgs) + if err != nil { + return nil, err + } + } + var ifat []Addr + for _, m := range msgs { + switch m := m.(type) { + case *syscall.InterfaceAddrMessage: + if index == 0 || index == int(m.Header.Index) { + if index == 0 { + var err error + ifi, err = interfaceByIndex(ift, int(m.Header.Index)) + if err != nil { + return nil, err + } + } + ifa, err := newAddr(ifi, m) + if err != nil { + return nil, err + } + if ifa != nil { + ifat = append(ifat, ifa) + } + } + } + } + return ifat, nil +} + +func newAddr(ifi *Interface, m *syscall.InterfaceAddrMessage) (Addr, error) { + sas, err := syscall.ParseRoutingSockaddr(m) + if err != nil { + return nil, os.NewSyscallError("route sockaddr", err) + } + ifa := &IPNet{} + for i, sa := range sas { + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + switch i { + case 0: + ifa.Mask = IPv4Mask(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]) + case 1: + ifa.IP = IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]) + } + case *syscall.SockaddrInet6: + switch i { + case 0: + ifa.Mask = make(IPMask, IPv6len) + copy(ifa.Mask, sa.Addr[:]) + case 1: + ifa.IP = make(IP, IPv6len) + copy(ifa.IP, sa.Addr[:]) + // NOTE: KAME based IPv6 protcol stack usually embeds + // the interface index in the interface-local or link- + // local address as the kernel-internal form. + if ifa.IP.IsLinkLocalUnicast() { + ifa.IP[2], ifa.IP[3] = 0, 0 + } + } + default: // Sockaddrs contain syscall.SockaddrDatalink on NetBSD + return nil, nil + } + } + return ifa, nil +} diff --git a/src/net/interface_bsd_test.go b/src/net/interface_bsd_test.go new file mode 100644 index 000000000..88daf7393 --- /dev/null +++ b/src/net/interface_bsd_test.go @@ -0,0 +1,52 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd netbsd openbsd + +package net + +import ( + "fmt" + "os/exec" +) + +func (ti *testInterface) setBroadcast(suffix int) error { + ti.name = fmt.Sprintf("vlan%d", suffix) + xname, err := exec.LookPath("ifconfig") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "create"}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "destroy"}, + }) + return nil +} + +func (ti *testInterface) setPointToPoint(suffix int, local, remote string) error { + ti.name = fmt.Sprintf("gif%d", suffix) + ti.local = local + ti.remote = remote + xname, err := exec.LookPath("ifconfig") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "create"}, + }) + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "inet", ti.local, ti.remote}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "destroy"}, + }) + return nil +} diff --git a/src/net/interface_darwin.go b/src/net/interface_darwin.go new file mode 100644 index 000000000..ad0937db0 --- /dev/null +++ b/src/net/interface_darwin.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST2, ifi.Index) + if err != nil { + return nil, os.NewSyscallError("route rib", err) + } + msgs, err := syscall.ParseRoutingMessage(tab) + if err != nil { + return nil, os.NewSyscallError("route message", err) + } + var ifmat []Addr + for _, m := range msgs { + switch m := m.(type) { + case *syscall.InterfaceMulticastAddrMessage: + if ifi.Index == int(m.Header.Index) { + ifma, err := newMulticastAddr(ifi, m) + if err != nil { + return nil, err + } + ifmat = append(ifmat, ifma...) + } + } + } + return ifmat, nil +} + +func newMulticastAddr(ifi *Interface, m *syscall.InterfaceMulticastAddrMessage) ([]Addr, error) { + sas, err := syscall.ParseRoutingSockaddr(m) + if err != nil { + return nil, os.NewSyscallError("route sockaddr", err) + } + var ifmat []Addr + for _, sa := range sas { + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + ifma := &IPAddr{IP: IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])} + ifmat = append(ifmat, ifma.toAddr()) + case *syscall.SockaddrInet6: + ifma := &IPAddr{IP: make(IP, IPv6len)} + copy(ifma.IP, sa.Addr[:]) + // NOTE: KAME based IPv6 protocol stack usually embeds + // the interface index in the interface-local or link- + // local address as the kernel-internal form. + if ifma.IP.IsInterfaceLocalMulticast() || ifma.IP.IsLinkLocalMulticast() { + ifma.IP[2], ifma.IP[3] = 0, 0 + } + ifmat = append(ifmat, ifma.toAddr()) + } + } + return ifmat, nil +} diff --git a/src/net/interface_dragonfly.go b/src/net/interface_dragonfly.go new file mode 100644 index 000000000..c9ce5a7ac --- /dev/null +++ b/src/net/interface_dragonfly.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + // TODO(mikio): Implement this like other platforms. + return nil, nil +} diff --git a/src/net/interface_freebsd.go b/src/net/interface_freebsd.go new file mode 100644 index 000000000..5df767910 --- /dev/null +++ b/src/net/interface_freebsd.go @@ -0,0 +1,63 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + tab, err := syscall.RouteRIB(syscall.NET_RT_IFMALIST, ifi.Index) + if err != nil { + return nil, os.NewSyscallError("route rib", err) + } + msgs, err := syscall.ParseRoutingMessage(tab) + if err != nil { + return nil, os.NewSyscallError("route message", err) + } + var ifmat []Addr + for _, m := range msgs { + switch m := m.(type) { + case *syscall.InterfaceMulticastAddrMessage: + if ifi.Index == int(m.Header.Index) { + ifma, err := newMulticastAddr(ifi, m) + if err != nil { + return nil, err + } + ifmat = append(ifmat, ifma...) + } + } + } + return ifmat, nil +} + +func newMulticastAddr(ifi *Interface, m *syscall.InterfaceMulticastAddrMessage) ([]Addr, error) { + sas, err := syscall.ParseRoutingSockaddr(m) + if err != nil { + return nil, os.NewSyscallError("route sockaddr", err) + } + var ifmat []Addr + for _, sa := range sas { + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + ifma := &IPAddr{IP: IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])} + ifmat = append(ifmat, ifma.toAddr()) + case *syscall.SockaddrInet6: + ifma := &IPAddr{IP: make(IP, IPv6len)} + copy(ifma.IP, sa.Addr[:]) + // NOTE: KAME based IPv6 protocol stack usually embeds + // the interface index in the interface-local or link- + // local address as the kernel-internal form. + if ifma.IP.IsInterfaceLocalMulticast() || ifma.IP.IsLinkLocalMulticast() { + ifma.IP[2], ifma.IP[3] = 0, 0 + } + ifmat = append(ifmat, ifma.toAddr()) + } + } + return ifmat, nil +} diff --git a/src/net/interface_linux.go b/src/net/interface_linux.go new file mode 100644 index 000000000..1115d0fc4 --- /dev/null +++ b/src/net/interface_linux.go @@ -0,0 +1,271 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" + "unsafe" +) + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otherwise it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, error) { + tab, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC) + if err != nil { + return nil, os.NewSyscallError("netlink rib", err) + } + msgs, err := syscall.ParseNetlinkMessage(tab) + if err != nil { + return nil, os.NewSyscallError("netlink message", err) + } + var ift []Interface +loop: + for _, m := range msgs { + switch m.Header.Type { + case syscall.NLMSG_DONE: + break loop + case syscall.RTM_NEWLINK: + ifim := (*syscall.IfInfomsg)(unsafe.Pointer(&m.Data[0])) + if ifindex == 0 || ifindex == int(ifim.Index) { + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return nil, os.NewSyscallError("netlink routeattr", err) + } + ift = append(ift, *newLink(ifim, attrs)) + if ifindex == int(ifim.Index) { + break loop + } + } + } + } + return ift, nil +} + +const ( + // See linux/if_arp.h. + // Note that Linux doesn't support IPv4 over IPv6 tunneling. + sysARPHardwareIPv4IPv4 = 768 // IPv4 over IPv4 tunneling + sysARPHardwareIPv6IPv6 = 769 // IPv6 over IPv6 tunneling + sysARPHardwareIPv6IPv4 = 776 // IPv6 over IPv4 tunneling + sysARPHardwareGREIPv4 = 778 // any over GRE over IPv4 tunneling + sysARPHardwareGREIPv6 = 823 // any over GRE over IPv6 tunneling +) + +func newLink(ifim *syscall.IfInfomsg, attrs []syscall.NetlinkRouteAttr) *Interface { + ifi := &Interface{Index: int(ifim.Index), Flags: linkFlags(ifim.Flags)} + for _, a := range attrs { + switch a.Attr.Type { + case syscall.IFLA_ADDRESS: + // We never return any /32 or /128 IP address + // prefix on any IP tunnel interface as the + // hardware address. + switch len(a.Value) { + case IPv4len: + switch ifim.Type { + case sysARPHardwareIPv4IPv4, sysARPHardwareGREIPv4, sysARPHardwareIPv6IPv4: + continue + } + case IPv6len: + switch ifim.Type { + case sysARPHardwareIPv6IPv6, sysARPHardwareGREIPv6: + continue + } + } + var nonzero bool + for _, b := range a.Value { + if b != 0 { + nonzero = true + break + } + } + if nonzero { + ifi.HardwareAddr = a.Value[:] + } + case syscall.IFLA_IFNAME: + ifi.Name = string(a.Value[:len(a.Value)-1]) + case syscall.IFLA_MTU: + ifi.MTU = int(*(*uint32)(unsafe.Pointer(&a.Value[:4][0]))) + } + } + return ifi +} + +func linkFlags(rawFlags uint32) Flags { + var f Flags + if rawFlags&syscall.IFF_UP != 0 { + f |= FlagUp + } + if rawFlags&syscall.IFF_BROADCAST != 0 { + f |= FlagBroadcast + } + if rawFlags&syscall.IFF_LOOPBACK != 0 { + f |= FlagLoopback + } + if rawFlags&syscall.IFF_POINTOPOINT != 0 { + f |= FlagPointToPoint + } + if rawFlags&syscall.IFF_MULTICAST != 0 { + f |= FlagMulticast + } + return f +} + +// If the ifi is nil, interfaceAddrTable returns addresses for all +// network interfaces. Otherwise it returns addresses for a specific +// interface. +func interfaceAddrTable(ifi *Interface) ([]Addr, error) { + tab, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC) + if err != nil { + return nil, os.NewSyscallError("netlink rib", err) + } + msgs, err := syscall.ParseNetlinkMessage(tab) + if err != nil { + return nil, os.NewSyscallError("netlink message", err) + } + var ift []Interface + if ifi == nil { + var err error + ift, err = interfaceTable(0) + if err != nil { + return nil, err + } + } + ifat, err := addrTable(ift, ifi, msgs) + if err != nil { + return nil, err + } + return ifat, nil +} + +func addrTable(ift []Interface, ifi *Interface, msgs []syscall.NetlinkMessage) ([]Addr, error) { + var ifat []Addr +loop: + for _, m := range msgs { + switch m.Header.Type { + case syscall.NLMSG_DONE: + break loop + case syscall.RTM_NEWADDR: + ifam := (*syscall.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) + if len(ift) != 0 || ifi.Index == int(ifam.Index) { + if len(ift) != 0 { + var err error + ifi, err = interfaceByIndex(ift, int(ifam.Index)) + if err != nil { + return nil, err + } + } + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return nil, os.NewSyscallError("netlink routeattr", err) + } + ifa := newAddr(ifi, ifam, attrs) + if ifa != nil { + ifat = append(ifat, ifa) + } + } + } + } + return ifat, nil +} + +func newAddr(ifi *Interface, ifam *syscall.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) Addr { + var ipPointToPoint bool + // Seems like we need to make sure whether the IP interface + // stack consists of IP point-to-point numbered or unnumbered + // addressing over point-to-point link encapsulation. + if ifi.Flags&FlagPointToPoint != 0 { + for _, a := range attrs { + if a.Attr.Type == syscall.IFA_LOCAL { + ipPointToPoint = true + break + } + } + } + for _, a := range attrs { + if ipPointToPoint && a.Attr.Type == syscall.IFA_ADDRESS || !ipPointToPoint && a.Attr.Type == syscall.IFA_LOCAL { + continue + } + switch ifam.Family { + case syscall.AF_INET: + return &IPNet{IP: IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3]), Mask: CIDRMask(int(ifam.Prefixlen), 8*IPv4len)} + case syscall.AF_INET6: + ifa := &IPNet{IP: make(IP, IPv6len), Mask: CIDRMask(int(ifam.Prefixlen), 8*IPv6len)} + copy(ifa.IP, a.Value[:]) + return ifa + } + } + return nil +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + ifmat4 := parseProcNetIGMP("/proc/net/igmp", ifi) + ifmat6 := parseProcNetIGMP6("/proc/net/igmp6", ifi) + return append(ifmat4, ifmat6...), nil +} + +func parseProcNetIGMP(path string, ifi *Interface) []Addr { + fd, err := open(path) + if err != nil { + return nil + } + defer fd.close() + var ( + ifmat []Addr + name string + ) + fd.readLine() // skip first line + b := make([]byte, IPv4len) + for l, ok := fd.readLine(); ok; l, ok = fd.readLine() { + f := splitAtBytes(l, " :\r\t\n") + if len(f) < 4 { + continue + } + switch { + case l[0] != ' ' && l[0] != '\t': // new interface line + name = f[1] + case len(f[0]) == 8: + if ifi == nil || name == ifi.Name { + // The Linux kernel puts the IP + // address in /proc/net/igmp in native + // endianness. + for i := 0; i+1 < len(f[0]); i += 2 { + b[i/2], _ = xtoi2(f[0][i:i+2], 0) + } + i := *(*uint32)(unsafe.Pointer(&b[:4][0])) + ifma := IPAddr{IP: IPv4(byte(i>>24), byte(i>>16), byte(i>>8), byte(i))} + ifmat = append(ifmat, ifma.toAddr()) + } + } + } + return ifmat +} + +func parseProcNetIGMP6(path string, ifi *Interface) []Addr { + fd, err := open(path) + if err != nil { + return nil + } + defer fd.close() + var ifmat []Addr + b := make([]byte, IPv6len) + for l, ok := fd.readLine(); ok; l, ok = fd.readLine() { + f := splitAtBytes(l, " \r\t\n") + if len(f) < 6 { + continue + } + if ifi == nil || f[1] == ifi.Name { + for i := 0; i+1 < len(f[2]); i += 2 { + b[i/2], _ = xtoi2(f[2][i:i+2], 0) + } + ifma := IPAddr{IP: IP{b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]}} + ifmat = append(ifmat, ifma.toAddr()) + } + } + return ifmat +} diff --git a/src/net/interface_linux_test.go b/src/net/interface_linux_test.go new file mode 100644 index 000000000..d8800bd0c --- /dev/null +++ b/src/net/interface_linux_test.go @@ -0,0 +1,102 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "fmt" + "os/exec" + "testing" +) + +func (ti *testInterface) setBroadcast(suffix int) error { + ti.name = fmt.Sprintf("gotest%d", suffix) + xname, err := exec.LookPath("ip") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "add", ti.name, "type", "dummy"}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "link", "delete", ti.name, "type", "dummy"}, + }) + return nil +} + +func (ti *testInterface) setPointToPoint(suffix int, local, remote string) error { + ti.name = fmt.Sprintf("gotest%d", suffix) + ti.local = local + ti.remote = remote + xname, err := exec.LookPath("ip") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "tunnel", "add", ti.name, "mode", "gre", "local", local, "remote", remote}, + }) + ti.teardownCmds = append(ti.teardownCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ip", "tunnel", "del", ti.name, "mode", "gre", "local", local, "remote", remote}, + }) + xname, err = exec.LookPath("ifconfig") + if err != nil { + return err + } + ti.setupCmds = append(ti.setupCmds, &exec.Cmd{ + Path: xname, + Args: []string{"ifconfig", ti.name, "inet", local, "dstaddr", remote}, + }) + return nil +} + +const ( + numOfTestIPv4MCAddrs = 14 + numOfTestIPv6MCAddrs = 18 +) + +var ( + igmpInterfaceTable = []Interface{ + {Name: "lo"}, + {Name: "eth0"}, {Name: "eth1"}, {Name: "eth2"}, + {Name: "eth0.100"}, {Name: "eth0.101"}, {Name: "eth0.102"}, {Name: "eth0.103"}, + {Name: "device1tap2"}, + } + igmp6InterfaceTable = []Interface{ + {Name: "lo"}, + {Name: "eth0"}, {Name: "eth1"}, {Name: "eth2"}, + {Name: "eth0.100"}, {Name: "eth0.101"}, {Name: "eth0.102"}, {Name: "eth0.103"}, + {Name: "device1tap2"}, + {Name: "pan0"}, + } +) + +func TestParseProcNet(t *testing.T) { + defer func() { + if p := recover(); p != nil { + t.Fatalf("parseProcNetIGMP or parseProtNetIGMP6 panicked: %v", p) + } + }() + + var ifmat4 []Addr + for _, ifi := range igmpInterfaceTable { + ifmat := parseProcNetIGMP("testdata/igmp", &ifi) + ifmat4 = append(ifmat4, ifmat...) + } + if len(ifmat4) != numOfTestIPv4MCAddrs { + t.Fatalf("parseProcNetIGMP returns %v addresses, expected %v", len(ifmat4), numOfTestIPv4MCAddrs) + } + + var ifmat6 []Addr + for _, ifi := range igmp6InterfaceTable { + ifmat := parseProcNetIGMP6("testdata/igmp6", &ifi) + ifmat6 = append(ifmat6, ifmat...) + } + if len(ifmat6) != numOfTestIPv6MCAddrs { + t.Fatalf("parseProcNetIGMP6 returns %v addresses, expected %v", len(ifmat6), numOfTestIPv6MCAddrs) + } +} diff --git a/src/net/interface_netbsd.go b/src/net/interface_netbsd.go new file mode 100644 index 000000000..c9ce5a7ac --- /dev/null +++ b/src/net/interface_netbsd.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + // TODO(mikio): Implement this like other platforms. + return nil, nil +} diff --git a/src/net/interface_openbsd.go b/src/net/interface_openbsd.go new file mode 100644 index 000000000..c9ce5a7ac --- /dev/null +++ b/src/net/interface_openbsd.go @@ -0,0 +1,12 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + // TODO(mikio): Implement this like other platforms. + return nil, nil +} diff --git a/src/net/interface_stub.go b/src/net/interface_stub.go new file mode 100644 index 000000000..c38fb7f76 --- /dev/null +++ b/src/net/interface_stub.go @@ -0,0 +1,27 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build nacl plan9 solaris + +package net + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otherwise it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, error) { + return nil, nil +} + +// If the ifi is nil, interfaceAddrTable returns addresses for all +// network interfaces. Otherwise it returns addresses for a specific +// interface. +func interfaceAddrTable(ifi *Interface) ([]Addr, error) { + return nil, nil +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + return nil, nil +} diff --git a/src/net/interface_test.go b/src/net/interface_test.go new file mode 100644 index 000000000..efabb5f3c --- /dev/null +++ b/src/net/interface_test.go @@ -0,0 +1,211 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "reflect" + "testing" +) + +// loopbackInterface returns an available logical network interface +// for loopback tests. It returns nil if no suitable interface is +// found. +func loopbackInterface() *Interface { + ift, err := Interfaces() + if err != nil { + return nil + } + for _, ifi := range ift { + if ifi.Flags&FlagLoopback != 0 && ifi.Flags&FlagUp != 0 { + return &ifi + } + } + return nil +} + +// ipv6LinkLocalUnicastAddr returns an IPv6 link-local unicast address +// on the given network interface for tests. It returns "" if no +// suitable address is found. +func ipv6LinkLocalUnicastAddr(ifi *Interface) string { + if ifi == nil { + return "" + } + ifat, err := ifi.Addrs() + if err != nil { + return "" + } + for _, ifa := range ifat { + switch ifa := ifa.(type) { + case *IPAddr: + if ifa.IP.To4() == nil && ifa.IP.IsLinkLocalUnicast() { + return ifa.IP.String() + } + case *IPNet: + if ifa.IP.To4() == nil && ifa.IP.IsLinkLocalUnicast() { + return ifa.IP.String() + } + } + } + return "" +} + +func TestInterfaces(t *testing.T) { + ift, err := Interfaces() + if err != nil { + t.Fatalf("Interfaces failed: %v", err) + } + t.Logf("table: len/cap = %v/%v", len(ift), cap(ift)) + + for _, ifi := range ift { + ifxi, err := InterfaceByIndex(ifi.Index) + if err != nil { + t.Fatalf("InterfaceByIndex(%v) failed: %v", ifi.Index, err) + } + if !reflect.DeepEqual(ifxi, &ifi) { + t.Fatalf("InterfaceByIndex(%v) = %v, want %v", ifi.Index, ifxi, ifi) + } + ifxn, err := InterfaceByName(ifi.Name) + if err != nil { + t.Fatalf("InterfaceByName(%q) failed: %v", ifi.Name, err) + } + if !reflect.DeepEqual(ifxn, &ifi) { + t.Fatalf("InterfaceByName(%q) = %v, want %v", ifi.Name, ifxn, ifi) + } + t.Logf("%q: flags %q, ifindex %v, mtu %v", ifi.Name, ifi.Flags.String(), ifi.Index, ifi.MTU) + t.Logf("\thardware address %q", ifi.HardwareAddr.String()) + testInterfaceAddrs(t, &ifi) + testInterfaceMulticastAddrs(t, &ifi) + } +} + +func TestInterfaceAddrs(t *testing.T) { + ifat, err := InterfaceAddrs() + if err != nil { + t.Fatalf("InterfaceAddrs failed: %v", err) + } + t.Logf("table: len/cap = %v/%v", len(ifat), cap(ifat)) + testAddrs(t, ifat) +} + +func testInterfaceAddrs(t *testing.T, ifi *Interface) { + ifat, err := ifi.Addrs() + if err != nil { + t.Fatalf("Interface.Addrs failed: %v", err) + } + testAddrs(t, ifat) +} + +func testInterfaceMulticastAddrs(t *testing.T, ifi *Interface) { + ifmat, err := ifi.MulticastAddrs() + if err != nil { + t.Fatalf("Interface.MulticastAddrs failed: %v", err) + } + testMulticastAddrs(t, ifmat) +} + +func testAddrs(t *testing.T, ifat []Addr) { + for _, ifa := range ifat { + switch ifa := ifa.(type) { + case *IPAddr: + if ifa == nil || ifa.IP == nil { + t.Errorf("\tunexpected value: %v, %v", ifa, ifa.IP) + } else { + t.Logf("\tinterface address %q", ifa.String()) + } + case *IPNet: + if ifa == nil || ifa.IP == nil || ifa.Mask == nil { + t.Errorf("\tunexpected value: %v, %v, %v", ifa, ifa.IP, ifa.Mask) + } else { + _, prefixLen := ifa.Mask.Size() + if ifa.IP.To4() != nil && prefixLen != 8*IPv4len || ifa.IP.To16() != nil && ifa.IP.To4() == nil && prefixLen != 8*IPv6len { + t.Errorf("\tunexpected value: %v, %v, %v, %v", ifa, ifa.IP, ifa.Mask, prefixLen) + } else { + t.Logf("\tinterface address %q", ifa.String()) + } + } + default: + t.Errorf("\tunexpected type: %T", ifa) + } + } +} + +func testMulticastAddrs(t *testing.T, ifmat []Addr) { + for _, ifma := range ifmat { + switch ifma := ifma.(type) { + case *IPAddr: + if ifma == nil { + t.Errorf("\tunexpected value: %v", ifma) + } else { + t.Logf("\tjoined group address %q", ifma.String()) + } + default: + t.Errorf("\tunexpected type: %T", ifma) + } + } +} + +func BenchmarkInterfaces(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := Interfaces(); err != nil { + b.Fatalf("Interfaces failed: %v", err) + } + } +} + +func BenchmarkInterfaceByIndex(b *testing.B) { + ifi := loopbackInterface() + if ifi == nil { + b.Skip("loopback interface not found") + } + for i := 0; i < b.N; i++ { + if _, err := InterfaceByIndex(ifi.Index); err != nil { + b.Fatalf("InterfaceByIndex failed: %v", err) + } + } +} + +func BenchmarkInterfaceByName(b *testing.B) { + ifi := loopbackInterface() + if ifi == nil { + b.Skip("loopback interface not found") + } + for i := 0; i < b.N; i++ { + if _, err := InterfaceByName(ifi.Name); err != nil { + b.Fatalf("InterfaceByName failed: %v", err) + } + } +} + +func BenchmarkInterfaceAddrs(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := InterfaceAddrs(); err != nil { + b.Fatalf("InterfaceAddrs failed: %v", err) + } + } +} + +func BenchmarkInterfacesAndAddrs(b *testing.B) { + ifi := loopbackInterface() + if ifi == nil { + b.Skip("loopback interface not found") + } + for i := 0; i < b.N; i++ { + if _, err := ifi.Addrs(); err != nil { + b.Fatalf("Interface.Addrs failed: %v", err) + } + } +} + +func BenchmarkInterfacesAndMulticastAddrs(b *testing.B) { + ifi := loopbackInterface() + if ifi == nil { + b.Skip("loopback interface not found") + } + for i := 0; i < b.N; i++ { + if _, err := ifi.MulticastAddrs(); err != nil { + b.Fatalf("Interface.MulticastAddrs failed: %v", err) + } + } +} diff --git a/src/net/interface_unix_test.go b/src/net/interface_unix_test.go new file mode 100644 index 000000000..01f609f15 --- /dev/null +++ b/src/net/interface_unix_test.go @@ -0,0 +1,151 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +package net + +import ( + "os" + "os/exec" + "runtime" + "testing" + "time" +) + +type testInterface struct { + name string + local string + remote string + setupCmds []*exec.Cmd + teardownCmds []*exec.Cmd +} + +func (ti *testInterface) setup() error { + for _, cmd := range ti.setupCmds { + if err := cmd.Run(); err != nil { + return err + } + } + return nil +} + +func (ti *testInterface) teardown() error { + for _, cmd := range ti.teardownCmds { + if err := cmd.Run(); err != nil { + return err + } + } + return nil +} + +func TestPointToPointInterface(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + switch { + case runtime.GOOS == "darwin": + t.Skipf("skipping read test on %q", runtime.GOOS) + } + if os.Getuid() != 0 { + t.Skip("skipping test; must be root") + } + + local, remote := "169.254.0.1", "169.254.0.254" + ip := ParseIP(remote) + for i := 0; i < 3; i++ { + ti := &testInterface{} + if err := ti.setPointToPoint(5963+i, local, remote); err != nil { + t.Skipf("test requries external command: %v", err) + } + if err := ti.setup(); err != nil { + t.Fatalf("testInterface.setup failed: %v", err) + } else { + time.Sleep(3 * time.Millisecond) + } + ift, err := Interfaces() + if err != nil { + ti.teardown() + t.Fatalf("Interfaces failed: %v", err) + } + for _, ifi := range ift { + if ti.name == ifi.Name { + ifat, err := ifi.Addrs() + if err != nil { + ti.teardown() + t.Fatalf("Interface.Addrs failed: %v", err) + } + for _, ifa := range ifat { + if ip.Equal(ifa.(*IPNet).IP) { + ti.teardown() + t.Fatalf("got %v; want %v", ip, local) + } + } + } + } + if err := ti.teardown(); err != nil { + t.Fatalf("testInterface.teardown failed: %v", err) + } else { + time.Sleep(3 * time.Millisecond) + } + } +} + +func TestInterfaceArrivalAndDeparture(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode") + } + if os.Getuid() != 0 { + t.Skip("skipping test; must be root") + } + + for i := 0; i < 3; i++ { + ift1, err := Interfaces() + if err != nil { + t.Fatalf("Interfaces failed: %v", err) + } + ti := &testInterface{} + if err := ti.setBroadcast(5682 + i); err != nil { + t.Skipf("test requires external command: %v", err) + } + if err := ti.setup(); err != nil { + t.Fatalf("testInterface.setup failed: %v", err) + } else { + time.Sleep(3 * time.Millisecond) + } + ift2, err := Interfaces() + if err != nil { + ti.teardown() + t.Fatalf("Interfaces failed: %v", err) + } + if len(ift2) <= len(ift1) { + for _, ifi := range ift1 { + t.Logf("before: %v", ifi) + } + for _, ifi := range ift2 { + t.Logf("after: %v", ifi) + } + ti.teardown() + t.Fatalf("got %v; want gt %v", len(ift2), len(ift1)) + } + if err := ti.teardown(); err != nil { + t.Fatalf("testInterface.teardown failed: %v", err) + } else { + time.Sleep(3 * time.Millisecond) + } + ift3, err := Interfaces() + if err != nil { + t.Fatalf("Interfaces failed: %v", err) + } + if len(ift3) >= len(ift2) { + for _, ifi := range ift2 { + t.Logf("before: %v", ifi) + } + for _, ifi := range ift3 { + t.Logf("after: %v", ifi) + } + t.Fatalf("got %v; want lt %v", len(ift3), len(ift2)) + } + } +} diff --git a/src/net/interface_windows.go b/src/net/interface_windows.go new file mode 100644 index 000000000..0759dc255 --- /dev/null +++ b/src/net/interface_windows.go @@ -0,0 +1,158 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" + "unsafe" +) + +func bytePtrToString(p *uint8) string { + a := (*[10000]uint8)(unsafe.Pointer(p)) + i := 0 + for a[i] != 0 { + i++ + } + return string(a[:i]) +} + +func getAdapterList() (*syscall.IpAdapterInfo, error) { + b := make([]byte, 1000) + l := uint32(len(b)) + a := (*syscall.IpAdapterInfo)(unsafe.Pointer(&b[0])) + // TODO(mikio): GetAdaptersInfo returns IP_ADAPTER_INFO that + // contains IPv4 address list only. We should use another API + // for fetching IPv6 stuff from the kernel. + err := syscall.GetAdaptersInfo(a, &l) + if err == syscall.ERROR_BUFFER_OVERFLOW { + b = make([]byte, l) + a = (*syscall.IpAdapterInfo)(unsafe.Pointer(&b[0])) + err = syscall.GetAdaptersInfo(a, &l) + } + if err != nil { + return nil, os.NewSyscallError("GetAdaptersInfo", err) + } + return a, nil +} + +func getInterfaceList() ([]syscall.InterfaceInfo, error) { + s, err := sysSocket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_UDP) + if err != nil { + return nil, os.NewSyscallError("Socket", err) + } + defer syscall.Closesocket(s) + + ii := [20]syscall.InterfaceInfo{} + ret := uint32(0) + size := uint32(unsafe.Sizeof(ii)) + err = syscall.WSAIoctl(s, syscall.SIO_GET_INTERFACE_LIST, nil, 0, (*byte)(unsafe.Pointer(&ii[0])), size, &ret, nil, 0) + if err != nil { + return nil, os.NewSyscallError("WSAIoctl", err) + } + c := ret / uint32(unsafe.Sizeof(ii[0])) + return ii[:c-1], nil +} + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otherwise it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, error) { + ai, err := getAdapterList() + if err != nil { + return nil, err + } + + ii, err := getInterfaceList() + if err != nil { + return nil, err + } + + var ift []Interface + for ; ai != nil; ai = ai.Next { + index := ai.Index + if ifindex == 0 || ifindex == int(index) { + var flags Flags + + row := syscall.MibIfRow{Index: index} + e := syscall.GetIfEntry(&row) + if e != nil { + return nil, os.NewSyscallError("GetIfEntry", e) + } + + for _, ii := range ii { + ip := (*syscall.RawSockaddrInet4)(unsafe.Pointer(&ii.Address)).Addr + ipv4 := IPv4(ip[0], ip[1], ip[2], ip[3]) + ipl := &ai.IpAddressList + for ipl != nil { + ips := bytePtrToString(&ipl.IpAddress.String[0]) + if ipv4.Equal(parseIPv4(ips)) { + break + } + ipl = ipl.Next + } + if ipl == nil { + continue + } + if ii.Flags&syscall.IFF_UP != 0 { + flags |= FlagUp + } + if ii.Flags&syscall.IFF_LOOPBACK != 0 { + flags |= FlagLoopback + } + if ii.Flags&syscall.IFF_BROADCAST != 0 { + flags |= FlagBroadcast + } + if ii.Flags&syscall.IFF_POINTTOPOINT != 0 { + flags |= FlagPointToPoint + } + if ii.Flags&syscall.IFF_MULTICAST != 0 { + flags |= FlagMulticast + } + } + + name := bytePtrToString(&ai.AdapterName[0]) + + ifi := Interface{ + Index: int(index), + MTU: int(row.Mtu), + Name: name, + HardwareAddr: HardwareAddr(row.PhysAddr[:row.PhysAddrLen]), + Flags: flags} + ift = append(ift, ifi) + } + } + return ift, nil +} + +// If the ifi is nil, interfaceAddrTable returns addresses for all +// network interfaces. Otherwise it returns addresses for a specific +// interface. +func interfaceAddrTable(ifi *Interface) ([]Addr, error) { + ai, err := getAdapterList() + if err != nil { + return nil, err + } + + var ifat []Addr + for ; ai != nil; ai = ai.Next { + index := ai.Index + if ifi == nil || ifi.Index == int(index) { + ipl := &ai.IpAddressList + for ; ipl != nil; ipl = ipl.Next { + ifa := IPAddr{IP: parseIPv4(bytePtrToString(&ipl.IpAddress.String[0]))} + ifat = append(ifat, ifa.toAddr()) + } + } + } + return ifat, nil +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + // TODO(mikio): Implement this like other platforms. + return nil, nil +} diff --git a/src/net/ip.go b/src/net/ip.go new file mode 100644 index 000000000..4a93e97b3 --- /dev/null +++ b/src/net/ip.go @@ -0,0 +1,689 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// IP address manipulations +// +// IPv4 addresses are 4 bytes; IPv6 addresses are 16 bytes. +// An IPv4 address can be converted to an IPv6 address by +// adding a canonical prefix (10 zeros, 2 0xFFs). +// This library accepts either size of byte slice but always +// returns 16-byte addresses. + +package net + +import "errors" + +// IP address lengths (bytes). +const ( + IPv4len = 4 + IPv6len = 16 +) + +// An IP is a single IP address, a slice of bytes. +// Functions in this package accept either 4-byte (IPv4) +// or 16-byte (IPv6) slices as input. +// +// Note that in this documentation, referring to an +// IP address as an IPv4 address or an IPv6 address +// is a semantic property of the address, not just the +// length of the byte slice: a 16-byte slice can still +// be an IPv4 address. +type IP []byte + +// An IP mask is an IP address. +type IPMask []byte + +// An IPNet represents an IP network. +type IPNet struct { + IP IP // network number + Mask IPMask // network mask +} + +// IPv4 returns the IP address (in 16-byte form) of the +// IPv4 address a.b.c.d. +func IPv4(a, b, c, d byte) IP { + p := make(IP, IPv6len) + copy(p, v4InV6Prefix) + p[12] = a + p[13] = b + p[14] = c + p[15] = d + return p +} + +var v4InV6Prefix = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff} + +// IPv4Mask returns the IP mask (in 4-byte form) of the +// IPv4 mask a.b.c.d. +func IPv4Mask(a, b, c, d byte) IPMask { + p := make(IPMask, IPv4len) + p[0] = a + p[1] = b + p[2] = c + p[3] = d + return p +} + +// CIDRMask returns an IPMask consisting of `ones' 1 bits +// followed by 0s up to a total length of `bits' bits. +// For a mask of this form, CIDRMask is the inverse of IPMask.Size. +func CIDRMask(ones, bits int) IPMask { + if bits != 8*IPv4len && bits != 8*IPv6len { + return nil + } + if ones < 0 || ones > bits { + return nil + } + l := bits / 8 + m := make(IPMask, l) + n := uint(ones) + for i := 0; i < l; i++ { + if n >= 8 { + m[i] = 0xff + n -= 8 + continue + } + m[i] = ^byte(0xff >> n) + n = 0 + } + return m +} + +// Well-known IPv4 addresses +var ( + IPv4bcast = IPv4(255, 255, 255, 255) // broadcast + IPv4allsys = IPv4(224, 0, 0, 1) // all systems + IPv4allrouter = IPv4(224, 0, 0, 2) // all routers + IPv4zero = IPv4(0, 0, 0, 0) // all zeros +) + +// Well-known IPv6 addresses +var ( + IPv6zero = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + IPv6unspecified = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + IPv6loopback = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1} + IPv6interfacelocalallnodes = IP{0xff, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01} + IPv6linklocalallnodes = IP{0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01} + IPv6linklocalallrouters = IP{0xff, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x02} +) + +// IsUnspecified returns true if ip is an unspecified address. +func (ip IP) IsUnspecified() bool { + if ip.Equal(IPv4zero) || ip.Equal(IPv6unspecified) { + return true + } + return false +} + +// IsLoopback returns true if ip is a loopback address. +func (ip IP) IsLoopback() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0] == 127 { + return true + } + return ip.Equal(IPv6loopback) +} + +// IsMulticast returns true if ip is a multicast address. +func (ip IP) IsMulticast() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0]&0xf0 == 0xe0 { + return true + } + return ip[0] == 0xff +} + +// IsInterfaceLinkLocalMulticast returns true if ip is +// an interface-local multicast address. +func (ip IP) IsInterfaceLocalMulticast() bool { + return len(ip) == IPv6len && ip[0] == 0xff && ip[1]&0x0f == 0x01 +} + +// IsLinkLocalMulticast returns true if ip is a link-local +// multicast address. +func (ip IP) IsLinkLocalMulticast() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0] == 224 && ip4[1] == 0 && ip4[2] == 0 { + return true + } + return ip[0] == 0xff && ip[1]&0x0f == 0x02 +} + +// IsLinkLocalUnicast returns true if ip is a link-local +// unicast address. +func (ip IP) IsLinkLocalUnicast() bool { + if ip4 := ip.To4(); ip4 != nil && ip4[0] == 169 && ip4[1] == 254 { + return true + } + return ip[0] == 0xfe && ip[1]&0xc0 == 0x80 +} + +// IsGlobalUnicast returns true if ip is a global unicast +// address. +func (ip IP) IsGlobalUnicast() bool { + return !ip.IsUnspecified() && + !ip.IsLoopback() && + !ip.IsMulticast() && + !ip.IsLinkLocalUnicast() +} + +// Is p all zeros? +func isZeros(p IP) bool { + for i := 0; i < len(p); i++ { + if p[i] != 0 { + return false + } + } + return true +} + +// To4 converts the IPv4 address ip to a 4-byte representation. +// If ip is not an IPv4 address, To4 returns nil. +func (ip IP) To4() IP { + if len(ip) == IPv4len { + return ip + } + if len(ip) == IPv6len && + isZeros(ip[0:10]) && + ip[10] == 0xff && + ip[11] == 0xff { + return ip[12:16] + } + return nil +} + +// To16 converts the IP address ip to a 16-byte representation. +// If ip is not an IP address (it is the wrong length), To16 returns nil. +func (ip IP) To16() IP { + if len(ip) == IPv4len { + return IPv4(ip[0], ip[1], ip[2], ip[3]) + } + if len(ip) == IPv6len { + return ip + } + return nil +} + +// Default route masks for IPv4. +var ( + classAMask = IPv4Mask(0xff, 0, 0, 0) + classBMask = IPv4Mask(0xff, 0xff, 0, 0) + classCMask = IPv4Mask(0xff, 0xff, 0xff, 0) +) + +// DefaultMask returns the default IP mask for the IP address ip. +// Only IPv4 addresses have default masks; DefaultMask returns +// nil if ip is not a valid IPv4 address. +func (ip IP) DefaultMask() IPMask { + if ip = ip.To4(); ip == nil { + return nil + } + switch true { + case ip[0] < 0x80: + return classAMask + case ip[0] < 0xC0: + return classBMask + default: + return classCMask + } +} + +func allFF(b []byte) bool { + for _, c := range b { + if c != 0xff { + return false + } + } + return true +} + +// Mask returns the result of masking the IP address ip with mask. +func (ip IP) Mask(mask IPMask) IP { + if len(mask) == IPv6len && len(ip) == IPv4len && allFF(mask[:12]) { + mask = mask[12:] + } + if len(mask) == IPv4len && len(ip) == IPv6len && bytesEqual(ip[:12], v4InV6Prefix) { + ip = ip[12:] + } + n := len(ip) + if n != len(mask) { + return nil + } + out := make(IP, n) + for i := 0; i < n; i++ { + out[i] = ip[i] & mask[i] + } + return out +} + +// String returns the string form of the IP address ip. +// If the address is an IPv4 address, the string representation +// is dotted decimal ("74.125.19.99"). Otherwise the representation +// is IPv6 ("2001:4860:0:2001::68"). +func (ip IP) String() string { + p := ip + + if len(ip) == 0 { + return "<nil>" + } + + // If IPv4, use dotted notation. + if p4 := p.To4(); len(p4) == IPv4len { + return itod(uint(p4[0])) + "." + + itod(uint(p4[1])) + "." + + itod(uint(p4[2])) + "." + + itod(uint(p4[3])) + } + if len(p) != IPv6len { + return "?" + } + + // Find longest run of zeros. + e0 := -1 + e1 := -1 + for i := 0; i < IPv6len; i += 2 { + j := i + for j < IPv6len && p[j] == 0 && p[j+1] == 0 { + j += 2 + } + if j > i && j-i > e1-e0 { + e0 = i + e1 = j + i = j + } + } + // The symbol "::" MUST NOT be used to shorten just one 16 bit 0 field. + if e1-e0 <= 2 { + e0 = -1 + e1 = -1 + } + + const maxLen = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") + b := make([]byte, 0, maxLen) + + // Print with possible :: in place of run of zeros + for i := 0; i < IPv6len; i += 2 { + if i == e0 { + b = append(b, ':', ':') + i = e1 + if i >= IPv6len { + break + } + } else if i > 0 { + b = append(b, ':') + } + b = appendHex(b, (uint32(p[i])<<8)|uint32(p[i+1])) + } + return string(b) +} + +// ipEmptyString is like ip.String except that it returns +// an empty string when ip is unset. +func ipEmptyString(ip IP) string { + if len(ip) == 0 { + return "" + } + return ip.String() +} + +// MarshalText implements the encoding.TextMarshaler interface. +// The encoding is the same as returned by String. +func (ip IP) MarshalText() ([]byte, error) { + if len(ip) == 0 { + return []byte(""), nil + } + if len(ip) != IPv4len && len(ip) != IPv6len { + return nil, errors.New("invalid IP address") + } + return []byte(ip.String()), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +// The IP address is expected in a form accepted by ParseIP. +func (ip *IP) UnmarshalText(text []byte) error { + if len(text) == 0 { + *ip = nil + return nil + } + s := string(text) + x := ParseIP(s) + if x == nil { + return &ParseError{"IP address", s} + } + *ip = x + return nil +} + +// Equal returns true if ip and x are the same IP address. +// An IPv4 address and that same address in IPv6 form are +// considered to be equal. +func (ip IP) Equal(x IP) bool { + if len(ip) == len(x) { + return bytesEqual(ip, x) + } + if len(ip) == IPv4len && len(x) == IPv6len { + return bytesEqual(x[0:12], v4InV6Prefix) && bytesEqual(ip, x[12:]) + } + if len(ip) == IPv6len && len(x) == IPv4len { + return bytesEqual(ip[0:12], v4InV6Prefix) && bytesEqual(ip[12:], x) + } + return false +} + +func bytesEqual(x, y []byte) bool { + if len(x) != len(y) { + return false + } + for i, b := range x { + if y[i] != b { + return false + } + } + return true +} + +// If mask is a sequence of 1 bits followed by 0 bits, +// return the number of 1 bits. +func simpleMaskLength(mask IPMask) int { + var n int + for i, v := range mask { + if v == 0xff { + n += 8 + continue + } + // found non-ff byte + // count 1 bits + for v&0x80 != 0 { + n++ + v <<= 1 + } + // rest must be 0 bits + if v != 0 { + return -1 + } + for i++; i < len(mask); i++ { + if mask[i] != 0 { + return -1 + } + } + break + } + return n +} + +// Size returns the number of leading ones and total bits in the mask. +// If the mask is not in the canonical form--ones followed by zeros--then +// Size returns 0, 0. +func (m IPMask) Size() (ones, bits int) { + ones, bits = simpleMaskLength(m), len(m)*8 + if ones == -1 { + return 0, 0 + } + return +} + +// String returns the hexadecimal form of m, with no punctuation. +func (m IPMask) String() string { + if len(m) == 0 { + return "<nil>" + } + buf := make([]byte, len(m)*2) + for i, b := range m { + buf[i*2], buf[i*2+1] = hexDigit[b>>4], hexDigit[b&0xf] + } + return string(buf) +} + +func networkNumberAndMask(n *IPNet) (ip IP, m IPMask) { + if ip = n.IP.To4(); ip == nil { + ip = n.IP + if len(ip) != IPv6len { + return nil, nil + } + } + m = n.Mask + switch len(m) { + case IPv4len: + if len(ip) != IPv4len { + return nil, nil + } + case IPv6len: + if len(ip) == IPv4len { + m = m[12:] + } + default: + return nil, nil + } + return +} + +// Contains reports whether the network includes ip. +func (n *IPNet) Contains(ip IP) bool { + nn, m := networkNumberAndMask(n) + if x := ip.To4(); x != nil { + ip = x + } + l := len(ip) + if l != len(nn) { + return false + } + for i := 0; i < l; i++ { + if nn[i]&m[i] != ip[i]&m[i] { + return false + } + } + return true +} + +// Network returns the address's network name, "ip+net". +func (n *IPNet) Network() string { return "ip+net" } + +// String returns the CIDR notation of n like "192.168.100.1/24" +// or "2001:DB8::/48" as defined in RFC 4632 and RFC 4291. +// If the mask is not in the canonical form, it returns the +// string which consists of an IP address, followed by a slash +// character and a mask expressed as hexadecimal form with no +// punctuation like "192.168.100.1/c000ff00". +func (n *IPNet) String() string { + nn, m := networkNumberAndMask(n) + if nn == nil || m == nil { + return "<nil>" + } + l := simpleMaskLength(m) + if l == -1 { + return nn.String() + "/" + m.String() + } + return nn.String() + "/" + itod(uint(l)) +} + +// Parse IPv4 address (d.d.d.d). +func parseIPv4(s string) IP { + var p [IPv4len]byte + i := 0 + for j := 0; j < IPv4len; j++ { + if i >= len(s) { + // Missing octets. + return nil + } + if j > 0 { + if s[i] != '.' { + return nil + } + i++ + } + var ( + n int + ok bool + ) + n, i, ok = dtoi(s, i) + if !ok || n > 0xFF { + return nil + } + p[j] = byte(n) + } + if i != len(s) { + return nil + } + return IPv4(p[0], p[1], p[2], p[3]) +} + +// parseIPv6 parses s as a literal IPv6 address described in RFC 4291 +// and RFC 5952. It can also parse a literal scoped IPv6 address with +// zone identifier which is described in RFC 4007 when zoneAllowed is +// true. +func parseIPv6(s string, zoneAllowed bool) (ip IP, zone string) { + ip = make(IP, IPv6len) + ellipsis := -1 // position of ellipsis in p + i := 0 // index in string s + + if zoneAllowed { + s, zone = splitHostZone(s) + } + + // Might have leading ellipsis + if len(s) >= 2 && s[0] == ':' && s[1] == ':' { + ellipsis = 0 + i = 2 + // Might be only ellipsis + if i == len(s) { + return ip, zone + } + } + + // Loop, parsing hex numbers followed by colon. + j := 0 + for j < IPv6len { + // Hex number. + n, i1, ok := xtoi(s, i) + if !ok || n > 0xFFFF { + return nil, zone + } + + // If followed by dot, might be in trailing IPv4. + if i1 < len(s) && s[i1] == '.' { + if ellipsis < 0 && j != IPv6len-IPv4len { + // Not the right place. + return nil, zone + } + if j+IPv4len > IPv6len { + // Not enough room. + return nil, zone + } + ip4 := parseIPv4(s[i:]) + if ip4 == nil { + return nil, zone + } + ip[j] = ip4[12] + ip[j+1] = ip4[13] + ip[j+2] = ip4[14] + ip[j+3] = ip4[15] + i = len(s) + j += IPv4len + break + } + + // Save this 16-bit chunk. + ip[j] = byte(n >> 8) + ip[j+1] = byte(n) + j += 2 + + // Stop at end of string. + i = i1 + if i == len(s) { + break + } + + // Otherwise must be followed by colon and more. + if s[i] != ':' || i+1 == len(s) { + return nil, zone + } + i++ + + // Look for ellipsis. + if s[i] == ':' { + if ellipsis >= 0 { // already have one + return nil, zone + } + ellipsis = j + if i++; i == len(s) { // can be at end + break + } + } + } + + // Must have used entire string. + if i != len(s) { + return nil, zone + } + + // If didn't parse enough, expand ellipsis. + if j < IPv6len { + if ellipsis < 0 { + return nil, zone + } + n := IPv6len - j + for k := j - 1; k >= ellipsis; k-- { + ip[k+n] = ip[k] + } + for k := ellipsis + n - 1; k >= ellipsis; k-- { + ip[k] = 0 + } + } else if ellipsis >= 0 { + // Ellipsis must represent at least one 0 group. + return nil, zone + } + return ip, zone +} + +// A ParseError represents a malformed text string and the type of string that was expected. +type ParseError struct { + Type string + Text string +} + +func (e *ParseError) Error() string { + return "invalid " + e.Type + ": " + e.Text +} + +// ParseIP parses s as an IP address, returning the result. +// The string s can be in dotted decimal ("74.125.19.99") +// or IPv6 ("2001:4860:0:2001::68") form. +// If s is not a valid textual representation of an IP address, +// ParseIP returns nil. +func ParseIP(s string) IP { + for i := 0; i < len(s); i++ { + switch s[i] { + case '.': + return parseIPv4(s) + case ':': + ip, _ := parseIPv6(s, false) + return ip + } + } + return nil +} + +// ParseCIDR parses s as a CIDR notation IP address and mask, +// like "192.168.100.1/24" or "2001:DB8::/48", as defined in +// RFC 4632 and RFC 4291. +// +// It returns the IP address and the network implied by the IP +// and mask. For example, ParseCIDR("192.168.100.1/16") returns +// the IP address 192.168.100.1 and the network 192.168.0.0/16. +func ParseCIDR(s string) (IP, *IPNet, error) { + i := byteIndex(s, '/') + if i < 0 { + return nil, nil, &ParseError{"CIDR address", s} + } + addr, mask := s[:i], s[i+1:] + iplen := IPv4len + ip := parseIPv4(addr) + if ip == nil { + iplen = IPv6len + ip, _ = parseIPv6(addr, false) + } + n, i, ok := dtoi(mask, 0) + if ip == nil || !ok || i != len(mask) || n < 0 || n > 8*iplen { + return nil, nil, &ParseError{"CIDR address", s} + } + m := CIDRMask(n, 8*iplen) + return ip, &IPNet{IP: ip.Mask(m), Mask: m}, nil +} diff --git a/src/net/ip_test.go b/src/net/ip_test.go new file mode 100644 index 000000000..485ff5115 --- /dev/null +++ b/src/net/ip_test.go @@ -0,0 +1,465 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "reflect" + "runtime" + "testing" +) + +var parseIPTests = []struct { + in string + out IP +}{ + {"127.0.1.2", IPv4(127, 0, 1, 2)}, + {"127.0.0.1", IPv4(127, 0, 0, 1)}, + {"127.0.0.256", nil}, + {"abc", nil}, + {"123:", nil}, + {"::ffff:127.0.0.1", IPv4(127, 0, 0, 1)}, + {"2001:4860:0:2001::68", IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68}}, + {"::ffff:4a7d:1363", IPv4(74, 125, 19, 99)}, + {"fe80::1%lo0", nil}, + {"fe80::1%911", nil}, + {"", nil}, + {"a1:a2:a3:a4::b1:b2:b3:b4", nil}, // Issue 6628 +} + +func TestParseIP(t *testing.T) { + for _, tt := range parseIPTests { + if out := ParseIP(tt.in); !reflect.DeepEqual(out, tt.out) { + t.Errorf("ParseIP(%q) = %v, want %v", tt.in, out, tt.out) + } + if tt.in == "" { + // Tested in TestMarshalEmptyIP below. + continue + } + var out IP + if err := out.UnmarshalText([]byte(tt.in)); !reflect.DeepEqual(out, tt.out) || (tt.out == nil) != (err != nil) { + t.Errorf("IP.UnmarshalText(%q) = %v, %v, want %v", tt.in, out, err, tt.out) + } + } +} + +func BenchmarkParseIP(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, tt := range parseIPTests { + ParseIP(tt.in) + } + } +} + +// Issue 6339 +func TestMarshalEmptyIP(t *testing.T) { + for _, in := range [][]byte{nil, []byte("")} { + var out = IP{1, 2, 3, 4} + if err := out.UnmarshalText(in); err != nil || out != nil { + t.Errorf("UnmarshalText(%v) = %v, %v; want nil, nil", in, out, err) + } + } + var ip IP + got, err := ip.MarshalText() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, []byte("")) { + t.Errorf(`got %#v, want []byte("")`, got) + } +} + +var ipStringTests = []struct { + in IP + out string // see RFC 5952 +}{ + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, "2001:db8::123:12:1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, "2001:db8:0:1:0:1:0:1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, "2001:db8:1:0:1:0:1:0"}, + {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001::1:0:0:1"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, "2001:db8:0:0:1::"}, + {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1:0:0:1"}, + {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, "2001:db8::a:b:c:d"}, + {IPv4(192, 168, 0, 1), "192.168.0.1"}, + {nil, ""}, +} + +func TestIPString(t *testing.T) { + for _, tt := range ipStringTests { + if tt.in != nil { + if out := tt.in.String(); out != tt.out { + t.Errorf("IP.String(%v) = %q, want %q", tt.in, out, tt.out) + } + } + if out, err := tt.in.MarshalText(); string(out) != tt.out || err != nil { + t.Errorf("IP.MarshalText(%v) = %q, %v, want %q, nil", tt.in, out, err, tt.out) + } + } +} + +func BenchmarkIPString(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, tt := range ipStringTests { + if tt.in != nil { + tt.in.String() + } + } + } +} + +var ipMaskTests = []struct { + in IP + mask IPMask + out IP +}{ + {IPv4(192, 168, 1, 127), IPv4Mask(255, 255, 255, 128), IPv4(192, 168, 1, 0)}, + {IPv4(192, 168, 1, 127), IPMask(ParseIP("255.255.255.192")), IPv4(192, 168, 1, 64)}, + {IPv4(192, 168, 1, 127), IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffe0")), IPv4(192, 168, 1, 96)}, + {IPv4(192, 168, 1, 127), IPv4Mask(255, 0, 255, 0), IPv4(192, 0, 1, 0)}, + {ParseIP("2001:db8::1"), IPMask(ParseIP("ffff:ff80::")), ParseIP("2001:d80::")}, + {ParseIP("2001:db8::1"), IPMask(ParseIP("f0f0:0f0f::")), ParseIP("2000:d08::")}, +} + +func TestIPMask(t *testing.T) { + for _, tt := range ipMaskTests { + if out := tt.in.Mask(tt.mask); out == nil || !tt.out.Equal(out) { + t.Errorf("IP(%v).Mask(%v) = %v, want %v", tt.in, tt.mask, out, tt.out) + } + } +} + +var ipMaskStringTests = []struct { + in IPMask + out string +}{ + {IPv4Mask(255, 255, 255, 240), "fffffff0"}, + {IPv4Mask(255, 0, 128, 0), "ff008000"}, + {IPMask(ParseIP("ffff:ff80::")), "ffffff80000000000000000000000000"}, + {IPMask(ParseIP("ef00:ff80::cafe:0")), "ef00ff800000000000000000cafe0000"}, + {nil, "<nil>"}, +} + +func TestIPMaskString(t *testing.T) { + for _, tt := range ipMaskStringTests { + if out := tt.in.String(); out != tt.out { + t.Errorf("IPMask.String(%v) = %q, want %q", tt.in, out, tt.out) + } + } +} + +func BenchmarkIPMaskString(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, tt := range ipMaskStringTests { + tt.in.String() + } + } +} + +var parseCIDRTests = []struct { + in string + ip IP + net *IPNet + err error +}{ + {"135.104.0.0/32", IPv4(135, 104, 0, 0), &IPNet{IP: IPv4(135, 104, 0, 0), Mask: IPv4Mask(255, 255, 255, 255)}, nil}, + {"0.0.0.0/24", IPv4(0, 0, 0, 0), &IPNet{IP: IPv4(0, 0, 0, 0), Mask: IPv4Mask(255, 255, 255, 0)}, nil}, + {"135.104.0.0/24", IPv4(135, 104, 0, 0), &IPNet{IP: IPv4(135, 104, 0, 0), Mask: IPv4Mask(255, 255, 255, 0)}, nil}, + {"135.104.0.1/32", IPv4(135, 104, 0, 1), &IPNet{IP: IPv4(135, 104, 0, 1), Mask: IPv4Mask(255, 255, 255, 255)}, nil}, + {"135.104.0.1/24", IPv4(135, 104, 0, 1), &IPNet{IP: IPv4(135, 104, 0, 0), Mask: IPv4Mask(255, 255, 255, 0)}, nil}, + {"::1/128", ParseIP("::1"), &IPNet{IP: ParseIP("::1"), Mask: IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"))}, nil}, + {"abcd:2345::/127", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2345::"), Mask: IPMask(ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:fffe"))}, nil}, + {"abcd:2345::/65", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2345::"), Mask: IPMask(ParseIP("ffff:ffff:ffff:ffff:8000::"))}, nil}, + {"abcd:2345::/64", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2345::"), Mask: IPMask(ParseIP("ffff:ffff:ffff:ffff::"))}, nil}, + {"abcd:2345::/63", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2345::"), Mask: IPMask(ParseIP("ffff:ffff:ffff:fffe::"))}, nil}, + {"abcd:2345::/33", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2345::"), Mask: IPMask(ParseIP("ffff:ffff:8000::"))}, nil}, + {"abcd:2345::/32", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2345::"), Mask: IPMask(ParseIP("ffff:ffff::"))}, nil}, + {"abcd:2344::/31", ParseIP("abcd:2344::"), &IPNet{IP: ParseIP("abcd:2344::"), Mask: IPMask(ParseIP("ffff:fffe::"))}, nil}, + {"abcd:2300::/24", ParseIP("abcd:2300::"), &IPNet{IP: ParseIP("abcd:2300::"), Mask: IPMask(ParseIP("ffff:ff00::"))}, nil}, + {"abcd:2345::/24", ParseIP("abcd:2345::"), &IPNet{IP: ParseIP("abcd:2300::"), Mask: IPMask(ParseIP("ffff:ff00::"))}, nil}, + {"2001:DB8::/48", ParseIP("2001:DB8::"), &IPNet{IP: ParseIP("2001:DB8::"), Mask: IPMask(ParseIP("ffff:ffff:ffff::"))}, nil}, + {"2001:DB8::1/48", ParseIP("2001:DB8::1"), &IPNet{IP: ParseIP("2001:DB8::"), Mask: IPMask(ParseIP("ffff:ffff:ffff::"))}, nil}, + {"192.168.1.1/255.255.255.0", nil, nil, &ParseError{"CIDR address", "192.168.1.1/255.255.255.0"}}, + {"192.168.1.1/35", nil, nil, &ParseError{"CIDR address", "192.168.1.1/35"}}, + {"2001:db8::1/-1", nil, nil, &ParseError{"CIDR address", "2001:db8::1/-1"}}, + {"", nil, nil, &ParseError{"CIDR address", ""}}, +} + +func TestParseCIDR(t *testing.T) { + for _, tt := range parseCIDRTests { + ip, net, err := ParseCIDR(tt.in) + if !reflect.DeepEqual(err, tt.err) { + t.Errorf("ParseCIDR(%q) = %v, %v; want %v, %v", tt.in, ip, net, tt.ip, tt.net) + } + if err == nil && (!tt.ip.Equal(ip) || !tt.net.IP.Equal(net.IP) || !reflect.DeepEqual(net.Mask, tt.net.Mask)) { + t.Errorf("ParseCIDR(%q) = %v, {%v, %v}; want %v, {%v, %v}", tt.in, ip, net.IP, net.Mask, tt.ip, tt.net.IP, tt.net.Mask) + } + } +} + +var ipNetContainsTests = []struct { + ip IP + net *IPNet + ok bool +}{ + {IPv4(172, 16, 1, 1), &IPNet{IP: IPv4(172, 16, 0, 0), Mask: CIDRMask(12, 32)}, true}, + {IPv4(172, 24, 0, 1), &IPNet{IP: IPv4(172, 16, 0, 0), Mask: CIDRMask(13, 32)}, false}, + {IPv4(192, 168, 0, 3), &IPNet{IP: IPv4(192, 168, 0, 0), Mask: IPv4Mask(0, 0, 255, 252)}, true}, + {IPv4(192, 168, 0, 4), &IPNet{IP: IPv4(192, 168, 0, 0), Mask: IPv4Mask(0, 255, 0, 252)}, false}, + {ParseIP("2001:db8:1:2::1"), &IPNet{IP: ParseIP("2001:db8:1::"), Mask: CIDRMask(47, 128)}, true}, + {ParseIP("2001:db8:1:2::1"), &IPNet{IP: ParseIP("2001:db8:2::"), Mask: CIDRMask(47, 128)}, false}, + {ParseIP("2001:db8:1:2::1"), &IPNet{IP: ParseIP("2001:db8:1::"), Mask: IPMask(ParseIP("ffff:0:ffff::"))}, true}, + {ParseIP("2001:db8:1:2::1"), &IPNet{IP: ParseIP("2001:db8:1::"), Mask: IPMask(ParseIP("0:0:0:ffff::"))}, false}, +} + +func TestIPNetContains(t *testing.T) { + for _, tt := range ipNetContainsTests { + if ok := tt.net.Contains(tt.ip); ok != tt.ok { + t.Errorf("IPNet(%v).Contains(%v) = %v, want %v", tt.net, tt.ip, ok, tt.ok) + } + } +} + +var ipNetStringTests = []struct { + in *IPNet + out string +}{ + {&IPNet{IP: IPv4(192, 168, 1, 0), Mask: CIDRMask(26, 32)}, "192.168.1.0/26"}, + {&IPNet{IP: IPv4(192, 168, 1, 0), Mask: IPv4Mask(255, 0, 255, 0)}, "192.168.1.0/ff00ff00"}, + {&IPNet{IP: ParseIP("2001:db8::"), Mask: CIDRMask(55, 128)}, "2001:db8::/55"}, + {&IPNet{IP: ParseIP("2001:db8::"), Mask: IPMask(ParseIP("8000:f123:0:cafe::"))}, "2001:db8::/8000f1230000cafe0000000000000000"}, +} + +func TestIPNetString(t *testing.T) { + for _, tt := range ipNetStringTests { + if out := tt.in.String(); out != tt.out { + t.Errorf("IPNet.String(%v) = %q, want %q", tt.in, out, tt.out) + } + } +} + +var cidrMaskTests = []struct { + ones int + bits int + out IPMask +}{ + {0, 32, IPv4Mask(0, 0, 0, 0)}, + {12, 32, IPv4Mask(255, 240, 0, 0)}, + {24, 32, IPv4Mask(255, 255, 255, 0)}, + {32, 32, IPv4Mask(255, 255, 255, 255)}, + {0, 128, IPMask{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {4, 128, IPMask{0xf0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {48, 128, IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}}, + {128, 128, IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {33, 32, nil}, + {32, 33, nil}, + {-1, 128, nil}, + {128, -1, nil}, +} + +func TestCIDRMask(t *testing.T) { + for _, tt := range cidrMaskTests { + if out := CIDRMask(tt.ones, tt.bits); !reflect.DeepEqual(out, tt.out) { + t.Errorf("CIDRMask(%v, %v) = %v, want %v", tt.ones, tt.bits, out, tt.out) + } + } +} + +var ( + v4addr = IP{192, 168, 0, 1} + v4mappedv6addr = IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 168, 0, 1} + v6addr = IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1} + v4mask = IPMask{255, 255, 255, 0} + v4mappedv6mask = IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 255, 255, 255, 0} + v6mask = IPMask{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0, 0, 0, 0, 0, 0, 0, 0} + badaddr = IP{192, 168, 0} + badmask = IPMask{255, 255, 0} + v4maskzero = IPMask{0, 0, 0, 0} +) + +var networkNumberAndMaskTests = []struct { + in IPNet + out IPNet +}{ + {IPNet{IP: v4addr, Mask: v4mask}, IPNet{IP: v4addr, Mask: v4mask}}, + {IPNet{IP: v4addr, Mask: v4mappedv6mask}, IPNet{IP: v4addr, Mask: v4mask}}, + {IPNet{IP: v4mappedv6addr, Mask: v4mappedv6mask}, IPNet{IP: v4addr, Mask: v4mask}}, + {IPNet{IP: v4mappedv6addr, Mask: v6mask}, IPNet{IP: v4addr, Mask: v4maskzero}}, + {IPNet{IP: v4addr, Mask: v6mask}, IPNet{IP: v4addr, Mask: v4maskzero}}, + {IPNet{IP: v6addr, Mask: v6mask}, IPNet{IP: v6addr, Mask: v6mask}}, + {IPNet{IP: v6addr, Mask: v4mappedv6mask}, IPNet{IP: v6addr, Mask: v4mappedv6mask}}, + {in: IPNet{IP: v6addr, Mask: v4mask}}, + {in: IPNet{IP: v4addr, Mask: badmask}}, + {in: IPNet{IP: v4mappedv6addr, Mask: badmask}}, + {in: IPNet{IP: v6addr, Mask: badmask}}, + {in: IPNet{IP: badaddr, Mask: v4mask}}, + {in: IPNet{IP: badaddr, Mask: v4mappedv6mask}}, + {in: IPNet{IP: badaddr, Mask: v6mask}}, + {in: IPNet{IP: badaddr, Mask: badmask}}, +} + +func TestNetworkNumberAndMask(t *testing.T) { + for _, tt := range networkNumberAndMaskTests { + ip, m := networkNumberAndMask(&tt.in) + out := &IPNet{IP: ip, Mask: m} + if !reflect.DeepEqual(&tt.out, out) { + t.Errorf("networkNumberAndMask(%v) = %v, want %v", tt.in, out, &tt.out) + } + } +} + +var splitJoinTests = []struct { + host string + port string + join string +}{ + {"www.google.com", "80", "www.google.com:80"}, + {"127.0.0.1", "1234", "127.0.0.1:1234"}, + {"::1", "80", "[::1]:80"}, + {"fe80::1%lo0", "80", "[fe80::1%lo0]:80"}, + {"localhost%lo0", "80", "[localhost%lo0]:80"}, + {"", "0", ":0"}, + + {"google.com", "https%foo", "google.com:https%foo"}, // Go 1.0 behavior + {"127.0.0.1", "", "127.0.0.1:"}, // Go 1.0 behaviour + {"www.google.com", "", "www.google.com:"}, // Go 1.0 behaviour +} + +var splitFailureTests = []struct { + hostPort string + err string +}{ + {"www.google.com", "missing port in address"}, + {"127.0.0.1", "missing port in address"}, + {"[::1]", "missing port in address"}, + {"[fe80::1%lo0]", "missing port in address"}, + {"[localhost%lo0]", "missing port in address"}, + {"localhost%lo0", "missing port in address"}, + + {"::1", "too many colons in address"}, + {"fe80::1%lo0", "too many colons in address"}, + {"fe80::1%lo0:80", "too many colons in address"}, + + {"localhost%lo0:80", "missing brackets in address"}, + + // Test cases that didn't fail in Go 1.0 + + {"[foo:bar]", "missing port in address"}, + {"[foo:bar]baz", "missing port in address"}, + {"[foo]bar:baz", "missing port in address"}, + + {"[foo]:[bar]:baz", "too many colons in address"}, + + {"[foo]:[bar]baz", "unexpected '[' in address"}, + {"foo[bar]:baz", "unexpected '[' in address"}, + + {"foo]bar:baz", "unexpected ']' in address"}, +} + +func TestSplitHostPort(t *testing.T) { + for _, tt := range splitJoinTests { + if host, port, err := SplitHostPort(tt.join); host != tt.host || port != tt.port || err != nil { + t.Errorf("SplitHostPort(%q) = %q, %q, %v; want %q, %q, nil", tt.join, host, port, err, tt.host, tt.port) + } + } + for _, tt := range splitFailureTests { + if _, _, err := SplitHostPort(tt.hostPort); err == nil { + t.Errorf("SplitHostPort(%q) should have failed", tt.hostPort) + } else { + e := err.(*AddrError) + if e.Err != tt.err { + t.Errorf("SplitHostPort(%q) = _, _, %q; want %q", tt.hostPort, e.Err, tt.err) + } + } + } +} + +func TestJoinHostPort(t *testing.T) { + for _, tt := range splitJoinTests { + if join := JoinHostPort(tt.host, tt.port); join != tt.join { + t.Errorf("JoinHostPort(%q, %q) = %q; want %q", tt.host, tt.port, join, tt.join) + } + } +} + +var ipAddrFamilyTests = []struct { + in IP + af4 bool + af6 bool +}{ + {IPv4bcast, true, false}, + {IPv4allsys, true, false}, + {IPv4allrouter, true, false}, + {IPv4zero, true, false}, + {IPv4(224, 0, 0, 1), true, false}, + {IPv4(127, 0, 0, 1), true, false}, + {IPv4(240, 0, 0, 1), true, false}, + {IPv6unspecified, false, true}, + {IPv6loopback, false, true}, + {IPv6interfacelocalallnodes, false, true}, + {IPv6linklocalallnodes, false, true}, + {IPv6linklocalallrouters, false, true}, + {ParseIP("ff05::a:b:c:d"), false, true}, + {ParseIP("fe80::1:2:3:4"), false, true}, + {ParseIP("2001:db8::123:12:1"), false, true}, +} + +func TestIPAddrFamily(t *testing.T) { + for _, tt := range ipAddrFamilyTests { + if af := tt.in.To4() != nil; af != tt.af4 { + t.Errorf("verifying IPv4 address family for %q = %v, want %v", tt.in, af, tt.af4) + } + if af := len(tt.in) == IPv6len && tt.in.To4() == nil; af != tt.af6 { + t.Errorf("verifying IPv6 address family for %q = %v, want %v", tt.in, af, tt.af6) + } + } +} + +var ipAddrScopeTests = []struct { + scope func(IP) bool + in IP + ok bool +}{ + {IP.IsUnspecified, IPv4zero, true}, + {IP.IsUnspecified, IPv4(127, 0, 0, 1), false}, + {IP.IsUnspecified, IPv6unspecified, true}, + {IP.IsUnspecified, IPv6interfacelocalallnodes, false}, + {IP.IsLoopback, IPv4(127, 0, 0, 1), true}, + {IP.IsLoopback, IPv4(127, 255, 255, 254), true}, + {IP.IsLoopback, IPv4(128, 1, 2, 3), false}, + {IP.IsLoopback, IPv6loopback, true}, + {IP.IsLoopback, IPv6linklocalallrouters, false}, + {IP.IsMulticast, IPv4(224, 0, 0, 0), true}, + {IP.IsMulticast, IPv4(239, 0, 0, 0), true}, + {IP.IsMulticast, IPv4(240, 0, 0, 0), false}, + {IP.IsMulticast, IPv6linklocalallnodes, true}, + {IP.IsMulticast, IP{0xff, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true}, + {IP.IsMulticast, IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsLinkLocalMulticast, IPv4(224, 0, 0, 0), true}, + {IP.IsLinkLocalMulticast, IPv4(239, 0, 0, 0), false}, + {IP.IsLinkLocalMulticast, IPv6linklocalallrouters, true}, + {IP.IsLinkLocalMulticast, IP{0xff, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsLinkLocalUnicast, IPv4(169, 254, 0, 0), true}, + {IP.IsLinkLocalUnicast, IPv4(169, 255, 0, 0), false}, + {IP.IsLinkLocalUnicast, IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true}, + {IP.IsLinkLocalUnicast, IP{0xfe, 0xc0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsGlobalUnicast, IPv4(240, 0, 0, 0), true}, + {IP.IsGlobalUnicast, IPv4(232, 0, 0, 0), false}, + {IP.IsGlobalUnicast, IPv4(169, 254, 0, 0), false}, + {IP.IsGlobalUnicast, IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, true}, + {IP.IsGlobalUnicast, IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, + {IP.IsGlobalUnicast, IP{0xff, 0x05, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false}, +} + +func name(f interface{}) string { + return runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() +} + +func TestIPAddrScope(t *testing.T) { + for _, tt := range ipAddrScopeTests { + if ok := tt.scope(tt.in); ok != tt.ok { + t.Errorf("%s(%q) = %v, want %v", name(tt.scope), tt.in, ok, tt.ok) + } + } +} diff --git a/src/net/ipraw_test.go b/src/net/ipraw_test.go new file mode 100644 index 000000000..0632dafc6 --- /dev/null +++ b/src/net/ipraw_test.go @@ -0,0 +1,289 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "fmt" + "os" + "reflect" + "runtime" + "testing" + "time" +) + +type resolveIPAddrTest struct { + net string + litAddrOrName string + addr *IPAddr + err error +} + +var resolveIPAddrTests = []resolveIPAddrTest{ + {"ip", "127.0.0.1", &IPAddr{IP: IPv4(127, 0, 0, 1)}, nil}, + {"ip4", "127.0.0.1", &IPAddr{IP: IPv4(127, 0, 0, 1)}, nil}, + {"ip4:icmp", "127.0.0.1", &IPAddr{IP: IPv4(127, 0, 0, 1)}, nil}, + + {"ip", "::1", &IPAddr{IP: ParseIP("::1")}, nil}, + {"ip6", "::1", &IPAddr{IP: ParseIP("::1")}, nil}, + {"ip6:ipv6-icmp", "::1", &IPAddr{IP: ParseIP("::1")}, nil}, + {"ip6:IPv6-ICMP", "::1", &IPAddr{IP: ParseIP("::1")}, nil}, + + {"ip", "::1%en0", &IPAddr{IP: ParseIP("::1"), Zone: "en0"}, nil}, + {"ip6", "::1%911", &IPAddr{IP: ParseIP("::1"), Zone: "911"}, nil}, + + {"", "127.0.0.1", &IPAddr{IP: IPv4(127, 0, 0, 1)}, nil}, // Go 1.0 behavior + {"", "::1", &IPAddr{IP: ParseIP("::1")}, nil}, // Go 1.0 behavior + + {"l2tp", "127.0.0.1", nil, UnknownNetworkError("l2tp")}, + {"l2tp:gre", "127.0.0.1", nil, UnknownNetworkError("l2tp:gre")}, + {"tcp", "1.2.3.4:123", nil, UnknownNetworkError("tcp")}, +} + +func init() { + if ifi := loopbackInterface(); ifi != nil { + index := fmt.Sprintf("%v", ifi.Index) + resolveIPAddrTests = append(resolveIPAddrTests, []resolveIPAddrTest{ + {"ip6", "fe80::1%" + ifi.Name, &IPAddr{IP: ParseIP("fe80::1"), Zone: zoneToString(ifi.Index)}, nil}, + {"ip6", "fe80::1%" + index, &IPAddr{IP: ParseIP("fe80::1"), Zone: index}, nil}, + }...) + } + if ips, err := LookupIP("localhost"); err == nil && len(ips) > 1 && supportsIPv4 && supportsIPv6 { + resolveIPAddrTests = append(resolveIPAddrTests, []resolveIPAddrTest{ + {"ip", "localhost", &IPAddr{IP: IPv4(127, 0, 0, 1)}, nil}, + {"ip4", "localhost", &IPAddr{IP: IPv4(127, 0, 0, 1)}, nil}, + {"ip6", "localhost", &IPAddr{IP: IPv6loopback}, nil}, + }...) + } +} + +func skipRawSocketTest(t *testing.T) (skip bool, skipmsg string) { + skip, skipmsg, err := skipRawSocketTests() + if err != nil { + t.Fatal(err) + } + return skip, skipmsg +} + +func TestResolveIPAddr(t *testing.T) { + for _, tt := range resolveIPAddrTests { + addr, err := ResolveIPAddr(tt.net, tt.litAddrOrName) + if err != tt.err { + t.Fatalf("ResolveIPAddr(%v, %v) failed: %v", tt.net, tt.litAddrOrName, err) + } else if !reflect.DeepEqual(addr, tt.addr) { + t.Fatalf("got %#v; expected %#v", addr, tt.addr) + } + } +} + +var icmpEchoTests = []struct { + net string + laddr string + raddr string +}{ + {"ip4:icmp", "0.0.0.0", "127.0.0.1"}, + {"ip6:ipv6-icmp", "::", "::1"}, +} + +func TestConnICMPEcho(t *testing.T) { + if skip, skipmsg := skipRawSocketTest(t); skip { + t.Skip(skipmsg) + } + + for i, tt := range icmpEchoTests { + net, _, err := parseNetwork(tt.net) + if err != nil { + t.Fatalf("parseNetwork failed: %v", err) + } + if net == "ip6" && !supportsIPv6 { + continue + } + + c, err := Dial(tt.net, tt.raddr) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + c.SetDeadline(time.Now().Add(100 * time.Millisecond)) + defer c.Close() + + typ := icmpv4EchoRequest + if net == "ip6" { + typ = icmpv6EchoRequest + } + xid, xseq := os.Getpid()&0xffff, i+1 + wb, err := (&icmpMessage{ + Type: typ, Code: 0, + Body: &icmpEcho{ + ID: xid, Seq: xseq, + Data: bytes.Repeat([]byte("Go Go Gadget Ping!!!"), 3), + }, + }).Marshal() + if err != nil { + t.Fatalf("icmpMessage.Marshal failed: %v", err) + } + if _, err := c.Write(wb); err != nil { + t.Fatalf("Conn.Write failed: %v", err) + } + var m *icmpMessage + rb := make([]byte, 20+len(wb)) + for { + if _, err := c.Read(rb); err != nil { + t.Fatalf("Conn.Read failed: %v", err) + } + if net == "ip4" { + rb = ipv4Payload(rb) + } + if m, err = parseICMPMessage(rb); err != nil { + t.Fatalf("parseICMPMessage failed: %v", err) + } + switch m.Type { + case icmpv4EchoRequest, icmpv6EchoRequest: + continue + } + break + } + switch p := m.Body.(type) { + case *icmpEcho: + if p.ID != xid || p.Seq != xseq { + t.Fatalf("got id=%v, seqnum=%v; expected id=%v, seqnum=%v", p.ID, p.Seq, xid, xseq) + } + default: + t.Fatalf("got type=%v, code=%v; expected type=%v, code=%v", m.Type, m.Code, typ, 0) + } + } +} + +func TestPacketConnICMPEcho(t *testing.T) { + if skip, skipmsg := skipRawSocketTest(t); skip { + t.Skip(skipmsg) + } + + for i, tt := range icmpEchoTests { + net, _, err := parseNetwork(tt.net) + if err != nil { + t.Fatalf("parseNetwork failed: %v", err) + } + if net == "ip6" && !supportsIPv6 { + continue + } + + c, err := ListenPacket(tt.net, tt.laddr) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + c.SetDeadline(time.Now().Add(100 * time.Millisecond)) + defer c.Close() + + ra, err := ResolveIPAddr(tt.net, tt.raddr) + if err != nil { + t.Fatalf("ResolveIPAddr failed: %v", err) + } + typ := icmpv4EchoRequest + if net == "ip6" { + typ = icmpv6EchoRequest + } + xid, xseq := os.Getpid()&0xffff, i+1 + wb, err := (&icmpMessage{ + Type: typ, Code: 0, + Body: &icmpEcho{ + ID: xid, Seq: xseq, + Data: bytes.Repeat([]byte("Go Go Gadget Ping!!!"), 3), + }, + }).Marshal() + if err != nil { + t.Fatalf("icmpMessage.Marshal failed: %v", err) + } + if _, err := c.WriteTo(wb, ra); err != nil { + t.Fatalf("PacketConn.WriteTo failed: %v", err) + } + var m *icmpMessage + rb := make([]byte, 20+len(wb)) + for { + if _, _, err := c.ReadFrom(rb); err != nil { + t.Fatalf("PacketConn.ReadFrom failed: %v", err) + } + // See BUG section. + //if net == "ip4" { + // rb = ipv4Payload(rb) + //} + if m, err = parseICMPMessage(rb); err != nil { + t.Fatalf("parseICMPMessage failed: %v", err) + } + switch m.Type { + case icmpv4EchoRequest, icmpv6EchoRequest: + continue + } + break + } + switch p := m.Body.(type) { + case *icmpEcho: + if p.ID != xid || p.Seq != xseq { + t.Fatalf("got id=%v, seqnum=%v; expected id=%v, seqnum=%v", p.ID, p.Seq, xid, xseq) + } + default: + t.Fatalf("got type=%v, code=%v; expected type=%v, code=%v", m.Type, m.Code, typ, 0) + } + } +} + +func ipv4Payload(b []byte) []byte { + if len(b) < 20 { + return b + } + hdrlen := int(b[0]&0x0f) << 2 + return b[hdrlen:] +} + +var ipConnLocalNameTests = []struct { + net string + laddr *IPAddr +}{ + {"ip4:icmp", &IPAddr{IP: IPv4(127, 0, 0, 1)}}, + {"ip4:icmp", &IPAddr{}}, + {"ip4:icmp", nil}, +} + +func TestIPConnLocalName(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + default: + if os.Getuid() != 0 { + t.Skip("skipping test; must be root") + } + } + + for _, tt := range ipConnLocalNameTests { + c, err := ListenIP(tt.net, tt.laddr) + if err != nil { + t.Fatalf("ListenIP failed: %v", err) + } + defer c.Close() + if la := c.LocalAddr(); la == nil { + t.Fatal("IPConn.LocalAddr failed") + } + } +} + +func TestIPConnRemoteName(t *testing.T) { + switch runtime.GOOS { + case "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + default: + if os.Getuid() != 0 { + t.Skip("skipping test; must be root") + } + } + + raddr := &IPAddr{IP: IPv4(127, 0, 0, 1).To4()} + c, err := DialIP("ip:tcp", &IPAddr{IP: IPv4(127, 0, 0, 1)}, raddr) + if err != nil { + t.Fatalf("DialIP failed: %v", err) + } + defer c.Close() + if !reflect.DeepEqual(raddr, c.RemoteAddr()) { + t.Fatalf("got %#v, expected %#v", c.RemoteAddr(), raddr) + } +} diff --git a/src/net/iprawsock.go b/src/net/iprawsock.go new file mode 100644 index 000000000..5cc361390 --- /dev/null +++ b/src/net/iprawsock.go @@ -0,0 +1,54 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// IPAddr represents the address of an IP end point. +type IPAddr struct { + IP IP + Zone string // IPv6 scoped addressing zone +} + +// Network returns the address's network name, "ip". +func (a *IPAddr) Network() string { return "ip" } + +func (a *IPAddr) String() string { + if a == nil { + return "<nil>" + } + if a.Zone != "" { + return a.IP.String() + "%" + a.Zone + } + return a.IP.String() +} + +func (a *IPAddr) toAddr() Addr { + if a == nil { + return nil + } + return a +} + +// ResolveIPAddr parses addr as an IP address of the form "host" or +// "ipv6-host%zone" and resolves the domain name on the network net, +// which must be "ip", "ip4" or "ip6". +func ResolveIPAddr(net, addr string) (*IPAddr, error) { + if net == "" { // a hint wildcard for Go 1.0 undocumented behavior + net = "ip" + } + afnet, _, err := parseNetwork(net) + if err != nil { + return nil, err + } + switch afnet { + case "ip", "ip4", "ip6": + default: + return nil, UnknownNetworkError(net) + } + a, err := resolveInternetAddr(afnet, addr, noDeadline) + if err != nil { + return nil, err + } + return a.toAddr().(*IPAddr), nil +} diff --git a/src/net/iprawsock_plan9.go b/src/net/iprawsock_plan9.go new file mode 100644 index 000000000..e62d116b8 --- /dev/null +++ b/src/net/iprawsock_plan9.go @@ -0,0 +1,82 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "syscall" + "time" +) + +// IPConn is the implementation of the Conn and PacketConn interfaces +// for IP network connections. +type IPConn struct { + conn +} + +// ReadFromIP reads an IP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { + return 0, nil, syscall.EPLAN9 +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { + return 0, nil, syscall.EPLAN9 +} + +// ReadMsgIP reads a packet from c, copying the payload into b and the +// associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet and the source address of the packet. +func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 +} + +// WriteToIP writes an IP packet to addr via c, copying the payload +// from b. +// +// WriteToIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { + return 0, syscall.EPLAN9 +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { + return 0, syscall.EPLAN9 +} + +// WriteMsgIP writes a packet to addr via c, copying the payload from +// b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. +func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 +} + +// DialIP connects to the remote address raddr on the network protocol +// netProto, which must be "ip", "ip4", or "ip6" followed by a colon +// and a protocol number or name. +func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + return dialIP(netProto, laddr, raddr, noDeadline) +} + +func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) { + return nil, syscall.EPLAN9 +} + +// ListenIP listens for incoming IP packets addressed to the local +// address laddr. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send IP packets with per-packet +// addressing. +func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { + return nil, syscall.EPLAN9 +} diff --git a/src/net/iprawsock_posix.go b/src/net/iprawsock_posix.go new file mode 100644 index 000000000..99b081ba8 --- /dev/null +++ b/src/net/iprawsock_posix.go @@ -0,0 +1,227 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "syscall" + "time" +) + +// BUG(mikio): On every POSIX platform, reads from the "ip4" network +// using the ReadFrom or ReadFromIP method might not return a complete +// IPv4 packet, including its header, even if there is space +// available. This can occur even in cases where Read or ReadMsgIP +// could return a complete packet. For this reason, it is recommended +// that you do not uses these methods if it is important to receive a +// full packet. +// +// The Go 1 compatibility guidelines make it impossible for us to +// change the behavior of these methods; use Read or ReadMsgIP +// instead. + +func sockaddrToIP(sa syscall.Sockaddr) Addr { + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + return &IPAddr{IP: sa.Addr[0:]} + case *syscall.SockaddrInet6: + return &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} + } + return nil +} + +func (a *IPAddr) family() int { + if a == nil || len(a.IP) <= IPv4len { + return syscall.AF_INET + } + if a.IP.To4() != nil { + return syscall.AF_INET + } + return syscall.AF_INET6 +} + +func (a *IPAddr) isWildcard() bool { + if a == nil || a.IP == nil { + return true + } + return a.IP.IsUnspecified() +} + +func (a *IPAddr) sockaddr(family int) (syscall.Sockaddr, error) { + if a == nil { + return nil, nil + } + return ipToSockaddr(family, a.IP, 0, a.Zone) +} + +// IPConn is the implementation of the Conn and PacketConn interfaces +// for IP network connections. +type IPConn struct { + conn +} + +func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } + +// ReadFromIP reads an IP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + // TODO(cw,rsc): consider using readv if we know the family + // type to avoid the header trim/copy + var addr *IPAddr + n, sa, err := c.fd.readFrom(b) + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + addr = &IPAddr{IP: sa.Addr[0:]} + if len(b) >= IPv4len { // discard ipv4 header + hsize := (int(b[0]) & 0xf) * 4 + copy(b, b[hsize:]) + n -= hsize + } + case *syscall.SockaddrInet6: + addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.ReadFromIP(b) + return n, addr.toAddr(), err +} + +// ReadMsgIP reads a packet from c, copying the payload into b and the +// associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet and the source address of the packet. +func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + var sa syscall.Sockaddr + n, oobn, flags, sa, err = c.fd.readMsg(b, oob) + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + addr = &IPAddr{IP: sa.Addr[0:]} + case *syscall.SockaddrInet6: + addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} + } + return +} + +// WriteToIP writes an IP packet to addr via c, copying the payload +// from b. +// +// WriteToIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + if c.fd.isConnected { + return 0, &OpError{Op: "write", Net: c.fd.net, Addr: addr, Err: ErrWriteToConnected} + } + if addr == nil { + return 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress} + } + sa, err := addr.sockaddr(c.fd.family) + if err != nil { + return 0, &OpError{"write", c.fd.net, addr, err} + } + return c.fd.writeTo(b, sa) +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*IPAddr) + if !ok { + return 0, &OpError{"write", c.fd.net, addr, syscall.EINVAL} + } + return c.WriteToIP(b, a) +} + +// WriteMsgIP writes a packet to addr via c, copying the payload from +// b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. +func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + if c.fd.isConnected { + return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: addr, Err: ErrWriteToConnected} + } + if addr == nil { + return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress} + } + sa, err := addr.sockaddr(c.fd.family) + if err != nil { + return 0, 0, &OpError{"write", c.fd.net, addr, err} + } + return c.fd.writeMsg(b, oob, sa) +} + +// DialIP connects to the remote address raddr on the network protocol +// netProto, which must be "ip", "ip4", or "ip6" followed by a colon +// and a protocol number or name. +func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + return dialIP(netProto, laddr, raddr, noDeadline) +} + +func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) { + net, proto, err := parseNetwork(netProto) + if err != nil { + return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: err} + } + switch net { + case "ip", "ip4", "ip6": + default: + return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: UnknownNetworkError(netProto)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: netProto, Addr: nil, Err: errMissingAddress} + } + fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_RAW, proto, "dial") + if err != nil { + return nil, &OpError{Op: "dial", Net: netProto, Addr: raddr, Err: err} + } + return newIPConn(fd), nil +} + +// ListenIP listens for incoming IP packets addressed to the local +// address laddr. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send IP packets with per-packet +// addressing. +func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { + net, proto, err := parseNetwork(netProto) + if err != nil { + return nil, &OpError{Op: "dial", Net: netProto, Addr: laddr, Err: err} + } + switch net { + case "ip", "ip4", "ip6": + default: + return nil, &OpError{Op: "listen", Net: netProto, Addr: laddr, Err: UnknownNetworkError(netProto)} + } + fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_RAW, proto, "listen") + if err != nil { + return nil, &OpError{Op: "listen", Net: netProto, Addr: laddr, Err: err} + } + return newIPConn(fd), nil +} diff --git a/src/net/ipsock.go b/src/net/ipsock.go new file mode 100644 index 000000000..dda857803 --- /dev/null +++ b/src/net/ipsock.go @@ -0,0 +1,318 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Internet protocol family sockets + +package net + +import ( + "errors" + "time" +) + +var ( + // supportsIPv4 reports whether the platform supports IPv4 + // networking functionality. + supportsIPv4 bool + + // supportsIPv6 reports whether the platform supports IPv6 + // networking functionality. + supportsIPv6 bool + + // supportsIPv4map reports whether the platform supports + // mapping an IPv4 address inside an IPv6 address at transport + // layer protocols. See RFC 4291, RFC 4038 and RFC 3493. + supportsIPv4map bool +) + +func init() { + sysInit() + supportsIPv4 = probeIPv4Stack() + supportsIPv6, supportsIPv4map = probeIPv6Stack() +} + +// A netaddr represents a network endpoint address or a list of +// network endpoint addresses. +type netaddr interface { + // toAddr returns the address represented in Addr interface. + // It returns a nil interface when the address is nil. + toAddr() Addr +} + +// An addrList represents a list of network endpoint addresses. +type addrList []netaddr + +func (al addrList) toAddr() Addr { + switch len(al) { + case 0: + return nil + case 1: + return al[0].toAddr() + default: + // For now, we'll roughly pick first one without + // considering dealing with any preferences such as + // DNS TTL, transport path quality, network routing + // information. + return al[0].toAddr() + } +} + +var errNoSuitableAddress = errors.New("no suitable address found") + +// firstFavoriteAddr returns an address or a list of addresses that +// implement the netaddr interface. Known filters are nil, ipv4only +// and ipv6only. It returns any address when filter is nil. The result +// contains at least one address when error is nil. +func firstFavoriteAddr(filter func(IP) IP, ips []IP, inetaddr func(IP) netaddr) (netaddr, error) { + if filter != nil { + return firstSupportedAddr(filter, ips, inetaddr) + } + var ( + ipv4, ipv6, swap bool + list addrList + ) + for _, ip := range ips { + // We'll take any IP address, but since the dialing + // code does not yet try multiple addresses + // effectively, prefer to use an IPv4 address if + // possible. This is especially relevant if localhost + // resolves to [ipv6-localhost, ipv4-localhost]. Too + // much code assumes localhost == ipv4-localhost. + if ip4 := ipv4only(ip); ip4 != nil && !ipv4 { + list = append(list, inetaddr(ip4)) + ipv4 = true + if ipv6 { + swap = true + } + } else if ip6 := ipv6only(ip); ip6 != nil && !ipv6 { + list = append(list, inetaddr(ip6)) + ipv6 = true + } + if ipv4 && ipv6 { + if swap { + list[0], list[1] = list[1], list[0] + } + break + } + } + switch len(list) { + case 0: + return nil, errNoSuitableAddress + case 1: + return list[0], nil + default: + return list, nil + } +} + +func firstSupportedAddr(filter func(IP) IP, ips []IP, inetaddr func(IP) netaddr) (netaddr, error) { + for _, ip := range ips { + if ip := filter(ip); ip != nil { + return inetaddr(ip), nil + } + } + return nil, errNoSuitableAddress +} + +// ipv4only returns IPv4 addresses that we can use with the kernel's +// IPv4 addressing modes. If ip is an IPv4 address, ipv4only returns ip. +// Otherwise it returns nil. +func ipv4only(ip IP) IP { + if supportsIPv4 && ip.To4() != nil { + return ip + } + return nil +} + +// ipv6only returns IPv6 addresses that we can use with the kernel's +// IPv6 addressing modes. It returns IPv4-mapped IPv6 addresses as +// nils and returns other IPv6 address types as IPv6 addresses. +func ipv6only(ip IP) IP { + if supportsIPv6 && len(ip) == IPv6len && ip.To4() == nil { + return ip + } + return nil +} + +// SplitHostPort splits a network address of the form "host:port", +// "[host]:port" or "[ipv6-host%zone]:port" into host or +// ipv6-host%zone and port. A literal address or host name for IPv6 +// must be enclosed in square brackets, as in "[::1]:80", +// "[ipv6-host]:http" or "[ipv6-host%zone]:80". +func SplitHostPort(hostport string) (host, port string, err error) { + j, k := 0, 0 + + // The port starts after the last colon. + i := last(hostport, ':') + if i < 0 { + goto missingPort + } + + if hostport[0] == '[' { + // Expect the first ']' just before the last ':'. + end := byteIndex(hostport, ']') + if end < 0 { + err = &AddrError{"missing ']' in address", hostport} + return + } + switch end + 1 { + case len(hostport): + // There can't be a ':' behind the ']' now. + goto missingPort + case i: + // The expected result. + default: + // Either ']' isn't followed by a colon, or it is + // followed by a colon that is not the last one. + if hostport[end+1] == ':' { + goto tooManyColons + } + goto missingPort + } + host = hostport[1:end] + j, k = 1, end+1 // there can't be a '[' resp. ']' before these positions + } else { + host = hostport[:i] + if byteIndex(host, ':') >= 0 { + goto tooManyColons + } + if byteIndex(host, '%') >= 0 { + goto missingBrackets + } + } + if byteIndex(hostport[j:], '[') >= 0 { + err = &AddrError{"unexpected '[' in address", hostport} + return + } + if byteIndex(hostport[k:], ']') >= 0 { + err = &AddrError{"unexpected ']' in address", hostport} + return + } + + port = hostport[i+1:] + return + +missingPort: + err = &AddrError{"missing port in address", hostport} + return + +tooManyColons: + err = &AddrError{"too many colons in address", hostport} + return + +missingBrackets: + err = &AddrError{"missing brackets in address", hostport} + return +} + +func splitHostZone(s string) (host, zone string) { + // The IPv6 scoped addressing zone identifier starts after the + // last percent sign. + if i := last(s, '%'); i > 0 { + host, zone = s[:i], s[i+1:] + } else { + host = s + } + return +} + +// JoinHostPort combines host and port into a network address of the +// form "host:port" or, if host contains a colon or a percent sign, +// "[host]:port". +func JoinHostPort(host, port string) string { + // If host has colons or a percent sign, have to bracket it. + if byteIndex(host, ':') >= 0 || byteIndex(host, '%') >= 0 { + return "[" + host + "]:" + port + } + return host + ":" + port +} + +// resolveInternetAddr resolves addr that is either a literal IP +// address or a DNS name and returns an internet protocol family +// address. It returns a list that contains a pair of different +// address family addresses when addr is a DNS name and the name has +// multiple address family records. The result contains at least one +// address when error is nil. +func resolveInternetAddr(net, addr string, deadline time.Time) (netaddr, error) { + var ( + err error + host, port, zone string + portnum int + ) + switch net { + case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": + if addr != "" { + if host, port, err = SplitHostPort(addr); err != nil { + return nil, err + } + if portnum, err = parsePort(net, port); err != nil { + return nil, err + } + } + case "ip", "ip4", "ip6": + if addr != "" { + host = addr + } + default: + return nil, UnknownNetworkError(net) + } + inetaddr := func(ip IP) netaddr { + switch net { + case "tcp", "tcp4", "tcp6": + return &TCPAddr{IP: ip, Port: portnum, Zone: zone} + case "udp", "udp4", "udp6": + return &UDPAddr{IP: ip, Port: portnum, Zone: zone} + case "ip", "ip4", "ip6": + return &IPAddr{IP: ip, Zone: zone} + default: + panic("unexpected network: " + net) + } + } + if host == "" { + return inetaddr(nil), nil + } + // Try as a literal IP address. + var ip IP + if ip = parseIPv4(host); ip != nil { + return inetaddr(ip), nil + } + if ip, zone = parseIPv6(host, true); ip != nil { + return inetaddr(ip), nil + } + // Try as a DNS name. + host, zone = splitHostZone(host) + ips, err := lookupIPDeadline(host, deadline) + if err != nil { + return nil, err + } + var filter func(IP) IP + if net != "" && net[len(net)-1] == '4' { + filter = ipv4only + } + if net != "" && net[len(net)-1] == '6' || zone != "" { + filter = ipv6only + } + return firstFavoriteAddr(filter, ips, inetaddr) +} + +func zoneToString(zone int) string { + if zone == 0 { + return "" + } + if ifi, err := InterfaceByIndex(zone); err == nil { + return ifi.Name + } + return itod(uint(zone)) +} + +func zoneToInt(zone string) int { + if zone == "" { + return 0 + } + if ifi, err := InterfaceByName(zone); err == nil { + return ifi.Index + } + n, _, _ := dtoi(zone, 0) + return n +} diff --git a/src/net/ipsock_plan9.go b/src/net/ipsock_plan9.go new file mode 100644 index 000000000..94ceea31b --- /dev/null +++ b/src/net/ipsock_plan9.go @@ -0,0 +1,228 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Internet protocol family sockets for Plan 9 + +package net + +import ( + "errors" + "os" + "syscall" +) + +func probe(filename, query string) bool { + var file *file + var err error + if file, err = open(filename); err != nil { + return false + } + + r := false + for line, ok := file.readLine(); ok && !r; line, ok = file.readLine() { + f := getFields(line) + if len(f) < 3 { + continue + } + for i := 0; i < len(f); i++ { + if query == f[i] { + r = true + break + } + } + } + file.close() + return r +} + +func probeIPv4Stack() bool { + return probe(netdir+"/iproute", "4i") +} + +// probeIPv6Stack returns two boolean values. If the first boolean +// value is true, kernel supports basic IPv6 functionality. If the +// second boolean value is true, kernel supports IPv6 IPv4-mapping. +func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { + // Plan 9 uses IPv6 natively, see ip(3). + r := probe(netdir+"/iproute", "6i") + v := false + if r { + v = probe(netdir+"/iproute", "4i") + } + return r, v +} + +// parsePlan9Addr parses address of the form [ip!]port (e.g. 127.0.0.1!80). +func parsePlan9Addr(s string) (ip IP, iport int, err error) { + addr := IPv4zero // address contains port only + i := byteIndex(s, '!') + if i >= 0 { + addr = ParseIP(s[:i]) + if addr == nil { + return nil, 0, errors.New("parsing IP failed") + } + } + p, _, ok := dtoi(s[i+1:], 0) + if !ok { + return nil, 0, errors.New("parsing port failed") + } + if p < 0 || p > 0xFFFF { + return nil, 0, &AddrError{"invalid port", string(p)} + } + return addr, p, nil +} + +func readPlan9Addr(proto, filename string) (addr Addr, err error) { + var buf [128]byte + + f, err := os.Open(filename) + if err != nil { + return + } + defer f.Close() + n, err := f.Read(buf[:]) + if err != nil { + return + } + ip, port, err := parsePlan9Addr(string(buf[:n])) + if err != nil { + return + } + switch proto { + case "tcp": + addr = &TCPAddr{IP: ip, Port: port} + case "udp": + addr = &UDPAddr{IP: ip, Port: port} + default: + return nil, errors.New("unknown protocol " + proto) + } + return addr, nil +} + +func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, err error) { + var ( + ip IP + port int + ) + switch a := addr.(type) { + case *TCPAddr: + proto = "tcp" + ip = a.IP + port = a.Port + case *UDPAddr: + proto = "udp" + ip = a.IP + port = a.Port + default: + err = UnknownNetworkError(net) + return + } + + clone, dest, err := queryCS1(proto, ip, port) + if err != nil { + return + } + f, err := os.OpenFile(clone, os.O_RDWR, 0) + if err != nil { + return + } + var buf [16]byte + n, err := f.Read(buf[:]) + if err != nil { + f.Close() + return + } + return f, dest, proto, string(buf[:n]), nil +} + +func netErr(e error) { + oe, ok := e.(*OpError) + if !ok { + return + } + if pe, ok := oe.Err.(*os.PathError); ok { + if _, ok = pe.Err.(syscall.ErrorString); ok { + oe.Err = pe.Err + } + } +} + +func dialPlan9(net string, laddr, raddr Addr) (fd *netFD, err error) { + defer func() { netErr(err) }() + f, dest, proto, name, err := startPlan9(net, raddr) + if err != nil { + return nil, &OpError{"dial", net, raddr, err} + } + _, err = f.WriteString("connect " + dest) + if err != nil { + f.Close() + return nil, &OpError{"dial", f.Name(), raddr, err} + } + data, err := os.OpenFile(netdir+"/"+proto+"/"+name+"/data", os.O_RDWR, 0) + if err != nil { + f.Close() + return nil, &OpError{"dial", net, raddr, err} + } + laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") + if err != nil { + data.Close() + f.Close() + return nil, &OpError{"dial", proto, raddr, err} + } + return newFD(proto, name, f, data, laddr, raddr) +} + +func listenPlan9(net string, laddr Addr) (fd *netFD, err error) { + defer func() { netErr(err) }() + f, dest, proto, name, err := startPlan9(net, laddr) + if err != nil { + return nil, &OpError{"listen", net, laddr, err} + } + _, err = f.WriteString("announce " + dest) + if err != nil { + f.Close() + return nil, &OpError{"announce", proto, laddr, err} + } + laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") + if err != nil { + f.Close() + return nil, &OpError{Op: "listen", Net: net, Err: err} + } + return newFD(proto, name, f, nil, laddr, nil) +} + +func (l *netFD) netFD() (*netFD, error) { + return newFD(l.proto, l.n, l.ctl, l.data, l.laddr, l.raddr) +} + +func (l *netFD) acceptPlan9() (fd *netFD, err error) { + defer func() { netErr(err) }() + if err := l.readLock(); err != nil { + return nil, err + } + defer l.readUnlock() + f, err := os.Open(l.dir + "/listen") + if err != nil { + return nil, &OpError{"accept", l.dir + "/listen", l.laddr, err} + } + var buf [16]byte + n, err := f.Read(buf[:]) + if err != nil { + f.Close() + return nil, &OpError{"accept", l.dir + "/listen", l.laddr, err} + } + name := string(buf[:n]) + data, err := os.OpenFile(netdir+"/"+l.proto+"/"+name+"/data", os.O_RDWR, 0) + if err != nil { + f.Close() + return nil, &OpError{"accept", l.proto, l.laddr, err} + } + raddr, err := readPlan9Addr(l.proto, netdir+"/"+l.proto+"/"+name+"/remote") + if err != nil { + data.Close() + f.Close() + return nil, &OpError{"accept", l.proto, l.laddr, err} + } + return newFD(l.proto, name, f, data, l.laddr, raddr) +} diff --git a/src/net/ipsock_posix.go b/src/net/ipsock_posix.go new file mode 100644 index 000000000..f9ebe40a2 --- /dev/null +++ b/src/net/ipsock_posix.go @@ -0,0 +1,177 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +// Internet protocol family sockets for POSIX + +package net + +import ( + "syscall" + "time" +) + +func probeIPv4Stack() bool { + s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + switch err { + case syscall.EAFNOSUPPORT, syscall.EPROTONOSUPPORT: + return false + case nil: + closesocket(s) + } + return true +} + +// Should we try to use the IPv4 socket interface if we're +// only dealing with IPv4 sockets? As long as the host system +// understands IPv6, it's okay to pass IPv4 addresses to the IPv6 +// interface. That simplifies our code and is most general. +// Unfortunately, we need to run on kernels built without IPv6 +// support too. So probe the kernel to figure it out. +// +// probeIPv6Stack probes both basic IPv6 capability and IPv6 IPv4- +// mapping capability which is controlled by IPV6_V6ONLY socket +// option and/or kernel state "net.inet6.ip6.v6only". +// It returns two boolean values. If the first boolean value is +// true, kernel supports basic IPv6 functionality. If the second +// boolean value is true, kernel supports IPv6 IPv4-mapping. +func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { + var probes = []struct { + laddr TCPAddr + value int + ok bool + }{ + // IPv6 communication capability + {laddr: TCPAddr{IP: ParseIP("::1")}, value: 1}, + // IPv6 IPv4-mapped address communication capability + {laddr: TCPAddr{IP: IPv4(127, 0, 0, 1)}, value: 0}, + } + + for i := range probes { + s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) + if err != nil { + continue + } + defer closesocket(s) + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, probes[i].value) + sa, err := probes[i].laddr.sockaddr(syscall.AF_INET6) + if err != nil { + continue + } + if err := syscall.Bind(s, sa); err != nil { + continue + } + probes[i].ok = true + } + + return probes[0].ok, probes[1].ok +} + +// favoriteAddrFamily returns the appropriate address family to +// the given net, laddr, raddr and mode. At first it figures +// address family out from the net. If mode indicates "listen" +// and laddr is a wildcard, it assumes that the user wants to +// make a passive connection with a wildcard address family, both +// AF_INET and AF_INET6, and a wildcard address like following: +// +// 1. A wild-wild listen, "tcp" + "" +// If the platform supports both IPv6 and IPv6 IPv4-mapping +// capabilities, we assume that the user want to listen on +// both IPv4 and IPv6 wildcard address over an AF_INET6 +// socket with IPV6_V6ONLY=0. Otherwise we prefer an IPv4 +// wildcard address listen over an AF_INET socket. +// +// 2. A wild-ipv4wild listen, "tcp" + "0.0.0.0" +// Same as 1. +// +// 3. A wild-ipv6wild listen, "tcp" + "[::]" +// Almost same as 1 but we prefer an IPv6 wildcard address +// listen over an AF_INET6 socket with IPV6_V6ONLY=0 when +// the platform supports IPv6 capability but not IPv6 IPv4- +// mapping capability. +// +// 4. A ipv4-ipv4wild listen, "tcp4" + "" or "0.0.0.0" +// We use an IPv4 (AF_INET) wildcard address listen. +// +// 5. A ipv6-ipv6wild listen, "tcp6" + "" or "[::]" +// We use an IPv6 (AF_INET6, IPV6_V6ONLY=1) wildcard address +// listen. +// +// Otherwise guess: if the addresses are IPv4 then returns AF_INET, +// or else returns AF_INET6. It also returns a boolean value what +// designates IPV6_V6ONLY option. +// +// Note that OpenBSD allows neither "net.inet6.ip6.v6only=1" change +// nor IPPROTO_IPV6 level IPV6_V6ONLY socket option setting. +func favoriteAddrFamily(net string, laddr, raddr sockaddr, mode string) (family int, ipv6only bool) { + switch net[len(net)-1] { + case '4': + return syscall.AF_INET, false + case '6': + return syscall.AF_INET6, true + } + + if mode == "listen" && (laddr == nil || laddr.isWildcard()) { + if supportsIPv4map { + return syscall.AF_INET6, false + } + if laddr == nil { + return syscall.AF_INET, false + } + return laddr.family(), false + } + + if (laddr == nil || laddr.family() == syscall.AF_INET) && + (raddr == nil || raddr.family() == syscall.AF_INET) { + return syscall.AF_INET, false + } + return syscall.AF_INET6, false +} + +// Internet sockets (TCP, UDP, IP) + +func internetSocket(net string, laddr, raddr sockaddr, deadline time.Time, sotype, proto int, mode string) (fd *netFD, err error) { + family, ipv6only := favoriteAddrFamily(net, laddr, raddr, mode) + return socket(net, family, sotype, proto, ipv6only, laddr, raddr, deadline) +} + +func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, error) { + switch family { + case syscall.AF_INET: + if len(ip) == 0 { + ip = IPv4zero + } + if ip = ip.To4(); ip == nil { + return nil, InvalidAddrError("non-IPv4 address") + } + sa := new(syscall.SockaddrInet4) + for i := 0; i < IPv4len; i++ { + sa.Addr[i] = ip[i] + } + sa.Port = port + return sa, nil + case syscall.AF_INET6: + if len(ip) == 0 { + ip = IPv6zero + } + // IPv4 callers use 0.0.0.0 to mean "announce on any available address". + // In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0", + // which it refuses to do. Rewrite to the IPv6 unspecified address. + if ip.Equal(IPv4zero) { + ip = IPv6zero + } + if ip = ip.To16(); ip == nil { + return nil, InvalidAddrError("non-IPv6 address") + } + sa := new(syscall.SockaddrInet6) + for i := 0; i < IPv6len; i++ { + sa.Addr[i] = ip[i] + } + sa.Port = port + sa.ZoneId = uint32(zoneToInt(zone)) + return sa, nil + } + return nil, InvalidAddrError("unexpected socket family") +} diff --git a/src/net/ipsock_test.go b/src/net/ipsock_test.go new file mode 100644 index 000000000..9ecaaec69 --- /dev/null +++ b/src/net/ipsock_test.go @@ -0,0 +1,193 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "reflect" + "testing" +) + +var testInetaddr = func(ip IP) netaddr { return &TCPAddr{IP: ip, Port: 5682} } + +var firstFavoriteAddrTests = []struct { + filter func(IP) IP + ips []IP + inetaddr func(IP) netaddr + addr netaddr + err error +}{ + { + nil, + []IP{ + IPv4(127, 0, 0, 1), + IPv6loopback, + }, + testInetaddr, + addrList{ + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + }, + nil, + }, + { + nil, + []IP{ + IPv6loopback, + IPv4(127, 0, 0, 1), + }, + testInetaddr, + addrList{ + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + }, + nil, + }, + { + nil, + []IP{ + IPv4(127, 0, 0, 1), + IPv4(192, 168, 0, 1), + }, + testInetaddr, + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + nil, + }, + { + nil, + []IP{ + IPv6loopback, + ParseIP("fe80::1"), + }, + testInetaddr, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + nil, + }, + { + nil, + []IP{ + IPv4(127, 0, 0, 1), + IPv4(192, 168, 0, 1), + IPv6loopback, + ParseIP("fe80::1"), + }, + testInetaddr, + addrList{ + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + }, + nil, + }, + { + nil, + []IP{ + IPv6loopback, + ParseIP("fe80::1"), + IPv4(127, 0, 0, 1), + IPv4(192, 168, 0, 1), + }, + testInetaddr, + addrList{ + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + }, + nil, + }, + { + nil, + []IP{ + IPv4(127, 0, 0, 1), + IPv6loopback, + IPv4(192, 168, 0, 1), + ParseIP("fe80::1"), + }, + testInetaddr, + addrList{ + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + }, + nil, + }, + { + nil, + []IP{ + IPv6loopback, + IPv4(127, 0, 0, 1), + ParseIP("fe80::1"), + IPv4(192, 168, 0, 1), + }, + testInetaddr, + addrList{ + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + }, + nil, + }, + + { + ipv4only, + []IP{ + IPv4(127, 0, 0, 1), + IPv6loopback, + }, + testInetaddr, + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + nil, + }, + { + ipv4only, + []IP{ + IPv6loopback, + IPv4(127, 0, 0, 1), + }, + testInetaddr, + &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5682}, + nil, + }, + + { + ipv6only, + []IP{ + IPv4(127, 0, 0, 1), + IPv6loopback, + }, + testInetaddr, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + nil, + }, + { + ipv6only, + []IP{ + IPv6loopback, + IPv4(127, 0, 0, 1), + }, + testInetaddr, + &TCPAddr{IP: IPv6loopback, Port: 5682}, + nil, + }, + + {nil, nil, testInetaddr, nil, errNoSuitableAddress}, + + {ipv4only, nil, testInetaddr, nil, errNoSuitableAddress}, + {ipv4only, []IP{IPv6loopback}, testInetaddr, nil, errNoSuitableAddress}, + + {ipv6only, nil, testInetaddr, nil, errNoSuitableAddress}, + {ipv6only, []IP{IPv4(127, 0, 0, 1)}, testInetaddr, nil, errNoSuitableAddress}, +} + +func TestFirstFavoriteAddr(t *testing.T) { + if !supportsIPv4 || !supportsIPv6 { + t.Skip("ipv4 or ipv6 is not supported") + } + + for i, tt := range firstFavoriteAddrTests { + addr, err := firstFavoriteAddr(tt.filter, tt.ips, tt.inetaddr) + if err != tt.err { + t.Errorf("#%v: got %v; expected %v", i, err, tt.err) + } + if !reflect.DeepEqual(addr, tt.addr) { + t.Errorf("#%v: got %v; expected %v", i, addr, tt.addr) + } + } +} diff --git a/src/net/lookup.go b/src/net/lookup.go new file mode 100644 index 000000000..20f20578c --- /dev/null +++ b/src/net/lookup.go @@ -0,0 +1,137 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "time" + +// protocols contains minimal mappings between internet protocol +// names and numbers for platforms that don't have a complete list of +// protocol numbers. +// +// See http://www.iana.org/assignments/protocol-numbers +var protocols = map[string]int{ + "icmp": 1, "ICMP": 1, + "igmp": 2, "IGMP": 2, + "tcp": 6, "TCP": 6, + "udp": 17, "UDP": 17, + "ipv6-icmp": 58, "IPV6-ICMP": 58, "IPv6-ICMP": 58, +} + +// LookupHost looks up the given host using the local resolver. +// It returns an array of that host's addresses. +func LookupHost(host string) (addrs []string, err error) { + return lookupHost(host) +} + +// LookupIP looks up host using the local resolver. +// It returns an array of that host's IPv4 and IPv6 addresses. +func LookupIP(host string) (addrs []IP, err error) { + return lookupIPMerge(host) +} + +var lookupGroup singleflight + +// lookupIPMerge wraps lookupIP, but makes sure that for any given +// host, only one lookup is in-flight at a time. The returned memory +// is always owned by the caller. +func lookupIPMerge(host string) (addrs []IP, err error) { + addrsi, err, shared := lookupGroup.Do(host, func() (interface{}, error) { + return lookupIP(host) + }) + if err != nil { + return nil, err + } + addrs = addrsi.([]IP) + if shared { + clone := make([]IP, len(addrs)) + copy(clone, addrs) + addrs = clone + } + return addrs, nil +} + +func lookupIPDeadline(host string, deadline time.Time) (addrs []IP, err error) { + if deadline.IsZero() { + return lookupIPMerge(host) + } + + // TODO(bradfitz): consider pushing the deadline down into the + // name resolution functions. But that involves fixing it for + // the native Go resolver, cgo, Windows, etc. + // + // In the meantime, just use a goroutine. Most users affected + // by http://golang.org/issue/2631 are due to TCP connections + // to unresponsive hosts, not DNS. + timeout := deadline.Sub(time.Now()) + if timeout <= 0 { + err = errTimeout + return + } + t := time.NewTimer(timeout) + defer t.Stop() + type res struct { + addrs []IP + err error + } + resc := make(chan res, 1) + go func() { + a, err := lookupIPMerge(host) + resc <- res{a, err} + }() + select { + case <-t.C: + err = errTimeout + case r := <-resc: + addrs, err = r.addrs, r.err + } + return +} + +// LookupPort looks up the port for the given network and service. +func LookupPort(network, service string) (port int, err error) { + return lookupPort(network, service) +} + +// LookupCNAME returns the canonical DNS host for the given name. +// Callers that do not care about the canonical name can call +// LookupHost or LookupIP directly; both take care of resolving +// the canonical name as part of the lookup. +func LookupCNAME(name string) (cname string, err error) { + return lookupCNAME(name) +} + +// LookupSRV tries to resolve an SRV query of the given service, +// protocol, and domain name. The proto is "tcp" or "udp". +// The returned records are sorted by priority and randomized +// by weight within a priority. +// +// LookupSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _service._proto.name. To accommodate services +// publishing SRV records under non-standard names, if both service +// and proto are empty strings, LookupSRV looks up name directly. +func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { + return lookupSRV(service, proto, name) +} + +// LookupMX returns the DNS MX records for the given domain name sorted by preference. +func LookupMX(name string) (mx []*MX, err error) { + return lookupMX(name) +} + +// LookupNS returns the DNS NS records for the given domain name. +func LookupNS(name string) (ns []*NS, err error) { + return lookupNS(name) +} + +// LookupTXT returns the DNS TXT records for the given domain name. +func LookupTXT(name string) (txt []string, err error) { + return lookupTXT(name) +} + +// LookupAddr performs a reverse lookup for the given address, returning a list +// of names mapping to that address. +func LookupAddr(addr string) (name []string, err error) { + return lookupAddr(addr) +} diff --git a/src/net/lookup_plan9.go b/src/net/lookup_plan9.go new file mode 100644 index 000000000..b80ac10e0 --- /dev/null +++ b/src/net/lookup_plan9.go @@ -0,0 +1,297 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "os" +) + +func query(filename, query string, bufSize int) (res []string, err error) { + file, err := os.OpenFile(filename, os.O_RDWR, 0) + if err != nil { + return + } + defer file.Close() + + _, err = file.Seek(0, 0) + if err != nil { + return + } + _, err = file.WriteString(query) + if err != nil { + return + } + _, err = file.Seek(0, 0) + if err != nil { + return + } + buf := make([]byte, bufSize) + for { + n, _ := file.Read(buf) + if n <= 0 { + break + } + res = append(res, string(buf[:n])) + } + return +} + +func queryCS(net, host, service string) (res []string, err error) { + switch net { + case "tcp4", "tcp6": + net = "tcp" + case "udp4", "udp6": + net = "udp" + } + if host == "" { + host = "*" + } + return query(netdir+"/cs", net+"!"+host+"!"+service, 128) +} + +func queryCS1(net string, ip IP, port int) (clone, dest string, err error) { + ips := "*" + if len(ip) != 0 && !ip.IsUnspecified() { + ips = ip.String() + } + lines, err := queryCS(net, ips, itoa(port)) + if err != nil { + return + } + f := getFields(lines[0]) + if len(f) < 2 { + return "", "", errors.New("bad response from ndb/cs") + } + clone, dest = f[0], f[1] + return +} + +func queryDNS(addr string, typ string) (res []string, err error) { + return query(netdir+"/dns", addr+" "+typ, 1024) +} + +// toLower returns a lower-case version of in. Restricting us to +// ASCII is sufficient to handle the IP protocol names and allow +// us to not depend on the strings and unicode packages. +func toLower(in string) string { + for _, c := range in { + if 'A' <= c && c <= 'Z' { + // Has upper case; need to fix. + out := []byte(in) + for i := 0; i < len(in); i++ { + c := in[i] + if 'A' <= c && c <= 'Z' { + c += 'a' - 'A' + } + out[i] = c + } + return string(out) + } + } + return in +} + +// lookupProtocol looks up IP protocol name and returns +// the corresponding protocol number. +func lookupProtocol(name string) (proto int, err error) { + lines, err := query(netdir+"/cs", "!protocol="+toLower(name), 128) + if err != nil { + return 0, err + } + unknownProtoError := errors.New("unknown IP protocol specified: " + name) + if len(lines) == 0 { + return 0, unknownProtoError + } + f := getFields(lines[0]) + if len(f) < 2 { + return 0, unknownProtoError + } + s := f[1] + if n, _, ok := dtoi(s, byteIndex(s, '=')+1); ok { + return n, nil + } + return 0, unknownProtoError +} + +func lookupHost(host string) (addrs []string, err error) { + // Use netdir/cs instead of netdir/dns because cs knows about + // host names in local network (e.g. from /lib/ndb/local) + lines, err := queryCS("net", host, "1") + if err != nil { + return + } +loop: + for _, line := range lines { + f := getFields(line) + if len(f) < 2 { + continue + } + addr := f[1] + if i := byteIndex(addr, '!'); i >= 0 { + addr = addr[:i] // remove port + } + if ParseIP(addr) == nil { + continue + } + // only return unique addresses + for _, a := range addrs { + if a == addr { + continue loop + } + } + addrs = append(addrs, addr) + } + return +} + +func lookupIP(host string) (ips []IP, err error) { + addrs, err := LookupHost(host) + if err != nil { + return + } + for _, addr := range addrs { + if ip := ParseIP(addr); ip != nil { + ips = append(ips, ip) + } + } + return +} + +func lookupPort(network, service string) (port int, err error) { + switch network { + case "tcp4", "tcp6": + network = "tcp" + case "udp4", "udp6": + network = "udp" + } + lines, err := queryCS(network, "127.0.0.1", service) + if err != nil { + return + } + unknownPortError := &AddrError{"unknown port", network + "/" + service} + if len(lines) == 0 { + return 0, unknownPortError + } + f := getFields(lines[0]) + if len(f) < 2 { + return 0, unknownPortError + } + s := f[1] + if i := byteIndex(s, '!'); i >= 0 { + s = s[i+1:] // remove address + } + if n, _, ok := dtoi(s, 0); ok { + return n, nil + } + return 0, unknownPortError +} + +func lookupCNAME(name string) (cname string, err error) { + lines, err := queryDNS(name, "cname") + if err != nil { + return + } + if len(lines) > 0 { + if f := getFields(lines[0]); len(f) >= 3 { + return f[2] + ".", nil + } + } + return "", errors.New("bad response from ndb/dns") +} + +func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { + var target string + if service == "" && proto == "" { + target = name + } else { + target = "_" + service + "._" + proto + "." + name + } + lines, err := queryDNS(target, "srv") + if err != nil { + return + } + for _, line := range lines { + f := getFields(line) + if len(f) < 6 { + continue + } + port, _, portOk := dtoi(f[4], 0) + priority, _, priorityOk := dtoi(f[3], 0) + weight, _, weightOk := dtoi(f[2], 0) + if !(portOk && priorityOk && weightOk) { + continue + } + addrs = append(addrs, &SRV{f[5], uint16(port), uint16(priority), uint16(weight)}) + cname = f[0] + } + byPriorityWeight(addrs).sort() + return +} + +func lookupMX(name string) (mx []*MX, err error) { + lines, err := queryDNS(name, "mx") + if err != nil { + return + } + for _, line := range lines { + f := getFields(line) + if len(f) < 4 { + continue + } + if pref, _, ok := dtoi(f[2], 0); ok { + mx = append(mx, &MX{f[3], uint16(pref)}) + } + } + byPref(mx).sort() + return +} + +func lookupNS(name string) (ns []*NS, err error) { + lines, err := queryDNS(name, "ns") + if err != nil { + return + } + for _, line := range lines { + f := getFields(line) + if len(f) < 3 { + continue + } + ns = append(ns, &NS{f[2]}) + } + return +} + +func lookupTXT(name string) (txt []string, err error) { + lines, err := queryDNS(name, "txt") + if err != nil { + return + } + for _, line := range lines { + if i := byteIndex(line, '\t'); i >= 0 { + txt = append(txt, line[i+1:]) + } + } + return +} + +func lookupAddr(addr string) (name []string, err error) { + arpa, err := reverseaddr(addr) + if err != nil { + return + } + lines, err := queryDNS(arpa, "ptr") + if err != nil { + return + } + for _, line := range lines { + f := getFields(line) + if len(f) < 3 { + continue + } + name = append(name, f[2]) + } + return +} diff --git a/src/net/lookup_test.go b/src/net/lookup_test.go new file mode 100644 index 000000000..057e1322b --- /dev/null +++ b/src/net/lookup_test.go @@ -0,0 +1,231 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TODO It would be nice to use a mock DNS server, to eliminate +// external dependencies. + +package net + +import ( + "flag" + "strings" + "testing" +) + +var testExternal = flag.Bool("external", true, "allow use of external networks during long test") + +var lookupGoogleSRVTests = []struct { + service, proto, name string + cname, target string +}{ + { + "xmpp-server", "tcp", "google.com", + ".google.com", ".google.com", + }, + { + "", "", "_xmpp-server._tcp.google.com", // non-standard back door + ".google.com", ".google.com", + }, +} + +func TestLookupGoogleSRV(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + for _, tt := range lookupGoogleSRVTests { + cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name) + if err != nil { + t.Fatal(err) + } + if len(srvs) == 0 { + t.Error("got no record") + } + if !strings.Contains(cname, tt.cname) { + t.Errorf("got %q; want %q", cname, tt.cname) + } + for _, srv := range srvs { + if !strings.Contains(srv.Target, tt.target) { + t.Errorf("got %v; want a record containing %q", srv, tt.target) + } + } + } +} + +func TestLookupGmailMX(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + mxs, err := LookupMX("gmail.com") + if err != nil { + t.Fatal(err) + } + if len(mxs) == 0 { + t.Error("got no record") + } + for _, mx := range mxs { + if !strings.Contains(mx.Host, ".google.com") { + t.Errorf("got %v; want a record containing .google.com.", mx) + } + } +} + +func TestLookupGmailNS(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + nss, err := LookupNS("gmail.com") + if err != nil { + t.Fatal(err) + } + if len(nss) == 0 { + t.Error("got no record") + } + for _, ns := range nss { + if !strings.Contains(ns.Host, ".google.com") { + t.Errorf("got %v; want a record containing .google.com.", ns) + } + } +} + +func TestLookupGmailTXT(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + txts, err := LookupTXT("gmail.com") + if err != nil { + t.Fatal(err) + } + if len(txts) == 0 { + t.Error("got no record") + } + for _, txt := range txts { + if !strings.Contains(txt, "spf") { + t.Errorf("got %q; want a spf record", txt) + } + } +} + +var lookupGooglePublicDNSAddrs = []struct { + addr string + name string +}{ + {"8.8.8.8", ".google.com."}, + {"8.8.4.4", ".google.com."}, + {"2001:4860:4860::8888", ".google.com."}, + {"2001:4860:4860::8844", ".google.com."}, +} + +func TestLookupGooglePublicDNSAddr(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + for _, tt := range lookupGooglePublicDNSAddrs { + names, err := LookupAddr(tt.addr) + if err != nil { + t.Fatal(err) + } + if len(names) == 0 { + t.Error("got no record") + } + for _, name := range names { + if !strings.HasSuffix(name, tt.name) { + t.Errorf("got %q; want a record containing %q", name, tt.name) + } + } + } +} + +func TestLookupIANACNAME(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + cname, err := LookupCNAME("www.iana.org") + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(cname, ".icann.org.") { + t.Errorf("got %q; want a record containing .icann.org.", cname) + } +} + +func TestLookupGoogleHost(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + addrs, err := LookupHost("google.com") + if err != nil { + t.Fatal(err) + } + if len(addrs) == 0 { + t.Error("got no record") + } + for _, addr := range addrs { + if ParseIP(addr) == nil { + t.Errorf("got %q; want a literal ip address", addr) + } + } +} + +func TestLookupGoogleIP(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + ips, err := LookupIP("google.com") + if err != nil { + t.Fatal(err) + } + if len(ips) == 0 { + t.Error("got no record") + } + for _, ip := range ips { + if ip.To4() == nil && ip.To16() == nil { + t.Errorf("got %v; want an ip address", ip) + } + } +} + +var revAddrTests = []struct { + Addr string + Reverse string + ErrPrefix string +}{ + {"1.2.3.4", "4.3.2.1.in-addr.arpa.", ""}, + {"245.110.36.114", "114.36.110.245.in-addr.arpa.", ""}, + {"::ffff:12.34.56.78", "78.56.34.12.in-addr.arpa.", ""}, + {"::1", "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa.", ""}, + {"1::", "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.1.0.0.0.ip6.arpa.", ""}, + {"1234:567::89a:bcde", "e.d.c.b.a.9.8.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.7.6.5.0.4.3.2.1.ip6.arpa.", ""}, + {"1234:567:fefe:bcbc:adad:9e4a:89a:bcde", "e.d.c.b.a.9.8.0.a.4.e.9.d.a.d.a.c.b.c.b.e.f.e.f.7.6.5.0.4.3.2.1.ip6.arpa.", ""}, + {"1.2.3", "", "unrecognized address"}, + {"1.2.3.4.5", "", "unrecognized address"}, + {"1234:567:bcbca::89a:bcde", "", "unrecognized address"}, + {"1234:567::bcbc:adad::89a:bcde", "", "unrecognized address"}, +} + +func TestReverseAddress(t *testing.T) { + for i, tt := range revAddrTests { + a, err := reverseaddr(tt.Addr) + if len(tt.ErrPrefix) > 0 && err == nil { + t.Errorf("#%d: expected %q, got <nil> (error)", i, tt.ErrPrefix) + continue + } + if len(tt.ErrPrefix) == 0 && err != nil { + t.Errorf("#%d: expected <nil>, got %q (error)", i, err) + } + if err != nil && err.(*DNSError).Err != tt.ErrPrefix { + t.Errorf("#%d: expected %q, got %q (mismatched error)", i, tt.ErrPrefix, err.(*DNSError).Err) + } + if a != tt.Reverse { + t.Errorf("#%d: expected %q, got %q (reverse address)", i, tt.Reverse, a) + } + } +} diff --git a/src/net/lookup_unix.go b/src/net/lookup_unix.go new file mode 100644 index 000000000..b1d2f8f31 --- /dev/null +++ b/src/net/lookup_unix.go @@ -0,0 +1,168 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +package net + +import ( + "errors" + "sync" +) + +var onceReadProtocols sync.Once + +// readProtocols loads contents of /etc/protocols into protocols map +// for quick access. +func readProtocols() { + if file, err := open("/etc/protocols"); err == nil { + for line, ok := file.readLine(); ok; line, ok = file.readLine() { + // tcp 6 TCP # transmission control protocol + if i := byteIndex(line, '#'); i >= 0 { + line = line[0:i] + } + f := getFields(line) + if len(f) < 2 { + continue + } + if proto, _, ok := dtoi(f[1], 0); ok { + if _, ok := protocols[f[0]]; !ok { + protocols[f[0]] = proto + } + for _, alias := range f[2:] { + if _, ok := protocols[alias]; !ok { + protocols[alias] = proto + } + } + } + } + file.close() + } +} + +// lookupProtocol looks up IP protocol name in /etc/protocols and +// returns correspondent protocol number. +func lookupProtocol(name string) (proto int, err error) { + onceReadProtocols.Do(readProtocols) + proto, found := protocols[name] + if !found { + return 0, errors.New("unknown IP protocol specified: " + name) + } + return +} + +func lookupHost(host string) (addrs []string, err error) { + addrs, err, ok := cgoLookupHost(host) + if !ok { + addrs, err = goLookupHost(host) + } + return +} + +func lookupIP(host string) (addrs []IP, err error) { + addrs, err, ok := cgoLookupIP(host) + if !ok { + addrs, err = goLookupIP(host) + } + return +} + +func lookupPort(network, service string) (port int, err error) { + port, err, ok := cgoLookupPort(network, service) + if !ok { + port, err = goLookupPort(network, service) + } + return +} + +func lookupCNAME(name string) (cname string, err error) { + cname, err, ok := cgoLookupCNAME(name) + if !ok { + cname, err = goLookupCNAME(name) + } + return +} + +func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { + var target string + if service == "" && proto == "" { + target = name + } else { + target = "_" + service + "._" + proto + "." + name + } + var records []dnsRR + cname, records, err = lookup(target, dnsTypeSRV) + if err != nil { + return + } + addrs = make([]*SRV, len(records)) + for i, rr := range records { + r := rr.(*dnsRR_SRV) + addrs[i] = &SRV{r.Target, r.Port, r.Priority, r.Weight} + } + byPriorityWeight(addrs).sort() + return +} + +func lookupMX(name string) (mx []*MX, err error) { + _, records, err := lookup(name, dnsTypeMX) + if err != nil { + return + } + mx = make([]*MX, len(records)) + for i, rr := range records { + r := rr.(*dnsRR_MX) + mx[i] = &MX{r.Mx, r.Pref} + } + byPref(mx).sort() + return +} + +func lookupNS(name string) (ns []*NS, err error) { + _, records, err := lookup(name, dnsTypeNS) + if err != nil { + return + } + ns = make([]*NS, len(records)) + for i, r := range records { + r := r.(*dnsRR_NS) + ns[i] = &NS{r.Ns} + } + return +} + +func lookupTXT(name string) (txt []string, err error) { + _, records, err := lookup(name, dnsTypeTXT) + if err != nil { + return + } + txt = make([]string, len(records)) + for i, r := range records { + txt[i] = r.(*dnsRR_TXT).Txt + } + return +} + +func lookupAddr(addr string) (name []string, err error) { + name = lookupStaticAddr(addr) + if len(name) > 0 { + return + } + var arpa string + arpa, err = reverseaddr(addr) + if err != nil { + return + } + var records []dnsRR + _, records, err = lookup(arpa, dnsTypePTR) + if err != nil { + return + } + name = make([]string, len(records)) + for i := range records { + r := records[i].(*dnsRR_PTR) + name[i] = r.Ptr + } + return +} diff --git a/src/net/lookup_windows.go b/src/net/lookup_windows.go new file mode 100644 index 000000000..6a925b0a7 --- /dev/null +++ b/src/net/lookup_windows.go @@ -0,0 +1,381 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "runtime" + "syscall" + "unsafe" +) + +var ( + lookupPort = oldLookupPort + lookupIP = oldLookupIP +) + +func getprotobyname(name string) (proto int, err error) { + p, err := syscall.GetProtoByName(name) + if err != nil { + return 0, os.NewSyscallError("GetProtoByName", err) + } + return int(p.Proto), nil +} + +// lookupProtocol looks up IP protocol name and returns correspondent protocol number. +func lookupProtocol(name string) (proto int, err error) { + // GetProtoByName return value is stored in thread local storage. + // Start new os thread before the call to prevent races. + type result struct { + proto int + err error + } + ch := make(chan result) + go func() { + acquireThread() + defer releaseThread() + runtime.LockOSThread() + defer runtime.UnlockOSThread() + proto, err := getprotobyname(name) + ch <- result{proto: proto, err: err} + }() + r := <-ch + if r.err != nil { + if proto, ok := protocols[name]; ok { + return proto, nil + } + } + return r.proto, r.err +} + +func lookupHost(name string) (addrs []string, err error) { + ips, err := LookupIP(name) + if err != nil { + return + } + addrs = make([]string, 0, len(ips)) + for _, ip := range ips { + addrs = append(addrs, ip.String()) + } + return +} + +func gethostbyname(name string) (addrs []IP, err error) { + // caller already acquired thread + h, err := syscall.GetHostByName(name) + if err != nil { + return nil, os.NewSyscallError("GetHostByName", err) + } + switch h.AddrType { + case syscall.AF_INET: + i := 0 + addrs = make([]IP, 100) // plenty of room to grow + for p := (*[100](*[4]byte))(unsafe.Pointer(h.AddrList)); i < cap(addrs) && p[i] != nil; i++ { + addrs[i] = IPv4(p[i][0], p[i][1], p[i][2], p[i][3]) + } + addrs = addrs[0:i] + default: // TODO(vcc): Implement non IPv4 address lookups. + return nil, os.NewSyscallError("LookupIP", syscall.EWINDOWS) + } + return addrs, nil +} + +func oldLookupIP(name string) (addrs []IP, err error) { + // GetHostByName return value is stored in thread local storage. + // Start new os thread before the call to prevent races. + type result struct { + addrs []IP + err error + } + ch := make(chan result) + go func() { + acquireThread() + defer releaseThread() + runtime.LockOSThread() + defer runtime.UnlockOSThread() + addrs, err := gethostbyname(name) + ch <- result{addrs: addrs, err: err} + }() + r := <-ch + return r.addrs, r.err +} + +func newLookupIP(name string) (addrs []IP, err error) { + acquireThread() + defer releaseThread() + hints := syscall.AddrinfoW{ + Family: syscall.AF_UNSPEC, + Socktype: syscall.SOCK_STREAM, + Protocol: syscall.IPPROTO_IP, + } + var result *syscall.AddrinfoW + e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result) + if e != nil { + return nil, os.NewSyscallError("GetAddrInfoW", e) + } + defer syscall.FreeAddrInfoW(result) + addrs = make([]IP, 0, 5) + for ; result != nil; result = result.Next { + addr := unsafe.Pointer(result.Addr) + switch result.Family { + case syscall.AF_INET: + a := (*syscall.RawSockaddrInet4)(addr).Addr + addrs = append(addrs, IPv4(a[0], a[1], a[2], a[3])) + case syscall.AF_INET6: + a := (*syscall.RawSockaddrInet6)(addr).Addr + addrs = append(addrs, IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]}) + default: + return nil, os.NewSyscallError("LookupIP", syscall.EWINDOWS) + } + } + return addrs, nil +} + +func getservbyname(network, service string) (port int, err error) { + acquireThread() + defer releaseThread() + switch network { + case "tcp4", "tcp6": + network = "tcp" + case "udp4", "udp6": + network = "udp" + } + s, err := syscall.GetServByName(service, network) + if err != nil { + return 0, os.NewSyscallError("GetServByName", err) + } + return int(syscall.Ntohs(s.Port)), nil +} + +func oldLookupPort(network, service string) (port int, err error) { + // GetServByName return value is stored in thread local storage. + // Start new os thread before the call to prevent races. + type result struct { + port int + err error + } + ch := make(chan result) + go func() { + acquireThread() + defer releaseThread() + runtime.LockOSThread() + defer runtime.UnlockOSThread() + port, err := getservbyname(network, service) + ch <- result{port: port, err: err} + }() + r := <-ch + return r.port, r.err +} + +func newLookupPort(network, service string) (port int, err error) { + acquireThread() + defer releaseThread() + var stype int32 + switch network { + case "tcp4", "tcp6": + stype = syscall.SOCK_STREAM + case "udp4", "udp6": + stype = syscall.SOCK_DGRAM + } + hints := syscall.AddrinfoW{ + Family: syscall.AF_UNSPEC, + Socktype: stype, + Protocol: syscall.IPPROTO_IP, + } + var result *syscall.AddrinfoW + e := syscall.GetAddrInfoW(nil, syscall.StringToUTF16Ptr(service), &hints, &result) + if e != nil { + return 0, os.NewSyscallError("GetAddrInfoW", e) + } + defer syscall.FreeAddrInfoW(result) + if result == nil { + return 0, os.NewSyscallError("LookupPort", syscall.EINVAL) + } + addr := unsafe.Pointer(result.Addr) + switch result.Family { + case syscall.AF_INET: + a := (*syscall.RawSockaddrInet4)(addr) + return int(syscall.Ntohs(a.Port)), nil + case syscall.AF_INET6: + a := (*syscall.RawSockaddrInet6)(addr) + return int(syscall.Ntohs(a.Port)), nil + } + return 0, os.NewSyscallError("LookupPort", syscall.EINVAL) +} + +func lookupCNAME(name string) (cname string, err error) { + acquireThread() + defer releaseThread() + var r *syscall.DNSRecord + e := syscall.DnsQuery(name, syscall.DNS_TYPE_CNAME, 0, nil, &r, nil) + // windows returns DNS_INFO_NO_RECORDS if there are no CNAME-s + if errno, ok := e.(syscall.Errno); ok && errno == syscall.DNS_INFO_NO_RECORDS { + // if there are no aliases, the canonical name is the input name + if name == "" || name[len(name)-1] != '.' { + return name + ".", nil + } + return name, nil + } + if e != nil { + return "", os.NewSyscallError("LookupCNAME", e) + } + defer syscall.DnsRecordListFree(r, 1) + + resolved := resolveCNAME(syscall.StringToUTF16Ptr(name), r) + cname = syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(resolved))[:]) + "." + return +} + +func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { + acquireThread() + defer releaseThread() + var target string + if service == "" && proto == "" { + target = name + } else { + target = "_" + service + "._" + proto + "." + name + } + var r *syscall.DNSRecord + e := syscall.DnsQuery(target, syscall.DNS_TYPE_SRV, 0, nil, &r, nil) + if e != nil { + return "", nil, os.NewSyscallError("LookupSRV", e) + } + defer syscall.DnsRecordListFree(r, 1) + + addrs = make([]*SRV, 0, 10) + for _, p := range validRecs(r, syscall.DNS_TYPE_SRV, target) { + v := (*syscall.DNSSRVData)(unsafe.Pointer(&p.Data[0])) + addrs = append(addrs, &SRV{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Target))[:]), v.Port, v.Priority, v.Weight}) + } + byPriorityWeight(addrs).sort() + return name, addrs, nil +} + +func lookupMX(name string) (mx []*MX, err error) { + acquireThread() + defer releaseThread() + var r *syscall.DNSRecord + e := syscall.DnsQuery(name, syscall.DNS_TYPE_MX, 0, nil, &r, nil) + if e != nil { + return nil, os.NewSyscallError("LookupMX", e) + } + defer syscall.DnsRecordListFree(r, 1) + + mx = make([]*MX, 0, 10) + for _, p := range validRecs(r, syscall.DNS_TYPE_MX, name) { + v := (*syscall.DNSMXData)(unsafe.Pointer(&p.Data[0])) + mx = append(mx, &MX{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.NameExchange))[:]) + ".", v.Preference}) + } + byPref(mx).sort() + return mx, nil +} + +func lookupNS(name string) (ns []*NS, err error) { + acquireThread() + defer releaseThread() + var r *syscall.DNSRecord + e := syscall.DnsQuery(name, syscall.DNS_TYPE_NS, 0, nil, &r, nil) + if e != nil { + return nil, os.NewSyscallError("LookupNS", e) + } + defer syscall.DnsRecordListFree(r, 1) + + ns = make([]*NS, 0, 10) + for _, p := range validRecs(r, syscall.DNS_TYPE_NS, name) { + v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0])) + ns = append(ns, &NS{syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:]) + "."}) + } + return ns, nil +} + +func lookupTXT(name string) (txt []string, err error) { + acquireThread() + defer releaseThread() + var r *syscall.DNSRecord + e := syscall.DnsQuery(name, syscall.DNS_TYPE_TEXT, 0, nil, &r, nil) + if e != nil { + return nil, os.NewSyscallError("LookupTXT", e) + } + defer syscall.DnsRecordListFree(r, 1) + + txt = make([]string, 0, 10) + for _, p := range validRecs(r, syscall.DNS_TYPE_TEXT, name) { + d := (*syscall.DNSTXTData)(unsafe.Pointer(&p.Data[0])) + for _, v := range (*[1 << 10]*uint16)(unsafe.Pointer(&(d.StringArray[0])))[:d.StringCount] { + s := syscall.UTF16ToString((*[1 << 20]uint16)(unsafe.Pointer(v))[:]) + txt = append(txt, s) + } + } + return +} + +func lookupAddr(addr string) (name []string, err error) { + acquireThread() + defer releaseThread() + arpa, err := reverseaddr(addr) + if err != nil { + return nil, err + } + var r *syscall.DNSRecord + e := syscall.DnsQuery(arpa, syscall.DNS_TYPE_PTR, 0, nil, &r, nil) + if e != nil { + return nil, os.NewSyscallError("LookupAddr", e) + } + defer syscall.DnsRecordListFree(r, 1) + + name = make([]string, 0, 10) + for _, p := range validRecs(r, syscall.DNS_TYPE_PTR, arpa) { + v := (*syscall.DNSPTRData)(unsafe.Pointer(&p.Data[0])) + name = append(name, syscall.UTF16ToString((*[256]uint16)(unsafe.Pointer(v.Host))[:])) + } + return name, nil +} + +const dnsSectionMask = 0x0003 + +// returns only results applicable to name and resolves CNAME entries +func validRecs(r *syscall.DNSRecord, dnstype uint16, name string) []*syscall.DNSRecord { + cname := syscall.StringToUTF16Ptr(name) + if dnstype != syscall.DNS_TYPE_CNAME { + cname = resolveCNAME(cname, r) + } + rec := make([]*syscall.DNSRecord, 0, 10) + for p := r; p != nil; p = p.Next { + if p.Dw&dnsSectionMask != syscall.DnsSectionAnswer { + continue + } + if p.Type != dnstype { + continue + } + if !syscall.DnsNameCompare(cname, p.Name) { + continue + } + rec = append(rec, p) + } + return rec +} + +// returns the last CNAME in chain +func resolveCNAME(name *uint16, r *syscall.DNSRecord) *uint16 { + // limit cname resolving to 10 in case of a infinite CNAME loop +Cname: + for cnameloop := 0; cnameloop < 10; cnameloop++ { + for p := r; p != nil; p = p.Next { + if p.Dw&dnsSectionMask != syscall.DnsSectionAnswer { + continue + } + if p.Type != syscall.DNS_TYPE_CNAME { + continue + } + if !syscall.DnsNameCompare(name, p.Name) { + continue + } + name = (*syscall.DNSPTRData)(unsafe.Pointer(&r.Data[0])).Host + continue Cname + } + break + } + return name +} diff --git a/src/net/lookup_windows_test.go b/src/net/lookup_windows_test.go new file mode 100644 index 000000000..7495b5b57 --- /dev/null +++ b/src/net/lookup_windows_test.go @@ -0,0 +1,243 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "encoding/json" + "errors" + "os/exec" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "testing" +) + +var nslookupTestServers = []string{"mail.golang.com", "gmail.com"} + +func toJson(v interface{}) string { + data, _ := json.Marshal(v) + return string(data) +} + +func TestLookupMX(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + for _, server := range nslookupTestServers { + mx, err := LookupMX(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if len(mx) == 0 { + t.Errorf("no results") + continue + } + expected, err := nslookupMX(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + } + sort.Sort(byPrefAndHost(expected)) + sort.Sort(byPrefAndHost(mx)) + if !reflect.DeepEqual(expected, mx) { + t.Errorf("different results %s:\texp:%v\tgot:%v", server, toJson(expected), toJson(mx)) + } + } +} + +func TestLookupCNAME(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + for _, server := range nslookupTestServers { + cname, err := LookupCNAME(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if cname == "" { + t.Errorf("no result %s", server) + } + expected, err := nslookupCNAME(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + continue + } + if expected != cname { + t.Errorf("different results %s:\texp:%v\tgot:%v", server, expected, cname) + } + } +} + +func TestLookupNS(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + for _, server := range nslookupTestServers { + ns, err := LookupNS(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if len(ns) == 0 { + t.Errorf("no results") + continue + } + expected, err := nslookupNS(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + continue + } + sort.Sort(byHost(expected)) + sort.Sort(byHost(ns)) + if !reflect.DeepEqual(expected, ns) { + t.Errorf("different results %s:\texp:%v\tgot:%v", toJson(server), toJson(expected), ns) + } + } +} + +func TestLookupTXT(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + for _, server := range nslookupTestServers { + txt, err := LookupTXT(server) + if err != nil { + t.Errorf("failed %s: %s", server, err) + continue + } + if len(txt) == 0 { + t.Errorf("no results") + continue + } + expected, err := nslookupTXT(server) + if err != nil { + t.Logf("skipping failed nslookup %s test: %s", server, err) + continue + } + sort.Strings(expected) + sort.Strings(txt) + if !reflect.DeepEqual(expected, txt) { + t.Errorf("different results %s:\texp:%v\tgot:%v", server, toJson(expected), toJson(txt)) + } + } +} + +type byPrefAndHost []*MX + +func (s byPrefAndHost) Len() int { return len(s) } +func (s byPrefAndHost) Less(i, j int) bool { + if s[i].Pref != s[j].Pref { + return s[i].Pref < s[j].Pref + } + return s[i].Host < s[j].Host +} +func (s byPrefAndHost) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type byHost []*NS + +func (s byHost) Len() int { return len(s) } +func (s byHost) Less(i, j int) bool { return s[i].Host < s[j].Host } +func (s byHost) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func fqdn(s string) string { + if len(s) == 0 || s[len(s)-1] != '.' { + return s + "." + } + return s +} + +func nslookup(qtype, name string) (string, error) { + var out bytes.Buffer + var err bytes.Buffer + cmd := exec.Command("nslookup", "-querytype="+qtype, name) + cmd.Stdout = &out + cmd.Stderr = &err + if err := cmd.Run(); err != nil { + return "", err + } + r := strings.Replace(out.String(), "\r\n", "\n", -1) + // nslookup stderr output contains also debug information such as + // "Non-authoritative answer" and it doesn't return the correct errcode + if strings.Contains(err.String(), "can't find") { + return r, errors.New(err.String()) + } + return r, nil +} + +func nslookupMX(name string) (mx []*MX, err error) { + var r string + if r, err = nslookup("mx", name); err != nil { + return + } + mx = make([]*MX, 0, 10) + // linux nslookup syntax + // golang.org mail exchanger = 2 alt1.aspmx.l.google.com. + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+mail exchanger\s*=\s*([0-9]+)\s*([a-z0-9.\-]+)$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + pref, _ := strconv.Atoi(ans[2]) + mx = append(mx, &MX{fqdn(ans[3]), uint16(pref)}) + } + // windows nslookup syntax + // gmail.com MX preference = 30, mail exchanger = alt3.gmail-smtp-in.l.google.com + rx = regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+MX preference\s*=\s*([0-9]+)\s*,\s*mail exchanger\s*=\s*([a-z0-9.\-]+)$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + pref, _ := strconv.Atoi(ans[2]) + mx = append(mx, &MX{fqdn(ans[3]), uint16(pref)}) + } + return +} + +func nslookupNS(name string) (ns []*NS, err error) { + var r string + if r, err = nslookup("ns", name); err != nil { + return + } + ns = make([]*NS, 0, 10) + // golang.org nameserver = ns1.google.com. + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+nameserver\s*=\s*([a-z0-9.\-]+)$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + ns = append(ns, &NS{fqdn(ans[2])}) + } + return +} + +func nslookupCNAME(name string) (cname string, err error) { + var r string + if r, err = nslookup("cname", name); err != nil { + return + } + // mail.golang.com canonical name = golang.org. + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+canonical name\s*=\s*([a-z0-9.\-]+)$`) + // assumes the last CNAME is the correct one + last := name + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + last = ans[2] + } + return fqdn(last), nil +} + +func nslookupTXT(name string) (txt []string, err error) { + var r string + if r, err = nslookup("txt", name); err != nil { + return + } + txt = make([]string, 0, 10) + // linux + // golang.org text = "v=spf1 redirect=_spf.google.com" + + // windows + // golang.org text = + // + // "v=spf1 redirect=_spf.google.com" + rx := regexp.MustCompile(`(?m)^([a-z0-9.\-]+)\s+text\s*=\s*"(.*)"$`) + for _, ans := range rx.FindAllStringSubmatch(r, -1) { + txt = append(txt, ans[2]) + } + return +} diff --git a/src/net/mac.go b/src/net/mac.go new file mode 100644 index 000000000..d616b1f68 --- /dev/null +++ b/src/net/mac.go @@ -0,0 +1,86 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// MAC address manipulations + +package net + +import "errors" + +const hexDigit = "0123456789abcdef" + +// A HardwareAddr represents a physical hardware address. +type HardwareAddr []byte + +func (a HardwareAddr) String() string { + if len(a) == 0 { + return "" + } + buf := make([]byte, 0, len(a)*3-1) + for i, b := range a { + if i > 0 { + buf = append(buf, ':') + } + buf = append(buf, hexDigit[b>>4]) + buf = append(buf, hexDigit[b&0xF]) + } + return string(buf) +} + +// ParseMAC parses s as an IEEE 802 MAC-48, EUI-48, or EUI-64 using one of the +// following formats: +// 01:23:45:67:89:ab +// 01:23:45:67:89:ab:cd:ef +// 01-23-45-67-89-ab +// 01-23-45-67-89-ab-cd-ef +// 0123.4567.89ab +// 0123.4567.89ab.cdef +func ParseMAC(s string) (hw HardwareAddr, err error) { + if len(s) < 14 { + goto error + } + + if s[2] == ':' || s[2] == '-' { + if (len(s)+1)%3 != 0 { + goto error + } + n := (len(s) + 1) / 3 + if n != 6 && n != 8 { + goto error + } + hw = make(HardwareAddr, n) + for x, i := 0, 0; i < n; i++ { + var ok bool + if hw[i], ok = xtoi2(s[x:], s[2]); !ok { + goto error + } + x += 3 + } + } else if s[4] == '.' { + if (len(s)+1)%5 != 0 { + goto error + } + n := 2 * (len(s) + 1) / 5 + if n != 6 && n != 8 { + goto error + } + hw = make(HardwareAddr, n) + for x, i := 0, 0; i < n; i += 2 { + var ok bool + if hw[i], ok = xtoi2(s[x:x+2], 0); !ok { + goto error + } + if hw[i+1], ok = xtoi2(s[x+2:], s[4]); !ok { + goto error + } + x += 5 + } + } else { + goto error + } + return hw, nil + +error: + return nil, errors.New("invalid MAC address: " + s) +} diff --git a/src/net/mac_test.go b/src/net/mac_test.go new file mode 100644 index 000000000..8f9dc6685 --- /dev/null +++ b/src/net/mac_test.go @@ -0,0 +1,66 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "reflect" + "strings" + "testing" +) + +var mactests = []struct { + in string + out HardwareAddr + err string +}{ + {"01:23:45:67:89:AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, + {"01-23-45-67-89-AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, + {"0123.4567.89AB", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab}, ""}, + {"ab:cd:ef:AB:CD:EF", HardwareAddr{0xab, 0xcd, 0xef, 0xab, 0xcd, 0xef}, ""}, + {"01.02.03.04.05.06", nil, "invalid MAC address"}, + {"01:02:03:04:05:06:", nil, "invalid MAC address"}, + {"x1:02:03:04:05:06", nil, "invalid MAC address"}, + {"01002:03:04:05:06", nil, "invalid MAC address"}, + {"01:02003:04:05:06", nil, "invalid MAC address"}, + {"01:02:03004:05:06", nil, "invalid MAC address"}, + {"01:02:03:04005:06", nil, "invalid MAC address"}, + {"01:02:03:04:05006", nil, "invalid MAC address"}, + {"01-02:03:04:05:06", nil, "invalid MAC address"}, + {"01:02-03-04-05-06", nil, "invalid MAC address"}, + {"0123:4567:89AF", nil, "invalid MAC address"}, + {"0123-4567-89AF", nil, "invalid MAC address"}, + {"01:23:45:67:89:AB:CD:EF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, + {"01-23-45-67-89-AB-CD-EF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, + {"0123.4567.89AB.CDEF", HardwareAddr{1, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, ""}, +} + +func match(err error, s string) bool { + if s == "" { + return err == nil + } + return err != nil && strings.Contains(err.Error(), s) +} + +func TestMACParseString(t *testing.T) { + for i, tt := range mactests { + out, err := ParseMAC(tt.in) + if !reflect.DeepEqual(out, tt.out) || !match(err, tt.err) { + t.Errorf("ParseMAC(%q) = %v, %v, want %v, %v", tt.in, out, err, tt.out, + tt.err) + } + if tt.err == "" { + // Verify that serialization works too, and that it round-trips. + s := out.String() + out2, err := ParseMAC(s) + if err != nil { + t.Errorf("%d. ParseMAC(%q) = %v", i, s, err) + continue + } + if !reflect.DeepEqual(out2, out) { + t.Errorf("%d. ParseMAC(%q) = %v, want %v", i, s, out2, out) + } + } + } +} diff --git a/src/net/mail/message.go b/src/net/mail/message.go new file mode 100644 index 000000000..ba0778caa --- /dev/null +++ b/src/net/mail/message.go @@ -0,0 +1,545 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package mail implements parsing of mail messages. + +For the most part, this package follows the syntax as specified by RFC 5322. +Notable divergences: + * Obsolete address formats are not parsed, including addresses with + embedded route information. + * Group addresses are not parsed. + * The full range of spacing (the CFWS syntax element) is not supported, + such as breaking addresses across lines. +*/ +package mail + +import ( + "bufio" + "bytes" + "encoding/base64" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/textproto" + "strconv" + "strings" + "time" +) + +var debug = debugT(false) + +type debugT bool + +func (d debugT) Printf(format string, args ...interface{}) { + if d { + log.Printf(format, args...) + } +} + +// A Message represents a parsed mail message. +type Message struct { + Header Header + Body io.Reader +} + +// ReadMessage reads a message from r. +// The headers are parsed, and the body of the message will be available +// for reading from r. +func ReadMessage(r io.Reader) (msg *Message, err error) { + tp := textproto.NewReader(bufio.NewReader(r)) + + hdr, err := tp.ReadMIMEHeader() + if err != nil { + return nil, err + } + + return &Message{ + Header: Header(hdr), + Body: tp.R, + }, nil +} + +// Layouts suitable for passing to time.Parse. +// These are tried in order. +var dateLayouts []string + +func init() { + // Generate layouts based on RFC 5322, section 3.3. + + dows := [...]string{"", "Mon, "} // day-of-week + days := [...]string{"2", "02"} // day = 1*2DIGIT + years := [...]string{"2006", "06"} // year = 4*DIGIT / 2*DIGIT + seconds := [...]string{":05", ""} // second + // "-0700 (MST)" is not in RFC 5322, but is common. + zones := [...]string{"-0700", "MST", "-0700 (MST)"} // zone = (("+" / "-") 4DIGIT) / "GMT" / ... + + for _, dow := range dows { + for _, day := range days { + for _, year := range years { + for _, second := range seconds { + for _, zone := range zones { + s := dow + day + " Jan " + year + " 15:04" + second + " " + zone + dateLayouts = append(dateLayouts, s) + } + } + } + } + } +} + +func parseDate(date string) (time.Time, error) { + for _, layout := range dateLayouts { + t, err := time.Parse(layout, date) + if err == nil { + return t, nil + } + } + return time.Time{}, errors.New("mail: header could not be parsed") +} + +// A Header represents the key-value pairs in a mail message header. +type Header map[string][]string + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +func (h Header) Get(key string) string { + return textproto.MIMEHeader(h).Get(key) +} + +var ErrHeaderNotPresent = errors.New("mail: header not in message") + +// Date parses the Date header field. +func (h Header) Date() (time.Time, error) { + hdr := h.Get("Date") + if hdr == "" { + return time.Time{}, ErrHeaderNotPresent + } + return parseDate(hdr) +} + +// AddressList parses the named header field as a list of addresses. +func (h Header) AddressList(key string) ([]*Address, error) { + hdr := h.Get(key) + if hdr == "" { + return nil, ErrHeaderNotPresent + } + return ParseAddressList(hdr) +} + +// Address represents a single mail address. +// An address such as "Barry Gibbs <bg@example.com>" is represented +// as Address{Name: "Barry Gibbs", Address: "bg@example.com"}. +type Address struct { + Name string // Proper name; may be empty. + Address string // user@domain +} + +// Parses a single RFC 5322 address, e.g. "Barry Gibbs <bg@example.com>" +func ParseAddress(address string) (*Address, error) { + return newAddrParser(address).parseAddress() +} + +// ParseAddressList parses the given string as a list of addresses. +func ParseAddressList(list string) ([]*Address, error) { + return newAddrParser(list).parseAddressList() +} + +// String formats the address as a valid RFC 5322 address. +// If the address's name contains non-ASCII characters +// the name will be rendered according to RFC 2047. +func (a *Address) String() string { + s := "<" + a.Address + ">" + if a.Name == "" { + return s + } + // If every character is printable ASCII, quoting is simple. + allPrintable := true + for i := 0; i < len(a.Name); i++ { + // isWSP here should actually be isFWS, + // but we don't support folding yet. + if !isVchar(a.Name[i]) && !isWSP(a.Name[i]) { + allPrintable = false + break + } + } + if allPrintable { + b := bytes.NewBufferString(`"`) + for i := 0; i < len(a.Name); i++ { + if !isQtext(a.Name[i]) && !isWSP(a.Name[i]) { + b.WriteByte('\\') + } + b.WriteByte(a.Name[i]) + } + b.WriteString(`" `) + b.WriteString(s) + return b.String() + } + + // UTF-8 "Q" encoding + b := bytes.NewBufferString("=?utf-8?q?") + for i := 0; i < len(a.Name); i++ { + switch c := a.Name[i]; { + case c == ' ': + b.WriteByte('_') + case isVchar(c) && c != '=' && c != '?' && c != '_': + b.WriteByte(c) + default: + fmt.Fprintf(b, "=%02X", c) + } + } + b.WriteString("?= ") + b.WriteString(s) + return b.String() +} + +type addrParser []byte + +func newAddrParser(s string) *addrParser { + p := addrParser(s) + return &p +} + +func (p *addrParser) parseAddressList() ([]*Address, error) { + var list []*Address + for { + p.skipSpace() + addr, err := p.parseAddress() + if err != nil { + return nil, err + } + list = append(list, addr) + + p.skipSpace() + if p.empty() { + break + } + if !p.consume(',') { + return nil, errors.New("mail: expected comma") + } + } + return list, nil +} + +// parseAddress parses a single RFC 5322 address at the start of p. +func (p *addrParser) parseAddress() (addr *Address, err error) { + debug.Printf("parseAddress: %q", *p) + p.skipSpace() + if p.empty() { + return nil, errors.New("mail: no address") + } + + // address = name-addr / addr-spec + // TODO(dsymonds): Support parsing group address. + + // addr-spec has a more restricted grammar than name-addr, + // so try parsing it first, and fallback to name-addr. + // TODO(dsymonds): Is this really correct? + spec, err := p.consumeAddrSpec() + if err == nil { + return &Address{ + Address: spec, + }, err + } + debug.Printf("parseAddress: not an addr-spec: %v", err) + debug.Printf("parseAddress: state is now %q", *p) + + // display-name + var displayName string + if p.peek() != '<' { + displayName, err = p.consumePhrase() + if err != nil { + return nil, err + } + } + debug.Printf("parseAddress: displayName=%q", displayName) + + // angle-addr = "<" addr-spec ">" + p.skipSpace() + if !p.consume('<') { + return nil, errors.New("mail: no angle-addr") + } + spec, err = p.consumeAddrSpec() + if err != nil { + return nil, err + } + if !p.consume('>') { + return nil, errors.New("mail: unclosed angle-addr") + } + debug.Printf("parseAddress: spec=%q", spec) + + return &Address{ + Name: displayName, + Address: spec, + }, nil +} + +// consumeAddrSpec parses a single RFC 5322 addr-spec at the start of p. +func (p *addrParser) consumeAddrSpec() (spec string, err error) { + debug.Printf("consumeAddrSpec: %q", *p) + + orig := *p + defer func() { + if err != nil { + *p = orig + } + }() + + // local-part = dot-atom / quoted-string + var localPart string + p.skipSpace() + if p.empty() { + return "", errors.New("mail: no addr-spec") + } + if p.peek() == '"' { + // quoted-string + debug.Printf("consumeAddrSpec: parsing quoted-string") + localPart, err = p.consumeQuotedString() + } else { + // dot-atom + debug.Printf("consumeAddrSpec: parsing dot-atom") + localPart, err = p.consumeAtom(true) + } + if err != nil { + debug.Printf("consumeAddrSpec: failed: %v", err) + return "", err + } + + if !p.consume('@') { + return "", errors.New("mail: missing @ in addr-spec") + } + + // domain = dot-atom / domain-literal + var domain string + p.skipSpace() + if p.empty() { + return "", errors.New("mail: no domain in addr-spec") + } + // TODO(dsymonds): Handle domain-literal + domain, err = p.consumeAtom(true) + if err != nil { + return "", err + } + + return localPart + "@" + domain, nil +} + +// consumePhrase parses the RFC 5322 phrase at the start of p. +func (p *addrParser) consumePhrase() (phrase string, err error) { + debug.Printf("consumePhrase: [%s]", *p) + // phrase = 1*word + var words []string + for { + // word = atom / quoted-string + var word string + p.skipSpace() + if p.empty() { + return "", errors.New("mail: missing phrase") + } + if p.peek() == '"' { + // quoted-string + word, err = p.consumeQuotedString() + } else { + // atom + // We actually parse dot-atom here to be more permissive + // than what RFC 5322 specifies. + word, err = p.consumeAtom(true) + } + + // RFC 2047 encoded-word starts with =?, ends with ?=, and has two other ?s. + if err == nil && strings.HasPrefix(word, "=?") && strings.HasSuffix(word, "?=") && strings.Count(word, "?") == 4 { + word, err = decodeRFC2047Word(word) + } + + if err != nil { + break + } + debug.Printf("consumePhrase: consumed %q", word) + words = append(words, word) + } + // Ignore any error if we got at least one word. + if err != nil && len(words) == 0 { + debug.Printf("consumePhrase: hit err: %v", err) + return "", fmt.Errorf("mail: missing word in phrase: %v", err) + } + phrase = strings.Join(words, " ") + return phrase, nil +} + +// consumeQuotedString parses the quoted string at the start of p. +func (p *addrParser) consumeQuotedString() (qs string, err error) { + // Assume first byte is '"'. + i := 1 + qsb := make([]byte, 0, 10) +Loop: + for { + if i >= p.len() { + return "", errors.New("mail: unclosed quoted-string") + } + switch c := (*p)[i]; { + case c == '"': + break Loop + case c == '\\': + if i+1 == p.len() { + return "", errors.New("mail: unclosed quoted-string") + } + qsb = append(qsb, (*p)[i+1]) + i += 2 + case isQtext(c), c == ' ' || c == '\t': + // qtext (printable US-ASCII excluding " and \), or + // FWS (almost; we're ignoring CRLF) + qsb = append(qsb, c) + i++ + default: + return "", fmt.Errorf("mail: bad character in quoted-string: %q", c) + } + } + *p = (*p)[i+1:] + return string(qsb), nil +} + +// consumeAtom parses an RFC 5322 atom at the start of p. +// If dot is true, consumeAtom parses an RFC 5322 dot-atom instead. +func (p *addrParser) consumeAtom(dot bool) (atom string, err error) { + if !isAtext(p.peek(), false) { + return "", errors.New("mail: invalid string") + } + i := 1 + for ; i < p.len() && isAtext((*p)[i], dot); i++ { + } + atom, *p = string((*p)[:i]), (*p)[i:] + return atom, nil +} + +func (p *addrParser) consume(c byte) bool { + if p.empty() || p.peek() != c { + return false + } + *p = (*p)[1:] + return true +} + +// skipSpace skips the leading space and tab characters. +func (p *addrParser) skipSpace() { + *p = bytes.TrimLeft(*p, " \t") +} + +func (p *addrParser) peek() byte { + return (*p)[0] +} + +func (p *addrParser) empty() bool { + return p.len() == 0 +} + +func (p *addrParser) len() int { + return len(*p) +} + +func decodeRFC2047Word(s string) (string, error) { + fields := strings.Split(s, "?") + if len(fields) != 5 || fields[0] != "=" || fields[4] != "=" { + return "", errors.New("address not RFC 2047 encoded") + } + charset, enc := strings.ToLower(fields[1]), strings.ToLower(fields[2]) + if charset != "iso-8859-1" && charset != "utf-8" { + return "", fmt.Errorf("charset not supported: %q", charset) + } + + in := bytes.NewBufferString(fields[3]) + var r io.Reader + switch enc { + case "b": + r = base64.NewDecoder(base64.StdEncoding, in) + case "q": + r = qDecoder{r: in} + default: + return "", fmt.Errorf("RFC 2047 encoding not supported: %q", enc) + } + + dec, err := ioutil.ReadAll(r) + if err != nil { + return "", err + } + + switch charset { + case "iso-8859-1": + b := new(bytes.Buffer) + for _, c := range dec { + b.WriteRune(rune(c)) + } + return b.String(), nil + case "utf-8": + return string(dec), nil + } + panic("unreachable") +} + +type qDecoder struct { + r io.Reader + scratch [2]byte +} + +func (qd qDecoder) Read(p []byte) (n int, err error) { + // This method writes at most one byte into p. + if len(p) == 0 { + return 0, nil + } + if _, err := qd.r.Read(qd.scratch[:1]); err != nil { + return 0, err + } + switch c := qd.scratch[0]; { + case c == '=': + if _, err := io.ReadFull(qd.r, qd.scratch[:2]); err != nil { + return 0, err + } + x, err := strconv.ParseInt(string(qd.scratch[:2]), 16, 64) + if err != nil { + return 0, fmt.Errorf("mail: invalid RFC 2047 encoding: %q", qd.scratch[:2]) + } + p[0] = byte(x) + case c == '_': + p[0] = ' ' + default: + p[0] = c + } + return 1, nil +} + +var atextChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + "abcdefghijklmnopqrstuvwxyz" + + "0123456789" + + "!#$%&'*+-/=?^_`{|}~") + +// isAtext returns true if c is an RFC 5322 atext character. +// If dot is true, period is included. +func isAtext(c byte, dot bool) bool { + if dot && c == '.' { + return true + } + return bytes.IndexByte(atextChars, c) >= 0 +} + +// isQtext returns true if c is an RFC 5322 qtext character. +func isQtext(c byte) bool { + // Printable US-ASCII, excluding backslash or quote. + if c == '\\' || c == '"' { + return false + } + return '!' <= c && c <= '~' +} + +// isVchar returns true if c is an RFC 5322 VCHAR character. +func isVchar(c byte) bool { + // Visible (printing) characters. + return '!' <= c && c <= '~' +} + +// isWSP returns true if c is a WSP (white space). +// WSP is a space or horizontal tab (RFC5234 Appendix B). +func isWSP(c byte) bool { + return c == ' ' || c == '\t' +} diff --git a/src/net/mail/message_test.go b/src/net/mail/message_test.go new file mode 100644 index 000000000..eb9c8cbdc --- /dev/null +++ b/src/net/mail/message_test.go @@ -0,0 +1,304 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mail + +import ( + "bytes" + "io/ioutil" + "reflect" + "strings" + "testing" + "time" +) + +var parseTests = []struct { + in string + header Header + body string +}{ + { + // RFC 5322, Appendix A.1.1 + in: `From: John Doe <jdoe@machine.example> +To: Mary Smith <mary@example.net> +Subject: Saying Hello +Date: Fri, 21 Nov 1997 09:55:06 -0600 +Message-ID: <1234@local.machine.example> + +This is a message just to say hello. +So, "Hello". +`, + header: Header{ + "From": []string{"John Doe <jdoe@machine.example>"}, + "To": []string{"Mary Smith <mary@example.net>"}, + "Subject": []string{"Saying Hello"}, + "Date": []string{"Fri, 21 Nov 1997 09:55:06 -0600"}, + "Message-Id": []string{"<1234@local.machine.example>"}, + }, + body: "This is a message just to say hello.\nSo, \"Hello\".\n", + }, +} + +func TestParsing(t *testing.T) { + for i, test := range parseTests { + msg, err := ReadMessage(bytes.NewBuffer([]byte(test.in))) + if err != nil { + t.Errorf("test #%d: Failed parsing message: %v", i, err) + continue + } + if !headerEq(msg.Header, test.header) { + t.Errorf("test #%d: Incorrectly parsed message header.\nGot:\n%+v\nWant:\n%+v", + i, msg.Header, test.header) + } + body, err := ioutil.ReadAll(msg.Body) + if err != nil { + t.Errorf("test #%d: Failed reading body: %v", i, err) + continue + } + bodyStr := string(body) + if bodyStr != test.body { + t.Errorf("test #%d: Incorrectly parsed message body.\nGot:\n%+v\nWant:\n%+v", + i, bodyStr, test.body) + } + } +} + +func headerEq(a, b Header) bool { + if len(a) != len(b) { + return false + } + for k, as := range a { + bs, ok := b[k] + if !ok { + return false + } + if !reflect.DeepEqual(as, bs) { + return false + } + } + return true +} + +func TestDateParsing(t *testing.T) { + tests := []struct { + dateStr string + exp time.Time + }{ + // RFC 5322, Appendix A.1.1 + { + "Fri, 21 Nov 1997 09:55:06 -0600", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + }, + // RFC5322, Appendix A.6.2 + // Obsolete date. + { + "21 Nov 97 09:55:06 GMT", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("GMT", 0)), + }, + // Commonly found format not specified by RFC 5322. + { + "Fri, 21 Nov 1997 09:55:06 -0600 (MDT)", + time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), + }, + } + for _, test := range tests { + hdr := Header{ + "Date": []string{test.dateStr}, + } + date, err := hdr.Date() + if err != nil { + t.Errorf("Failed parsing %q: %v", test.dateStr, err) + continue + } + if !date.Equal(test.exp) { + t.Errorf("Parse of %q: got %+v, want %+v", test.dateStr, date, test.exp) + } + } +} + +func TestAddressParsingError(t *testing.T) { + const txt = "=?iso-8859-2?Q?Bogl=E1rka_Tak=E1cs?= <unknown@gmail.com>" + _, err := ParseAddress(txt) + if err == nil || !strings.Contains(err.Error(), "charset not supported") { + t.Errorf(`mail.ParseAddress(%q) err: %q, want ".*charset not supported.*"`, txt, err) + } +} + +func TestAddressParsing(t *testing.T) { + tests := []struct { + addrsStr string + exp []*Address + }{ + // Bare address + { + `jdoe@machine.example`, + []*Address{{ + Address: "jdoe@machine.example", + }}, + }, + // RFC 5322, Appendix A.1.1 + { + `John Doe <jdoe@machine.example>`, + []*Address{{ + Name: "John Doe", + Address: "jdoe@machine.example", + }}, + }, + // RFC 5322, Appendix A.1.2 + { + `"Joe Q. Public" <john.q.public@example.com>`, + []*Address{{ + Name: "Joe Q. Public", + Address: "john.q.public@example.com", + }}, + }, + { + `Mary Smith <mary@x.test>, jdoe@example.org, Who? <one@y.test>`, + []*Address{ + { + Name: "Mary Smith", + Address: "mary@x.test", + }, + { + Address: "jdoe@example.org", + }, + { + Name: "Who?", + Address: "one@y.test", + }, + }, + }, + { + `<boss@nil.test>, "Giant; \"Big\" Box" <sysservices@example.net>`, + []*Address{ + { + Address: "boss@nil.test", + }, + { + Name: `Giant; "Big" Box`, + Address: "sysservices@example.net", + }, + }, + }, + // RFC 5322, Appendix A.1.3 + // TODO(dsymonds): Group addresses. + + // RFC 2047 "Q"-encoded ISO-8859-1 address. + { + `=?iso-8859-1?q?J=F6rg_Doe?= <joerg@example.com>`, + []*Address{ + { + Name: `Jörg Doe`, + Address: "joerg@example.com", + }, + }, + }, + // RFC 2047 "Q"-encoded UTF-8 address. + { + `=?utf-8?q?J=C3=B6rg_Doe?= <joerg@example.com>`, + []*Address{ + { + Name: `Jörg Doe`, + Address: "joerg@example.com", + }, + }, + }, + // RFC 2047, Section 8. + { + `=?ISO-8859-1?Q?Andr=E9?= Pirard <PIRARD@vm1.ulg.ac.be>`, + []*Address{ + { + Name: `André Pirard`, + Address: "PIRARD@vm1.ulg.ac.be", + }, + }, + }, + // Custom example of RFC 2047 "B"-encoded ISO-8859-1 address. + { + `=?ISO-8859-1?B?SvZyZw==?= <joerg@example.com>`, + []*Address{ + { + Name: `Jörg`, + Address: "joerg@example.com", + }, + }, + }, + // Custom example of RFC 2047 "B"-encoded UTF-8 address. + { + `=?UTF-8?B?SsO2cmc=?= <joerg@example.com>`, + []*Address{ + { + Name: `Jörg`, + Address: "joerg@example.com", + }, + }, + }, + // Custom example with "." in name. For issue 4938 + { + `Asem H. <noreply@example.com>`, + []*Address{ + { + Name: `Asem H.`, + Address: "noreply@example.com", + }, + }, + }, + } + for _, test := range tests { + if len(test.exp) == 1 { + addr, err := ParseAddress(test.addrsStr) + if err != nil { + t.Errorf("Failed parsing (single) %q: %v", test.addrsStr, err) + continue + } + if !reflect.DeepEqual([]*Address{addr}, test.exp) { + t.Errorf("Parse (single) of %q: got %+v, want %+v", test.addrsStr, addr, test.exp) + } + } + + addrs, err := ParseAddressList(test.addrsStr) + if err != nil { + t.Errorf("Failed parsing (list) %q: %v", test.addrsStr, err) + continue + } + if !reflect.DeepEqual(addrs, test.exp) { + t.Errorf("Parse (list) of %q: got %+v, want %+v", test.addrsStr, addrs, test.exp) + } + } +} + +func TestAddressFormatting(t *testing.T) { + tests := []struct { + addr *Address + exp string + }{ + { + &Address{Address: "bob@example.com"}, + "<bob@example.com>", + }, + { + &Address{Name: "Bob", Address: "bob@example.com"}, + `"Bob" <bob@example.com>`, + }, + { + // note the ö (o with an umlaut) + &Address{Name: "Böb", Address: "bob@example.com"}, + `=?utf-8?q?B=C3=B6b?= <bob@example.com>`, + }, + { + &Address{Name: "Bob Jane", Address: "bob@example.com"}, + `"Bob Jane" <bob@example.com>`, + }, + { + &Address{Name: "Böb Jacöb", Address: "bob@example.com"}, + `=?utf-8?q?B=C3=B6b_Jac=C3=B6b?= <bob@example.com>`, + }, + } + for _, test := range tests { + s := test.addr.String() + if s != test.exp { + t.Errorf("Address%+v.String() = %v, want %v", *test.addr, s, test.exp) + } + } +} diff --git a/src/net/mockicmp_test.go b/src/net/mockicmp_test.go new file mode 100644 index 000000000..e742365ea --- /dev/null +++ b/src/net/mockicmp_test.go @@ -0,0 +1,116 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "errors" + +const ( + icmpv4EchoRequest = 8 + icmpv4EchoReply = 0 + icmpv6EchoRequest = 128 + icmpv6EchoReply = 129 +) + +// icmpMessage represents an ICMP message. +type icmpMessage struct { + Type int // type + Code int // code + Checksum int // checksum + Body icmpMessageBody // body +} + +// icmpMessageBody represents an ICMP message body. +type icmpMessageBody interface { + Len() int + Marshal() ([]byte, error) +} + +// Marshal returns the binary enconding of the ICMP echo request or +// reply message m. +func (m *icmpMessage) Marshal() ([]byte, error) { + b := []byte{byte(m.Type), byte(m.Code), 0, 0} + if m.Body != nil && m.Body.Len() != 0 { + mb, err := m.Body.Marshal() + if err != nil { + return nil, err + } + b = append(b, mb...) + } + switch m.Type { + case icmpv6EchoRequest, icmpv6EchoReply: + return b, nil + } + csumcv := len(b) - 1 // checksum coverage + s := uint32(0) + for i := 0; i < csumcv; i += 2 { + s += uint32(b[i+1])<<8 | uint32(b[i]) + } + if csumcv&1 == 0 { + s += uint32(b[csumcv]) + } + s = s>>16 + s&0xffff + s = s + s>>16 + // Place checksum back in header; using ^= avoids the + // assumption the checksum bytes are zero. + b[2] ^= byte(^s) + b[3] ^= byte(^s >> 8) + return b, nil +} + +// parseICMPMessage parses b as an ICMP message. +func parseICMPMessage(b []byte) (*icmpMessage, error) { + msglen := len(b) + if msglen < 4 { + return nil, errors.New("message too short") + } + m := &icmpMessage{Type: int(b[0]), Code: int(b[1]), Checksum: int(b[2])<<8 | int(b[3])} + if msglen > 4 { + var err error + switch m.Type { + case icmpv4EchoRequest, icmpv4EchoReply, icmpv6EchoRequest, icmpv6EchoReply: + m.Body, err = parseICMPEcho(b[4:]) + if err != nil { + return nil, err + } + } + } + return m, nil +} + +// imcpEcho represenets an ICMP echo request or reply message body. +type icmpEcho struct { + ID int // identifier + Seq int // sequence number + Data []byte // data +} + +func (p *icmpEcho) Len() int { + if p == nil { + return 0 + } + return 4 + len(p.Data) +} + +// Marshal returns the binary enconding of the ICMP echo request or +// reply message body p. +func (p *icmpEcho) Marshal() ([]byte, error) { + b := make([]byte, 4+len(p.Data)) + b[0], b[1] = byte(p.ID>>8), byte(p.ID) + b[2], b[3] = byte(p.Seq>>8), byte(p.Seq) + copy(b[4:], p.Data) + return b, nil +} + +// parseICMPEcho parses b as an ICMP echo request or reply message +// body. +func parseICMPEcho(b []byte) (*icmpEcho, error) { + bodylen := len(b) + p := &icmpEcho{ID: int(b[0])<<8 | int(b[1]), Seq: int(b[2])<<8 | int(b[3])} + if bodylen > 4 { + p.Data = make([]byte, bodylen-4) + copy(p.Data, b[4:]) + } + return p, nil +} diff --git a/src/net/mockserver_test.go b/src/net/mockserver_test.go new file mode 100644 index 000000000..68ded5d75 --- /dev/null +++ b/src/net/mockserver_test.go @@ -0,0 +1,82 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "sync" + +type streamListener struct { + net, addr string + ln Listener +} + +type dualStackServer struct { + lnmu sync.RWMutex + lns []streamListener + port string + + cmu sync.RWMutex + cs []Conn // established connections at the passive open side +} + +func (dss *dualStackServer) buildup(server func(*dualStackServer, Listener)) error { + for i := range dss.lns { + go server(dss, dss.lns[i].ln) + } + return nil +} + +func (dss *dualStackServer) putConn(c Conn) error { + dss.cmu.Lock() + dss.cs = append(dss.cs, c) + dss.cmu.Unlock() + return nil +} + +func (dss *dualStackServer) teardownNetwork(net string) error { + dss.lnmu.Lock() + for i := range dss.lns { + if net == dss.lns[i].net && dss.lns[i].ln != nil { + dss.lns[i].ln.Close() + dss.lns[i].ln = nil + } + } + dss.lnmu.Unlock() + return nil +} + +func (dss *dualStackServer) teardown() error { + dss.lnmu.Lock() + for i := range dss.lns { + if dss.lns[i].ln != nil { + dss.lns[i].ln.Close() + } + } + dss.lnmu.Unlock() + dss.cmu.Lock() + for _, c := range dss.cs { + c.Close() + } + dss.cmu.Unlock() + return nil +} + +func newDualStackServer(lns []streamListener) (*dualStackServer, error) { + dss := &dualStackServer{lns: lns, port: "0"} + for i := range dss.lns { + ln, err := Listen(dss.lns[i].net, dss.lns[i].addr+":"+dss.port) + if err != nil { + dss.teardown() + return nil, err + } + dss.lns[i].ln = ln + if dss.port == "0" { + if _, dss.port, err = SplitHostPort(ln.Addr().String()); err != nil { + dss.teardown() + return nil, err + } + } + } + return dss, nil +} diff --git a/src/net/multicast_test.go b/src/net/multicast_test.go new file mode 100644 index 000000000..5f253f44a --- /dev/null +++ b/src/net/multicast_test.go @@ -0,0 +1,188 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "fmt" + "os" + "runtime" + "testing" +) + +var ipv4MulticastListenerTests = []struct { + net string + gaddr *UDPAddr // see RFC 4727 +}{ + {"udp", &UDPAddr{IP: IPv4(224, 0, 0, 254), Port: 12345}}, + + {"udp4", &UDPAddr{IP: IPv4(224, 0, 0, 254), Port: 12345}}, +} + +// TestIPv4MulticastListener tests both single and double listen to a +// test listener with same address family, same group address and same +// port. +func TestIPv4MulticastListener(t *testing.T) { + switch runtime.GOOS { + case "android", "nacl", "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + case "solaris": + t.Skipf("skipping test on solaris, see issue 7399") + } + + closer := func(cs []*UDPConn) { + for _, c := range cs { + if c != nil { + c.Close() + } + } + } + + for _, ifi := range []*Interface{loopbackInterface(), nil} { + // Note that multicast interface assignment by system + // is not recommended because it usually relies on + // routing stuff for finding out an appropriate + // nexthop containing both network and link layer + // adjacencies. + if ifi == nil && !*testExternal { + continue + } + for _, tt := range ipv4MulticastListenerTests { + var err error + cs := make([]*UDPConn, 2) + if cs[0], err = ListenMulticastUDP(tt.net, ifi, tt.gaddr); err != nil { + t.Fatalf("First ListenMulticastUDP on %v failed: %v", ifi, err) + } + if err := checkMulticastListener(cs[0], tt.gaddr.IP); err != nil { + closer(cs) + t.Fatal(err) + } + if cs[1], err = ListenMulticastUDP(tt.net, ifi, tt.gaddr); err != nil { + closer(cs) + t.Fatalf("Second ListenMulticastUDP on %v failed: %v", ifi, err) + } + if err := checkMulticastListener(cs[1], tt.gaddr.IP); err != nil { + closer(cs) + t.Fatal(err) + } + closer(cs) + } + } +} + +var ipv6MulticastListenerTests = []struct { + net string + gaddr *UDPAddr // see RFC 4727 +}{ + {"udp", &UDPAddr{IP: ParseIP("ff01::114"), Port: 12345}}, + {"udp", &UDPAddr{IP: ParseIP("ff02::114"), Port: 12345}}, + {"udp", &UDPAddr{IP: ParseIP("ff04::114"), Port: 12345}}, + {"udp", &UDPAddr{IP: ParseIP("ff05::114"), Port: 12345}}, + {"udp", &UDPAddr{IP: ParseIP("ff08::114"), Port: 12345}}, + {"udp", &UDPAddr{IP: ParseIP("ff0e::114"), Port: 12345}}, + + {"udp6", &UDPAddr{IP: ParseIP("ff01::114"), Port: 12345}}, + {"udp6", &UDPAddr{IP: ParseIP("ff02::114"), Port: 12345}}, + {"udp6", &UDPAddr{IP: ParseIP("ff04::114"), Port: 12345}}, + {"udp6", &UDPAddr{IP: ParseIP("ff05::114"), Port: 12345}}, + {"udp6", &UDPAddr{IP: ParseIP("ff08::114"), Port: 12345}}, + {"udp6", &UDPAddr{IP: ParseIP("ff0e::114"), Port: 12345}}, +} + +// TestIPv6MulticastListener tests both single and double listen to a +// test listener with same address family, same group address and same +// port. +func TestIPv6MulticastListener(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + case "solaris": + t.Skipf("skipping test on solaris, see issue 7399") + } + if !supportsIPv6 { + t.Skip("ipv6 is not supported") + } + if os.Getuid() != 0 { + t.Skip("skipping test; must be root") + } + + closer := func(cs []*UDPConn) { + for _, c := range cs { + if c != nil { + c.Close() + } + } + } + + for _, ifi := range []*Interface{loopbackInterface(), nil} { + // Note that multicast interface assignment by system + // is not recommended because it usually relies on + // routing stuff for finding out an appropriate + // nexthop containing both network and link layer + // adjacencies. + if ifi == nil && (!*testExternal || !*testIPv6) { + continue + } + for _, tt := range ipv6MulticastListenerTests { + var err error + cs := make([]*UDPConn, 2) + if cs[0], err = ListenMulticastUDP(tt.net, ifi, tt.gaddr); err != nil { + t.Fatalf("First ListenMulticastUDP on %v failed: %v", ifi, err) + } + if err := checkMulticastListener(cs[0], tt.gaddr.IP); err != nil { + closer(cs) + t.Fatal(err) + } + if cs[1], err = ListenMulticastUDP(tt.net, ifi, tt.gaddr); err != nil { + closer(cs) + t.Fatalf("Second ListenMulticastUDP on %v failed: %v", ifi, err) + } + if err := checkMulticastListener(cs[1], tt.gaddr.IP); err != nil { + closer(cs) + t.Fatal(err) + } + closer(cs) + } + } +} + +func checkMulticastListener(c *UDPConn, ip IP) error { + if ok, err := multicastRIBContains(ip); err != nil { + return err + } else if !ok { + return fmt.Errorf("%q not found in multicast RIB", ip.String()) + } + la := c.LocalAddr() + if la, ok := la.(*UDPAddr); !ok || la.Port == 0 { + return fmt.Errorf("got %v; expected a proper address with non-zero port number", la) + } + return nil +} + +func multicastRIBContains(ip IP) (bool, error) { + switch runtime.GOOS { + case "dragonfly", "netbsd", "openbsd", "plan9", "solaris", "windows": + return true, nil // not implemented yet + case "linux": + if runtime.GOARCH == "arm" || runtime.GOARCH == "alpha" { + return true, nil // not implemented yet + } + } + ift, err := Interfaces() + if err != nil { + return false, err + } + for _, ifi := range ift { + ifmat, err := ifi.MulticastAddrs() + if err != nil { + return false, err + } + for _, ifma := range ifmat { + if ifma.(*IPAddr).IP.Equal(ip) { + return true, nil + } + } + } + return false, nil +} diff --git a/src/net/net.go b/src/net/net.go new file mode 100644 index 000000000..ca56af54f --- /dev/null +++ b/src/net/net.go @@ -0,0 +1,426 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package net provides a portable interface for network I/O, including +TCP/IP, UDP, domain name resolution, and Unix domain sockets. + +Although the package provides access to low-level networking +primitives, most clients will need only the basic interface provided +by the Dial, Listen, and Accept functions and the associated +Conn and Listener interfaces. The crypto/tls package uses +the same interfaces and similar Dial and Listen functions. + +The Dial function connects to a server: + + conn, err := net.Dial("tcp", "google.com:80") + if err != nil { + // handle error + } + fmt.Fprintf(conn, "GET / HTTP/1.0\r\n\r\n") + status, err := bufio.NewReader(conn).ReadString('\n') + // ... + +The Listen function creates servers: + + ln, err := net.Listen("tcp", ":8080") + if err != nil { + // handle error + } + for { + conn, err := ln.Accept() + if err != nil { + // handle error + continue + } + go handleConnection(conn) + } +*/ +package net + +// TODO(rsc): +// support for raw ethernet sockets + +import ( + "errors" + "io" + "os" + "syscall" + "time" +) + +// Addr represents a network end point address. +type Addr interface { + Network() string // name of the network + String() string // string form of address +} + +// Conn is a generic stream-oriented network connection. +// +// Multiple goroutines may invoke methods on a Conn simultaneously. +type Conn interface { + // Read reads data from the connection. + // Read can be made to time out and return a Error with Timeout() == true + // after a fixed time limit; see SetDeadline and SetReadDeadline. + Read(b []byte) (n int, err error) + + // Write writes data to the connection. + // Write can be made to time out and return a Error with Timeout() == true + // after a fixed time limit; see SetDeadline and SetWriteDeadline. + Write(b []byte) (n int, err error) + + // Close closes the connection. + // Any blocked Read or Write operations will be unblocked and return errors. + Close() error + + // LocalAddr returns the local network address. + LocalAddr() Addr + + // RemoteAddr returns the remote network address. + RemoteAddr() Addr + + // SetDeadline sets the read and write deadlines associated + // with the connection. It is equivalent to calling both + // SetReadDeadline and SetWriteDeadline. + // + // A deadline is an absolute time after which I/O operations + // fail with a timeout (see type Error) instead of + // blocking. The deadline applies to all future I/O, not just + // the immediately following call to Read or Write. + // + // An idle timeout can be implemented by repeatedly extending + // the deadline after successful Read or Write calls. + // + // A zero value for t means I/O operations will not time out. + SetDeadline(t time.Time) error + + // SetReadDeadline sets the deadline for future Read calls. + // A zero value for t means Read will not time out. + SetReadDeadline(t time.Time) error + + // SetWriteDeadline sets the deadline for future Write calls. + // Even if write times out, it may return n > 0, indicating that + // some of the data was successfully written. + // A zero value for t means Write will not time out. + SetWriteDeadline(t time.Time) error +} + +type conn struct { + fd *netFD +} + +func (c *conn) ok() bool { return c != nil && c.fd != nil } + +// Implementation of the Conn interface. + +// Read implements the Conn Read method. +func (c *conn) Read(b []byte) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + return c.fd.Read(b) +} + +// Write implements the Conn Write method. +func (c *conn) Write(b []byte) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + return c.fd.Write(b) +} + +// Close closes the connection. +func (c *conn) Close() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.Close() +} + +// LocalAddr returns the local network address. +func (c *conn) LocalAddr() Addr { + if !c.ok() { + return nil + } + return c.fd.laddr +} + +// RemoteAddr returns the remote network address. +func (c *conn) RemoteAddr() Addr { + if !c.ok() { + return nil + } + return c.fd.raddr +} + +// SetDeadline implements the Conn SetDeadline method. +func (c *conn) SetDeadline(t time.Time) error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.setDeadline(t) +} + +// SetReadDeadline implements the Conn SetReadDeadline method. +func (c *conn) SetReadDeadline(t time.Time) error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.setReadDeadline(t) +} + +// SetWriteDeadline implements the Conn SetWriteDeadline method. +func (c *conn) SetWriteDeadline(t time.Time) error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.setWriteDeadline(t) +} + +// SetReadBuffer sets the size of the operating system's +// receive buffer associated with the connection. +func (c *conn) SetReadBuffer(bytes int) error { + if !c.ok() { + return syscall.EINVAL + } + return setReadBuffer(c.fd, bytes) +} + +// SetWriteBuffer sets the size of the operating system's +// transmit buffer associated with the connection. +func (c *conn) SetWriteBuffer(bytes int) error { + if !c.ok() { + return syscall.EINVAL + } + return setWriteBuffer(c.fd, bytes) +} + +// File sets the underlying os.File to blocking mode and returns a copy. +// It is the caller's responsibility to close f when finished. +// Closing c does not affect f, and closing f does not affect c. +// +// The returned os.File's file descriptor is different from the connection's. +// Attempting to change properties of the original using this duplicate +// may or may not have the desired effect. +func (c *conn) File() (f *os.File, err error) { return c.fd.dup() } + +// An Error represents a network error. +type Error interface { + error + Timeout() bool // Is the error a timeout? + Temporary() bool // Is the error temporary? +} + +// PacketConn is a generic packet-oriented network connection. +// +// Multiple goroutines may invoke methods on a PacketConn simultaneously. +type PacketConn interface { + // ReadFrom reads a packet from the connection, + // copying the payload into b. It returns the number of + // bytes copied into b and the return address that + // was on the packet. + // ReadFrom can be made to time out and return + // an error with Timeout() == true after a fixed time limit; + // see SetDeadline and SetReadDeadline. + ReadFrom(b []byte) (n int, addr Addr, err error) + + // WriteTo writes a packet with payload b to addr. + // WriteTo can be made to time out and return + // an error with Timeout() == true after a fixed time limit; + // see SetDeadline and SetWriteDeadline. + // On packet-oriented connections, write timeouts are rare. + WriteTo(b []byte, addr Addr) (n int, err error) + + // Close closes the connection. + // Any blocked ReadFrom or WriteTo operations will be unblocked and return errors. + Close() error + + // LocalAddr returns the local network address. + LocalAddr() Addr + + // SetDeadline sets the read and write deadlines associated + // with the connection. + SetDeadline(t time.Time) error + + // SetReadDeadline sets the deadline for future Read calls. + // If the deadline is reached, Read will fail with a timeout + // (see type Error) instead of blocking. + // A zero value for t means Read will not time out. + SetReadDeadline(t time.Time) error + + // SetWriteDeadline sets the deadline for future Write calls. + // If the deadline is reached, Write will fail with a timeout + // (see type Error) instead of blocking. + // A zero value for t means Write will not time out. + // Even if write times out, it may return n > 0, indicating that + // some of the data was successfully written. + SetWriteDeadline(t time.Time) error +} + +var listenerBacklog = maxListenerBacklog() + +// A Listener is a generic network listener for stream-oriented protocols. +// +// Multiple goroutines may invoke methods on a Listener simultaneously. +type Listener interface { + // Accept waits for and returns the next connection to the listener. + Accept() (c Conn, err error) + + // Close closes the listener. + // Any blocked Accept operations will be unblocked and return errors. + Close() error + + // Addr returns the listener's network address. + Addr() Addr +} + +// Various errors contained in OpError. +var ( + // For connection setup and write operations. + errMissingAddress = errors.New("missing address") + + // For both read and write operations. + errTimeout error = &timeoutError{} + errClosing = errors.New("use of closed network connection") + ErrWriteToConnected = errors.New("use of WriteTo with pre-connected connection") +) + +// OpError is the error type usually returned by functions in the net +// package. It describes the operation, network type, and address of +// an error. +type OpError struct { + // Op is the operation which caused the error, such as + // "read" or "write". + Op string + + // Net is the network type on which this error occurred, + // such as "tcp" or "udp6". + Net string + + // Addr is the network address on which this error occurred. + Addr Addr + + // Err is the error that occurred during the operation. + Err error +} + +func (e *OpError) Error() string { + if e == nil { + return "<nil>" + } + s := e.Op + if e.Net != "" { + s += " " + e.Net + } + if e.Addr != nil { + s += " " + e.Addr.String() + } + s += ": " + e.Err.Error() + return s +} + +type temporary interface { + Temporary() bool +} + +func (e *OpError) Temporary() bool { + t, ok := e.Err.(temporary) + return ok && t.Temporary() +} + +var noDeadline = time.Time{} + +type timeout interface { + Timeout() bool +} + +func (e *OpError) Timeout() bool { + t, ok := e.Err.(timeout) + return ok && t.Timeout() +} + +type timeoutError struct{} + +func (e *timeoutError) Error() string { return "i/o timeout" } +func (e *timeoutError) Timeout() bool { return true } +func (e *timeoutError) Temporary() bool { return true } + +type AddrError struct { + Err string + Addr string +} + +func (e *AddrError) Error() string { + if e == nil { + return "<nil>" + } + s := e.Err + if e.Addr != "" { + s += " " + e.Addr + } + return s +} + +func (e *AddrError) Temporary() bool { + return false +} + +func (e *AddrError) Timeout() bool { + return false +} + +type UnknownNetworkError string + +func (e UnknownNetworkError) Error() string { return "unknown network " + string(e) } +func (e UnknownNetworkError) Temporary() bool { return false } +func (e UnknownNetworkError) Timeout() bool { return false } + +type InvalidAddrError string + +func (e InvalidAddrError) Error() string { return string(e) } +func (e InvalidAddrError) Timeout() bool { return false } +func (e InvalidAddrError) Temporary() bool { return false } + +// DNSConfigError represents an error reading the machine's DNS configuration. +type DNSConfigError struct { + Err error +} + +func (e *DNSConfigError) Error() string { + return "error reading DNS config: " + e.Err.Error() +} + +func (e *DNSConfigError) Timeout() bool { return false } +func (e *DNSConfigError) Temporary() bool { return false } + +type writerOnly struct { + io.Writer +} + +// Fallback implementation of io.ReaderFrom's ReadFrom, when sendfile isn't +// applicable. +func genericReadFrom(w io.Writer, r io.Reader) (n int64, err error) { + // Use wrapper to hide existing r.ReadFrom from io.Copy. + return io.Copy(writerOnly{w}, r) +} + +// Limit the number of concurrent cgo-using goroutines, because +// each will block an entire operating system thread. The usual culprit +// is resolving many DNS names in separate goroutines but the DNS +// server is not responding. Then the many lookups each use a different +// thread, and the system or the program runs out of threads. + +var threadLimit = make(chan struct{}, 500) + +// Using send for acquire is fine here because we are not using this +// to protect any memory. All we care about is the number of goroutines +// making calls at a time. + +func acquireThread() { + threadLimit <- struct{}{} +} + +func releaseThread() { + <-threadLimit +} diff --git a/src/net/net_test.go b/src/net/net_test.go new file mode 100644 index 000000000..bfed4d657 --- /dev/null +++ b/src/net/net_test.go @@ -0,0 +1,263 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "io/ioutil" + "os" + "runtime" + "testing" + "time" +) + +func TestShutdown(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skipf("skipping test on %q", runtime.GOOS) + } + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + if ln, err = Listen("tcp6", "[::1]:0"); err != nil { + t.Fatalf("ListenTCP on :0: %v", err) + } + } + + go func() { + defer ln.Close() + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + var buf [10]byte + n, err := c.Read(buf[:]) + if n != 0 || err != io.EOF { + t.Errorf("server Read = %d, %v; want 0, io.EOF", n, err) + return + } + c.Write([]byte("response")) + c.Close() + }() + + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + + err = c.(*TCPConn).CloseWrite() + if err != nil { + t.Fatalf("CloseWrite: %v", err) + } + var buf [10]byte + n, err := c.Read(buf[:]) + if err != nil { + t.Fatalf("client Read: %d, %v", n, err) + } + got := string(buf[:n]) + if got != "response" { + t.Errorf("read = %q, want \"response\"", got) + } +} + +func TestShutdownUnix(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + } + f, err := ioutil.TempFile("", "go_net_unixtest") + if err != nil { + t.Fatalf("TempFile: %s", err) + } + f.Close() + tmpname := f.Name() + os.Remove(tmpname) + ln, err := Listen("unix", tmpname) + if err != nil { + t.Fatalf("ListenUnix on %s: %s", tmpname, err) + } + defer func() { + ln.Close() + os.Remove(tmpname) + }() + + go func() { + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + var buf [10]byte + n, err := c.Read(buf[:]) + if n != 0 || err != io.EOF { + t.Errorf("server Read = %d, %v; want 0, io.EOF", n, err) + return + } + c.Write([]byte("response")) + c.Close() + }() + + c, err := Dial("unix", tmpname) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + + err = c.(*UnixConn).CloseWrite() + if err != nil { + t.Fatalf("CloseWrite: %v", err) + } + var buf [10]byte + n, err := c.Read(buf[:]) + if err != nil { + t.Fatalf("client Read: %d, %v", n, err) + } + got := string(buf[:n]) + if got != "response" { + t.Errorf("read = %q, want \"response\"", got) + } +} + +func TestTCPListenClose(t *testing.T) { + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + + done := make(chan bool, 1) + go func() { + time.Sleep(100 * time.Millisecond) + ln.Close() + }() + go func() { + c, err := ln.Accept() + if err == nil { + c.Close() + t.Error("Accept succeeded") + } else { + t.Logf("Accept timeout error: %s (any error is fine)", err) + } + done <- true + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for TCP close") + } +} + +func TestUDPListenClose(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + ln, err := ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + + buf := make([]byte, 1000) + done := make(chan bool, 1) + go func() { + time.Sleep(100 * time.Millisecond) + ln.Close() + }() + go func() { + _, _, err = ln.ReadFrom(buf) + if err == nil { + t.Error("ReadFrom succeeded") + } else { + t.Logf("ReadFrom timeout error: %s (any error is fine)", err) + } + done <- true + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("timeout waiting for UDP close") + } +} + +func TestTCPClose(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + l, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + + read := func(r io.Reader) error { + var m [1]byte + _, err := r.Read(m[:]) + return err + } + + go func() { + c, err := Dial("tcp", l.Addr().String()) + if err != nil { + t.Errorf("Dial: %v", err) + return + } + + go read(c) + + time.Sleep(10 * time.Millisecond) + c.Close() + }() + + c, err := l.Accept() + if err != nil { + t.Fatal(err) + } + defer c.Close() + + for err == nil { + err = read(c) + } + if err != nil && err != io.EOF { + t.Fatal(err) + } +} + +func TestErrorNil(t *testing.T) { + c, err := Dial("tcp", "127.0.0.1:65535") + if err == nil { + t.Fatal("Dial 127.0.0.1:65535 succeeded") + } + if c != nil { + t.Fatalf("Dial returned non-nil interface %T(%v) with err != nil", c, c) + } + + // Make Listen fail by relistening on the same address. + l, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen 127.0.0.1:0: %v", err) + } + defer l.Close() + l1, err := Listen("tcp", l.Addr().String()) + if err == nil { + t.Fatalf("second Listen %v: %v", l.Addr(), err) + } + if l1 != nil { + t.Fatalf("Listen returned non-nil interface %T(%v) with err != nil", l1, l1) + } + + // Make ListenPacket fail by relistening on the same address. + lp, err := ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen 127.0.0.1:0: %v", err) + } + defer lp.Close() + lp1, err := ListenPacket("udp", lp.LocalAddr().String()) + if err == nil { + t.Fatalf("second Listen %v: %v", lp.LocalAddr(), err) + } + if lp1 != nil { + t.Fatalf("ListenPacket returned non-nil interface %T(%v) with err != nil", lp1, lp1) + } +} diff --git a/src/net/net_windows_test.go b/src/net/net_windows_test.go new file mode 100644 index 000000000..750a4304b --- /dev/null +++ b/src/net/net_windows_test.go @@ -0,0 +1,148 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bufio" + "fmt" + "io" + "os" + "os/exec" + "syscall" + "testing" + "time" +) + +func TestAcceptIgnoreSomeErrors(t *testing.T) { + t.Skip("skipping temporarily, see issue 8662") + + recv := func(ln Listener) (string, error) { + c, err := ln.Accept() + if err != nil { + // Display windows errno in error message. + operr, ok := err.(*OpError) + if !ok { + return "", err + } + errno, ok := operr.Err.(syscall.Errno) + if !ok { + return "", err + } + return "", fmt.Errorf("%v (windows errno=%d)", err, errno) + } + defer c.Close() + + b := make([]byte, 100) + n, err := c.Read(b) + if err != nil && err != io.EOF { + return "", err + } + return string(b[:n]), nil + } + + send := func(addr string, data string) error { + c, err := Dial("tcp", addr) + if err != nil { + return err + } + defer c.Close() + + b := []byte(data) + n, err := c.Write(b) + if err != nil { + return err + } + if n != len(b) { + return fmt.Errorf(`Only %d chars of string "%s" sent`, n, data) + } + return nil + } + + if envaddr := os.Getenv("GOTEST_DIAL_ADDR"); envaddr != "" { + // In child process. + c, err := Dial("tcp", envaddr) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + fmt.Printf("sleeping\n") + time.Sleep(time.Minute) // process will be killed here + c.Close() + } + + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer ln.Close() + + // Start child process that connects to our listener. + cmd := exec.Command(os.Args[0], "-test.run=TestAcceptIgnoreSomeErrors") + cmd.Env = append(os.Environ(), "GOTEST_DIAL_ADDR="+ln.Addr().String()) + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatalf("cmd.StdoutPipe failed: %v", err) + } + err = cmd.Start() + if err != nil { + t.Fatalf("cmd.Start failed: %v\n", err) + } + outReader := bufio.NewReader(stdout) + for { + s, err := outReader.ReadString('\n') + if err != nil { + t.Fatalf("reading stdout failed: %v", err) + } + if s == "sleeping\n" { + break + } + } + defer cmd.Wait() // ignore error - we know it is getting killed + + const alittle = 100 * time.Millisecond + time.Sleep(alittle) + cmd.Process.Kill() // the only way to trigger the errors + time.Sleep(alittle) + + // Send second connection data (with delay in a separate goroutine). + result := make(chan error) + go func() { + time.Sleep(alittle) + err := send(ln.Addr().String(), "abc") + if err != nil { + result <- err + } + result <- nil + }() + defer func() { + err := <-result + if err != nil { + t.Fatalf("send failed: %v", err) + } + }() + + // Receive first or second connection. + s, err := recv(ln) + if err != nil { + t.Fatalf("recv failed: %v", err) + } + switch s { + case "": + // First connection data is received, lets get second connection data. + case "abc": + // First connection is lost forever, but that is ok. + return + default: + t.Fatalf(`"%s" received from recv, but "" or "abc" expected`, s) + } + + // Get second connection data. + s, err = recv(ln) + if err != nil { + t.Fatalf("recv failed: %v", err) + } + if s != "abc" { + t.Fatalf(`"%s" received from recv, but "abc" expected`, s) + } +} diff --git a/src/net/netgo_unix_test.go b/src/net/netgo_unix_test.go new file mode 100644 index 000000000..9fb2a567d --- /dev/null +++ b/src/net/netgo_unix_test.go @@ -0,0 +1,24 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !cgo netgo +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package net + +import "testing" + +func TestGoLookupIP(t *testing.T) { + host := "localhost" + _, err, ok := cgoLookupIP(host) + if ok { + t.Errorf("cgoLookupIP must be a placeholder") + } + if err != nil { + t.Errorf("cgoLookupIP failed: %v", err) + } + if _, err := goLookupIP(host); err != nil { + t.Errorf("goLookupIP failed: %v", err) + } +} diff --git a/src/net/packetconn_test.go b/src/net/packetconn_test.go new file mode 100644 index 000000000..b6e4e76f9 --- /dev/null +++ b/src/net/packetconn_test.go @@ -0,0 +1,186 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements API tests across platforms and will never have a build +// tag. + +package net + +import ( + "os" + "runtime" + "strings" + "testing" + "time" +) + +func packetConnTestData(t *testing.T, net string, i int) ([]byte, func()) { + switch net { + case "udp": + return []byte("UDP PACKETCONN TEST"), nil + case "ip": + if skip, skipmsg := skipRawSocketTest(t); skip { + return nil, func() { + t.Logf(skipmsg) + } + } + b, err := (&icmpMessage{ + Type: icmpv4EchoRequest, Code: 0, + Body: &icmpEcho{ + ID: os.Getpid() & 0xffff, Seq: i + 1, + Data: []byte("IP PACKETCONN TEST"), + }, + }).Marshal() + if err != nil { + return nil, func() { + t.Fatalf("icmpMessage.Marshal failed: %v", err) + } + } + return b, nil + case "unixgram": + switch runtime.GOOS { + case "nacl", "plan9", "windows": + return nil, func() { + t.Logf("skipping %q test on %q", net, runtime.GOOS) + } + default: + return []byte("UNIXGRAM PACKETCONN TEST"), nil + } + default: + return nil, func() { + t.Logf("skipping %q test", net) + } + } +} + +var packetConnTests = []struct { + net string + addr1 string + addr2 string +}{ + {"udp", "127.0.0.1:0", "127.0.0.1:0"}, + {"ip:icmp", "127.0.0.1", "127.0.0.1"}, + {"unixgram", testUnixAddr(), testUnixAddr()}, +} + +func TestPacketConn(t *testing.T) { + closer := func(c PacketConn, net, addr1, addr2 string) { + c.Close() + switch net { + case "unixgram": + os.Remove(addr1) + os.Remove(addr2) + } + } + + for i, tt := range packetConnTests { + netstr := strings.Split(tt.net, ":") + wb, skipOrFatalFn := packetConnTestData(t, netstr[0], i) + if skipOrFatalFn != nil { + skipOrFatalFn() + continue + } + + c1, err := ListenPacket(tt.net, tt.addr1) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + defer closer(c1, netstr[0], tt.addr1, tt.addr2) + c1.LocalAddr() + c1.SetDeadline(time.Now().Add(100 * time.Millisecond)) + c1.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + c1.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) + + c2, err := ListenPacket(tt.net, tt.addr2) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + defer closer(c2, netstr[0], tt.addr1, tt.addr2) + c2.LocalAddr() + c2.SetDeadline(time.Now().Add(100 * time.Millisecond)) + c2.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + c2.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) + + if _, err := c1.WriteTo(wb, c2.LocalAddr()); err != nil { + t.Fatalf("PacketConn.WriteTo failed: %v", err) + } + rb2 := make([]byte, 128) + if _, _, err := c2.ReadFrom(rb2); err != nil { + t.Fatalf("PacketConn.ReadFrom failed: %v", err) + } + if _, err := c2.WriteTo(wb, c1.LocalAddr()); err != nil { + t.Fatalf("PacketConn.WriteTo failed: %v", err) + } + rb1 := make([]byte, 128) + if _, _, err := c1.ReadFrom(rb1); err != nil { + t.Fatalf("PacketConn.ReadFrom failed: %v", err) + } + } +} + +func TestConnAndPacketConn(t *testing.T) { + closer := func(c PacketConn, net, addr1, addr2 string) { + c.Close() + switch net { + case "unixgram": + os.Remove(addr1) + os.Remove(addr2) + } + } + + for i, tt := range packetConnTests { + var wb []byte + netstr := strings.Split(tt.net, ":") + wb, skipOrFatalFn := packetConnTestData(t, netstr[0], i) + if skipOrFatalFn != nil { + skipOrFatalFn() + continue + } + + c1, err := ListenPacket(tt.net, tt.addr1) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + defer closer(c1, netstr[0], tt.addr1, tt.addr2) + c1.LocalAddr() + c1.SetDeadline(time.Now().Add(100 * time.Millisecond)) + c1.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + c1.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) + + c2, err := Dial(tt.net, c1.LocalAddr().String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c2.Close() + c2.LocalAddr() + c2.RemoteAddr() + c2.SetDeadline(time.Now().Add(100 * time.Millisecond)) + c2.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + c2.SetWriteDeadline(time.Now().Add(100 * time.Millisecond)) + + if _, err := c2.Write(wb); err != nil { + t.Fatalf("Conn.Write failed: %v", err) + } + rb1 := make([]byte, 128) + if _, _, err := c1.ReadFrom(rb1); err != nil { + t.Fatalf("PacketConn.ReadFrom failed: %v", err) + } + var dst Addr + switch netstr[0] { + case "ip": + dst = &IPAddr{IP: IPv4(127, 0, 0, 1)} + case "unixgram": + continue + default: + dst = c2.LocalAddr() + } + if _, err := c1.WriteTo(wb, dst); err != nil { + t.Fatalf("PacketConn.WriteTo failed: %v", err) + } + rb2 := make([]byte, 128) + if _, err := c2.Read(rb2); err != nil { + t.Fatalf("Conn.Read failed: %v", err) + } + } +} diff --git a/src/net/parse.go b/src/net/parse.go new file mode 100644 index 000000000..e1d0130c9 --- /dev/null +++ b/src/net/parse.go @@ -0,0 +1,247 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Simple file i/o and string manipulation, to avoid +// depending on strconv and bufio and strings. + +package net + +import ( + "io" + "os" +) + +type file struct { + file *os.File + data []byte + atEOF bool +} + +func (f *file) close() { f.file.Close() } + +func (f *file) getLineFromData() (s string, ok bool) { + data := f.data + i := 0 + for i = 0; i < len(data); i++ { + if data[i] == '\n' { + s = string(data[0:i]) + ok = true + // move data + i++ + n := len(data) - i + copy(data[0:], data[i:]) + f.data = data[0:n] + return + } + } + if f.atEOF && len(f.data) > 0 { + // EOF, return all we have + s = string(data) + f.data = f.data[0:0] + ok = true + } + return +} + +func (f *file) readLine() (s string, ok bool) { + if s, ok = f.getLineFromData(); ok { + return + } + if len(f.data) < cap(f.data) { + ln := len(f.data) + n, err := io.ReadFull(f.file, f.data[ln:cap(f.data)]) + if n >= 0 { + f.data = f.data[0 : ln+n] + } + if err == io.EOF || err == io.ErrUnexpectedEOF { + f.atEOF = true + } + } + s, ok = f.getLineFromData() + return +} + +func open(name string) (*file, error) { + fd, err := os.Open(name) + if err != nil { + return nil, err + } + return &file{fd, make([]byte, 0, os.Getpagesize()), false}, nil +} + +func byteIndex(s string, c byte) int { + for i := 0; i < len(s); i++ { + if s[i] == c { + return i + } + } + return -1 +} + +// Count occurrences in s of any bytes in t. +func countAnyByte(s string, t string) int { + n := 0 + for i := 0; i < len(s); i++ { + if byteIndex(t, s[i]) >= 0 { + n++ + } + } + return n +} + +// Split s at any bytes in t. +func splitAtBytes(s string, t string) []string { + a := make([]string, 1+countAnyByte(s, t)) + n := 0 + last := 0 + for i := 0; i < len(s); i++ { + if byteIndex(t, s[i]) >= 0 { + if last < i { + a[n] = string(s[last:i]) + n++ + } + last = i + 1 + } + } + if last < len(s) { + a[n] = string(s[last:]) + n++ + } + return a[0:n] +} + +func getFields(s string) []string { return splitAtBytes(s, " \r\t\n") } + +// Bigger than we need, not too big to worry about overflow +const big = 0xFFFFFF + +// Decimal to integer starting at &s[i0]. +// Returns number, new offset, success. +func dtoi(s string, i0 int) (n int, i int, ok bool) { + n = 0 + for i = i0; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + n = n*10 + int(s[i]-'0') + if n >= big { + return 0, i, false + } + } + if i == i0 { + return 0, i, false + } + return n, i, true +} + +// Hexadecimal to integer starting at &s[i0]. +// Returns number, new offset, success. +func xtoi(s string, i0 int) (n int, i int, ok bool) { + n = 0 + for i = i0; i < len(s); i++ { + if '0' <= s[i] && s[i] <= '9' { + n *= 16 + n += int(s[i] - '0') + } else if 'a' <= s[i] && s[i] <= 'f' { + n *= 16 + n += int(s[i]-'a') + 10 + } else if 'A' <= s[i] && s[i] <= 'F' { + n *= 16 + n += int(s[i]-'A') + 10 + } else { + break + } + if n >= big { + return 0, i, false + } + } + if i == i0 { + return 0, i, false + } + return n, i, true +} + +// xtoi2 converts the next two hex digits of s into a byte. +// If s is longer than 2 bytes then the third byte must be e. +// If the first two bytes of s are not hex digits or the third byte +// does not match e, false is returned. +func xtoi2(s string, e byte) (byte, bool) { + if len(s) > 2 && s[2] != e { + return 0, false + } + n, ei, ok := xtoi(s[:2], 0) + return byte(n), ok && ei == 2 +} + +// Integer to decimal. +func itoa(i int) string { + var buf [30]byte + n := len(buf) + neg := false + if i < 0 { + i = -i + neg = true + } + ui := uint(i) + for ui > 0 || n == len(buf) { + n-- + buf[n] = byte('0' + ui%10) + ui /= 10 + } + if neg { + n-- + buf[n] = '-' + } + return string(buf[n:]) +} + +// Convert i to decimal string. +func itod(i uint) string { + if i == 0 { + return "0" + } + + // Assemble decimal in reverse order. + var b [32]byte + bp := len(b) + for ; i > 0; i /= 10 { + bp-- + b[bp] = byte(i%10) + '0' + } + + return string(b[bp:]) +} + +// Convert i to a hexadecimal string. Leading zeros are not printed. +func appendHex(dst []byte, i uint32) []byte { + if i == 0 { + return append(dst, '0') + } + for j := 7; j >= 0; j-- { + v := i >> uint(j*4) + if v > 0 { + dst = append(dst, hexDigit[v&0xf]) + } + } + return dst +} + +// Number of occurrences of b in s. +func count(s string, b byte) int { + n := 0 + for i := 0; i < len(s); i++ { + if s[i] == b { + n++ + } + } + return n +} + +// Index of rightmost occurrence of b in s. +func last(s string, b byte) int { + i := len(s) + for i--; i >= 0; i-- { + if s[i] == b { + break + } + } + return i +} diff --git a/src/net/parse_test.go b/src/net/parse_test.go new file mode 100644 index 000000000..7b213b75b --- /dev/null +++ b/src/net/parse_test.go @@ -0,0 +1,53 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bufio" + "os" + "runtime" + "testing" +) + +func TestReadLine(t *testing.T) { + // /etc/services file does not exist on android, plan9, windows. + switch runtime.GOOS { + case "android", "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + } + filename := "/etc/services" // a nice big file + + fd, err := os.Open(filename) + if err != nil { + t.Fatalf("open %s: %v", filename, err) + } + defer fd.Close() + br := bufio.NewReader(fd) + + file, err := open(filename) + if file == nil { + t.Fatalf("net.open(%s) = nil", filename) + } + defer file.close() + + lineno := 1 + byteno := 0 + for { + bline, berr := br.ReadString('\n') + if n := len(bline); n > 0 { + bline = bline[0 : n-1] + } + line, ok := file.readLine() + if (berr != nil) != !ok || bline != line { + t.Fatalf("%s:%d (#%d)\nbufio => %q, %v\nnet => %q, %v", + filename, lineno, byteno, bline, berr, line, ok) + } + if !ok { + break + } + lineno++ + byteno += len(line) + 1 + } +} diff --git a/src/net/pipe.go b/src/net/pipe.go new file mode 100644 index 000000000..f1a2eca4e --- /dev/null +++ b/src/net/pipe.go @@ -0,0 +1,67 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "io" + "time" +) + +// Pipe creates a synchronous, in-memory, full duplex +// network connection; both ends implement the Conn interface. +// Reads on one end are matched with writes on the other, +// copying data directly between the two; there is no internal +// buffering. +func Pipe() (Conn, Conn) { + r1, w1 := io.Pipe() + r2, w2 := io.Pipe() + + return &pipe{r1, w2}, &pipe{r2, w1} +} + +type pipe struct { + *io.PipeReader + *io.PipeWriter +} + +type pipeAddr int + +func (pipeAddr) Network() string { + return "pipe" +} + +func (pipeAddr) String() string { + return "pipe" +} + +func (p *pipe) Close() error { + err := p.PipeReader.Close() + err1 := p.PipeWriter.Close() + if err == nil { + err = err1 + } + return err +} + +func (p *pipe) LocalAddr() Addr { + return pipeAddr(0) +} + +func (p *pipe) RemoteAddr() Addr { + return pipeAddr(0) +} + +func (p *pipe) SetDeadline(t time.Time) error { + return errors.New("net.Pipe does not support deadlines") +} + +func (p *pipe) SetReadDeadline(t time.Time) error { + return errors.New("net.Pipe does not support deadlines") +} + +func (p *pipe) SetWriteDeadline(t time.Time) error { + return errors.New("net.Pipe does not support deadlines") +} diff --git a/src/net/pipe_test.go b/src/net/pipe_test.go new file mode 100644 index 000000000..afe4f2408 --- /dev/null +++ b/src/net/pipe_test.go @@ -0,0 +1,56 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "io" + "testing" +) + +func checkWrite(t *testing.T, w io.Writer, data []byte, c chan int) { + n, err := w.Write(data) + if err != nil { + t.Errorf("write: %v", err) + } + if n != len(data) { + t.Errorf("short write: %d != %d", n, len(data)) + } + c <- 0 +} + +func checkRead(t *testing.T, r io.Reader, data []byte, wantErr error) { + buf := make([]byte, len(data)+10) + n, err := r.Read(buf) + if err != wantErr { + t.Errorf("read: %v", err) + return + } + if n != len(data) || !bytes.Equal(buf[0:n], data) { + t.Errorf("bad read: got %q", buf[0:n]) + return + } +} + +// Test a simple read/write/close sequence. +// Assumes that the underlying io.Pipe implementation +// is solid and we're just testing the net wrapping. + +func TestPipe(t *testing.T) { + c := make(chan int) + cli, srv := Pipe() + go checkWrite(t, cli, []byte("hello, world"), c) + checkRead(t, srv, []byte("hello, world"), nil) + <-c + go checkWrite(t, srv, []byte("line 2"), c) + checkRead(t, cli, []byte("line 2"), nil) + <-c + go checkWrite(t, cli, []byte("a third line"), c) + checkRead(t, srv, []byte("a third line"), nil) + <-c + go srv.Close() + checkRead(t, cli, nil, io.EOF) + cli.Close() +} diff --git a/src/net/port.go b/src/net/port.go new file mode 100644 index 000000000..c24f4ed5b --- /dev/null +++ b/src/net/port.go @@ -0,0 +1,24 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Network service port manipulations + +package net + +// parsePort parses port as a network service port number for both +// TCP and UDP. +func parsePort(net, port string) (int, error) { + p, i, ok := dtoi(port, 0) + if !ok || i != len(port) { + var err error + p, err = LookupPort(net, port) + if err != nil { + return 0, err + } + } + if p < 0 || p > 0xFFFF { + return 0, &AddrError{"invalid port", port} + } + return p, nil +} diff --git a/src/net/port_test.go b/src/net/port_test.go new file mode 100644 index 000000000..9e8968f35 --- /dev/null +++ b/src/net/port_test.go @@ -0,0 +1,53 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "testing" +) + +type portTest struct { + netw string + name string + port int + ok bool +} + +var porttests = []portTest{ + {"tcp", "echo", 7, true}, + {"tcp", "discard", 9, true}, + {"tcp", "systat", 11, true}, + {"tcp", "daytime", 13, true}, + {"tcp", "chargen", 19, true}, + {"tcp", "ftp-data", 20, true}, + {"tcp", "ftp", 21, true}, + {"tcp", "telnet", 23, true}, + {"tcp", "smtp", 25, true}, + {"tcp", "time", 37, true}, + {"tcp", "domain", 53, true}, + {"tcp", "finger", 79, true}, + + {"udp", "echo", 7, true}, + {"udp", "tftp", 69, true}, + {"udp", "bootpc", 68, true}, + {"udp", "bootps", 67, true}, + {"udp", "domain", 53, true}, + {"udp", "ntp", 123, true}, + {"udp", "snmp", 161, true}, + {"udp", "syslog", 514, true}, + + {"--badnet--", "zzz", 0, false}, + {"tcp", "--badport--", 0, false}, +} + +func TestLookupPort(t *testing.T) { + for i := 0; i < len(porttests); i++ { + tt := porttests[i] + if port, err := LookupPort(tt.netw, tt.name); port != tt.port || (err == nil) != tt.ok { + t.Errorf("LookupPort(%q, %q) = %v, %v; want %v", + tt.netw, tt.name, port, err, tt.port) + } + } +} diff --git a/src/net/port_unix.go b/src/net/port_unix.go new file mode 100644 index 000000000..89558c1f0 --- /dev/null +++ b/src/net/port_unix.go @@ -0,0 +1,73 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris + +// Read system port mappings from /etc/services + +package net + +import "sync" + +// services contains minimal mappings between services names and port +// numbers for platforms that don't have a complete list of port numbers +// (some Solaris distros). +var services = map[string]map[string]int{ + "tcp": {"http": 80}, +} +var servicesError error +var onceReadServices sync.Once + +func readServices() { + var file *file + if file, servicesError = open("/etc/services"); servicesError != nil { + return + } + for line, ok := file.readLine(); ok; line, ok = file.readLine() { + // "http 80/tcp www www-http # World Wide Web HTTP" + if i := byteIndex(line, '#'); i >= 0 { + line = line[0:i] + } + f := getFields(line) + if len(f) < 2 { + continue + } + portnet := f[1] // "80/tcp" + port, j, ok := dtoi(portnet, 0) + if !ok || port <= 0 || j >= len(portnet) || portnet[j] != '/' { + continue + } + netw := portnet[j+1:] // "tcp" + m, ok1 := services[netw] + if !ok1 { + m = make(map[string]int) + services[netw] = m + } + for i := 0; i < len(f); i++ { + if i != 1 { // f[1] was port/net + m[f[i]] = port + } + } + } + file.close() +} + +// goLookupPort is the native Go implementation of LookupPort. +func goLookupPort(network, service string) (port int, err error) { + onceReadServices.Do(readServices) + + switch network { + case "tcp4", "tcp6": + network = "tcp" + case "udp4", "udp6": + network = "udp" + } + + if m, ok := services[network]; ok { + if port, ok = m[service]; ok { + return + } + } + return 0, &AddrError{"unknown port", network + "/" + service} +} diff --git a/src/net/protoconn_test.go b/src/net/protoconn_test.go new file mode 100644 index 000000000..12856b6c3 --- /dev/null +++ b/src/net/protoconn_test.go @@ -0,0 +1,386 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements API tests across platforms and will never have a build +// tag. + +package net + +import ( + "io/ioutil" + "os" + "runtime" + "testing" + "time" +) + +// testUnixAddr uses ioutil.TempFile to get a name that is unique. It +// also uses /tmp directory in case it is prohibited to create UNIX +// sockets in TMPDIR. +func testUnixAddr() string { + f, err := ioutil.TempFile("", "nettest") + if err != nil { + panic(err) + } + addr := f.Name() + f.Close() + os.Remove(addr) + return addr +} + +var condFatalf = func() func(*testing.T, string, ...interface{}) { + // A few APIs are not implemented yet on both Plan 9 and Windows. + switch runtime.GOOS { + case "plan9", "windows": + return (*testing.T).Logf + } + return (*testing.T).Fatalf +}() + +func TestTCPListenerSpecificMethods(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + la, err := ResolveTCPAddr("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("ResolveTCPAddr failed: %v", err) + } + ln, err := ListenTCP("tcp4", la) + if err != nil { + t.Fatalf("ListenTCP failed: %v", err) + } + defer ln.Close() + ln.Addr() + ln.SetDeadline(time.Now().Add(30 * time.Nanosecond)) + + if c, err := ln.Accept(); err != nil { + if !err.(Error).Timeout() { + t.Fatalf("TCPListener.Accept failed: %v", err) + } + } else { + c.Close() + } + if c, err := ln.AcceptTCP(); err != nil { + if !err.(Error).Timeout() { + t.Fatalf("TCPListener.AcceptTCP failed: %v", err) + } + } else { + c.Close() + } + + if f, err := ln.File(); err != nil { + condFatalf(t, "TCPListener.File failed: %v", err) + } else { + f.Close() + } +} + +func TestTCPConnSpecificMethods(t *testing.T) { + la, err := ResolveTCPAddr("tcp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("ResolveTCPAddr failed: %v", err) + } + ln, err := ListenTCP("tcp4", la) + if err != nil { + t.Fatalf("ListenTCP failed: %v", err) + } + defer ln.Close() + ln.Addr() + + done := make(chan int) + go transponder(t, ln, done) + + ra, err := ResolveTCPAddr("tcp4", ln.Addr().String()) + if err != nil { + t.Fatalf("ResolveTCPAddr failed: %v", err) + } + c, err := DialTCP("tcp4", nil, ra) + if err != nil { + t.Fatalf("DialTCP failed: %v", err) + } + defer c.Close() + c.SetKeepAlive(false) + c.SetKeepAlivePeriod(3 * time.Second) + c.SetLinger(0) + c.SetNoDelay(false) + c.LocalAddr() + c.RemoteAddr() + c.SetDeadline(time.Now().Add(someTimeout)) + c.SetReadDeadline(time.Now().Add(someTimeout)) + c.SetWriteDeadline(time.Now().Add(someTimeout)) + + if _, err := c.Write([]byte("TCPCONN TEST")); err != nil { + t.Fatalf("TCPConn.Write failed: %v", err) + } + rb := make([]byte, 128) + if _, err := c.Read(rb); err != nil { + t.Fatalf("TCPConn.Read failed: %v", err) + } + + <-done +} + +func TestUDPConnSpecificMethods(t *testing.T) { + la, err := ResolveUDPAddr("udp4", "127.0.0.1:0") + if err != nil { + t.Fatalf("ResolveUDPAddr failed: %v", err) + } + c, err := ListenUDP("udp4", la) + if err != nil { + t.Fatalf("ListenUDP failed: %v", err) + } + defer c.Close() + c.LocalAddr() + c.RemoteAddr() + c.SetDeadline(time.Now().Add(someTimeout)) + c.SetReadDeadline(time.Now().Add(someTimeout)) + c.SetWriteDeadline(time.Now().Add(someTimeout)) + c.SetReadBuffer(2048) + c.SetWriteBuffer(2048) + + wb := []byte("UDPCONN TEST") + rb := make([]byte, 128) + if _, err := c.WriteToUDP(wb, c.LocalAddr().(*UDPAddr)); err != nil { + t.Fatalf("UDPConn.WriteToUDP failed: %v", err) + } + if _, _, err := c.ReadFromUDP(rb); err != nil { + t.Fatalf("UDPConn.ReadFromUDP failed: %v", err) + } + if _, _, err := c.WriteMsgUDP(wb, nil, c.LocalAddr().(*UDPAddr)); err != nil { + condFatalf(t, "UDPConn.WriteMsgUDP failed: %v", err) + } + if _, _, _, _, err := c.ReadMsgUDP(rb, nil); err != nil { + condFatalf(t, "UDPConn.ReadMsgUDP failed: %v", err) + } + + if f, err := c.File(); err != nil { + condFatalf(t, "UDPConn.File failed: %v", err) + } else { + f.Close() + } + + defer func() { + if p := recover(); p != nil { + t.Fatalf("UDPConn.WriteToUDP or WriteMsgUDP panicked: %v", p) + } + }() + + c.WriteToUDP(wb, nil) + c.WriteMsgUDP(wb, nil, nil) +} + +func TestIPConnSpecificMethods(t *testing.T) { + if skip, skipmsg := skipRawSocketTest(t); skip { + t.Skip(skipmsg) + } + + la, err := ResolveIPAddr("ip4", "127.0.0.1") + if err != nil { + t.Fatalf("ResolveIPAddr failed: %v", err) + } + c, err := ListenIP("ip4:icmp", la) + if err != nil { + t.Fatalf("ListenIP failed: %v", err) + } + defer c.Close() + c.LocalAddr() + c.RemoteAddr() + c.SetDeadline(time.Now().Add(someTimeout)) + c.SetReadDeadline(time.Now().Add(someTimeout)) + c.SetWriteDeadline(time.Now().Add(someTimeout)) + c.SetReadBuffer(2048) + c.SetWriteBuffer(2048) + + wb, err := (&icmpMessage{ + Type: icmpv4EchoRequest, Code: 0, + Body: &icmpEcho{ + ID: os.Getpid() & 0xffff, Seq: 1, + Data: []byte("IPCONN TEST "), + }, + }).Marshal() + if err != nil { + t.Fatalf("icmpMessage.Marshal failed: %v", err) + } + rb := make([]byte, 20+len(wb)) + if _, err := c.WriteToIP(wb, c.LocalAddr().(*IPAddr)); err != nil { + t.Fatalf("IPConn.WriteToIP failed: %v", err) + } + if _, _, err := c.ReadFromIP(rb); err != nil { + t.Fatalf("IPConn.ReadFromIP failed: %v", err) + } + if _, _, err := c.WriteMsgIP(wb, nil, c.LocalAddr().(*IPAddr)); err != nil { + condFatalf(t, "IPConn.WriteMsgIP failed: %v", err) + } + if _, _, _, _, err := c.ReadMsgIP(rb, nil); err != nil { + condFatalf(t, "IPConn.ReadMsgIP failed: %v", err) + } + + if f, err := c.File(); err != nil { + condFatalf(t, "IPConn.File failed: %v", err) + } else { + f.Close() + } + + defer func() { + if p := recover(); p != nil { + t.Fatalf("IPConn.WriteToIP or WriteMsgIP panicked: %v", p) + } + }() + + c.WriteToIP(wb, nil) + c.WriteMsgIP(wb, nil, nil) +} + +func TestUnixListenerSpecificMethods(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + addr := testUnixAddr() + la, err := ResolveUnixAddr("unix", addr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + ln, err := ListenUnix("unix", la) + if err != nil { + t.Fatalf("ListenUnix failed: %v", err) + } + defer ln.Close() + defer os.Remove(addr) + ln.Addr() + ln.SetDeadline(time.Now().Add(30 * time.Nanosecond)) + + if c, err := ln.Accept(); err != nil { + if !err.(Error).Timeout() { + t.Fatalf("UnixListener.Accept failed: %v", err) + } + } else { + c.Close() + } + if c, err := ln.AcceptUnix(); err != nil { + if !err.(Error).Timeout() { + t.Fatalf("UnixListener.AcceptUnix failed: %v", err) + } + } else { + c.Close() + } + + if f, err := ln.File(); err != nil { + t.Fatalf("UnixListener.File failed: %v", err) + } else { + f.Close() + } +} + +func TestUnixConnSpecificMethods(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9", "windows": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + addr1, addr2, addr3 := testUnixAddr(), testUnixAddr(), testUnixAddr() + + a1, err := ResolveUnixAddr("unixgram", addr1) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c1, err := DialUnix("unixgram", a1, nil) + if err != nil { + t.Fatalf("DialUnix failed: %v", err) + } + defer c1.Close() + defer os.Remove(addr1) + c1.LocalAddr() + c1.RemoteAddr() + c1.SetDeadline(time.Now().Add(someTimeout)) + c1.SetReadDeadline(time.Now().Add(someTimeout)) + c1.SetWriteDeadline(time.Now().Add(someTimeout)) + c1.SetReadBuffer(2048) + c1.SetWriteBuffer(2048) + + a2, err := ResolveUnixAddr("unixgram", addr2) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c2, err := DialUnix("unixgram", a2, nil) + if err != nil { + t.Fatalf("DialUnix failed: %v", err) + } + defer c2.Close() + defer os.Remove(addr2) + c2.LocalAddr() + c2.RemoteAddr() + c2.SetDeadline(time.Now().Add(someTimeout)) + c2.SetReadDeadline(time.Now().Add(someTimeout)) + c2.SetWriteDeadline(time.Now().Add(someTimeout)) + c2.SetReadBuffer(2048) + c2.SetWriteBuffer(2048) + + a3, err := ResolveUnixAddr("unixgram", addr3) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c3, err := ListenUnixgram("unixgram", a3) + if err != nil { + t.Fatalf("ListenUnixgram failed: %v", err) + } + defer c3.Close() + defer os.Remove(addr3) + c3.LocalAddr() + c3.RemoteAddr() + c3.SetDeadline(time.Now().Add(someTimeout)) + c3.SetReadDeadline(time.Now().Add(someTimeout)) + c3.SetWriteDeadline(time.Now().Add(someTimeout)) + c3.SetReadBuffer(2048) + c3.SetWriteBuffer(2048) + + wb := []byte("UNIXCONN TEST") + rb1 := make([]byte, 128) + rb2 := make([]byte, 128) + rb3 := make([]byte, 128) + if _, _, err := c1.WriteMsgUnix(wb, nil, a2); err != nil { + t.Fatalf("UnixConn.WriteMsgUnix failed: %v", err) + } + if _, _, _, _, err := c2.ReadMsgUnix(rb2, nil); err != nil { + t.Fatalf("UnixConn.ReadMsgUnix failed: %v", err) + } + if _, err := c2.WriteToUnix(wb, a1); err != nil { + t.Fatalf("UnixConn.WriteToUnix failed: %v", err) + } + if _, _, err := c1.ReadFromUnix(rb1); err != nil { + t.Fatalf("UnixConn.ReadFromUnix failed: %v", err) + } + if _, err := c3.WriteToUnix(wb, a1); err != nil { + t.Fatalf("UnixConn.WriteToUnix failed: %v", err) + } + if _, _, err := c1.ReadFromUnix(rb1); err != nil { + t.Fatalf("UnixConn.ReadFromUnix failed: %v", err) + } + if _, err := c2.WriteToUnix(wb, a3); err != nil { + t.Fatalf("UnixConn.WriteToUnix failed: %v", err) + } + if _, _, err := c3.ReadFromUnix(rb3); err != nil { + t.Fatalf("UnixConn.ReadFromUnix failed: %v", err) + } + + if f, err := c1.File(); err != nil { + t.Fatalf("UnixConn.File failed: %v", err) + } else { + f.Close() + } + + defer func() { + if p := recover(); p != nil { + t.Fatalf("UnixConn.WriteToUnix or WriteMsgUnix panicked: %v", p) + } + }() + + c1.WriteToUnix(wb, nil) + c1.WriteMsgUnix(wb, nil, nil) + c3.WriteToUnix(wb, nil) + c3.WriteMsgUnix(wb, nil, nil) +} diff --git a/src/net/race.go b/src/net/race.go new file mode 100644 index 000000000..2f02a6c22 --- /dev/null +++ b/src/net/race.go @@ -0,0 +1,31 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build race +// +build windows + +package net + +import ( + "runtime" + "unsafe" +) + +const raceenabled = true + +func raceAcquire(addr unsafe.Pointer) { + runtime.RaceAcquire(addr) +} + +func raceReleaseMerge(addr unsafe.Pointer) { + runtime.RaceReleaseMerge(addr) +} + +func raceReadRange(addr unsafe.Pointer, len int) { + runtime.RaceReadRange(addr, len) +} + +func raceWriteRange(addr unsafe.Pointer, len int) { + runtime.RaceWriteRange(addr, len) +} diff --git a/src/net/race0.go b/src/net/race0.go new file mode 100644 index 000000000..f50429779 --- /dev/null +++ b/src/net/race0.go @@ -0,0 +1,26 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !race +// +build windows + +package net + +import ( + "unsafe" +) + +const raceenabled = false + +func raceAcquire(addr unsafe.Pointer) { +} + +func raceReleaseMerge(addr unsafe.Pointer) { +} + +func raceReadRange(addr unsafe.Pointer, len int) { +} + +func raceWriteRange(addr unsafe.Pointer, len int) { +} diff --git a/src/net/rpc/client.go b/src/net/rpc/client.go new file mode 100644 index 000000000..21f79b068 --- /dev/null +++ b/src/net/rpc/client.go @@ -0,0 +1,317 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rpc + +import ( + "bufio" + "encoding/gob" + "errors" + "io" + "log" + "net" + "net/http" + "sync" +) + +// ServerError represents an error that has been returned from +// the remote side of the RPC connection. +type ServerError string + +func (e ServerError) Error() string { + return string(e) +} + +var ErrShutdown = errors.New("connection is shut down") + +// Call represents an active RPC. +type Call struct { + ServiceMethod string // The name of the service and method to call. + Args interface{} // The argument to the function (*struct). + Reply interface{} // The reply from the function (*struct). + Error error // After completion, the error status. + Done chan *Call // Strobes when call is complete. +} + +// Client represents an RPC Client. +// There may be multiple outstanding Calls associated +// with a single Client, and a Client may be used by +// multiple goroutines simultaneously. +type Client struct { + codec ClientCodec + + sending sync.Mutex + + mutex sync.Mutex // protects following + request Request + seq uint64 + pending map[uint64]*Call + closing bool // user has called Close + shutdown bool // server has told us to stop +} + +// A ClientCodec implements writing of RPC requests and +// reading of RPC responses for the client side of an RPC session. +// The client calls WriteRequest to write a request to the connection +// and calls ReadResponseHeader and ReadResponseBody in pairs +// to read responses. The client calls Close when finished with the +// connection. ReadResponseBody may be called with a nil +// argument to force the body of the response to be read and then +// discarded. +type ClientCodec interface { + // WriteRequest must be safe for concurrent use by multiple goroutines. + WriteRequest(*Request, interface{}) error + ReadResponseHeader(*Response) error + ReadResponseBody(interface{}) error + + Close() error +} + +func (client *Client) send(call *Call) { + client.sending.Lock() + defer client.sending.Unlock() + + // Register this call. + client.mutex.Lock() + if client.shutdown || client.closing { + call.Error = ErrShutdown + client.mutex.Unlock() + call.done() + return + } + seq := client.seq + client.seq++ + client.pending[seq] = call + client.mutex.Unlock() + + // Encode and send the request. + client.request.Seq = seq + client.request.ServiceMethod = call.ServiceMethod + err := client.codec.WriteRequest(&client.request, call.Args) + if err != nil { + client.mutex.Lock() + call = client.pending[seq] + delete(client.pending, seq) + client.mutex.Unlock() + if call != nil { + call.Error = err + call.done() + } + } +} + +func (client *Client) input() { + var err error + var response Response + for err == nil { + response = Response{} + err = client.codec.ReadResponseHeader(&response) + if err != nil { + break + } + seq := response.Seq + client.mutex.Lock() + call := client.pending[seq] + delete(client.pending, seq) + client.mutex.Unlock() + + switch { + case call == nil: + // We've got no pending call. That usually means that + // WriteRequest partially failed, and call was already + // removed; response is a server telling us about an + // error reading request body. We should still attempt + // to read error body, but there's no one to give it to. + err = client.codec.ReadResponseBody(nil) + if err != nil { + err = errors.New("reading error body: " + err.Error()) + } + case response.Error != "": + // We've got an error response. Give this to the request; + // any subsequent requests will get the ReadResponseBody + // error if there is one. + call.Error = ServerError(response.Error) + err = client.codec.ReadResponseBody(nil) + if err != nil { + err = errors.New("reading error body: " + err.Error()) + } + call.done() + default: + err = client.codec.ReadResponseBody(call.Reply) + if err != nil { + call.Error = errors.New("reading body " + err.Error()) + } + call.done() + } + } + // Terminate pending calls. + client.sending.Lock() + client.mutex.Lock() + client.shutdown = true + closing := client.closing + if err == io.EOF { + if closing { + err = ErrShutdown + } else { + err = io.ErrUnexpectedEOF + } + } + for _, call := range client.pending { + call.Error = err + call.done() + } + client.mutex.Unlock() + client.sending.Unlock() + if debugLog && err != io.EOF && !closing { + log.Println("rpc: client protocol error:", err) + } +} + +func (call *Call) done() { + select { + case call.Done <- call: + // ok + default: + // We don't want to block here. It is the caller's responsibility to make + // sure the channel has enough buffer space. See comment in Go(). + if debugLog { + log.Println("rpc: discarding Call reply due to insufficient Done chan capacity") + } + } +} + +// NewClient returns a new Client to handle requests to the +// set of services at the other end of the connection. +// It adds a buffer to the write side of the connection so +// the header and payload are sent as a unit. +func NewClient(conn io.ReadWriteCloser) *Client { + encBuf := bufio.NewWriter(conn) + client := &gobClientCodec{conn, gob.NewDecoder(conn), gob.NewEncoder(encBuf), encBuf} + return NewClientWithCodec(client) +} + +// NewClientWithCodec is like NewClient but uses the specified +// codec to encode requests and decode responses. +func NewClientWithCodec(codec ClientCodec) *Client { + client := &Client{ + codec: codec, + pending: make(map[uint64]*Call), + } + go client.input() + return client +} + +type gobClientCodec struct { + rwc io.ReadWriteCloser + dec *gob.Decoder + enc *gob.Encoder + encBuf *bufio.Writer +} + +func (c *gobClientCodec) WriteRequest(r *Request, body interface{}) (err error) { + if err = c.enc.Encode(r); err != nil { + return + } + if err = c.enc.Encode(body); err != nil { + return + } + return c.encBuf.Flush() +} + +func (c *gobClientCodec) ReadResponseHeader(r *Response) error { + return c.dec.Decode(r) +} + +func (c *gobClientCodec) ReadResponseBody(body interface{}) error { + return c.dec.Decode(body) +} + +func (c *gobClientCodec) Close() error { + return c.rwc.Close() +} + +// DialHTTP connects to an HTTP RPC server at the specified network address +// listening on the default HTTP RPC path. +func DialHTTP(network, address string) (*Client, error) { + return DialHTTPPath(network, address, DefaultRPCPath) +} + +// DialHTTPPath connects to an HTTP RPC server +// at the specified network address and path. +func DialHTTPPath(network, address, path string) (*Client, error) { + var err error + conn, err := net.Dial(network, address) + if err != nil { + return nil, err + } + io.WriteString(conn, "CONNECT "+path+" HTTP/1.0\n\n") + + // Require successful HTTP response + // before switching to RPC protocol. + resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: "CONNECT"}) + if err == nil && resp.Status == connected { + return NewClient(conn), nil + } + if err == nil { + err = errors.New("unexpected HTTP response: " + resp.Status) + } + conn.Close() + return nil, &net.OpError{ + Op: "dial-http", + Net: network + " " + address, + Addr: nil, + Err: err, + } +} + +// Dial connects to an RPC server at the specified network address. +func Dial(network, address string) (*Client, error) { + conn, err := net.Dial(network, address) + if err != nil { + return nil, err + } + return NewClient(conn), nil +} + +func (client *Client) Close() error { + client.mutex.Lock() + if client.closing { + client.mutex.Unlock() + return ErrShutdown + } + client.closing = true + client.mutex.Unlock() + return client.codec.Close() +} + +// Go invokes the function asynchronously. It returns the Call structure representing +// the invocation. The done channel will signal when the call is complete by returning +// the same Call object. If done is nil, Go will allocate a new channel. +// If non-nil, done must be buffered or Go will deliberately crash. +func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call { + call := new(Call) + call.ServiceMethod = serviceMethod + call.Args = args + call.Reply = reply + if done == nil { + done = make(chan *Call, 10) // buffered. + } else { + // If caller passes done != nil, it must arrange that + // done has enough buffer for the number of simultaneous + // RPCs that will be using that channel. If the channel + // is totally unbuffered, it's best not to run at all. + if cap(done) == 0 { + log.Panic("rpc: done channel is unbuffered") + } + } + call.Done = done + client.send(call) + return call +} + +// Call invokes the named function, waits for it to complete, and returns its error status. +func (client *Client) Call(serviceMethod string, args interface{}, reply interface{}) error { + call := <-client.Go(serviceMethod, args, reply, make(chan *Call, 1)).Done + return call.Error +} diff --git a/src/net/rpc/client_test.go b/src/net/rpc/client_test.go new file mode 100644 index 000000000..bbfc1ec3a --- /dev/null +++ b/src/net/rpc/client_test.go @@ -0,0 +1,36 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rpc + +import ( + "errors" + "testing" +) + +type shutdownCodec struct { + responded chan int + closed bool +} + +func (c *shutdownCodec) WriteRequest(*Request, interface{}) error { return nil } +func (c *shutdownCodec) ReadResponseBody(interface{}) error { return nil } +func (c *shutdownCodec) ReadResponseHeader(*Response) error { + c.responded <- 1 + return errors.New("shutdownCodec ReadResponseHeader") +} +func (c *shutdownCodec) Close() error { + c.closed = true + return nil +} + +func TestCloseCodec(t *testing.T) { + codec := &shutdownCodec{responded: make(chan int)} + client := NewClientWithCodec(codec) + <-codec.responded + client.Close() + if !codec.closed { + t.Error("client.Close did not close codec") + } +} diff --git a/src/net/rpc/debug.go b/src/net/rpc/debug.go new file mode 100644 index 000000000..98b2c1c6c --- /dev/null +++ b/src/net/rpc/debug.go @@ -0,0 +1,93 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rpc + +/* + Some HTML presented at http://machine:port/debug/rpc + Lists services, their methods, and some statistics, still rudimentary. +*/ + +import ( + "fmt" + "html/template" + "net/http" + "sort" +) + +const debugText = `<html> + <body> + <title>Services</title> + {{range .}} + <hr> + Service {{.Name}} + <hr> + <table> + <th align=center>Method</th><th align=center>Calls</th> + {{range .Method}} + <tr> + <td align=left font=fixed>{{.Name}}({{.Type.ArgType}}, {{.Type.ReplyType}}) error</td> + <td align=center>{{.Type.NumCalls}}</td> + </tr> + {{end}} + </table> + {{end}} + </body> + </html>` + +var debug = template.Must(template.New("RPC debug").Parse(debugText)) + +// If set, print log statements for internal and I/O errors. +var debugLog = false + +type debugMethod struct { + Type *methodType + Name string +} + +type methodArray []debugMethod + +type debugService struct { + Service *service + Name string + Method methodArray +} + +type serviceArray []debugService + +func (s serviceArray) Len() int { return len(s) } +func (s serviceArray) Less(i, j int) bool { return s[i].Name < s[j].Name } +func (s serviceArray) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func (m methodArray) Len() int { return len(m) } +func (m methodArray) Less(i, j int) bool { return m[i].Name < m[j].Name } +func (m methodArray) Swap(i, j int) { m[i], m[j] = m[j], m[i] } + +type debugHTTP struct { + *Server +} + +// Runs at /debug/rpc +func (server debugHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) { + // Build a sorted version of the data. + var services = make(serviceArray, len(server.serviceMap)) + i := 0 + server.mu.Lock() + for sname, service := range server.serviceMap { + services[i] = debugService{service, sname, make(methodArray, len(service.method))} + j := 0 + for mname, method := range service.method { + services[i].Method[j] = debugMethod{method, mname} + j++ + } + sort.Sort(services[i].Method) + i++ + } + server.mu.Unlock() + sort.Sort(services) + err := debug.Execute(w, services) + if err != nil { + fmt.Fprintln(w, "rpc: error executing template:", err.Error()) + } +} diff --git a/src/net/rpc/jsonrpc/all_test.go b/src/net/rpc/jsonrpc/all_test.go new file mode 100644 index 000000000..a433a365e --- /dev/null +++ b/src/net/rpc/jsonrpc/all_test.go @@ -0,0 +1,296 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package jsonrpc + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net" + "net/rpc" + "strings" + "testing" +) + +type Args struct { + A, B int +} + +type Reply struct { + C int +} + +type Arith int + +type ArithAddResp struct { + Id interface{} `json:"id"` + Result Reply `json:"result"` + Error interface{} `json:"error"` +} + +func (t *Arith) Add(args *Args, reply *Reply) error { + reply.C = args.A + args.B + return nil +} + +func (t *Arith) Mul(args *Args, reply *Reply) error { + reply.C = args.A * args.B + return nil +} + +func (t *Arith) Div(args *Args, reply *Reply) error { + if args.B == 0 { + return errors.New("divide by zero") + } + reply.C = args.A / args.B + return nil +} + +func (t *Arith) Error(args *Args, reply *Reply) error { + panic("ERROR") +} + +func init() { + rpc.Register(new(Arith)) +} + +func TestServerNoParams(t *testing.T) { + cli, srv := net.Pipe() + defer cli.Close() + go ServeConn(srv) + dec := json.NewDecoder(cli) + + fmt.Fprintf(cli, `{"method": "Arith.Add", "id": "123"}`) + var resp ArithAddResp + if err := dec.Decode(&resp); err != nil { + t.Fatalf("Decode after no params: %s", err) + } + if resp.Error == nil { + t.Fatalf("Expected error, got nil") + } +} + +func TestServerEmptyMessage(t *testing.T) { + cli, srv := net.Pipe() + defer cli.Close() + go ServeConn(srv) + dec := json.NewDecoder(cli) + + fmt.Fprintf(cli, "{}") + var resp ArithAddResp + if err := dec.Decode(&resp); err != nil { + t.Fatalf("Decode after empty: %s", err) + } + if resp.Error == nil { + t.Fatalf("Expected error, got nil") + } +} + +func TestServer(t *testing.T) { + cli, srv := net.Pipe() + defer cli.Close() + go ServeConn(srv) + dec := json.NewDecoder(cli) + + // Send hand-coded requests to server, parse responses. + for i := 0; i < 10; i++ { + fmt.Fprintf(cli, `{"method": "Arith.Add", "id": "\u%04d", "params": [{"A": %d, "B": %d}]}`, i, i, i+1) + var resp ArithAddResp + err := dec.Decode(&resp) + if err != nil { + t.Fatalf("Decode: %s", err) + } + if resp.Error != nil { + t.Fatalf("resp.Error: %s", resp.Error) + } + if resp.Id.(string) != string(i) { + t.Fatalf("resp: bad id %q want %q", resp.Id.(string), string(i)) + } + if resp.Result.C != 2*i+1 { + t.Fatalf("resp: bad result: %d+%d=%d", i, i+1, resp.Result.C) + } + } +} + +func TestClient(t *testing.T) { + // Assume server is okay (TestServer is above). + // Test client against server. + cli, srv := net.Pipe() + go ServeConn(srv) + + client := NewClient(cli) + defer client.Close() + + // Synchronous calls + args := &Args{7, 8} + reply := new(Reply) + err := client.Call("Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: got %d expected %d", reply.C, args.A+args.B) + } + + args = &Args{7, 8} + reply = new(Reply) + err = client.Call("Arith.Mul", args, reply) + if err != nil { + t.Errorf("Mul: expected no error but got string %q", err.Error()) + } + if reply.C != args.A*args.B { + t.Errorf("Mul: got %d expected %d", reply.C, args.A*args.B) + } + + // Out of order. + args = &Args{7, 8} + mulReply := new(Reply) + mulCall := client.Go("Arith.Mul", args, mulReply, nil) + addReply := new(Reply) + addCall := client.Go("Arith.Add", args, addReply, nil) + + addCall = <-addCall.Done + if addCall.Error != nil { + t.Errorf("Add: expected no error but got string %q", addCall.Error.Error()) + } + if addReply.C != args.A+args.B { + t.Errorf("Add: got %d expected %d", addReply.C, args.A+args.B) + } + + mulCall = <-mulCall.Done + if mulCall.Error != nil { + t.Errorf("Mul: expected no error but got string %q", mulCall.Error.Error()) + } + if mulReply.C != args.A*args.B { + t.Errorf("Mul: got %d expected %d", mulReply.C, args.A*args.B) + } + + // Error test + args = &Args{7, 0} + reply = new(Reply) + err = client.Call("Arith.Div", args, reply) + // expect an error: zero divide + if err == nil { + t.Error("Div: expected error") + } else if err.Error() != "divide by zero" { + t.Error("Div: expected divide by zero error; got", err) + } +} + +func TestMalformedInput(t *testing.T) { + cli, srv := net.Pipe() + go cli.Write([]byte(`{id:1}`)) // invalid json + ServeConn(srv) // must return, not loop +} + +func TestMalformedOutput(t *testing.T) { + cli, srv := net.Pipe() + go srv.Write([]byte(`{"id":0,"result":null,"error":null}`)) + go ioutil.ReadAll(srv) + + client := NewClient(cli) + defer client.Close() + + args := &Args{7, 8} + reply := new(Reply) + err := client.Call("Arith.Add", args, reply) + if err == nil { + t.Error("expected error") + } +} + +func TestServerErrorHasNullResult(t *testing.T) { + var out bytes.Buffer + sc := NewServerCodec(struct { + io.Reader + io.Writer + io.Closer + }{ + Reader: strings.NewReader(`{"method": "Arith.Add", "id": "123", "params": []}`), + Writer: &out, + Closer: ioutil.NopCloser(nil), + }) + r := new(rpc.Request) + if err := sc.ReadRequestHeader(r); err != nil { + t.Fatal(err) + } + const valueText = "the value we don't want to see" + const errorText = "some error" + err := sc.WriteResponse(&rpc.Response{ + ServiceMethod: "Method", + Seq: 1, + Error: errorText, + }, valueText) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), errorText) { + t.Fatalf("Response didn't contain expected error %q: %s", errorText, &out) + } + if strings.Contains(out.String(), valueText) { + t.Errorf("Response contains both an error and value: %s", &out) + } +} + +func TestUnexpectedError(t *testing.T) { + cli, srv := myPipe() + go cli.PipeWriter.CloseWithError(errors.New("unexpected error!")) // reader will get this error + ServeConn(srv) // must return, not loop +} + +// Copied from package net. +func myPipe() (*pipe, *pipe) { + r1, w1 := io.Pipe() + r2, w2 := io.Pipe() + + return &pipe{r1, w2}, &pipe{r2, w1} +} + +type pipe struct { + *io.PipeReader + *io.PipeWriter +} + +type pipeAddr int + +func (pipeAddr) Network() string { + return "pipe" +} + +func (pipeAddr) String() string { + return "pipe" +} + +func (p *pipe) Close() error { + err := p.PipeReader.Close() + err1 := p.PipeWriter.Close() + if err == nil { + err = err1 + } + return err +} + +func (p *pipe) LocalAddr() net.Addr { + return pipeAddr(0) +} + +func (p *pipe) RemoteAddr() net.Addr { + return pipeAddr(0) +} + +func (p *pipe) SetTimeout(nsec int64) error { + return errors.New("net.Pipe does not support timeouts") +} + +func (p *pipe) SetReadTimeout(nsec int64) error { + return errors.New("net.Pipe does not support timeouts") +} + +func (p *pipe) SetWriteTimeout(nsec int64) error { + return errors.New("net.Pipe does not support timeouts") +} diff --git a/src/net/rpc/jsonrpc/client.go b/src/net/rpc/jsonrpc/client.go new file mode 100644 index 000000000..2194f2125 --- /dev/null +++ b/src/net/rpc/jsonrpc/client.go @@ -0,0 +1,123 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package jsonrpc implements a JSON-RPC ClientCodec and ServerCodec +// for the rpc package. +package jsonrpc + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/rpc" + "sync" +) + +type clientCodec struct { + dec *json.Decoder // for reading JSON values + enc *json.Encoder // for writing JSON values + c io.Closer + + // temporary work space + req clientRequest + resp clientResponse + + // JSON-RPC responses include the request id but not the request method. + // Package rpc expects both. + // We save the request method in pending when sending a request + // and then look it up by request ID when filling out the rpc Response. + mutex sync.Mutex // protects pending + pending map[uint64]string // map request id to method name +} + +// NewClientCodec returns a new rpc.ClientCodec using JSON-RPC on conn. +func NewClientCodec(conn io.ReadWriteCloser) rpc.ClientCodec { + return &clientCodec{ + dec: json.NewDecoder(conn), + enc: json.NewEncoder(conn), + c: conn, + pending: make(map[uint64]string), + } +} + +type clientRequest struct { + Method string `json:"method"` + Params [1]interface{} `json:"params"` + Id uint64 `json:"id"` +} + +func (c *clientCodec) WriteRequest(r *rpc.Request, param interface{}) error { + c.mutex.Lock() + c.pending[r.Seq] = r.ServiceMethod + c.mutex.Unlock() + c.req.Method = r.ServiceMethod + c.req.Params[0] = param + c.req.Id = r.Seq + return c.enc.Encode(&c.req) +} + +type clientResponse struct { + Id uint64 `json:"id"` + Result *json.RawMessage `json:"result"` + Error interface{} `json:"error"` +} + +func (r *clientResponse) reset() { + r.Id = 0 + r.Result = nil + r.Error = nil +} + +func (c *clientCodec) ReadResponseHeader(r *rpc.Response) error { + c.resp.reset() + if err := c.dec.Decode(&c.resp); err != nil { + return err + } + + c.mutex.Lock() + r.ServiceMethod = c.pending[c.resp.Id] + delete(c.pending, c.resp.Id) + c.mutex.Unlock() + + r.Error = "" + r.Seq = c.resp.Id + if c.resp.Error != nil || c.resp.Result == nil { + x, ok := c.resp.Error.(string) + if !ok { + return fmt.Errorf("invalid error %v", c.resp.Error) + } + if x == "" { + x = "unspecified error" + } + r.Error = x + } + return nil +} + +func (c *clientCodec) ReadResponseBody(x interface{}) error { + if x == nil { + return nil + } + return json.Unmarshal(*c.resp.Result, x) +} + +func (c *clientCodec) Close() error { + return c.c.Close() +} + +// NewClient returns a new rpc.Client to handle requests to the +// set of services at the other end of the connection. +func NewClient(conn io.ReadWriteCloser) *rpc.Client { + return rpc.NewClientWithCodec(NewClientCodec(conn)) +} + +// Dial connects to a JSON-RPC server at the specified network address. +func Dial(network, address string) (*rpc.Client, error) { + conn, err := net.Dial(network, address) + if err != nil { + return nil, err + } + return NewClient(conn), err +} diff --git a/src/net/rpc/jsonrpc/server.go b/src/net/rpc/jsonrpc/server.go new file mode 100644 index 000000000..e6d37cfa6 --- /dev/null +++ b/src/net/rpc/jsonrpc/server.go @@ -0,0 +1,134 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package jsonrpc + +import ( + "encoding/json" + "errors" + "io" + "net/rpc" + "sync" +) + +var errMissingParams = errors.New("jsonrpc: request body missing params") + +type serverCodec struct { + dec *json.Decoder // for reading JSON values + enc *json.Encoder // for writing JSON values + c io.Closer + + // temporary work space + req serverRequest + + // JSON-RPC clients can use arbitrary json values as request IDs. + // Package rpc expects uint64 request IDs. + // We assign uint64 sequence numbers to incoming requests + // but save the original request ID in the pending map. + // When rpc responds, we use the sequence number in + // the response to find the original request ID. + mutex sync.Mutex // protects seq, pending + seq uint64 + pending map[uint64]*json.RawMessage +} + +// NewServerCodec returns a new rpc.ServerCodec using JSON-RPC on conn. +func NewServerCodec(conn io.ReadWriteCloser) rpc.ServerCodec { + return &serverCodec{ + dec: json.NewDecoder(conn), + enc: json.NewEncoder(conn), + c: conn, + pending: make(map[uint64]*json.RawMessage), + } +} + +type serverRequest struct { + Method string `json:"method"` + Params *json.RawMessage `json:"params"` + Id *json.RawMessage `json:"id"` +} + +func (r *serverRequest) reset() { + r.Method = "" + r.Params = nil + r.Id = nil +} + +type serverResponse struct { + Id *json.RawMessage `json:"id"` + Result interface{} `json:"result"` + Error interface{} `json:"error"` +} + +func (c *serverCodec) ReadRequestHeader(r *rpc.Request) error { + c.req.reset() + if err := c.dec.Decode(&c.req); err != nil { + return err + } + r.ServiceMethod = c.req.Method + + // JSON request id can be any JSON value; + // RPC package expects uint64. Translate to + // internal uint64 and save JSON on the side. + c.mutex.Lock() + c.seq++ + c.pending[c.seq] = c.req.Id + c.req.Id = nil + r.Seq = c.seq + c.mutex.Unlock() + + return nil +} + +func (c *serverCodec) ReadRequestBody(x interface{}) error { + if x == nil { + return nil + } + if c.req.Params == nil { + return errMissingParams + } + // JSON params is array value. + // RPC params is struct. + // Unmarshal into array containing struct for now. + // Should think about making RPC more general. + var params [1]interface{} + params[0] = x + return json.Unmarshal(*c.req.Params, ¶ms) +} + +var null = json.RawMessage([]byte("null")) + +func (c *serverCodec) WriteResponse(r *rpc.Response, x interface{}) error { + c.mutex.Lock() + b, ok := c.pending[r.Seq] + if !ok { + c.mutex.Unlock() + return errors.New("invalid sequence number in response") + } + delete(c.pending, r.Seq) + c.mutex.Unlock() + + if b == nil { + // Invalid request so no id. Use JSON null. + b = &null + } + resp := serverResponse{Id: b} + if r.Error == "" { + resp.Result = x + } else { + resp.Error = r.Error + } + return c.enc.Encode(resp) +} + +func (c *serverCodec) Close() error { + return c.c.Close() +} + +// ServeConn runs the JSON-RPC server on a single connection. +// ServeConn blocks, serving the connection until the client hangs up. +// The caller typically invokes ServeConn in a go statement. +func ServeConn(conn io.ReadWriteCloser) { + rpc.ServeCodec(NewServerCodec(conn)) +} diff --git a/src/net/rpc/server.go b/src/net/rpc/server.go new file mode 100644 index 000000000..6b264b46b --- /dev/null +++ b/src/net/rpc/server.go @@ -0,0 +1,686 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* + Package rpc provides access to the exported methods of an object across a + network or other I/O connection. A server registers an object, making it visible + as a service with the name of the type of the object. After registration, exported + methods of the object will be accessible remotely. A server may register multiple + objects (services) of different types but it is an error to register multiple + objects of the same type. + + Only methods that satisfy these criteria will be made available for remote access; + other methods will be ignored: + + - the method is exported. + - the method has two arguments, both exported (or builtin) types. + - the method's second argument is a pointer. + - the method has return type error. + + In effect, the method must look schematically like + + func (t *T) MethodName(argType T1, replyType *T2) error + + where T, T1 and T2 can be marshaled by encoding/gob. + These requirements apply even if a different codec is used. + (In the future, these requirements may soften for custom codecs.) + + The method's first argument represents the arguments provided by the caller; the + second argument represents the result parameters to be returned to the caller. + The method's return value, if non-nil, is passed back as a string that the client + sees as if created by errors.New. If an error is returned, the reply parameter + will not be sent back to the client. + + The server may handle requests on a single connection by calling ServeConn. More + typically it will create a network listener and call Accept or, for an HTTP + listener, HandleHTTP and http.Serve. + + A client wishing to use the service establishes a connection and then invokes + NewClient on the connection. The convenience function Dial (DialHTTP) performs + both steps for a raw network connection (an HTTP connection). The resulting + Client object has two methods, Call and Go, that specify the service and method to + call, a pointer containing the arguments, and a pointer to receive the result + parameters. + + The Call method waits for the remote call to complete while the Go method + launches the call asynchronously and signals completion using the Call + structure's Done channel. + + Unless an explicit codec is set up, package encoding/gob is used to + transport the data. + + Here is a simple example. A server wishes to export an object of type Arith: + + package server + + type Args struct { + A, B int + } + + type Quotient struct { + Quo, Rem int + } + + type Arith int + + func (t *Arith) Multiply(args *Args, reply *int) error { + *reply = args.A * args.B + return nil + } + + func (t *Arith) Divide(args *Args, quo *Quotient) error { + if args.B == 0 { + return errors.New("divide by zero") + } + quo.Quo = args.A / args.B + quo.Rem = args.A % args.B + return nil + } + + The server calls (for HTTP service): + + arith := new(Arith) + rpc.Register(arith) + rpc.HandleHTTP() + l, e := net.Listen("tcp", ":1234") + if e != nil { + log.Fatal("listen error:", e) + } + go http.Serve(l, nil) + + At this point, clients can see a service "Arith" with methods "Arith.Multiply" and + "Arith.Divide". To invoke one, a client first dials the server: + + client, err := rpc.DialHTTP("tcp", serverAddress + ":1234") + if err != nil { + log.Fatal("dialing:", err) + } + + Then it can make a remote call: + + // Synchronous call + args := &server.Args{7,8} + var reply int + err = client.Call("Arith.Multiply", args, &reply) + if err != nil { + log.Fatal("arith error:", err) + } + fmt.Printf("Arith: %d*%d=%d", args.A, args.B, reply) + + or + + // Asynchronous call + quotient := new(Quotient) + divCall := client.Go("Arith.Divide", args, quotient, nil) + replyCall := <-divCall.Done // will be equal to divCall + // check errors, print, etc. + + A server implementation will often provide a simple, type-safe wrapper for the + client. +*/ +package rpc + +import ( + "bufio" + "encoding/gob" + "errors" + "io" + "log" + "net" + "net/http" + "reflect" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +const ( + // Defaults used by HandleHTTP + DefaultRPCPath = "/_goRPC_" + DefaultDebugPath = "/debug/rpc" +) + +// Precompute the reflect type for error. Can't use error directly +// because Typeof takes an empty interface value. This is annoying. +var typeOfError = reflect.TypeOf((*error)(nil)).Elem() + +type methodType struct { + sync.Mutex // protects counters + method reflect.Method + ArgType reflect.Type + ReplyType reflect.Type + numCalls uint +} + +type service struct { + name string // name of service + rcvr reflect.Value // receiver of methods for the service + typ reflect.Type // type of the receiver + method map[string]*methodType // registered methods +} + +// Request is a header written before every RPC call. It is used internally +// but documented here as an aid to debugging, such as when analyzing +// network traffic. +type Request struct { + ServiceMethod string // format: "Service.Method" + Seq uint64 // sequence number chosen by client + next *Request // for free list in Server +} + +// Response is a header written before every RPC return. It is used internally +// but documented here as an aid to debugging, such as when analyzing +// network traffic. +type Response struct { + ServiceMethod string // echoes that of the Request + Seq uint64 // echoes that of the request + Error string // error, if any. + next *Response // for free list in Server +} + +// Server represents an RPC Server. +type Server struct { + mu sync.RWMutex // protects the serviceMap + serviceMap map[string]*service + reqLock sync.Mutex // protects freeReq + freeReq *Request + respLock sync.Mutex // protects freeResp + freeResp *Response +} + +// NewServer returns a new Server. +func NewServer() *Server { + return &Server{serviceMap: make(map[string]*service)} +} + +// DefaultServer is the default instance of *Server. +var DefaultServer = NewServer() + +// Is this an exported - upper case - name? +func isExported(name string) bool { + rune, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(rune) +} + +// Is this type exported or a builtin? +func isExportedOrBuiltinType(t reflect.Type) bool { + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + // PkgPath will be non-empty even for an exported type, + // so we need to check the type name as well. + return isExported(t.Name()) || t.PkgPath() == "" +} + +// Register publishes in the server the set of methods of the +// receiver value that satisfy the following conditions: +// - exported method +// - two arguments, both of exported type +// - the second argument is a pointer +// - one return value, of type error +// It returns an error if the receiver is not an exported type or has +// no suitable methods. It also logs the error using package log. +// The client accesses each method using a string of the form "Type.Method", +// where Type is the receiver's concrete type. +func (server *Server) Register(rcvr interface{}) error { + return server.register(rcvr, "", false) +} + +// RegisterName is like Register but uses the provided name for the type +// instead of the receiver's concrete type. +func (server *Server) RegisterName(name string, rcvr interface{}) error { + return server.register(rcvr, name, true) +} + +func (server *Server) register(rcvr interface{}, name string, useName bool) error { + server.mu.Lock() + defer server.mu.Unlock() + if server.serviceMap == nil { + server.serviceMap = make(map[string]*service) + } + s := new(service) + s.typ = reflect.TypeOf(rcvr) + s.rcvr = reflect.ValueOf(rcvr) + sname := reflect.Indirect(s.rcvr).Type().Name() + if useName { + sname = name + } + if sname == "" { + s := "rpc.Register: no service name for type " + s.typ.String() + log.Print(s) + return errors.New(s) + } + if !isExported(sname) && !useName { + s := "rpc.Register: type " + sname + " is not exported" + log.Print(s) + return errors.New(s) + } + if _, present := server.serviceMap[sname]; present { + return errors.New("rpc: service already defined: " + sname) + } + s.name = sname + + // Install the methods + s.method = suitableMethods(s.typ, true) + + if len(s.method) == 0 { + str := "" + + // To help the user, see if a pointer receiver would work. + method := suitableMethods(reflect.PtrTo(s.typ), false) + if len(method) != 0 { + str = "rpc.Register: type " + sname + " has no exported methods of suitable type (hint: pass a pointer to value of that type)" + } else { + str = "rpc.Register: type " + sname + " has no exported methods of suitable type" + } + log.Print(str) + return errors.New(str) + } + server.serviceMap[s.name] = s + return nil +} + +// suitableMethods returns suitable Rpc methods of typ, it will report +// error using log if reportErr is true. +func suitableMethods(typ reflect.Type, reportErr bool) map[string]*methodType { + methods := make(map[string]*methodType) + for m := 0; m < typ.NumMethod(); m++ { + method := typ.Method(m) + mtype := method.Type + mname := method.Name + // Method must be exported. + if method.PkgPath != "" { + continue + } + // Method needs three ins: receiver, *args, *reply. + if mtype.NumIn() != 3 { + if reportErr { + log.Println("method", mname, "has wrong number of ins:", mtype.NumIn()) + } + continue + } + // First arg need not be a pointer. + argType := mtype.In(1) + if !isExportedOrBuiltinType(argType) { + if reportErr { + log.Println(mname, "argument type not exported:", argType) + } + continue + } + // Second arg must be a pointer. + replyType := mtype.In(2) + if replyType.Kind() != reflect.Ptr { + if reportErr { + log.Println("method", mname, "reply type not a pointer:", replyType) + } + continue + } + // Reply type must be exported. + if !isExportedOrBuiltinType(replyType) { + if reportErr { + log.Println("method", mname, "reply type not exported:", replyType) + } + continue + } + // Method needs one out. + if mtype.NumOut() != 1 { + if reportErr { + log.Println("method", mname, "has wrong number of outs:", mtype.NumOut()) + } + continue + } + // The return type of the method must be error. + if returnType := mtype.Out(0); returnType != typeOfError { + if reportErr { + log.Println("method", mname, "returns", returnType.String(), "not error") + } + continue + } + methods[mname] = &methodType{method: method, ArgType: argType, ReplyType: replyType} + } + return methods +} + +// A value sent as a placeholder for the server's response value when the server +// receives an invalid request. It is never decoded by the client since the Response +// contains an error when it is used. +var invalidRequest = struct{}{} + +func (server *Server) sendResponse(sending *sync.Mutex, req *Request, reply interface{}, codec ServerCodec, errmsg string) { + resp := server.getResponse() + // Encode the response header + resp.ServiceMethod = req.ServiceMethod + if errmsg != "" { + resp.Error = errmsg + reply = invalidRequest + } + resp.Seq = req.Seq + sending.Lock() + err := codec.WriteResponse(resp, reply) + if debugLog && err != nil { + log.Println("rpc: writing response:", err) + } + sending.Unlock() + server.freeResponse(resp) +} + +func (m *methodType) NumCalls() (n uint) { + m.Lock() + n = m.numCalls + m.Unlock() + return n +} + +func (s *service) call(server *Server, sending *sync.Mutex, mtype *methodType, req *Request, argv, replyv reflect.Value, codec ServerCodec) { + mtype.Lock() + mtype.numCalls++ + mtype.Unlock() + function := mtype.method.Func + // Invoke the method, providing a new value for the reply. + returnValues := function.Call([]reflect.Value{s.rcvr, argv, replyv}) + // The return value for the method is an error. + errInter := returnValues[0].Interface() + errmsg := "" + if errInter != nil { + errmsg = errInter.(error).Error() + } + server.sendResponse(sending, req, replyv.Interface(), codec, errmsg) + server.freeRequest(req) +} + +type gobServerCodec struct { + rwc io.ReadWriteCloser + dec *gob.Decoder + enc *gob.Encoder + encBuf *bufio.Writer +} + +func (c *gobServerCodec) ReadRequestHeader(r *Request) error { + return c.dec.Decode(r) +} + +func (c *gobServerCodec) ReadRequestBody(body interface{}) error { + return c.dec.Decode(body) +} + +func (c *gobServerCodec) WriteResponse(r *Response, body interface{}) (err error) { + if err = c.enc.Encode(r); err != nil { + return + } + if err = c.enc.Encode(body); err != nil { + return + } + return c.encBuf.Flush() +} + +func (c *gobServerCodec) Close() error { + return c.rwc.Close() +} + +// ServeConn runs the server on a single connection. +// ServeConn blocks, serving the connection until the client hangs up. +// The caller typically invokes ServeConn in a go statement. +// ServeConn uses the gob wire format (see package gob) on the +// connection. To use an alternate codec, use ServeCodec. +func (server *Server) ServeConn(conn io.ReadWriteCloser) { + buf := bufio.NewWriter(conn) + srv := &gobServerCodec{conn, gob.NewDecoder(conn), gob.NewEncoder(buf), buf} + server.ServeCodec(srv) +} + +// ServeCodec is like ServeConn but uses the specified codec to +// decode requests and encode responses. +func (server *Server) ServeCodec(codec ServerCodec) { + sending := new(sync.Mutex) + for { + service, mtype, req, argv, replyv, keepReading, err := server.readRequest(codec) + if err != nil { + if debugLog && err != io.EOF { + log.Println("rpc:", err) + } + if !keepReading { + break + } + // send a response if we actually managed to read a header. + if req != nil { + server.sendResponse(sending, req, invalidRequest, codec, err.Error()) + server.freeRequest(req) + } + continue + } + go service.call(server, sending, mtype, req, argv, replyv, codec) + } + codec.Close() +} + +// ServeRequest is like ServeCodec but synchronously serves a single request. +// It does not close the codec upon completion. +func (server *Server) ServeRequest(codec ServerCodec) error { + sending := new(sync.Mutex) + service, mtype, req, argv, replyv, keepReading, err := server.readRequest(codec) + if err != nil { + if !keepReading { + return err + } + // send a response if we actually managed to read a header. + if req != nil { + server.sendResponse(sending, req, invalidRequest, codec, err.Error()) + server.freeRequest(req) + } + return err + } + service.call(server, sending, mtype, req, argv, replyv, codec) + return nil +} + +func (server *Server) getRequest() *Request { + server.reqLock.Lock() + req := server.freeReq + if req == nil { + req = new(Request) + } else { + server.freeReq = req.next + *req = Request{} + } + server.reqLock.Unlock() + return req +} + +func (server *Server) freeRequest(req *Request) { + server.reqLock.Lock() + req.next = server.freeReq + server.freeReq = req + server.reqLock.Unlock() +} + +func (server *Server) getResponse() *Response { + server.respLock.Lock() + resp := server.freeResp + if resp == nil { + resp = new(Response) + } else { + server.freeResp = resp.next + *resp = Response{} + } + server.respLock.Unlock() + return resp +} + +func (server *Server) freeResponse(resp *Response) { + server.respLock.Lock() + resp.next = server.freeResp + server.freeResp = resp + server.respLock.Unlock() +} + +func (server *Server) readRequest(codec ServerCodec) (service *service, mtype *methodType, req *Request, argv, replyv reflect.Value, keepReading bool, err error) { + service, mtype, req, keepReading, err = server.readRequestHeader(codec) + if err != nil { + if !keepReading { + return + } + // discard body + codec.ReadRequestBody(nil) + return + } + + // Decode the argument value. + argIsValue := false // if true, need to indirect before calling. + if mtype.ArgType.Kind() == reflect.Ptr { + argv = reflect.New(mtype.ArgType.Elem()) + } else { + argv = reflect.New(mtype.ArgType) + argIsValue = true + } + // argv guaranteed to be a pointer now. + if err = codec.ReadRequestBody(argv.Interface()); err != nil { + return + } + if argIsValue { + argv = argv.Elem() + } + + replyv = reflect.New(mtype.ReplyType.Elem()) + return +} + +func (server *Server) readRequestHeader(codec ServerCodec) (service *service, mtype *methodType, req *Request, keepReading bool, err error) { + // Grab the request header. + req = server.getRequest() + err = codec.ReadRequestHeader(req) + if err != nil { + req = nil + if err == io.EOF || err == io.ErrUnexpectedEOF { + return + } + err = errors.New("rpc: server cannot decode request: " + err.Error()) + return + } + + // We read the header successfully. If we see an error now, + // we can still recover and move on to the next request. + keepReading = true + + dot := strings.LastIndex(req.ServiceMethod, ".") + if dot < 0 { + err = errors.New("rpc: service/method request ill-formed: " + req.ServiceMethod) + return + } + serviceName := req.ServiceMethod[:dot] + methodName := req.ServiceMethod[dot+1:] + + // Look up the request. + server.mu.RLock() + service = server.serviceMap[serviceName] + server.mu.RUnlock() + if service == nil { + err = errors.New("rpc: can't find service " + req.ServiceMethod) + return + } + mtype = service.method[methodName] + if mtype == nil { + err = errors.New("rpc: can't find method " + req.ServiceMethod) + } + return +} + +// Accept accepts connections on the listener and serves requests +// for each incoming connection. Accept blocks; the caller typically +// invokes it in a go statement. +func (server *Server) Accept(lis net.Listener) { + for { + conn, err := lis.Accept() + if err != nil { + log.Fatal("rpc.Serve: accept:", err.Error()) // TODO(r): exit? + } + go server.ServeConn(conn) + } +} + +// Register publishes the receiver's methods in the DefaultServer. +func Register(rcvr interface{}) error { return DefaultServer.Register(rcvr) } + +// RegisterName is like Register but uses the provided name for the type +// instead of the receiver's concrete type. +func RegisterName(name string, rcvr interface{}) error { + return DefaultServer.RegisterName(name, rcvr) +} + +// A ServerCodec implements reading of RPC requests and writing of +// RPC responses for the server side of an RPC session. +// The server calls ReadRequestHeader and ReadRequestBody in pairs +// to read requests from the connection, and it calls WriteResponse to +// write a response back. The server calls Close when finished with the +// connection. ReadRequestBody may be called with a nil +// argument to force the body of the request to be read and discarded. +type ServerCodec interface { + ReadRequestHeader(*Request) error + ReadRequestBody(interface{}) error + // WriteResponse must be safe for concurrent use by multiple goroutines. + WriteResponse(*Response, interface{}) error + + Close() error +} + +// ServeConn runs the DefaultServer on a single connection. +// ServeConn blocks, serving the connection until the client hangs up. +// The caller typically invokes ServeConn in a go statement. +// ServeConn uses the gob wire format (see package gob) on the +// connection. To use an alternate codec, use ServeCodec. +func ServeConn(conn io.ReadWriteCloser) { + DefaultServer.ServeConn(conn) +} + +// ServeCodec is like ServeConn but uses the specified codec to +// decode requests and encode responses. +func ServeCodec(codec ServerCodec) { + DefaultServer.ServeCodec(codec) +} + +// ServeRequest is like ServeCodec but synchronously serves a single request. +// It does not close the codec upon completion. +func ServeRequest(codec ServerCodec) error { + return DefaultServer.ServeRequest(codec) +} + +// Accept accepts connections on the listener and serves requests +// to DefaultServer for each incoming connection. +// Accept blocks; the caller typically invokes it in a go statement. +func Accept(lis net.Listener) { DefaultServer.Accept(lis) } + +// Can connect to RPC service using HTTP CONNECT to rpcPath. +var connected = "200 Connected to Go RPC" + +// ServeHTTP implements an http.Handler that answers RPC requests. +func (server *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != "CONNECT" { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + w.WriteHeader(http.StatusMethodNotAllowed) + io.WriteString(w, "405 must CONNECT\n") + return + } + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + log.Print("rpc hijacking ", req.RemoteAddr, ": ", err.Error()) + return + } + io.WriteString(conn, "HTTP/1.0 "+connected+"\n\n") + server.ServeConn(conn) +} + +// HandleHTTP registers an HTTP handler for RPC messages on rpcPath, +// and a debugging handler on debugPath. +// It is still necessary to invoke http.Serve(), typically in a go statement. +func (server *Server) HandleHTTP(rpcPath, debugPath string) { + http.Handle(rpcPath, server) + http.Handle(debugPath, debugHTTP{server}) +} + +// HandleHTTP registers an HTTP handler for RPC messages to DefaultServer +// on DefaultRPCPath and a debugging handler on DefaultDebugPath. +// It is still necessary to invoke http.Serve(), typically in a go statement. +func HandleHTTP() { + DefaultServer.HandleHTTP(DefaultRPCPath, DefaultDebugPath) +} diff --git a/src/net/rpc/server_test.go b/src/net/rpc/server_test.go new file mode 100644 index 000000000..0dc4ddc2d --- /dev/null +++ b/src/net/rpc/server_test.go @@ -0,0 +1,683 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package rpc + +import ( + "errors" + "fmt" + "io" + "log" + "net" + "net/http/httptest" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +var ( + newServer *Server + serverAddr, newServerAddr string + httpServerAddr string + once, newOnce, httpOnce sync.Once +) + +const ( + newHttpPath = "/foo" +) + +type Args struct { + A, B int +} + +type Reply struct { + C int +} + +type Arith int + +// Some of Arith's methods have value args, some have pointer args. That's deliberate. + +func (t *Arith) Add(args Args, reply *Reply) error { + reply.C = args.A + args.B + return nil +} + +func (t *Arith) Mul(args *Args, reply *Reply) error { + reply.C = args.A * args.B + return nil +} + +func (t *Arith) Div(args Args, reply *Reply) error { + if args.B == 0 { + return errors.New("divide by zero") + } + reply.C = args.A / args.B + return nil +} + +func (t *Arith) String(args *Args, reply *string) error { + *reply = fmt.Sprintf("%d+%d=%d", args.A, args.B, args.A+args.B) + return nil +} + +func (t *Arith) Scan(args string, reply *Reply) (err error) { + _, err = fmt.Sscan(args, &reply.C) + return +} + +func (t *Arith) Error(args *Args, reply *Reply) error { + panic("ERROR") +} + +func listenTCP() (net.Listener, string) { + l, e := net.Listen("tcp", "127.0.0.1:0") // any available address + if e != nil { + log.Fatalf("net.Listen tcp :0: %v", e) + } + return l, l.Addr().String() +} + +func startServer() { + Register(new(Arith)) + RegisterName("net.rpc.Arith", new(Arith)) + + var l net.Listener + l, serverAddr = listenTCP() + log.Println("Test RPC server listening on", serverAddr) + go Accept(l) + + HandleHTTP() + httpOnce.Do(startHttpServer) +} + +func startNewServer() { + newServer = NewServer() + newServer.Register(new(Arith)) + newServer.RegisterName("net.rpc.Arith", new(Arith)) + newServer.RegisterName("newServer.Arith", new(Arith)) + + var l net.Listener + l, newServerAddr = listenTCP() + log.Println("NewServer test RPC server listening on", newServerAddr) + go newServer.Accept(l) + + newServer.HandleHTTP(newHttpPath, "/bar") + httpOnce.Do(startHttpServer) +} + +func startHttpServer() { + server := httptest.NewServer(nil) + httpServerAddr = server.Listener.Addr().String() + log.Println("Test HTTP RPC server listening on", httpServerAddr) +} + +func TestRPC(t *testing.T) { + once.Do(startServer) + testRPC(t, serverAddr) + newOnce.Do(startNewServer) + testRPC(t, newServerAddr) + testNewServerRPC(t, newServerAddr) +} + +func testRPC(t *testing.T, addr string) { + client, err := Dial("tcp", addr) + if err != nil { + t.Fatal("dialing", err) + } + defer client.Close() + + // Synchronous calls + args := &Args{7, 8} + reply := new(Reply) + err = client.Call("Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) + } + + // Nonexistent method + args = &Args{7, 0} + reply = new(Reply) + err = client.Call("Arith.BadOperation", args, reply) + // expect an error + if err == nil { + t.Error("BadOperation: expected error") + } else if !strings.HasPrefix(err.Error(), "rpc: can't find method ") { + t.Errorf("BadOperation: expected can't find method error; got %q", err) + } + + // Unknown service + args = &Args{7, 8} + reply = new(Reply) + err = client.Call("Arith.Unknown", args, reply) + if err == nil { + t.Error("expected error calling unknown service") + } else if strings.Index(err.Error(), "method") < 0 { + t.Error("expected error about method; got", err) + } + + // Out of order. + args = &Args{7, 8} + mulReply := new(Reply) + mulCall := client.Go("Arith.Mul", args, mulReply, nil) + addReply := new(Reply) + addCall := client.Go("Arith.Add", args, addReply, nil) + + addCall = <-addCall.Done + if addCall.Error != nil { + t.Errorf("Add: expected no error but got string %q", addCall.Error.Error()) + } + if addReply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", addReply.C, args.A+args.B) + } + + mulCall = <-mulCall.Done + if mulCall.Error != nil { + t.Errorf("Mul: expected no error but got string %q", mulCall.Error.Error()) + } + if mulReply.C != args.A*args.B { + t.Errorf("Mul: expected %d got %d", mulReply.C, args.A*args.B) + } + + // Error test + args = &Args{7, 0} + reply = new(Reply) + err = client.Call("Arith.Div", args, reply) + // expect an error: zero divide + if err == nil { + t.Error("Div: expected error") + } else if err.Error() != "divide by zero" { + t.Error("Div: expected divide by zero error; got", err) + } + + // Bad type. + reply = new(Reply) + err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use + if err == nil { + t.Error("expected error calling Arith.Add with wrong arg type") + } else if strings.Index(err.Error(), "type") < 0 { + t.Error("expected error about type; got", err) + } + + // Non-struct argument + const Val = 12345 + str := fmt.Sprint(Val) + reply = new(Reply) + err = client.Call("Arith.Scan", &str, reply) + if err != nil { + t.Errorf("Scan: expected no error but got string %q", err.Error()) + } else if reply.C != Val { + t.Errorf("Scan: expected %d got %d", Val, reply.C) + } + + // Non-struct reply + args = &Args{27, 35} + str = "" + err = client.Call("Arith.String", args, &str) + if err != nil { + t.Errorf("String: expected no error but got string %q", err.Error()) + } + expect := fmt.Sprintf("%d+%d=%d", args.A, args.B, args.A+args.B) + if str != expect { + t.Errorf("String: expected %s got %s", expect, str) + } + + args = &Args{7, 8} + reply = new(Reply) + err = client.Call("Arith.Mul", args, reply) + if err != nil { + t.Errorf("Mul: expected no error but got string %q", err.Error()) + } + if reply.C != args.A*args.B { + t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B) + } + + // ServiceName contain "." character + args = &Args{7, 8} + reply = new(Reply) + err = client.Call("net.rpc.Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) + } +} + +func testNewServerRPC(t *testing.T, addr string) { + client, err := Dial("tcp", addr) + if err != nil { + t.Fatal("dialing", err) + } + defer client.Close() + + // Synchronous calls + args := &Args{7, 8} + reply := new(Reply) + err = client.Call("newServer.Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) + } +} + +func TestHTTP(t *testing.T) { + once.Do(startServer) + testHTTPRPC(t, "") + newOnce.Do(startNewServer) + testHTTPRPC(t, newHttpPath) +} + +func testHTTPRPC(t *testing.T, path string) { + var client *Client + var err error + if path == "" { + client, err = DialHTTP("tcp", httpServerAddr) + } else { + client, err = DialHTTPPath("tcp", httpServerAddr, path) + } + if err != nil { + t.Fatal("dialing", err) + } + defer client.Close() + + // Synchronous calls + args := &Args{7, 8} + reply := new(Reply) + err = client.Call("Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) + } +} + +// CodecEmulator provides a client-like api and a ServerCodec interface. +// Can be used to test ServeRequest. +type CodecEmulator struct { + server *Server + serviceMethod string + args *Args + reply *Reply + err error +} + +func (codec *CodecEmulator) Call(serviceMethod string, args *Args, reply *Reply) error { + codec.serviceMethod = serviceMethod + codec.args = args + codec.reply = reply + codec.err = nil + var serverError error + if codec.server == nil { + serverError = ServeRequest(codec) + } else { + serverError = codec.server.ServeRequest(codec) + } + if codec.err == nil && serverError != nil { + codec.err = serverError + } + return codec.err +} + +func (codec *CodecEmulator) ReadRequestHeader(req *Request) error { + req.ServiceMethod = codec.serviceMethod + req.Seq = 0 + return nil +} + +func (codec *CodecEmulator) ReadRequestBody(argv interface{}) error { + if codec.args == nil { + return io.ErrUnexpectedEOF + } + *(argv.(*Args)) = *codec.args + return nil +} + +func (codec *CodecEmulator) WriteResponse(resp *Response, reply interface{}) error { + if resp.Error != "" { + codec.err = errors.New(resp.Error) + } else { + *codec.reply = *(reply.(*Reply)) + } + return nil +} + +func (codec *CodecEmulator) Close() error { + return nil +} + +func TestServeRequest(t *testing.T) { + once.Do(startServer) + testServeRequest(t, nil) + newOnce.Do(startNewServer) + testServeRequest(t, newServer) +} + +func testServeRequest(t *testing.T, server *Server) { + client := CodecEmulator{server: server} + defer client.Close() + + args := &Args{7, 8} + reply := new(Reply) + err := client.Call("Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) + } + + err = client.Call("Arith.Add", nil, reply) + if err == nil { + t.Errorf("expected error calling Arith.Add with nil arg") + } +} + +type ReplyNotPointer int +type ArgNotPublic int +type ReplyNotPublic int +type NeedsPtrType int +type local struct{} + +func (t *ReplyNotPointer) ReplyNotPointer(args *Args, reply Reply) error { + return nil +} + +func (t *ArgNotPublic) ArgNotPublic(args *local, reply *Reply) error { + return nil +} + +func (t *ReplyNotPublic) ReplyNotPublic(args *Args, reply *local) error { + return nil +} + +func (t *NeedsPtrType) NeedsPtrType(args *Args, reply *Reply) error { + return nil +} + +// Check that registration handles lots of bad methods and a type with no suitable methods. +func TestRegistrationError(t *testing.T) { + err := Register(new(ReplyNotPointer)) + if err == nil { + t.Error("expected error registering ReplyNotPointer") + } + err = Register(new(ArgNotPublic)) + if err == nil { + t.Error("expected error registering ArgNotPublic") + } + err = Register(new(ReplyNotPublic)) + if err == nil { + t.Error("expected error registering ReplyNotPublic") + } + err = Register(NeedsPtrType(0)) + if err == nil { + t.Error("expected error registering NeedsPtrType") + } else if !strings.Contains(err.Error(), "pointer") { + t.Error("expected hint when registering NeedsPtrType") + } +} + +type WriteFailCodec int + +func (WriteFailCodec) WriteRequest(*Request, interface{}) error { + // the panic caused by this error used to not unlock a lock. + return errors.New("fail") +} + +func (WriteFailCodec) ReadResponseHeader(*Response) error { + select {} +} + +func (WriteFailCodec) ReadResponseBody(interface{}) error { + select {} +} + +func (WriteFailCodec) Close() error { + return nil +} + +func TestSendDeadlock(t *testing.T) { + client := NewClientWithCodec(WriteFailCodec(0)) + defer client.Close() + + done := make(chan bool) + go func() { + testSendDeadlock(client) + testSendDeadlock(client) + done <- true + }() + select { + case <-done: + return + case <-time.After(5 * time.Second): + t.Fatal("deadlock") + } +} + +func testSendDeadlock(client *Client) { + defer func() { + recover() + }() + args := &Args{7, 8} + reply := new(Reply) + client.Call("Arith.Add", args, reply) +} + +func dialDirect() (*Client, error) { + return Dial("tcp", serverAddr) +} + +func dialHTTP() (*Client, error) { + return DialHTTP("tcp", httpServerAddr) +} + +func countMallocs(dial func() (*Client, error), t *testing.T) float64 { + once.Do(startServer) + client, err := dial() + if err != nil { + t.Fatal("error dialing", err) + } + defer client.Close() + + args := &Args{7, 8} + reply := new(Reply) + return testing.AllocsPerRun(100, func() { + err := client.Call("Arith.Add", args, reply) + if err != nil { + t.Errorf("Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B) + } + }) +} + +func TestCountMallocs(t *testing.T) { + if testing.Short() { + t.Skip("skipping malloc count in short mode") + } + if runtime.GOMAXPROCS(0) > 1 { + t.Skip("skipping; GOMAXPROCS>1") + } + fmt.Printf("mallocs per rpc round trip: %v\n", countMallocs(dialDirect, t)) +} + +func TestCountMallocsOverHTTP(t *testing.T) { + if testing.Short() { + t.Skip("skipping malloc count in short mode") + } + if runtime.GOMAXPROCS(0) > 1 { + t.Skip("skipping; GOMAXPROCS>1") + } + fmt.Printf("mallocs per HTTP rpc round trip: %v\n", countMallocs(dialHTTP, t)) +} + +type writeCrasher struct { + done chan bool +} + +func (writeCrasher) Close() error { + return nil +} + +func (w *writeCrasher) Read(p []byte) (int, error) { + <-w.done + return 0, io.EOF +} + +func (writeCrasher) Write(p []byte) (int, error) { + return 0, errors.New("fake write failure") +} + +func TestClientWriteError(t *testing.T) { + w := &writeCrasher{done: make(chan bool)} + c := NewClient(w) + defer c.Close() + + res := false + err := c.Call("foo", 1, &res) + if err == nil { + t.Fatal("expected error") + } + if err.Error() != "fake write failure" { + t.Error("unexpected value of error:", err) + } + w.done <- true +} + +func TestTCPClose(t *testing.T) { + once.Do(startServer) + + client, err := dialHTTP() + if err != nil { + t.Fatalf("dialing: %v", err) + } + defer client.Close() + + args := Args{17, 8} + var reply Reply + err = client.Call("Arith.Mul", args, &reply) + if err != nil { + t.Fatal("arith error:", err) + } + t.Logf("Arith: %d*%d=%d\n", args.A, args.B, reply) + if reply.C != args.A*args.B { + t.Errorf("Add: expected %d got %d", reply.C, args.A*args.B) + } +} + +func TestErrorAfterClientClose(t *testing.T) { + once.Do(startServer) + + client, err := dialHTTP() + if err != nil { + t.Fatalf("dialing: %v", err) + } + err = client.Close() + if err != nil { + t.Fatal("close error:", err) + } + err = client.Call("Arith.Add", &Args{7, 9}, new(Reply)) + if err != ErrShutdown { + t.Errorf("Forever: expected ErrShutdown got %v", err) + } +} + +func benchmarkEndToEnd(dial func() (*Client, error), b *testing.B) { + once.Do(startServer) + client, err := dial() + if err != nil { + b.Fatal("error dialing:", err) + } + defer client.Close() + + // Synchronous calls + args := &Args{7, 8} + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + reply := new(Reply) + for pb.Next() { + err := client.Call("Arith.Add", args, reply) + if err != nil { + b.Fatalf("rpc error: Add: expected no error but got string %q", err.Error()) + } + if reply.C != args.A+args.B { + b.Fatalf("rpc error: Add: expected %d got %d", reply.C, args.A+args.B) + } + } + }) +} + +func benchmarkEndToEndAsync(dial func() (*Client, error), b *testing.B) { + const MaxConcurrentCalls = 100 + once.Do(startServer) + client, err := dial() + if err != nil { + b.Fatal("error dialing:", err) + } + defer client.Close() + + // Asynchronous calls + args := &Args{7, 8} + procs := 4 * runtime.GOMAXPROCS(-1) + send := int32(b.N) + recv := int32(b.N) + var wg sync.WaitGroup + wg.Add(procs) + gate := make(chan bool, MaxConcurrentCalls) + res := make(chan *Call, MaxConcurrentCalls) + b.ResetTimer() + + for p := 0; p < procs; p++ { + go func() { + for atomic.AddInt32(&send, -1) >= 0 { + gate <- true + reply := new(Reply) + client.Go("Arith.Add", args, reply, res) + } + }() + go func() { + for call := range res { + A := call.Args.(*Args).A + B := call.Args.(*Args).B + C := call.Reply.(*Reply).C + if A+B != C { + b.Fatalf("incorrect reply: Add: expected %d got %d", A+B, C) + } + <-gate + if atomic.AddInt32(&recv, -1) == 0 { + close(res) + } + } + wg.Done() + }() + } + wg.Wait() +} + +func BenchmarkEndToEnd(b *testing.B) { + benchmarkEndToEnd(dialDirect, b) +} + +func BenchmarkEndToEndHTTP(b *testing.B) { + benchmarkEndToEnd(dialHTTP, b) +} + +func BenchmarkEndToEndAsync(b *testing.B) { + benchmarkEndToEndAsync(dialDirect, b) +} + +func BenchmarkEndToEndAsyncHTTP(b *testing.B) { + benchmarkEndToEndAsync(dialHTTP, b) +} diff --git a/src/net/sendfile_dragonfly.go b/src/net/sendfile_dragonfly.go new file mode 100644 index 000000000..bc88fd3b9 --- /dev/null +++ b/src/net/sendfile_dragonfly.go @@ -0,0 +1,103 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" +) + +// maxSendfileSize is the largest chunk size we ask the kernel to copy +// at a time. +const maxSendfileSize int = 4 << 20 + +// sendFile copies the contents of r to c using the sendfile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number of bytes copied and any +// non-EOF error. +// +// if handled == false, sendFile performed no work. +func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { + // DragonFly uses 0 as the "until EOF" value. If you pass in more bytes than the + // file contains, it will loop back to the beginning ad nauseam until it's sent + // exactly the number of bytes told to. As such, we need to know exactly how many + // bytes to send. + var remain int64 = 0 + + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, nil, true + } + } + f, ok := r.(*os.File) + if !ok { + return 0, nil, false + } + + if remain == 0 { + fi, err := f.Stat() + if err != nil { + return 0, err, false + } + + remain = fi.Size() + } + + // The other quirk with DragonFly's sendfile implementation is that it doesn't + // use the current position of the file -- if you pass it offset 0, it starts + // from offset 0. There's no way to tell it "start from current position", so + // we have to manage that explicitly. + pos, err := f.Seek(0, os.SEEK_CUR) + if err != nil { + return 0, err, false + } + + if err := c.writeLock(); err != nil { + return 0, err, true + } + defer c.writeUnlock() + + dst := c.sysfd + src := int(f.Fd()) + for remain > 0 { + n := maxSendfileSize + if int64(n) > remain { + n = int(remain) + } + pos1 := pos + n, err1 := syscall.Sendfile(dst, src, &pos1, n) + if n > 0 { + pos += int64(n) + written += int64(n) + remain -= int64(n) + } + if n == 0 && err1 == nil { + break + } + if err1 == syscall.EAGAIN { + if err1 = c.pd.WaitWrite(); err1 == nil { + continue + } + } + if err1 == syscall.EINTR { + continue + } + if err1 != nil { + // This includes syscall.ENOSYS (no kernel + // support) and syscall.EINVAL (fd types which + // don't implement sendfile together) + err = &OpError{"sendfile", c.net, c.raddr, err1} + break + } + } + if lr != nil { + lr.N = remain + } + return written, err, written > 0 +} diff --git a/src/net/sendfile_freebsd.go b/src/net/sendfile_freebsd.go new file mode 100644 index 000000000..ffc147262 --- /dev/null +++ b/src/net/sendfile_freebsd.go @@ -0,0 +1,103 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" +) + +// maxSendfileSize is the largest chunk size we ask the kernel to copy +// at a time. +const maxSendfileSize int = 4 << 20 + +// sendFile copies the contents of r to c using the sendfile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number of bytes copied and any +// non-EOF error. +// +// if handled == false, sendFile performed no work. +func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { + // FreeBSD uses 0 as the "until EOF" value. If you pass in more bytes than the + // file contains, it will loop back to the beginning ad nauseam until it's sent + // exactly the number of bytes told to. As such, we need to know exactly how many + // bytes to send. + var remain int64 = 0 + + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, nil, true + } + } + f, ok := r.(*os.File) + if !ok { + return 0, nil, false + } + + if remain == 0 { + fi, err := f.Stat() + if err != nil { + return 0, err, false + } + + remain = fi.Size() + } + + // The other quirk with FreeBSD's sendfile implementation is that it doesn't + // use the current position of the file -- if you pass it offset 0, it starts + // from offset 0. There's no way to tell it "start from current position", so + // we have to manage that explicitly. + pos, err := f.Seek(0, os.SEEK_CUR) + if err != nil { + return 0, err, false + } + + if err := c.writeLock(); err != nil { + return 0, err, true + } + defer c.writeUnlock() + + dst := c.sysfd + src := int(f.Fd()) + for remain > 0 { + n := maxSendfileSize + if int64(n) > remain { + n = int(remain) + } + pos1 := pos + n, err1 := syscall.Sendfile(dst, src, &pos1, n) + if n > 0 { + pos += int64(n) + written += int64(n) + remain -= int64(n) + } + if n == 0 && err1 == nil { + break + } + if err1 == syscall.EAGAIN { + if err1 = c.pd.WaitWrite(); err1 == nil { + continue + } + } + if err1 == syscall.EINTR { + continue + } + if err1 != nil { + // This includes syscall.ENOSYS (no kernel + // support) and syscall.EINVAL (fd types which + // don't implement sendfile together) + err = &OpError{"sendfile", c.net, c.raddr, err1} + break + } + } + if lr != nil { + lr.N = remain + } + return written, err, written > 0 +} diff --git a/src/net/sendfile_linux.go b/src/net/sendfile_linux.go new file mode 100644 index 000000000..5e117636a --- /dev/null +++ b/src/net/sendfile_linux.go @@ -0,0 +1,76 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" +) + +// maxSendfileSize is the largest chunk size we ask the kernel to copy +// at a time. +const maxSendfileSize int = 4 << 20 + +// sendFile copies the contents of r to c using the sendfile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number of bytes copied and any +// non-EOF error. +// +// if handled == false, sendFile performed no work. +func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { + var remain int64 = 1 << 62 // by default, copy until EOF + + lr, ok := r.(*io.LimitedReader) + if ok { + remain, r = lr.N, lr.R + if remain <= 0 { + return 0, nil, true + } + } + f, ok := r.(*os.File) + if !ok { + return 0, nil, false + } + + if err := c.writeLock(); err != nil { + return 0, err, true + } + defer c.writeUnlock() + + dst := c.sysfd + src := int(f.Fd()) + for remain > 0 { + n := maxSendfileSize + if int64(n) > remain { + n = int(remain) + } + n, err1 := syscall.Sendfile(dst, src, nil, n) + if n > 0 { + written += int64(n) + remain -= int64(n) + } + if n == 0 && err1 == nil { + break + } + if err1 == syscall.EAGAIN { + if err1 = c.pd.WaitWrite(); err1 == nil { + continue + } + } + if err1 != nil { + // This includes syscall.ENOSYS (no kernel + // support) and syscall.EINVAL (fd types which + // don't implement sendfile together) + err = &OpError{"sendfile", c.net, c.raddr, err1} + break + } + } + if lr != nil { + lr.N = remain + } + return written, err, written > 0 +} diff --git a/src/net/sendfile_stub.go b/src/net/sendfile_stub.go new file mode 100644 index 000000000..03426ef0d --- /dev/null +++ b/src/net/sendfile_stub.go @@ -0,0 +1,13 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin nacl netbsd openbsd solaris + +package net + +import "io" + +func sendFile(c *netFD, r io.Reader) (n int64, err error, handled bool) { + return 0, nil, false +} diff --git a/src/net/sendfile_windows.go b/src/net/sendfile_windows.go new file mode 100644 index 000000000..b128ba27b --- /dev/null +++ b/src/net/sendfile_windows.go @@ -0,0 +1,55 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" +) + +// sendFile copies the contents of r to c using the TransmitFile +// system call to minimize copies. +// +// if handled == true, sendFile returns the number of bytes copied and any +// non-EOF error. +// +// if handled == false, sendFile performed no work. +// +// Note that sendfile for windows does not suppport >2GB file. +func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { + var n int64 = 0 // by default, copy until EOF + + lr, ok := r.(*io.LimitedReader) + if ok { + n, r = lr.N, lr.R + if n <= 0 { + return 0, nil, true + } + } + f, ok := r.(*os.File) + if !ok { + return 0, nil, false + } + + if err := fd.writeLock(); err != nil { + return 0, err, true + } + defer fd.writeUnlock() + + o := &fd.wop + o.qty = uint32(n) + o.handle = syscall.Handle(f.Fd()) + done, err := wsrv.ExecIO(o, "TransmitFile", func(o *operation) error { + return syscall.TransmitFile(o.fd.sysfd, o.handle, o.qty, 0, &o.o, nil, syscall.TF_WRITE_BEHIND) + }) + if err != nil { + return 0, err, false + } + if lr != nil { + lr.N -= int64(done) + } + return int64(done), nil, true +} diff --git a/src/net/server_test.go b/src/net/server_test.go new file mode 100644 index 000000000..6a2bb9243 --- /dev/null +++ b/src/net/server_test.go @@ -0,0 +1,461 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "flag" + "io" + "os" + "runtime" + "testing" + "time" +) + +func skipServerTest(net, unixsotype, addr string, ipv6, ipv4map, linuxOnly bool) bool { + switch runtime.GOOS { + case "linux": + case "nacl", "plan9", "windows": + // "unix" sockets are not supported on Windows and Plan 9. + if net == unixsotype { + return true + } + default: + if net == unixsotype && linuxOnly { + return true + } + } + switch addr { + case "", "0.0.0.0", "[::ffff:0.0.0.0]", "[::]": + if testing.Short() || !*testExternal { + return true + } + } + if ipv6 && !supportsIPv6 { + return true + } + if ipv4map && !supportsIPv4map { + return true + } + return false +} + +var streamConnServerTests = []struct { + snet string // server side + saddr string + cnet string // client side + caddr string + ipv6 bool // test with underlying AF_INET6 socket + ipv4map bool // test with IPv6 IPv4-mapping functionality + empty bool // test with empty data + linuxOnly bool // test with abstract unix domain socket, a Linux-ism +}{ + {snet: "tcp", saddr: "", cnet: "tcp", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "0.0.0.0", cnet: "tcp", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "[::ffff:0.0.0.0]", cnet: "tcp", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "[::]", cnet: "tcp", caddr: "[::1]", ipv6: true}, + + {snet: "tcp", saddr: "", cnet: "tcp", caddr: "[::1]", ipv4map: true}, + {snet: "tcp", saddr: "0.0.0.0", cnet: "tcp", caddr: "[::1]", ipv4map: true}, + {snet: "tcp", saddr: "[::ffff:0.0.0.0]", cnet: "tcp", caddr: "[::1]", ipv4map: true}, + {snet: "tcp", saddr: "[::]", cnet: "tcp", caddr: "127.0.0.1", ipv4map: true}, + + {snet: "tcp", saddr: "", cnet: "tcp4", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "0.0.0.0", cnet: "tcp4", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "[::ffff:0.0.0.0]", cnet: "tcp4", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "[::]", cnet: "tcp6", caddr: "[::1]", ipv6: true}, + + {snet: "tcp", saddr: "", cnet: "tcp6", caddr: "[::1]", ipv4map: true}, + {snet: "tcp", saddr: "0.0.0.0", cnet: "tcp6", caddr: "[::1]", ipv4map: true}, + {snet: "tcp", saddr: "[::ffff:0.0.0.0]", cnet: "tcp6", caddr: "[::1]", ipv4map: true}, + {snet: "tcp", saddr: "[::]", cnet: "tcp4", caddr: "127.0.0.1", ipv4map: true}, + + {snet: "tcp", saddr: "127.0.0.1", cnet: "tcp", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "[::ffff:127.0.0.1]", cnet: "tcp", caddr: "127.0.0.1"}, + {snet: "tcp", saddr: "[::1]", cnet: "tcp", caddr: "[::1]", ipv6: true}, + + {snet: "tcp4", saddr: "", cnet: "tcp4", caddr: "127.0.0.1"}, + {snet: "tcp4", saddr: "0.0.0.0", cnet: "tcp4", caddr: "127.0.0.1"}, + {snet: "tcp4", saddr: "[::ffff:0.0.0.0]", cnet: "tcp4", caddr: "127.0.0.1"}, + + {snet: "tcp4", saddr: "127.0.0.1", cnet: "tcp4", caddr: "127.0.0.1"}, + + {snet: "tcp6", saddr: "", cnet: "tcp6", caddr: "[::1]", ipv6: true}, + {snet: "tcp6", saddr: "[::]", cnet: "tcp6", caddr: "[::1]", ipv6: true}, + + {snet: "tcp6", saddr: "[::1]", cnet: "tcp6", caddr: "[::1]", ipv6: true}, + + {snet: "unix", saddr: testUnixAddr(), cnet: "unix", caddr: testUnixAddr()}, + {snet: "unix", saddr: "@gotest2/net", cnet: "unix", caddr: "@gotest2/net.local", linuxOnly: true}, +} + +func TestStreamConnServer(t *testing.T) { + for _, tt := range streamConnServerTests { + if skipServerTest(tt.snet, "unix", tt.saddr, tt.ipv6, tt.ipv4map, tt.linuxOnly) { + continue + } + + listening := make(chan string) + done := make(chan int) + switch tt.snet { + case "tcp", "tcp4", "tcp6": + tt.saddr += ":0" + case "unix": + os.Remove(tt.saddr) + os.Remove(tt.caddr) + } + + go runStreamConnServer(t, tt.snet, tt.saddr, listening, done) + taddr := <-listening // wait for server to start + + switch tt.cnet { + case "tcp", "tcp4", "tcp6": + _, port, err := SplitHostPort(taddr) + if err != nil { + t.Fatalf("SplitHostPort(%q) failed: %v", taddr, err) + } + taddr = tt.caddr + ":" + port + } + + runStreamConnClient(t, tt.cnet, taddr, tt.empty) + <-done // make sure server stopped + + switch tt.snet { + case "unix": + os.Remove(tt.saddr) + os.Remove(tt.caddr) + } + } +} + +var seqpacketConnServerTests = []struct { + net string + saddr string // server address + caddr string // client address + empty bool // test with empty data + linuxOnly bool // test with abstract unix domain socket, a Linux-ism +}{ + {net: "unixpacket", saddr: testUnixAddr(), caddr: testUnixAddr()}, + {net: "unixpacket", saddr: "@gotest4/net", caddr: "@gotest4/net.local", linuxOnly: true}, +} + +func TestSeqpacketConnServer(t *testing.T) { + switch runtime.GOOS { + case "darwin", "nacl", "openbsd", "plan9", "windows": + fallthrough + case "freebsd": // FreeBSD 8 doesn't support unixpacket + t.Skipf("skipping test on %q", runtime.GOOS) + } + + for _, tt := range seqpacketConnServerTests { + if runtime.GOOS != "linux" && tt.linuxOnly { + continue + } + listening := make(chan string) + done := make(chan int) + switch tt.net { + case "unixpacket": + os.Remove(tt.saddr) + os.Remove(tt.caddr) + } + + go runStreamConnServer(t, tt.net, tt.saddr, listening, done) + taddr := <-listening // wait for server to start + + runStreamConnClient(t, tt.net, taddr, tt.empty) + <-done // make sure server stopped + + switch tt.net { + case "unixpacket": + os.Remove(tt.saddr) + os.Remove(tt.caddr) + } + } +} + +func runStreamConnServer(t *testing.T, net, laddr string, listening chan<- string, done chan<- int) { + defer close(done) + l, err := Listen(net, laddr) + if err != nil { + t.Errorf("Listen(%q, %q) failed: %v", net, laddr, err) + listening <- "<nil>" + return + } + defer l.Close() + listening <- l.Addr().String() + + echo := func(rw io.ReadWriter, done chan<- int) { + buf := make([]byte, 1024) + for { + n, err := rw.Read(buf[0:]) + if err != nil || n == 0 || string(buf[:n]) == "END" { + break + } + rw.Write(buf[0:n]) + } + close(done) + } + +run: + for { + c, err := l.Accept() + if err != nil { + t.Logf("Accept failed: %v", err) + continue run + } + echodone := make(chan int) + go echo(c, echodone) + <-echodone // make sure echo stopped + c.Close() + break run + } +} + +func runStreamConnClient(t *testing.T, net, taddr string, isEmpty bool) { + c, err := Dial(net, taddr) + if err != nil { + t.Fatalf("Dial(%q, %q) failed: %v", net, taddr, err) + } + defer c.Close() + c.SetReadDeadline(time.Now().Add(1 * time.Second)) + + var wb []byte + if !isEmpty { + wb = []byte("StreamConnClient by Dial\n") + } + if n, err := c.Write(wb); err != nil || n != len(wb) { + t.Fatalf("Write failed: %v, %v; want %v, <nil>", n, err, len(wb)) + } + + rb := make([]byte, 1024) + if n, err := c.Read(rb[0:]); err != nil || n != len(wb) { + t.Fatalf("Read failed: %v, %v; want %v, <nil>", n, err, len(wb)) + } + + // Send explicit ending for unixpacket. + // Older Linux kernels do not stop reads on close. + switch net { + case "unixpacket": + c.Write([]byte("END")) + } +} + +// Do not test empty datagrams by default. +// It causes unexplained timeouts on some systems, +// including Snow Leopard. I think that the kernel +// doesn't quite expect them. +var testDatagram = flag.Bool("datagram", false, "whether to test udp and unixgram") + +var datagramPacketConnServerTests = []struct { + snet string // server side + saddr string + cnet string // client side + caddr string + ipv6 bool // test with underlying AF_INET6 socket + ipv4map bool // test with IPv6 IPv4-mapping functionality + dial bool // test with Dial or DialUnix + empty bool // test with empty data + linuxOnly bool // test with abstract unix domain socket, a Linux-ism +}{ + {snet: "udp", saddr: "", cnet: "udp", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "0.0.0.0", cnet: "udp", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "[::ffff:0.0.0.0]", cnet: "udp", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "[::]", cnet: "udp", caddr: "[::1]", ipv6: true}, + + {snet: "udp", saddr: "", cnet: "udp", caddr: "[::1]", ipv4map: true}, + {snet: "udp", saddr: "0.0.0.0", cnet: "udp", caddr: "[::1]", ipv4map: true}, + {snet: "udp", saddr: "[::ffff:0.0.0.0]", cnet: "udp", caddr: "[::1]", ipv4map: true}, + {snet: "udp", saddr: "[::]", cnet: "udp", caddr: "127.0.0.1", ipv4map: true}, + + {snet: "udp", saddr: "", cnet: "udp4", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "0.0.0.0", cnet: "udp4", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "[::ffff:0.0.0.0]", cnet: "udp4", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "[::]", cnet: "udp6", caddr: "[::1]", ipv6: true}, + + {snet: "udp", saddr: "", cnet: "udp6", caddr: "[::1]", ipv4map: true}, + {snet: "udp", saddr: "0.0.0.0", cnet: "udp6", caddr: "[::1]", ipv4map: true}, + {snet: "udp", saddr: "[::ffff:0.0.0.0]", cnet: "udp6", caddr: "[::1]", ipv4map: true}, + {snet: "udp", saddr: "[::]", cnet: "udp4", caddr: "127.0.0.1", ipv4map: true}, + + {snet: "udp", saddr: "127.0.0.1", cnet: "udp", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "[::ffff:127.0.0.1]", cnet: "udp", caddr: "127.0.0.1"}, + {snet: "udp", saddr: "[::1]", cnet: "udp", caddr: "[::1]", ipv6: true}, + + {snet: "udp4", saddr: "", cnet: "udp4", caddr: "127.0.0.1"}, + {snet: "udp4", saddr: "0.0.0.0", cnet: "udp4", caddr: "127.0.0.1"}, + {snet: "udp4", saddr: "[::ffff:0.0.0.0]", cnet: "udp4", caddr: "127.0.0.1"}, + + {snet: "udp4", saddr: "127.0.0.1", cnet: "udp4", caddr: "127.0.0.1"}, + + {snet: "udp6", saddr: "", cnet: "udp6", caddr: "[::1]", ipv6: true}, + {snet: "udp6", saddr: "[::]", cnet: "udp6", caddr: "[::1]", ipv6: true}, + + {snet: "udp6", saddr: "[::1]", cnet: "udp6", caddr: "[::1]", ipv6: true}, + + {snet: "udp", saddr: "127.0.0.1", cnet: "udp", caddr: "127.0.0.1", dial: true}, + {snet: "udp", saddr: "127.0.0.1", cnet: "udp", caddr: "127.0.0.1", empty: true}, + {snet: "udp", saddr: "127.0.0.1", cnet: "udp", caddr: "127.0.0.1", dial: true, empty: true}, + + {snet: "udp", saddr: "[::1]", cnet: "udp", caddr: "[::1]", ipv6: true, dial: true}, + {snet: "udp", saddr: "[::1]", cnet: "udp", caddr: "[::1]", ipv6: true, empty: true}, + {snet: "udp", saddr: "[::1]", cnet: "udp", caddr: "[::1]", ipv6: true, dial: true, empty: true}, + + {snet: "unixgram", saddr: testUnixAddr(), cnet: "unixgram", caddr: testUnixAddr()}, + {snet: "unixgram", saddr: testUnixAddr(), cnet: "unixgram", caddr: testUnixAddr(), dial: true}, + {snet: "unixgram", saddr: testUnixAddr(), cnet: "unixgram", caddr: testUnixAddr(), empty: true}, + {snet: "unixgram", saddr: testUnixAddr(), cnet: "unixgram", caddr: testUnixAddr(), dial: true, empty: true}, + + {snet: "unixgram", saddr: "@gotest6/net", cnet: "unixgram", caddr: "@gotest6/net.local", linuxOnly: true}, +} + +func TestDatagramPacketConnServer(t *testing.T) { + if !*testDatagram { + return + } + + for _, tt := range datagramPacketConnServerTests { + if skipServerTest(tt.snet, "unixgram", tt.saddr, tt.ipv6, tt.ipv4map, tt.linuxOnly) { + continue + } + + listening := make(chan string) + done := make(chan int) + switch tt.snet { + case "udp", "udp4", "udp6": + tt.saddr += ":0" + case "unixgram": + os.Remove(tt.saddr) + os.Remove(tt.caddr) + } + + go runDatagramPacketConnServer(t, tt.snet, tt.saddr, listening, done) + taddr := <-listening // wait for server to start + + switch tt.cnet { + case "udp", "udp4", "udp6": + _, port, err := SplitHostPort(taddr) + if err != nil { + t.Fatalf("SplitHostPort(%q) failed: %v", taddr, err) + } + taddr = tt.caddr + ":" + port + tt.caddr += ":0" + } + if tt.dial { + runDatagramConnClient(t, tt.cnet, tt.caddr, taddr, tt.empty) + } else { + runDatagramPacketConnClient(t, tt.cnet, tt.caddr, taddr, tt.empty) + } + <-done // tell server to stop + <-done // make sure server stopped + + switch tt.snet { + case "unixgram": + os.Remove(tt.saddr) + os.Remove(tt.caddr) + } + } +} + +func runDatagramPacketConnServer(t *testing.T, net, laddr string, listening chan<- string, done chan<- int) { + c, err := ListenPacket(net, laddr) + if err != nil { + t.Errorf("ListenPacket(%q, %q) failed: %v", net, laddr, err) + listening <- "<nil>" + done <- 1 + return + } + defer c.Close() + listening <- c.LocalAddr().String() + + buf := make([]byte, 1024) +run: + for { + c.SetReadDeadline(time.Now().Add(10 * time.Millisecond)) + n, ra, err := c.ReadFrom(buf[0:]) + if nerr, ok := err.(Error); ok && nerr.Timeout() { + select { + case done <- 1: + break run + default: + continue run + } + } + if err != nil { + break run + } + if _, err = c.WriteTo(buf[0:n], ra); err != nil { + t.Errorf("WriteTo(%v) failed: %v", ra, err) + break run + } + } + done <- 1 +} + +func runDatagramConnClient(t *testing.T, net, laddr, taddr string, isEmpty bool) { + var c Conn + var err error + switch net { + case "udp", "udp4", "udp6": + c, err = Dial(net, taddr) + if err != nil { + t.Fatalf("Dial(%q, %q) failed: %v", net, taddr, err) + } + case "unixgram": + c, err = DialUnix(net, &UnixAddr{Name: laddr, Net: net}, &UnixAddr{Name: taddr, Net: net}) + if err != nil { + t.Fatalf("DialUnix(%q, {%q, %q}) failed: %v", net, laddr, taddr, err) + } + } + defer c.Close() + c.SetReadDeadline(time.Now().Add(1 * time.Second)) + + var wb []byte + if !isEmpty { + wb = []byte("DatagramConnClient by Dial\n") + } + if n, err := c.Write(wb[0:]); err != nil || n != len(wb) { + t.Fatalf("Write failed: %v, %v; want %v, <nil>", n, err, len(wb)) + } + + rb := make([]byte, 1024) + if n, err := c.Read(rb[0:]); err != nil || n != len(wb) { + t.Fatalf("Read failed: %v, %v; want %v, <nil>", n, err, len(wb)) + } +} + +func runDatagramPacketConnClient(t *testing.T, net, laddr, taddr string, isEmpty bool) { + var ra Addr + var err error + switch net { + case "udp", "udp4", "udp6": + ra, err = ResolveUDPAddr(net, taddr) + if err != nil { + t.Fatalf("ResolveUDPAddr(%q, %q) failed: %v", net, taddr, err) + } + case "unixgram": + ra, err = ResolveUnixAddr(net, taddr) + if err != nil { + t.Fatalf("ResolveUxixAddr(%q, %q) failed: %v", net, taddr, err) + } + } + c, err := ListenPacket(net, laddr) + if err != nil { + t.Fatalf("ListenPacket(%q, %q) faild: %v", net, laddr, err) + } + defer c.Close() + c.SetReadDeadline(time.Now().Add(1 * time.Second)) + + var wb []byte + if !isEmpty { + wb = []byte("DatagramPacketConnClient by ListenPacket\n") + } + if n, err := c.WriteTo(wb[0:], ra); err != nil || n != len(wb) { + t.Fatalf("WriteTo(%v) failed: %v, %v; want %v, <nil>", ra, n, err, len(wb)) + } + + rb := make([]byte, 1024) + if n, _, err := c.ReadFrom(rb[0:]); err != nil || n != len(wb) { + t.Fatalf("ReadFrom failed: %v, %v; want %v, <nil>", n, err, len(wb)) + } +} diff --git a/src/net/singleflight.go b/src/net/singleflight.go new file mode 100644 index 000000000..dc58affda --- /dev/null +++ b/src/net/singleflight.go @@ -0,0 +1,53 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "sync" + +// call is an in-flight or completed singleflight.Do call +type call struct { + wg sync.WaitGroup + val interface{} + err error + dups int +} + +// singleflight represents a class of work and forms a namespace in +// which units of work can be executed with duplicate suppression. +type singleflight struct { + mu sync.Mutex // protects m + m map[string]*call // lazily initialized +} + +// Do executes and returns the results of the given function, making +// sure that only one execution is in-flight for a given key at a +// time. If a duplicate comes in, the duplicate caller waits for the +// original to complete and receives the same results. +// The return value shared indicates whether v was given to multiple callers. +func (g *singleflight) Do(key string, fn func() (interface{}, error)) (v interface{}, err error, shared bool) { + g.mu.Lock() + if g.m == nil { + g.m = make(map[string]*call) + } + if c, ok := g.m[key]; ok { + c.dups++ + g.mu.Unlock() + c.wg.Wait() + return c.val, c.err, true + } + c := new(call) + c.wg.Add(1) + g.m[key] = c + g.mu.Unlock() + + c.val, c.err = fn() + c.wg.Done() + + g.mu.Lock() + delete(g.m, key) + g.mu.Unlock() + + return c.val, c.err, c.dups > 0 +} diff --git a/src/net/smtp/auth.go b/src/net/smtp/auth.go new file mode 100644 index 000000000..3f1339ebc --- /dev/null +++ b/src/net/smtp/auth.go @@ -0,0 +1,107 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package smtp + +import ( + "crypto/hmac" + "crypto/md5" + "errors" + "fmt" +) + +// Auth is implemented by an SMTP authentication mechanism. +type Auth interface { + // Start begins an authentication with a server. + // It returns the name of the authentication protocol + // and optionally data to include in the initial AUTH message + // sent to the server. It can return proto == "" to indicate + // that the authentication should be skipped. + // If it returns a non-nil error, the SMTP client aborts + // the authentication attempt and closes the connection. + Start(server *ServerInfo) (proto string, toServer []byte, err error) + + // Next continues the authentication. The server has just sent + // the fromServer data. If more is true, the server expects a + // response, which Next should return as toServer; otherwise + // Next should return toServer == nil. + // If Next returns a non-nil error, the SMTP client aborts + // the authentication attempt and closes the connection. + Next(fromServer []byte, more bool) (toServer []byte, err error) +} + +// ServerInfo records information about an SMTP server. +type ServerInfo struct { + Name string // SMTP server name + TLS bool // using TLS, with valid certificate for Name + Auth []string // advertised authentication mechanisms +} + +type plainAuth struct { + identity, username, password string + host string +} + +// PlainAuth returns an Auth that implements the PLAIN authentication +// mechanism as defined in RFC 4616. +// The returned Auth uses the given username and password to authenticate +// on TLS connections to host and act as identity. Usually identity will be +// left blank to act as username. +func PlainAuth(identity, username, password, host string) Auth { + return &plainAuth{identity, username, password, host} +} + +func (a *plainAuth) Start(server *ServerInfo) (string, []byte, error) { + if !server.TLS { + advertised := false + for _, mechanism := range server.Auth { + if mechanism == "PLAIN" { + advertised = true + break + } + } + if !advertised { + return "", nil, errors.New("unencrypted connection") + } + } + if server.Name != a.host { + return "", nil, errors.New("wrong host name") + } + resp := []byte(a.identity + "\x00" + a.username + "\x00" + a.password) + return "PLAIN", resp, nil +} + +func (a *plainAuth) Next(fromServer []byte, more bool) ([]byte, error) { + if more { + // We've already sent everything. + return nil, errors.New("unexpected server challenge") + } + return nil, nil +} + +type cramMD5Auth struct { + username, secret string +} + +// CRAMMD5Auth returns an Auth that implements the CRAM-MD5 authentication +// mechanism as defined in RFC 2195. +// The returned Auth uses the given username and secret to authenticate +// to the server using the challenge-response mechanism. +func CRAMMD5Auth(username, secret string) Auth { + return &cramMD5Auth{username, secret} +} + +func (a *cramMD5Auth) Start(server *ServerInfo) (string, []byte, error) { + return "CRAM-MD5", nil, nil +} + +func (a *cramMD5Auth) Next(fromServer []byte, more bool) ([]byte, error) { + if more { + d := hmac.New(md5.New, []byte(a.secret)) + d.Write(fromServer) + s := make([]byte, 0, d.Size()) + return []byte(fmt.Sprintf("%s %x", a.username, d.Sum(s))), nil + } + return nil, nil +} diff --git a/src/net/smtp/example_test.go b/src/net/smtp/example_test.go new file mode 100644 index 000000000..d551e365a --- /dev/null +++ b/src/net/smtp/example_test.go @@ -0,0 +1,61 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package smtp_test + +import ( + "fmt" + "log" + "net/smtp" +) + +func Example() { + // Connect to the remote SMTP server. + c, err := smtp.Dial("mail.example.com:25") + if err != nil { + log.Fatal(err) + } + + // Set the sender and recipient first + if err := c.Mail("sender@example.org"); err != nil { + log.Fatal(err) + } + if err := c.Rcpt("recipient@example.net"); err != nil { + log.Fatal(err) + } + + // Send the email body. + wc, err := c.Data() + if err != nil { + log.Fatal(err) + } + _, err = fmt.Fprintf(wc, "This is the email body") + if err != nil { + log.Fatal(err) + } + err = wc.Close() + if err != nil { + log.Fatal(err) + } + + // Send the QUIT command and close the connection. + err = c.Quit() + if err != nil { + log.Fatal(err) + } +} + +func ExamplePlainAuth() { + // Set up authentication information. + auth := smtp.PlainAuth("", "user@example.com", "password", "mail.example.com") + + // Connect to the server, authenticate, set the sender and recipient, + // and send the email all in one step. + to := []string{"recipient@example.net"} + msg := []byte("This is the email body.") + err := smtp.SendMail("mail.example.com:25", auth, "sender@example.org", to, msg) + if err != nil { + log.Fatal(err) + } +} diff --git a/src/net/smtp/smtp.go b/src/net/smtp/smtp.go new file mode 100644 index 000000000..87dea442c --- /dev/null +++ b/src/net/smtp/smtp.go @@ -0,0 +1,357 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package smtp implements the Simple Mail Transfer Protocol as defined in RFC 5321. +// It also implements the following extensions: +// 8BITMIME RFC 1652 +// AUTH RFC 2554 +// STARTTLS RFC 3207 +// Additional extensions may be handled by clients. +package smtp + +import ( + "crypto/tls" + "encoding/base64" + "errors" + "io" + "net" + "net/textproto" + "strings" +) + +// A Client represents a client connection to an SMTP server. +type Client struct { + // Text is the textproto.Conn used by the Client. It is exported to allow for + // clients to add extensions. + Text *textproto.Conn + // keep a reference to the connection so it can be used to create a TLS + // connection later + conn net.Conn + // whether the Client is using TLS + tls bool + serverName string + // map of supported extensions + ext map[string]string + // supported auth mechanisms + auth []string + localName string // the name to use in HELO/EHLO + didHello bool // whether we've said HELO/EHLO + helloError error // the error from the hello +} + +// Dial returns a new Client connected to an SMTP server at addr. +// The addr must include a port number. +func Dial(addr string) (*Client, error) { + conn, err := net.Dial("tcp", addr) + if err != nil { + return nil, err + } + host, _, _ := net.SplitHostPort(addr) + return NewClient(conn, host) +} + +// NewClient returns a new Client using an existing connection and host as a +// server name to be used when authenticating. +func NewClient(conn net.Conn, host string) (*Client, error) { + text := textproto.NewConn(conn) + _, _, err := text.ReadResponse(220) + if err != nil { + text.Close() + return nil, err + } + c := &Client{Text: text, conn: conn, serverName: host, localName: "localhost"} + return c, nil +} + +// Close closes the connection. +func (c *Client) Close() error { + return c.Text.Close() +} + +// hello runs a hello exchange if needed. +func (c *Client) hello() error { + if !c.didHello { + c.didHello = true + err := c.ehlo() + if err != nil { + c.helloError = c.helo() + } + } + return c.helloError +} + +// Hello sends a HELO or EHLO to the server as the given host name. +// Calling this method is only necessary if the client needs control +// over the host name used. The client will introduce itself as "localhost" +// automatically otherwise. If Hello is called, it must be called before +// any of the other methods. +func (c *Client) Hello(localName string) error { + if c.didHello { + return errors.New("smtp: Hello called after other methods") + } + c.localName = localName + return c.hello() +} + +// cmd is a convenience function that sends a command and returns the response +func (c *Client) cmd(expectCode int, format string, args ...interface{}) (int, string, error) { + id, err := c.Text.Cmd(format, args...) + if err != nil { + return 0, "", err + } + c.Text.StartResponse(id) + defer c.Text.EndResponse(id) + code, msg, err := c.Text.ReadResponse(expectCode) + return code, msg, err +} + +// helo sends the HELO greeting to the server. It should be used only when the +// server does not support ehlo. +func (c *Client) helo() error { + c.ext = nil + _, _, err := c.cmd(250, "HELO %s", c.localName) + return err +} + +// ehlo sends the EHLO (extended hello) greeting to the server. It +// should be the preferred greeting for servers that support it. +func (c *Client) ehlo() error { + _, msg, err := c.cmd(250, "EHLO %s", c.localName) + if err != nil { + return err + } + ext := make(map[string]string) + extList := strings.Split(msg, "\n") + if len(extList) > 1 { + extList = extList[1:] + for _, line := range extList { + args := strings.SplitN(line, " ", 2) + if len(args) > 1 { + ext[args[0]] = args[1] + } else { + ext[args[0]] = "" + } + } + } + if mechs, ok := ext["AUTH"]; ok { + c.auth = strings.Split(mechs, " ") + } + c.ext = ext + return err +} + +// StartTLS sends the STARTTLS command and encrypts all further communication. +// Only servers that advertise the STARTTLS extension support this function. +func (c *Client) StartTLS(config *tls.Config) error { + if err := c.hello(); err != nil { + return err + } + _, _, err := c.cmd(220, "STARTTLS") + if err != nil { + return err + } + c.conn = tls.Client(c.conn, config) + c.Text = textproto.NewConn(c.conn) + c.tls = true + return c.ehlo() +} + +// Verify checks the validity of an email address on the server. +// If Verify returns nil, the address is valid. A non-nil return +// does not necessarily indicate an invalid address. Many servers +// will not verify addresses for security reasons. +func (c *Client) Verify(addr string) error { + if err := c.hello(); err != nil { + return err + } + _, _, err := c.cmd(250, "VRFY %s", addr) + return err +} + +// Auth authenticates a client using the provided authentication mechanism. +// A failed authentication closes the connection. +// Only servers that advertise the AUTH extension support this function. +func (c *Client) Auth(a Auth) error { + if err := c.hello(); err != nil { + return err + } + encoding := base64.StdEncoding + mech, resp, err := a.Start(&ServerInfo{c.serverName, c.tls, c.auth}) + if err != nil { + c.Quit() + return err + } + resp64 := make([]byte, encoding.EncodedLen(len(resp))) + encoding.Encode(resp64, resp) + code, msg64, err := c.cmd(0, "AUTH %s %s", mech, resp64) + for err == nil { + var msg []byte + switch code { + case 334: + msg, err = encoding.DecodeString(msg64) + case 235: + // the last message isn't base64 because it isn't a challenge + msg = []byte(msg64) + default: + err = &textproto.Error{Code: code, Msg: msg64} + } + if err == nil { + resp, err = a.Next(msg, code == 334) + } + if err != nil { + // abort the AUTH + c.cmd(501, "*") + c.Quit() + break + } + if resp == nil { + break + } + resp64 = make([]byte, encoding.EncodedLen(len(resp))) + encoding.Encode(resp64, resp) + code, msg64, err = c.cmd(0, string(resp64)) + } + return err +} + +// Mail issues a MAIL command to the server using the provided email address. +// If the server supports the 8BITMIME extension, Mail adds the BODY=8BITMIME +// parameter. +// This initiates a mail transaction and is followed by one or more Rcpt calls. +func (c *Client) Mail(from string) error { + if err := c.hello(); err != nil { + return err + } + cmdStr := "MAIL FROM:<%s>" + if c.ext != nil { + if _, ok := c.ext["8BITMIME"]; ok { + cmdStr += " BODY=8BITMIME" + } + } + _, _, err := c.cmd(250, cmdStr, from) + return err +} + +// Rcpt issues a RCPT command to the server using the provided email address. +// A call to Rcpt must be preceded by a call to Mail and may be followed by +// a Data call or another Rcpt call. +func (c *Client) Rcpt(to string) error { + _, _, err := c.cmd(25, "RCPT TO:<%s>", to) + return err +} + +type dataCloser struct { + c *Client + io.WriteCloser +} + +func (d *dataCloser) Close() error { + d.WriteCloser.Close() + _, _, err := d.c.Text.ReadResponse(250) + return err +} + +// Data issues a DATA command to the server and returns a writer that +// can be used to write the data. The caller should close the writer +// before calling any more methods on c. +// A call to Data must be preceded by one or more calls to Rcpt. +func (c *Client) Data() (io.WriteCloser, error) { + _, _, err := c.cmd(354, "DATA") + if err != nil { + return nil, err + } + return &dataCloser{c, c.Text.DotWriter()}, nil +} + +var testHookStartTLS func(*tls.Config) // nil, except for tests + +// SendMail connects to the server at addr, switches to TLS if +// possible, authenticates with the optional mechanism a if possible, +// and then sends an email from address from, to addresses to, with +// message msg. +func SendMail(addr string, a Auth, from string, to []string, msg []byte) error { + c, err := Dial(addr) + if err != nil { + return err + } + defer c.Close() + if err = c.hello(); err != nil { + return err + } + if ok, _ := c.Extension("STARTTLS"); ok { + config := &tls.Config{ServerName: c.serverName} + if testHookStartTLS != nil { + testHookStartTLS(config) + } + if err = c.StartTLS(config); err != nil { + return err + } + } + if a != nil && c.ext != nil { + if _, ok := c.ext["AUTH"]; ok { + if err = c.Auth(a); err != nil { + return err + } + } + } + if err = c.Mail(from); err != nil { + return err + } + for _, addr := range to { + if err = c.Rcpt(addr); err != nil { + return err + } + } + w, err := c.Data() + if err != nil { + return err + } + _, err = w.Write(msg) + if err != nil { + return err + } + err = w.Close() + if err != nil { + return err + } + return c.Quit() +} + +// Extension reports whether an extension is support by the server. +// The extension name is case-insensitive. If the extension is supported, +// Extension also returns a string that contains any parameters the +// server specifies for the extension. +func (c *Client) Extension(ext string) (bool, string) { + if err := c.hello(); err != nil { + return false, "" + } + if c.ext == nil { + return false, "" + } + ext = strings.ToUpper(ext) + param, ok := c.ext[ext] + return ok, param +} + +// Reset sends the RSET command to the server, aborting the current mail +// transaction. +func (c *Client) Reset() error { + if err := c.hello(); err != nil { + return err + } + _, _, err := c.cmd(250, "RSET") + return err +} + +// Quit sends the QUIT command and closes the connection to the server. +func (c *Client) Quit() error { + if err := c.hello(); err != nil { + return err + } + _, _, err := c.cmd(221, "QUIT") + if err != nil { + return err + } + return c.Text.Close() +} diff --git a/src/net/smtp/smtp_test.go b/src/net/smtp/smtp_test.go new file mode 100644 index 000000000..5c659e8a0 --- /dev/null +++ b/src/net/smtp/smtp_test.go @@ -0,0 +1,694 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package smtp + +import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" + "io" + "net" + "net/textproto" + "strings" + "testing" + "time" +) + +type authTest struct { + auth Auth + challenges []string + name string + responses []string +} + +var authTests = []authTest{ + {PlainAuth("", "user", "pass", "testserver"), []string{}, "PLAIN", []string{"\x00user\x00pass"}}, + {PlainAuth("foo", "bar", "baz", "testserver"), []string{}, "PLAIN", []string{"foo\x00bar\x00baz"}}, + {CRAMMD5Auth("user", "pass"), []string{"<123456.1322876914@testserver>"}, "CRAM-MD5", []string{"", "user 287eb355114cf5c471c26a875f1ca4ae"}}, +} + +func TestAuth(t *testing.T) { +testLoop: + for i, test := range authTests { + name, resp, err := test.auth.Start(&ServerInfo{"testserver", true, nil}) + if name != test.name { + t.Errorf("#%d got name %s, expected %s", i, name, test.name) + } + if !bytes.Equal(resp, []byte(test.responses[0])) { + t.Errorf("#%d got response %s, expected %s", i, resp, test.responses[0]) + } + if err != nil { + t.Errorf("#%d error: %s", i, err) + } + for j := range test.challenges { + challenge := []byte(test.challenges[j]) + expected := []byte(test.responses[j+1]) + resp, err := test.auth.Next(challenge, true) + if err != nil { + t.Errorf("#%d error: %s", i, err) + continue testLoop + } + if !bytes.Equal(resp, expected) { + t.Errorf("#%d got %s, expected %s", i, resp, expected) + continue testLoop + } + } + } +} + +func TestAuthPlain(t *testing.T) { + auth := PlainAuth("foo", "bar", "baz", "servername") + + tests := []struct { + server *ServerInfo + err string + }{ + { + server: &ServerInfo{Name: "servername", TLS: true}, + }, + { + // Okay; explicitly advertised by server. + server: &ServerInfo{Name: "servername", Auth: []string{"PLAIN"}}, + }, + { + server: &ServerInfo{Name: "servername", Auth: []string{"CRAM-MD5"}}, + err: "unencrypted connection", + }, + { + server: &ServerInfo{Name: "attacker", TLS: true}, + err: "wrong host name", + }, + } + for i, tt := range tests { + _, _, err := auth.Start(tt.server) + got := "" + if err != nil { + got = err.Error() + } + if got != tt.err { + t.Errorf("%d. got error = %q; want %q", i, got, tt.err) + } + } +} + +type faker struct { + io.ReadWriter +} + +func (f faker) Close() error { return nil } +func (f faker) LocalAddr() net.Addr { return nil } +func (f faker) RemoteAddr() net.Addr { return nil } +func (f faker) SetDeadline(time.Time) error { return nil } +func (f faker) SetReadDeadline(time.Time) error { return nil } +func (f faker) SetWriteDeadline(time.Time) error { return nil } + +func TestBasic(t *testing.T) { + server := strings.Join(strings.Split(basicServer, "\n"), "\r\n") + client := strings.Join(strings.Split(basicClient, "\n"), "\r\n") + + var cmdbuf bytes.Buffer + bcmdbuf := bufio.NewWriter(&cmdbuf) + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c := &Client{Text: textproto.NewConn(fake), localName: "localhost"} + + if err := c.helo(); err != nil { + t.Fatalf("HELO failed: %s", err) + } + if err := c.ehlo(); err == nil { + t.Fatalf("Expected first EHLO to fail") + } + if err := c.ehlo(); err != nil { + t.Fatalf("Second EHLO failed: %s", err) + } + + c.didHello = true + if ok, args := c.Extension("aUtH"); !ok || args != "LOGIN PLAIN" { + t.Fatalf("Expected AUTH supported") + } + if ok, _ := c.Extension("DSN"); ok { + t.Fatalf("Shouldn't support DSN") + } + + if err := c.Mail("user@gmail.com"); err == nil { + t.Fatalf("MAIL should require authentication") + } + + if err := c.Verify("user1@gmail.com"); err == nil { + t.Fatalf("First VRFY: expected no verification") + } + if err := c.Verify("user2@gmail.com"); err != nil { + t.Fatalf("Second VRFY: expected verification, got %s", err) + } + + // fake TLS so authentication won't complain + c.tls = true + c.serverName = "smtp.google.com" + if err := c.Auth(PlainAuth("", "user", "pass", "smtp.google.com")); err != nil { + t.Fatalf("AUTH failed: %s", err) + } + + if err := c.Mail("user@gmail.com"); err != nil { + t.Fatalf("MAIL failed: %s", err) + } + if err := c.Rcpt("golang-nuts@googlegroups.com"); err != nil { + t.Fatalf("RCPT failed: %s", err) + } + msg := `From: user@gmail.com +To: golang-nuts@googlegroups.com +Subject: Hooray for Go + +Line 1 +.Leading dot line . +Goodbye.` + w, err := c.Data() + if err != nil { + t.Fatalf("DATA failed: %s", err) + } + if _, err := w.Write([]byte(msg)); err != nil { + t.Fatalf("Data write failed: %s", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Bad data response: %s", err) + } + + if err := c.Quit(); err != nil { + t.Fatalf("QUIT failed: %s", err) + } + + bcmdbuf.Flush() + actualcmds := cmdbuf.String() + if client != actualcmds { + t.Fatalf("Got:\n%s\nExpected:\n%s", actualcmds, client) + } +} + +var basicServer = `250 mx.google.com at your service +502 Unrecognized command. +250-mx.google.com at your service +250-SIZE 35651584 +250-AUTH LOGIN PLAIN +250 8BITMIME +530 Authentication required +252 Send some mail, I'll try my best +250 User is valid +235 Accepted +250 Sender OK +250 Receiver OK +354 Go ahead +250 Data OK +221 OK +` + +var basicClient = `HELO localhost +EHLO localhost +EHLO localhost +MAIL FROM:<user@gmail.com> BODY=8BITMIME +VRFY user1@gmail.com +VRFY user2@gmail.com +AUTH PLAIN AHVzZXIAcGFzcw== +MAIL FROM:<user@gmail.com> BODY=8BITMIME +RCPT TO:<golang-nuts@googlegroups.com> +DATA +From: user@gmail.com +To: golang-nuts@googlegroups.com +Subject: Hooray for Go + +Line 1 +..Leading dot line . +Goodbye. +. +QUIT +` + +func TestNewClient(t *testing.T) { + server := strings.Join(strings.Split(newClientServer, "\n"), "\r\n") + client := strings.Join(strings.Split(newClientClient, "\n"), "\r\n") + + var cmdbuf bytes.Buffer + bcmdbuf := bufio.NewWriter(&cmdbuf) + out := func() string { + bcmdbuf.Flush() + return cmdbuf.String() + } + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c, err := NewClient(fake, "fake.host") + if err != nil { + t.Fatalf("NewClient: %v\n(after %v)", err, out()) + } + defer c.Close() + if ok, args := c.Extension("aUtH"); !ok || args != "LOGIN PLAIN" { + t.Fatalf("Expected AUTH supported") + } + if ok, _ := c.Extension("DSN"); ok { + t.Fatalf("Shouldn't support DSN") + } + if err := c.Quit(); err != nil { + t.Fatalf("QUIT failed: %s", err) + } + + actualcmds := out() + if client != actualcmds { + t.Fatalf("Got:\n%s\nExpected:\n%s", actualcmds, client) + } +} + +var newClientServer = `220 hello world +250-mx.google.com at your service +250-SIZE 35651584 +250-AUTH LOGIN PLAIN +250 8BITMIME +221 OK +` + +var newClientClient = `EHLO localhost +QUIT +` + +func TestNewClient2(t *testing.T) { + server := strings.Join(strings.Split(newClient2Server, "\n"), "\r\n") + client := strings.Join(strings.Split(newClient2Client, "\n"), "\r\n") + + var cmdbuf bytes.Buffer + bcmdbuf := bufio.NewWriter(&cmdbuf) + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c, err := NewClient(fake, "fake.host") + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer c.Close() + if ok, _ := c.Extension("DSN"); ok { + t.Fatalf("Shouldn't support DSN") + } + if err := c.Quit(); err != nil { + t.Fatalf("QUIT failed: %s", err) + } + + bcmdbuf.Flush() + actualcmds := cmdbuf.String() + if client != actualcmds { + t.Fatalf("Got:\n%s\nExpected:\n%s", actualcmds, client) + } +} + +var newClient2Server = `220 hello world +502 EH? +250-mx.google.com at your service +250-SIZE 35651584 +250-AUTH LOGIN PLAIN +250 8BITMIME +221 OK +` + +var newClient2Client = `EHLO localhost +HELO localhost +QUIT +` + +func TestHello(t *testing.T) { + + if len(helloServer) != len(helloClient) { + t.Fatalf("Hello server and client size mismatch") + } + + for i := 0; i < len(helloServer); i++ { + server := strings.Join(strings.Split(baseHelloServer+helloServer[i], "\n"), "\r\n") + client := strings.Join(strings.Split(baseHelloClient+helloClient[i], "\n"), "\r\n") + var cmdbuf bytes.Buffer + bcmdbuf := bufio.NewWriter(&cmdbuf) + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c, err := NewClient(fake, "fake.host") + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer c.Close() + c.localName = "customhost" + err = nil + + switch i { + case 0: + err = c.Hello("customhost") + case 1: + err = c.StartTLS(nil) + if err.Error() == "502 Not implemented" { + err = nil + } + case 2: + err = c.Verify("test@example.com") + case 3: + c.tls = true + c.serverName = "smtp.google.com" + err = c.Auth(PlainAuth("", "user", "pass", "smtp.google.com")) + case 4: + err = c.Mail("test@example.com") + case 5: + ok, _ := c.Extension("feature") + if ok { + t.Errorf("Expected FEATURE not to be supported") + } + case 6: + err = c.Reset() + case 7: + err = c.Quit() + case 8: + err = c.Verify("test@example.com") + if err != nil { + err = c.Hello("customhost") + if err != nil { + t.Errorf("Want error, got none") + } + } + default: + t.Fatalf("Unhandled command") + } + + if err != nil { + t.Errorf("Command %d failed: %v", i, err) + } + + bcmdbuf.Flush() + actualcmds := cmdbuf.String() + if client != actualcmds { + t.Errorf("Got:\n%s\nExpected:\n%s", actualcmds, client) + } + } +} + +var baseHelloServer = `220 hello world +502 EH? +250-mx.google.com at your service +250 FEATURE +` + +var helloServer = []string{ + "", + "502 Not implemented\n", + "250 User is valid\n", + "235 Accepted\n", + "250 Sender ok\n", + "", + "250 Reset ok\n", + "221 Goodbye\n", + "250 Sender ok\n", +} + +var baseHelloClient = `EHLO customhost +HELO customhost +` + +var helloClient = []string{ + "", + "STARTTLS\n", + "VRFY test@example.com\n", + "AUTH PLAIN AHVzZXIAcGFzcw==\n", + "MAIL FROM:<test@example.com>\n", + "", + "RSET\n", + "QUIT\n", + "VRFY test@example.com\n", +} + +func TestSendMail(t *testing.T) { + server := strings.Join(strings.Split(sendMailServer, "\n"), "\r\n") + client := strings.Join(strings.Split(sendMailClient, "\n"), "\r\n") + var cmdbuf bytes.Buffer + bcmdbuf := bufio.NewWriter(&cmdbuf) + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Unable to to create listener: %v", err) + } + defer l.Close() + + // prevent data race on bcmdbuf + var done = make(chan struct{}) + go func(data []string) { + + defer close(done) + + conn, err := l.Accept() + if err != nil { + t.Errorf("Accept error: %v", err) + return + } + defer conn.Close() + + tc := textproto.NewConn(conn) + for i := 0; i < len(data) && data[i] != ""; i++ { + tc.PrintfLine(data[i]) + for len(data[i]) >= 4 && data[i][3] == '-' { + i++ + tc.PrintfLine(data[i]) + } + if data[i] == "221 Goodbye" { + return + } + read := false + for !read || data[i] == "354 Go ahead" { + msg, err := tc.ReadLine() + bcmdbuf.Write([]byte(msg + "\r\n")) + read = true + if err != nil { + t.Errorf("Read error: %v", err) + return + } + if data[i] == "354 Go ahead" && msg == "." { + break + } + } + } + }(strings.Split(server, "\r\n")) + + err = SendMail(l.Addr().String(), nil, "test@example.com", []string{"other@example.com"}, []byte(strings.Replace(`From: test@example.com +To: other@example.com +Subject: SendMail test + +SendMail is working for me. +`, "\n", "\r\n", -1))) + + if err != nil { + t.Errorf("%v", err) + } + + <-done + bcmdbuf.Flush() + actualcmds := cmdbuf.String() + if client != actualcmds { + t.Errorf("Got:\n%s\nExpected:\n%s", actualcmds, client) + } +} + +var sendMailServer = `220 hello world +502 EH? +250 mx.google.com at your service +250 Sender ok +250 Receiver ok +354 Go ahead +250 Data ok +221 Goodbye +` + +var sendMailClient = `EHLO localhost +HELO localhost +MAIL FROM:<test@example.com> +RCPT TO:<other@example.com> +DATA +From: test@example.com +To: other@example.com +Subject: SendMail test + +SendMail is working for me. +. +QUIT +` + +func TestAuthFailed(t *testing.T) { + server := strings.Join(strings.Split(authFailedServer, "\n"), "\r\n") + client := strings.Join(strings.Split(authFailedClient, "\n"), "\r\n") + var cmdbuf bytes.Buffer + bcmdbuf := bufio.NewWriter(&cmdbuf) + var fake faker + fake.ReadWriter = bufio.NewReadWriter(bufio.NewReader(strings.NewReader(server)), bcmdbuf) + c, err := NewClient(fake, "fake.host") + if err != nil { + t.Fatalf("NewClient: %v", err) + } + defer c.Close() + + c.tls = true + c.serverName = "smtp.google.com" + err = c.Auth(PlainAuth("", "user", "pass", "smtp.google.com")) + + if err == nil { + t.Error("Auth: expected error; got none") + } else if err.Error() != "535 Invalid credentials\nplease see www.example.com" { + t.Errorf("Auth: got error: %v, want: %s", err, "535 Invalid credentials\nplease see www.example.com") + } + + bcmdbuf.Flush() + actualcmds := cmdbuf.String() + if client != actualcmds { + t.Errorf("Got:\n%s\nExpected:\n%s", actualcmds, client) + } +} + +var authFailedServer = `220 hello world +250-mx.google.com at your service +250 AUTH LOGIN PLAIN +535-Invalid credentials +535 please see www.example.com +221 Goodbye +` + +var authFailedClient = `EHLO localhost +AUTH PLAIN AHVzZXIAcGFzcw== +* +QUIT +` + +func TestTLSClient(t *testing.T) { + ln := newLocalListener(t) + defer ln.Close() + errc := make(chan error) + go func() { + errc <- sendMail(ln.Addr().String()) + }() + conn, err := ln.Accept() + if err != nil { + t.Fatalf("failed to accept connection: %v", err) + } + defer conn.Close() + if err := serverHandle(conn, t); err != nil { + t.Fatalf("failed to handle connection: %v", err) + } + if err := <-errc; err != nil { + t.Fatalf("client error: %v", err) + } +} + +func newLocalListener(t *testing.T) net.Listener { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ln, err = net.Listen("tcp6", "[::1]:0") + } + if err != nil { + t.Fatal(err) + } + return ln +} + +type smtpSender struct { + w io.Writer +} + +func (s smtpSender) send(f string) { + s.w.Write([]byte(f + "\r\n")) +} + +// smtp server, finely tailored to deal with our own client only! +func serverHandle(c net.Conn, t *testing.T) error { + send := smtpSender{c}.send + send("220 127.0.0.1 ESMTP service ready") + s := bufio.NewScanner(c) + for s.Scan() { + switch s.Text() { + case "EHLO localhost": + send("250-127.0.0.1 ESMTP offers a warm hug of welcome") + send("250-STARTTLS") + send("250 Ok") + case "STARTTLS": + send("220 Go ahead") + keypair, err := tls.X509KeyPair(localhostCert, localhostKey) + if err != nil { + return err + } + config := &tls.Config{Certificates: []tls.Certificate{keypair}} + c = tls.Server(c, config) + defer c.Close() + return serverHandleTLS(c, t) + default: + t.Fatalf("unrecognized command: %q", s.Text()) + } + } + return s.Err() +} + +func serverHandleTLS(c net.Conn, t *testing.T) error { + send := smtpSender{c}.send + s := bufio.NewScanner(c) + for s.Scan() { + switch s.Text() { + case "EHLO localhost": + send("250 Ok") + case "MAIL FROM:<joe1@example.com>": + send("250 Ok") + case "RCPT TO:<joe2@example.com>": + send("250 Ok") + case "DATA": + send("354 send the mail data, end with .") + send("250 Ok") + case "Subject: test": + case "": + case "howdy!": + case ".": + case "QUIT": + send("221 127.0.0.1 Service closing transmission channel") + return nil + default: + t.Fatalf("unrecognized command during TLS: %q", s.Text()) + } + } + return s.Err() +} + +func init() { + testRootCAs := x509.NewCertPool() + testRootCAs.AppendCertsFromPEM(localhostCert) + testHookStartTLS = func(config *tls.Config) { + config.RootCAs = testRootCAs + } +} + +func sendMail(hostPort string) error { + host, _, err := net.SplitHostPort(hostPort) + if err != nil { + return err + } + auth := PlainAuth("", "", "", host) + from := "joe1@example.com" + to := []string{"joe2@example.com"} + return SendMail(hostPort, auth, from, to, []byte("Subject: test\n\nhowdy!")) +} + +// (copied from net/http/httptest) +// localhostCert is a PEM-encoded TLS cert with SAN IPs +// "127.0.0.1" and "[::1]", expiring at the last second of 2049 (the end +// of ASN.1 time). +// generated from src/crypto/tls: +// go run generate_cert.go --rsa-bits 512 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h +var localhostCert = []byte(`-----BEGIN CERTIFICATE----- +MIIBdzCCASOgAwIBAgIBADALBgkqhkiG9w0BAQUwEjEQMA4GA1UEChMHQWNtZSBD +bzAeFw03MDAxMDEwMDAwMDBaFw00OTEyMzEyMzU5NTlaMBIxEDAOBgNVBAoTB0Fj +bWUgQ28wWjALBgkqhkiG9w0BAQEDSwAwSAJBAN55NcYKZeInyTuhcCwFMhDHCmwa +IUSdtXdcbItRB/yfXGBhiex00IaLXQnSU+QZPRZWYqeTEbFSgihqi1PUDy8CAwEA +AaNoMGYwDgYDVR0PAQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud +EwEB/wQFMAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAA +AAAAAAAAAAAAAAEwCwYJKoZIhvcNAQEFA0EAAoQn/ytgqpiLcZu9XKbCJsJcvkgk +Se6AbGXgSlq+ZCEVo0qIwSgeBqmsJxUu7NCSOwVJLYNEBO2DtIxoYVk+MA== +-----END CERTIFICATE-----`) + +// localhostKey is the private key for localhostCert. +var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIBPAIBAAJBAN55NcYKZeInyTuhcCwFMhDHCmwaIUSdtXdcbItRB/yfXGBhiex0 +0IaLXQnSU+QZPRZWYqeTEbFSgihqi1PUDy8CAwEAAQJBAQdUx66rfh8sYsgfdcvV +NoafYpnEcB5s4m/vSVe6SU7dCK6eYec9f9wpT353ljhDUHq3EbmE4foNzJngh35d +AekCIQDhRQG5Li0Wj8TM4obOnnXUXf1jRv0UkzE9AHWLG5q3AwIhAPzSjpYUDjVW +MCUXgckTpKCuGwbJk7424Nb8bLzf3kllAiA5mUBgjfr/WtFSJdWcPQ4Zt9KTMNKD +EUO0ukpTwEIl6wIhAMbGqZK3zAAFdq8DD2jPx+UJXnh0rnOkZBzDtJ6/iN69AiEA +1Aq8MJgTaYsDQWyU/hDq5YkDJc9e9DSCvUIzqxQWMQE= +-----END RSA PRIVATE KEY-----`) diff --git a/src/net/sock_bsd.go b/src/net/sock_bsd.go new file mode 100644 index 000000000..48fb78527 --- /dev/null +++ b/src/net/sock_bsd.go @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd nacl netbsd openbsd + +package net + +import ( + "runtime" + "syscall" +) + +func maxListenerBacklog() int { + var ( + n uint32 + err error + ) + switch runtime.GOOS { + case "darwin", "freebsd": + n, err = syscall.SysctlUint32("kern.ipc.somaxconn") + case "netbsd": + // NOTE: NetBSD has no somaxconn-like kernel state so far + case "openbsd": + n, err = syscall.SysctlUint32("kern.somaxconn") + } + if n == 0 || err != nil { + return syscall.SOMAXCONN + } + // FreeBSD stores the backlog in a uint16, as does Linux. + // Assume the other BSDs do too. Truncate number to avoid wrapping. + // See issue 5030. + if n > 1<<16-1 { + n = 1<<16 - 1 + } + return int(n) +} diff --git a/src/net/sock_cloexec.go b/src/net/sock_cloexec.go new file mode 100644 index 000000000..dec81855b --- /dev/null +++ b/src/net/sock_cloexec.go @@ -0,0 +1,78 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements sysSocket and accept for platforms that +// provide a fast path for setting SetNonblock and CloseOnExec. + +// +build freebsd linux + +package net + +import "syscall" + +// Wrapper around the socket system call that marks the returned file +// descriptor as nonblocking and close-on-exec. +func sysSocket(family, sotype, proto int) (int, error) { + s, err := syscall.Socket(family, sotype|syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC, proto) + // On Linux the SOCK_NONBLOCK and SOCK_CLOEXEC flags were + // introduced in 2.6.27 kernel and on FreeBSD both flags were + // introduced in 10 kernel. If we get an EINVAL error on Linux + // or EPROTONOSUPPORT error on FreeBSD, fall back to using + // socket without them. + if err == nil || (err != syscall.EPROTONOSUPPORT && err != syscall.EINVAL) { + return s, err + } + + // See ../syscall/exec_unix.go for description of ForkLock. + syscall.ForkLock.RLock() + s, err = syscall.Socket(family, sotype, proto) + if err == nil { + syscall.CloseOnExec(s) + } + syscall.ForkLock.RUnlock() + if err != nil { + return -1, err + } + if err = syscall.SetNonblock(s, true); err != nil { + syscall.Close(s) + return -1, err + } + return s, nil +} + +// Wrapper around the accept system call that marks the returned file +// descriptor as nonblocking and close-on-exec. +func accept(s int) (int, syscall.Sockaddr, error) { + ns, sa, err := syscall.Accept4(s, syscall.SOCK_NONBLOCK|syscall.SOCK_CLOEXEC) + // On Linux the accept4 system call was introduced in 2.6.28 + // kernel and on FreeBSD it was introduced in 10 kernel. If we + // get an ENOSYS error on both Linux and FreeBSD, or EINVAL + // error on Linux, fall back to using accept. + switch err { + default: // nil and errors other than the ones listed + return ns, sa, err + case syscall.ENOSYS: // syscall missing + case syscall.EINVAL: // some Linux use this instead of ENOSYS + case syscall.EACCES: // some Linux use this instead of ENOSYS + case syscall.EFAULT: // some Linux use this instead of ENOSYS + } + + // See ../syscall/exec_unix.go for description of ForkLock. + // It is probably okay to hold the lock across syscall.Accept + // because we have put fd.sysfd into non-blocking mode. + // However, a call to the File method will put it back into + // blocking mode. We can't take that risk, so no use of ForkLock here. + ns, sa, err = syscall.Accept(s) + if err == nil { + syscall.CloseOnExec(ns) + } + if err != nil { + return -1, nil, err + } + if err = syscall.SetNonblock(ns, true); err != nil { + syscall.Close(ns) + return -1, nil, err + } + return ns, sa, nil +} diff --git a/src/net/sock_linux.go b/src/net/sock_linux.go new file mode 100644 index 000000000..cc5ce153b --- /dev/null +++ b/src/net/sock_linux.go @@ -0,0 +1,31 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "syscall" + +func maxListenerBacklog() int { + fd, err := open("/proc/sys/net/core/somaxconn") + if err != nil { + return syscall.SOMAXCONN + } + defer fd.close() + l, ok := fd.readLine() + if !ok { + return syscall.SOMAXCONN + } + f := getFields(l) + n, _, ok := dtoi(f[0], 0) + if n == 0 || !ok { + return syscall.SOMAXCONN + } + // Linux stores the backlog in a uint16. + // Truncate number to avoid wrapping. + // See issue 5030. + if n > 1<<16-1 { + n = 1<<16 - 1 + } + return n +} diff --git a/src/net/sock_plan9.go b/src/net/sock_plan9.go new file mode 100644 index 000000000..88d9ed15c --- /dev/null +++ b/src/net/sock_plan9.go @@ -0,0 +1,10 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +func maxListenerBacklog() int { + // /sys/include/ape/sys/socket.h:/SOMAXCONN + return 5 +} diff --git a/src/net/sock_posix.go b/src/net/sock_posix.go new file mode 100644 index 000000000..3f956df65 --- /dev/null +++ b/src/net/sock_posix.go @@ -0,0 +1,216 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "os" + "syscall" + "time" +) + +// A sockaddr represents a TCP, UDP, IP or Unix network endpoint +// address that can be converted into a syscall.Sockaddr. +type sockaddr interface { + Addr + + netaddr + + // family returns the platform-dependent address family + // identifier. + family() int + + // isWildcard reports whether the address is a wildcard + // address. + isWildcard() bool + + // sockaddr returns the address converted into a syscall + // sockaddr type that implements syscall.Sockaddr + // interface. It returns a nil interface when the address is + // nil. + sockaddr(family int) (syscall.Sockaddr, error) +} + +// socket returns a network file descriptor that is ready for +// asynchronous I/O using the network poller. +func socket(net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr, deadline time.Time) (fd *netFD, err error) { + s, err := sysSocket(family, sotype, proto) + if err != nil { + return nil, err + } + if err = setDefaultSockopts(s, family, sotype, ipv6only); err != nil { + closesocket(s) + return nil, err + } + if fd, err = newFD(s, family, sotype, net); err != nil { + closesocket(s) + return nil, err + } + + // This function makes a network file descriptor for the + // following applications: + // + // - An endpoint holder that opens a passive stream + // connenction, known as a stream listener + // + // - An endpoint holder that opens a destination-unspecific + // datagram connection, known as a datagram listener + // + // - An endpoint holder that opens an active stream or a + // destination-specific datagram connection, known as a + // dialer + // + // - An endpoint holder that opens the other connection, such + // as talking to the protocol stack inside the kernel + // + // For stream and datagram listeners, they will only require + // named sockets, so we can assume that it's just a request + // from stream or datagram listeners when laddr is not nil but + // raddr is nil. Otherwise we assume it's just for dialers or + // the other connection holders. + + if laddr != nil && raddr == nil { + switch sotype { + case syscall.SOCK_STREAM, syscall.SOCK_SEQPACKET: + if err := fd.listenStream(laddr, listenerBacklog); err != nil { + fd.Close() + return nil, err + } + return fd, nil + case syscall.SOCK_DGRAM: + if err := fd.listenDatagram(laddr); err != nil { + fd.Close() + return nil, err + } + return fd, nil + } + } + if err := fd.dial(laddr, raddr, deadline); err != nil { + fd.Close() + return nil, err + } + return fd, nil +} + +func (fd *netFD) addrFunc() func(syscall.Sockaddr) Addr { + switch fd.family { + case syscall.AF_INET, syscall.AF_INET6: + switch fd.sotype { + case syscall.SOCK_STREAM: + return sockaddrToTCP + case syscall.SOCK_DGRAM: + return sockaddrToUDP + case syscall.SOCK_RAW: + return sockaddrToIP + } + case syscall.AF_UNIX: + switch fd.sotype { + case syscall.SOCK_STREAM: + return sockaddrToUnix + case syscall.SOCK_DGRAM: + return sockaddrToUnixgram + case syscall.SOCK_SEQPACKET: + return sockaddrToUnixpacket + } + } + return func(syscall.Sockaddr) Addr { return nil } +} + +func (fd *netFD) dial(laddr, raddr sockaddr, deadline time.Time) error { + var err error + var lsa syscall.Sockaddr + if laddr != nil { + if lsa, err = laddr.sockaddr(fd.family); err != nil { + return err + } else if lsa != nil { + if err := syscall.Bind(fd.sysfd, lsa); err != nil { + return os.NewSyscallError("bind", err) + } + } + } + var rsa syscall.Sockaddr + if raddr != nil { + if rsa, err = raddr.sockaddr(fd.family); err != nil { + return err + } + if err := fd.connect(lsa, rsa, deadline); err != nil { + return err + } + fd.isConnected = true + } else { + if err := fd.init(); err != nil { + return err + } + } + lsa, _ = syscall.Getsockname(fd.sysfd) + if rsa, _ = syscall.Getpeername(fd.sysfd); rsa != nil { + fd.setAddr(fd.addrFunc()(lsa), fd.addrFunc()(rsa)) + } else { + fd.setAddr(fd.addrFunc()(lsa), raddr) + } + return nil +} + +func (fd *netFD) listenStream(laddr sockaddr, backlog int) error { + if err := setDefaultListenerSockopts(fd.sysfd); err != nil { + return err + } + if lsa, err := laddr.sockaddr(fd.family); err != nil { + return err + } else if lsa != nil { + if err := syscall.Bind(fd.sysfd, lsa); err != nil { + return os.NewSyscallError("bind", err) + } + } + if err := syscall.Listen(fd.sysfd, backlog); err != nil { + return os.NewSyscallError("listen", err) + } + if err := fd.init(); err != nil { + return err + } + lsa, _ := syscall.Getsockname(fd.sysfd) + fd.setAddr(fd.addrFunc()(lsa), nil) + return nil +} + +func (fd *netFD) listenDatagram(laddr sockaddr) error { + switch addr := laddr.(type) { + case *UDPAddr: + // We provide a socket that listens to a wildcard + // address with reusable UDP port when the given laddr + // is an appropriate UDP multicast address prefix. + // This makes it possible for a single UDP listener to + // join multiple different group addresses, for + // multiple UDP listeners that listen on the same UDP + // port to join the same group address. + if addr.IP != nil && addr.IP.IsMulticast() { + if err := setDefaultMulticastSockopts(fd.sysfd); err != nil { + return err + } + addr := *addr + switch fd.family { + case syscall.AF_INET: + addr.IP = IPv4zero + case syscall.AF_INET6: + addr.IP = IPv6unspecified + } + laddr = &addr + } + } + if lsa, err := laddr.sockaddr(fd.family); err != nil { + return err + } else if lsa != nil { + if err := syscall.Bind(fd.sysfd, lsa); err != nil { + return os.NewSyscallError("bind", err) + } + } + if err := fd.init(); err != nil { + return err + } + lsa, _ := syscall.Getsockname(fd.sysfd) + fd.setAddr(fd.addrFunc()(lsa), nil) + return nil +} diff --git a/src/net/sock_solaris.go b/src/net/sock_solaris.go new file mode 100644 index 000000000..90fe9de89 --- /dev/null +++ b/src/net/sock_solaris.go @@ -0,0 +1,13 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "syscall" + +func maxListenerBacklog() int { + // TODO: Implement this + // NOTE: Never return a number bigger than 1<<16 - 1. See issue 5030. + return syscall.SOMAXCONN +} diff --git a/src/net/sock_windows.go b/src/net/sock_windows.go new file mode 100644 index 000000000..6ccde3a24 --- /dev/null +++ b/src/net/sock_windows.go @@ -0,0 +1,24 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "syscall" + +func maxListenerBacklog() int { + // TODO: Implement this + // NOTE: Never return a number bigger than 1<<16 - 1. See issue 5030. + return syscall.SOMAXCONN +} + +func sysSocket(f, t, p int) (syscall.Handle, error) { + // See ../syscall/exec_unix.go for description of ForkLock. + syscall.ForkLock.RLock() + s, err := syscall.Socket(f, t, p) + if err == nil { + syscall.CloseOnExec(s) + } + syscall.ForkLock.RUnlock() + return s, err +} diff --git a/src/net/sockopt_bsd.go b/src/net/sockopt_bsd.go new file mode 100644 index 000000000..2d36a5595 --- /dev/null +++ b/src/net/sockopt_bsd.go @@ -0,0 +1,54 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd nacl netbsd openbsd + +package net + +import ( + "os" + "runtime" + "syscall" +) + +func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { + if runtime.GOOS == "dragonfly" && sotype != syscall.SOCK_RAW { + // On DragonFly BSD, we adjust the ephemeral port + // range because unlike other BSD systems its default + // port range doesn't conform to IANA recommendation + // as described in RFC 6056 and is pretty narrow. + switch family { + case syscall.AF_INET: + syscall.SetsockoptInt(s, syscall.IPPROTO_IP, syscall.IP_PORTRANGE, syscall.IP_PORTRANGE_HIGH) + case syscall.AF_INET6: + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_PORTRANGE, syscall.IPV6_PORTRANGE_HIGH) + } + } + if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { + // Allow both IP versions even if the OS default + // is otherwise. Note that some operating systems + // never admit this option. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) + } + // Allow broadcast. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)) +} + +func setDefaultListenerSockopts(s int) error { + // Allow reuse of recently-used addresses. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)) +} + +func setDefaultMulticastSockopts(s int) error { + // Allow multicast UDP and raw IP datagram sockets to listen + // concurrently across multiple listeners. + if err := syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1); err != nil { + return os.NewSyscallError("setsockopt", err) + } + // Allow reuse of recently-used ports. + // This option is supported only in descendants of 4.4BSD, + // to make an effective multicast application that requires + // quick draw possible. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEPORT, 1)) +} diff --git a/src/net/sockopt_linux.go b/src/net/sockopt_linux.go new file mode 100644 index 000000000..54c20b140 --- /dev/null +++ b/src/net/sockopt_linux.go @@ -0,0 +1,32 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { + if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { + // Allow both IP versions even if the OS default + // is otherwise. Note that some operating systems + // never admit this option. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) + } + // Allow broadcast. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)) +} + +func setDefaultListenerSockopts(s int) error { + // Allow reuse of recently-used addresses. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)) +} + +func setDefaultMulticastSockopts(s int) error { + // Allow multicast UDP and raw IP datagram sockets to listen + // concurrently across multiple listeners. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)) +} diff --git a/src/net/sockopt_plan9.go b/src/net/sockopt_plan9.go new file mode 100644 index 000000000..8bc689b6c --- /dev/null +++ b/src/net/sockopt_plan9.go @@ -0,0 +1,13 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +func setKeepAlive(fd *netFD, keepalive bool) error { + if keepalive { + _, e := fd.ctl.WriteAt([]byte("keepalive"), 0) + return e + } + return nil +} diff --git a/src/net/sockopt_posix.go b/src/net/sockopt_posix.go new file mode 100644 index 000000000..921918c37 --- /dev/null +++ b/src/net/sockopt_posix.go @@ -0,0 +1,141 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "os" + "syscall" +) + +// Boolean to int. +func boolint(b bool) int { + if b { + return 1 + } + return 0 +} + +func ipv4AddrToInterface(ip IP) (*Interface, error) { + ift, err := Interfaces() + if err != nil { + return nil, err + } + for _, ifi := range ift { + ifat, err := ifi.Addrs() + if err != nil { + return nil, err + } + for _, ifa := range ifat { + switch v := ifa.(type) { + case *IPAddr: + if ip.Equal(v.IP) { + return &ifi, nil + } + case *IPNet: + if ip.Equal(v.IP) { + return &ifi, nil + } + } + } + } + if ip.Equal(IPv4zero) { + return nil, nil + } + return nil, errNoSuchInterface +} + +func interfaceToIPv4Addr(ifi *Interface) (IP, error) { + if ifi == nil { + return IPv4zero, nil + } + ifat, err := ifi.Addrs() + if err != nil { + return nil, err + } + for _, ifa := range ifat { + switch v := ifa.(type) { + case *IPAddr: + if v.IP.To4() != nil { + return v.IP, nil + } + case *IPNet: + if v.IP.To4() != nil { + return v.IP, nil + } + } + } + return nil, errNoSuchInterface +} + +func setIPv4MreqToInterface(mreq *syscall.IPMreq, ifi *Interface) error { + if ifi == nil { + return nil + } + ifat, err := ifi.Addrs() + if err != nil { + return err + } + for _, ifa := range ifat { + switch v := ifa.(type) { + case *IPAddr: + if a := v.IP.To4(); a != nil { + copy(mreq.Interface[:], a) + goto done + } + case *IPNet: + if a := v.IP.To4(); a != nil { + copy(mreq.Interface[:], a) + goto done + } + } + } +done: + if bytesEqual(mreq.Multiaddr[:], IPv4zero.To4()) { + return errNoSuchMulticastInterface + } + return nil +} + +func setReadBuffer(fd *netFD, bytes int) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes)) +} + +func setWriteBuffer(fd *netFD, bytes int) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes)) +} + +func setKeepAlive(fd *netFD, keepalive bool) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, boolint(keepalive))) +} + +func setLinger(fd *netFD, sec int) error { + var l syscall.Linger + if sec >= 0 { + l.Onoff = 1 + l.Linger = int32(sec) + } else { + l.Onoff = 0 + l.Linger = 0 + } + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptLinger(fd.sysfd, syscall.SOL_SOCKET, syscall.SO_LINGER, &l)) +} diff --git a/src/net/sockopt_solaris.go b/src/net/sockopt_solaris.go new file mode 100644 index 000000000..54c20b140 --- /dev/null +++ b/src/net/sockopt_solaris.go @@ -0,0 +1,32 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { + if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { + // Allow both IP versions even if the OS default + // is otherwise. Note that some operating systems + // never admit this option. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) + } + // Allow broadcast. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)) +} + +func setDefaultListenerSockopts(s int) error { + // Allow reuse of recently-used addresses. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)) +} + +func setDefaultMulticastSockopts(s int) error { + // Allow multicast UDP and raw IP datagram sockets to listen + // concurrently across multiple listeners. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)) +} diff --git a/src/net/sockopt_windows.go b/src/net/sockopt_windows.go new file mode 100644 index 000000000..cb64a40c6 --- /dev/null +++ b/src/net/sockopt_windows.go @@ -0,0 +1,38 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +func setDefaultSockopts(s syscall.Handle, family, sotype int, ipv6only bool) error { + if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { + // Allow both IP versions even if the OS default + // is otherwise. Note that some operating systems + // never admit this option. + syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) + } + // Allow broadcast. + syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + return nil +} + +func setDefaultListenerSockopts(s syscall.Handle) error { + // Windows will reuse recently-used addresses by default. + // SO_REUSEADDR should not be used here, as it allows + // a socket to forcibly bind to a port in use by another socket. + // This could lead to a non-deterministic behavior, where + // connection requests over the port cannot be guaranteed + // to be handled by the correct socket. + return nil +} + +func setDefaultMulticastSockopts(s syscall.Handle) error { + // Allow multicast UDP and raw IP datagram sockets to listen + // concurrently across multiple listeners. + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)) +} diff --git a/src/net/sockoptip_bsd.go b/src/net/sockoptip_bsd.go new file mode 100644 index 000000000..87132f0f4 --- /dev/null +++ b/src/net/sockoptip_bsd.go @@ -0,0 +1,34 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd nacl netbsd openbsd + +package net + +import ( + "os" + "syscall" +) + +func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { + ip, err := interfaceToIPv4Addr(ifi) + if err != nil { + return os.NewSyscallError("setsockopt", err) + } + var a [4]byte + copy(a[:], ip.To4()) + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInet4Addr(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, a)) +} + +func setIPv4MulticastLoopback(fd *netFD, v bool) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptByte(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, byte(boolint(v)))) +} diff --git a/src/net/sockoptip_linux.go b/src/net/sockoptip_linux.go new file mode 100644 index 000000000..a69b778e4 --- /dev/null +++ b/src/net/sockoptip_linux.go @@ -0,0 +1,31 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" +) + +func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { + var v int32 + if ifi != nil { + v = int32(ifi.Index) + } + mreq := &syscall.IPMreqn{Ifindex: v} + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreqn(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, mreq)) +} + +func setIPv4MulticastLoopback(fd *netFD, v bool) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v))) +} diff --git a/src/net/sockoptip_posix.go b/src/net/sockoptip_posix.go new file mode 100644 index 000000000..b5c80e449 --- /dev/null +++ b/src/net/sockoptip_posix.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd windows + +package net + +import ( + "os" + "syscall" +) + +func joinIPv4Group(fd *netFD, ifi *Interface, ip IP) error { + mreq := &syscall.IPMreq{Multiaddr: [4]byte{ip[0], ip[1], ip[2], ip[3]}} + if err := setIPv4MreqToInterface(mreq, ifi); err != nil { + return err + } + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptIPMreq(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_ADD_MEMBERSHIP, mreq)) +} + +func setIPv6MulticastInterface(fd *netFD, ifi *Interface) error { + var v int + if ifi != nil { + v = ifi.Index + } + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_IF, v)) +} + +func setIPv6MulticastLoopback(fd *netFD, v bool) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_MULTICAST_LOOP, boolint(v))) +} + +func joinIPv6Group(fd *netFD, ifi *Interface, ip IP) error { + mreq := &syscall.IPv6Mreq{} + copy(mreq.Multiaddr[:], ip) + if ifi != nil { + mreq.Interface = uint32(ifi.Index) + } + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptIPv6Mreq(fd.sysfd, syscall.IPPROTO_IPV6, syscall.IPV6_JOIN_GROUP, mreq)) +} diff --git a/src/net/sockoptip_stub.go b/src/net/sockoptip_stub.go new file mode 100644 index 000000000..dcd3a22b5 --- /dev/null +++ b/src/net/sockoptip_stub.go @@ -0,0 +1,39 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build solaris + +package net + +import "syscall" + +func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { + // See golang.org/issue/7399. + return syscall.EINVAL +} + +func setIPv4MulticastLoopback(fd *netFD, v bool) error { + // See golang.org/issue/7399. + return syscall.EINVAL +} + +func joinIPv4Group(fd *netFD, ifi *Interface, ip IP) error { + // See golang.org/issue/7399. + return syscall.EINVAL +} + +func setIPv6MulticastInterface(fd *netFD, ifi *Interface) error { + // See golang.org/issue/7399. + return syscall.EINVAL +} + +func setIPv6MulticastLoopback(fd *netFD, v bool) error { + // See golang.org/issue/7399. + return syscall.EINVAL +} + +func joinIPv6Group(fd *netFD, ifi *Interface, ip IP) error { + // See golang.org/issue/7399. + return syscall.EINVAL +} diff --git a/src/net/sockoptip_windows.go b/src/net/sockoptip_windows.go new file mode 100644 index 000000000..7b11f207a --- /dev/null +++ b/src/net/sockoptip_windows.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" + "unsafe" +) + +func setIPv4MulticastInterface(fd *netFD, ifi *Interface) error { + ip, err := interfaceToIPv4Addr(ifi) + if err != nil { + return os.NewSyscallError("setsockopt", err) + } + var a [4]byte + copy(a[:], ip.To4()) + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.Setsockopt(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_IF, (*byte)(unsafe.Pointer(&a[0])), 4)) +} + +func setIPv4MulticastLoopback(fd *netFD, v bool) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_IP, syscall.IP_MULTICAST_LOOP, boolint(v))) +} diff --git a/src/net/sys_cloexec.go b/src/net/sys_cloexec.go new file mode 100644 index 000000000..898fb7c0c --- /dev/null +++ b/src/net/sys_cloexec.go @@ -0,0 +1,54 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements sysSocket and accept for platforms that do not +// provide a fast path for setting SetNonblock and CloseOnExec. + +// +build darwin dragonfly nacl netbsd openbsd solaris + +package net + +import "syscall" + +// Wrapper around the socket system call that marks the returned file +// descriptor as nonblocking and close-on-exec. +func sysSocket(family, sotype, proto int) (int, error) { + // See ../syscall/exec_unix.go for description of ForkLock. + syscall.ForkLock.RLock() + s, err := syscall.Socket(family, sotype, proto) + if err == nil { + syscall.CloseOnExec(s) + } + syscall.ForkLock.RUnlock() + if err != nil { + return -1, err + } + if err = syscall.SetNonblock(s, true); err != nil { + syscall.Close(s) + return -1, err + } + return s, nil +} + +// Wrapper around the accept system call that marks the returned file +// descriptor as nonblocking and close-on-exec. +func accept(s int) (int, syscall.Sockaddr, error) { + // See ../syscall/exec_unix.go for description of ForkLock. + // It is probably okay to hold the lock across syscall.Accept + // because we have put fd.sysfd into non-blocking mode. + // However, a call to the File method will put it back into + // blocking mode. We can't take that risk, so no use of ForkLock here. + ns, sa, err := syscall.Accept(s) + if err == nil { + syscall.CloseOnExec(ns) + } + if err != nil { + return -1, nil, err + } + if err = syscall.SetNonblock(ns, true); err != nil { + syscall.Close(ns) + return -1, nil, err + } + return ns, sa, nil +} diff --git a/src/net/tcp_test.go b/src/net/tcp_test.go new file mode 100644 index 000000000..c04198ea0 --- /dev/null +++ b/src/net/tcp_test.go @@ -0,0 +1,611 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "fmt" + "io" + "reflect" + "runtime" + "sync" + "testing" + "time" +) + +func BenchmarkTCP4OneShot(b *testing.B) { + benchmarkTCP(b, false, false, "127.0.0.1:0") +} + +func BenchmarkTCP4OneShotTimeout(b *testing.B) { + benchmarkTCP(b, false, true, "127.0.0.1:0") +} + +func BenchmarkTCP4Persistent(b *testing.B) { + benchmarkTCP(b, true, false, "127.0.0.1:0") +} + +func BenchmarkTCP4PersistentTimeout(b *testing.B) { + benchmarkTCP(b, true, true, "127.0.0.1:0") +} + +func BenchmarkTCP6OneShot(b *testing.B) { + if !supportsIPv6 { + b.Skip("ipv6 is not supported") + } + benchmarkTCP(b, false, false, "[::1]:0") +} + +func BenchmarkTCP6OneShotTimeout(b *testing.B) { + if !supportsIPv6 { + b.Skip("ipv6 is not supported") + } + benchmarkTCP(b, false, true, "[::1]:0") +} + +func BenchmarkTCP6Persistent(b *testing.B) { + if !supportsIPv6 { + b.Skip("ipv6 is not supported") + } + benchmarkTCP(b, true, false, "[::1]:0") +} + +func BenchmarkTCP6PersistentTimeout(b *testing.B) { + if !supportsIPv6 { + b.Skip("ipv6 is not supported") + } + benchmarkTCP(b, true, true, "[::1]:0") +} + +func benchmarkTCP(b *testing.B, persistent, timeout bool, laddr string) { + const msgLen = 512 + conns := b.N + numConcurrent := runtime.GOMAXPROCS(-1) * 2 + msgs := 1 + if persistent { + conns = numConcurrent + msgs = b.N / conns + if msgs == 0 { + msgs = 1 + } + if conns > b.N { + conns = b.N + } + } + sendMsg := func(c Conn, buf []byte) bool { + n, err := c.Write(buf) + if n != len(buf) || err != nil { + b.Logf("Write failed: %v", err) + return false + } + return true + } + recvMsg := func(c Conn, buf []byte) bool { + for read := 0; read != len(buf); { + n, err := c.Read(buf) + read += n + if err != nil { + b.Logf("Read failed: %v", err) + return false + } + } + return true + } + ln, err := Listen("tcp", laddr) + if err != nil { + b.Fatalf("Listen failed: %v", err) + } + defer ln.Close() + serverSem := make(chan bool, numConcurrent) + // Acceptor. + go func() { + for { + c, err := ln.Accept() + if err != nil { + break + } + serverSem <- true + // Server connection. + go func(c Conn) { + defer func() { + c.Close() + <-serverSem + }() + if timeout { + c.SetDeadline(time.Now().Add(time.Hour)) // Not intended to fire. + } + var buf [msgLen]byte + for m := 0; m < msgs; m++ { + if !recvMsg(c, buf[:]) || !sendMsg(c, buf[:]) { + break + } + } + }(c) + } + }() + clientSem := make(chan bool, numConcurrent) + for i := 0; i < conns; i++ { + clientSem <- true + // Client connection. + go func() { + defer func() { + <-clientSem + }() + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + b.Logf("Dial failed: %v", err) + return + } + defer c.Close() + if timeout { + c.SetDeadline(time.Now().Add(time.Hour)) // Not intended to fire. + } + var buf [msgLen]byte + for m := 0; m < msgs; m++ { + if !sendMsg(c, buf[:]) || !recvMsg(c, buf[:]) { + break + } + } + }() + } + for i := 0; i < numConcurrent; i++ { + clientSem <- true + serverSem <- true + } +} + +func BenchmarkTCP4ConcurrentReadWrite(b *testing.B) { + benchmarkTCPConcurrentReadWrite(b, "127.0.0.1:0") +} + +func BenchmarkTCP6ConcurrentReadWrite(b *testing.B) { + if !supportsIPv6 { + b.Skip("ipv6 is not supported") + } + benchmarkTCPConcurrentReadWrite(b, "[::1]:0") +} + +func benchmarkTCPConcurrentReadWrite(b *testing.B, laddr string) { + // The benchmark creates GOMAXPROCS client/server pairs. + // Each pair creates 4 goroutines: client reader/writer and server reader/writer. + // The benchmark stresses concurrent reading and writing to the same connection. + // Such pattern is used in net/http and net/rpc. + + b.StopTimer() + + P := runtime.GOMAXPROCS(0) + N := b.N / P + W := 1000 + + // Setup P client/server connections. + clients := make([]Conn, P) + servers := make([]Conn, P) + ln, err := Listen("tcp", laddr) + if err != nil { + b.Fatalf("Listen failed: %v", err) + } + defer ln.Close() + done := make(chan bool) + go func() { + for p := 0; p < P; p++ { + s, err := ln.Accept() + if err != nil { + b.Errorf("Accept failed: %v", err) + return + } + servers[p] = s + } + done <- true + }() + for p := 0; p < P; p++ { + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + b.Fatalf("Dial failed: %v", err) + } + clients[p] = c + } + <-done + + b.StartTimer() + + var wg sync.WaitGroup + wg.Add(4 * P) + for p := 0; p < P; p++ { + // Client writer. + go func(c Conn) { + defer wg.Done() + var buf [1]byte + for i := 0; i < N; i++ { + v := byte(i) + for w := 0; w < W; w++ { + v *= v + } + buf[0] = v + _, err := c.Write(buf[:]) + if err != nil { + b.Errorf("Write failed: %v", err) + return + } + } + }(clients[p]) + + // Pipe between server reader and server writer. + pipe := make(chan byte, 128) + + // Server reader. + go func(s Conn) { + defer wg.Done() + var buf [1]byte + for i := 0; i < N; i++ { + _, err := s.Read(buf[:]) + if err != nil { + b.Errorf("Read failed: %v", err) + return + } + pipe <- buf[0] + } + }(servers[p]) + + // Server writer. + go func(s Conn) { + defer wg.Done() + var buf [1]byte + for i := 0; i < N; i++ { + v := <-pipe + for w := 0; w < W; w++ { + v *= v + } + buf[0] = v + _, err := s.Write(buf[:]) + if err != nil { + b.Errorf("Write failed: %v", err) + return + } + } + s.Close() + }(servers[p]) + + // Client reader. + go func(c Conn) { + defer wg.Done() + var buf [1]byte + for i := 0; i < N; i++ { + _, err := c.Read(buf[:]) + if err != nil { + b.Errorf("Read failed: %v", err) + return + } + } + c.Close() + }(clients[p]) + } + wg.Wait() +} + +type resolveTCPAddrTest struct { + net string + litAddrOrName string + addr *TCPAddr + err error +} + +var resolveTCPAddrTests = []resolveTCPAddrTest{ + {"tcp", "127.0.0.1:0", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 0}, nil}, + {"tcp4", "127.0.0.1:65535", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 65535}, nil}, + + {"tcp", "[::1]:1", &TCPAddr{IP: ParseIP("::1"), Port: 1}, nil}, + {"tcp6", "[::1]:65534", &TCPAddr{IP: ParseIP("::1"), Port: 65534}, nil}, + + {"tcp", "[::1%en0]:1", &TCPAddr{IP: ParseIP("::1"), Port: 1, Zone: "en0"}, nil}, + {"tcp6", "[::1%911]:2", &TCPAddr{IP: ParseIP("::1"), Port: 2, Zone: "911"}, nil}, + + {"", "127.0.0.1:0", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 0}, nil}, // Go 1.0 behavior + {"", "[::1]:0", &TCPAddr{IP: ParseIP("::1"), Port: 0}, nil}, // Go 1.0 behavior + + {"tcp", ":12345", &TCPAddr{Port: 12345}, nil}, + + {"http", "127.0.0.1:0", nil, UnknownNetworkError("http")}, +} + +func init() { + if ifi := loopbackInterface(); ifi != nil { + index := fmt.Sprintf("%v", ifi.Index) + resolveTCPAddrTests = append(resolveTCPAddrTests, []resolveTCPAddrTest{ + {"tcp6", "[fe80::1%" + ifi.Name + "]:3", &TCPAddr{IP: ParseIP("fe80::1"), Port: 3, Zone: zoneToString(ifi.Index)}, nil}, + {"tcp6", "[fe80::1%" + index + "]:4", &TCPAddr{IP: ParseIP("fe80::1"), Port: 4, Zone: index}, nil}, + }...) + } + if ips, err := LookupIP("localhost"); err == nil && len(ips) > 1 && supportsIPv4 && supportsIPv6 { + resolveTCPAddrTests = append(resolveTCPAddrTests, []resolveTCPAddrTest{ + {"tcp", "localhost:5", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 5}, nil}, + {"tcp4", "localhost:6", &TCPAddr{IP: IPv4(127, 0, 0, 1), Port: 6}, nil}, + {"tcp6", "localhost:7", &TCPAddr{IP: IPv6loopback, Port: 7}, nil}, + }...) + } +} + +func TestResolveTCPAddr(t *testing.T) { + for _, tt := range resolveTCPAddrTests { + addr, err := ResolveTCPAddr(tt.net, tt.litAddrOrName) + if err != tt.err { + t.Fatalf("ResolveTCPAddr(%q, %q) failed: %v", tt.net, tt.litAddrOrName, err) + } + if !reflect.DeepEqual(addr, tt.addr) { + t.Fatalf("ResolveTCPAddr(%q, %q) = %#v, want %#v", tt.net, tt.litAddrOrName, addr, tt.addr) + } + if err == nil { + str := addr.String() + addr1, err := ResolveTCPAddr(tt.net, str) + if err != nil { + t.Fatalf("ResolveTCPAddr(%q, %q) [from %q]: %v", tt.net, str, tt.litAddrOrName, err) + } + if !reflect.DeepEqual(addr1, addr) { + t.Fatalf("ResolveTCPAddr(%q, %q) [from %q] = %#v, want %#v", tt.net, str, tt.litAddrOrName, addr1, addr) + } + } + } +} + +var tcpListenerNameTests = []struct { + net string + laddr *TCPAddr +}{ + {"tcp4", &TCPAddr{IP: IPv4(127, 0, 0, 1)}}, + {"tcp4", &TCPAddr{}}, + {"tcp4", nil}, +} + +func TestTCPListenerName(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + for _, tt := range tcpListenerNameTests { + ln, err := ListenTCP(tt.net, tt.laddr) + if err != nil { + t.Fatalf("ListenTCP failed: %v", err) + } + defer ln.Close() + la := ln.Addr() + if a, ok := la.(*TCPAddr); !ok || a.Port == 0 { + t.Fatalf("got %v; expected a proper address with non-zero port number", la) + } + } +} + +func TestIPv6LinkLocalUnicastTCP(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + if !supportsIPv6 { + t.Skip("ipv6 is not supported") + } + ifi := loopbackInterface() + if ifi == nil { + t.Skip("loopback interface not found") + } + laddr := ipv6LinkLocalUnicastAddr(ifi) + if laddr == "" { + t.Skip("ipv6 unicast address on loopback not found") + } + + type test struct { + net, addr string + nameLookup bool + } + var tests = []test{ + {"tcp", "[" + laddr + "%" + ifi.Name + "]:0", false}, + {"tcp6", "[" + laddr + "%" + ifi.Name + "]:0", false}, + } + switch runtime.GOOS { + case "darwin", "freebsd", "openbsd", "netbsd": + tests = append(tests, []test{ + {"tcp", "[localhost%" + ifi.Name + "]:0", true}, + {"tcp6", "[localhost%" + ifi.Name + "]:0", true}, + }...) + case "linux": + tests = append(tests, []test{ + {"tcp", "[ip6-localhost%" + ifi.Name + "]:0", true}, + {"tcp6", "[ip6-localhost%" + ifi.Name + "]:0", true}, + }...) + } + for _, tt := range tests { + ln, err := Listen(tt.net, tt.addr) + if err != nil { + // It might return "LookupHost returned no + // suitable address" error on some platforms. + t.Logf("Listen failed: %v", err) + continue + } + defer ln.Close() + if la, ok := ln.Addr().(*TCPAddr); !ok || !tt.nameLookup && la.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", la) + } + + done := make(chan int) + go transponder(t, ln, done) + + c, err := Dial(tt.net, ln.Addr().String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c.Close() + if la, ok := c.LocalAddr().(*TCPAddr); !ok || !tt.nameLookup && la.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", la) + } + if ra, ok := c.RemoteAddr().(*TCPAddr); !ok || !tt.nameLookup && ra.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", ra) + } + + if _, err := c.Write([]byte("TCP OVER IPV6 LINKLOCAL TEST")); err != nil { + t.Fatalf("Conn.Write failed: %v", err) + } + b := make([]byte, 32) + if _, err := c.Read(b); err != nil { + t.Fatalf("Conn.Read failed: %v", err) + } + + <-done + } +} + +func TestTCPConcurrentAccept(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + const N = 10 + var wg sync.WaitGroup + wg.Add(N) + for i := 0; i < N; i++ { + go func() { + for { + c, err := ln.Accept() + if err != nil { + break + } + c.Close() + } + wg.Done() + }() + } + attempts := 10 * N + fails := 0 + d := &Dialer{Timeout: 200 * time.Millisecond} + for i := 0; i < attempts; i++ { + c, err := d.Dial("tcp", ln.Addr().String()) + if err != nil { + fails++ + } else { + c.Close() + } + } + ln.Close() + wg.Wait() + if fails > attempts/9 { // see issues 7400 and 7541 + t.Fatalf("too many Dial failed: %v", fails) + } + if fails > 0 { + t.Logf("# of failed Dials: %v", fails) + } +} + +func TestTCPReadWriteMallocs(t *testing.T) { + if testing.Short() { + t.Skip("skipping malloc count in short mode") + } + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer ln.Close() + var server Conn + errc := make(chan error) + go func() { + var err error + server, err = ln.Accept() + errc <- err + }() + client, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + if err := <-errc; err != nil { + t.Fatalf("Accept failed: %v", err) + } + defer server.Close() + var buf [128]byte + mallocs := testing.AllocsPerRun(1000, func() { + _, err := server.Write(buf[:]) + if err != nil { + t.Fatalf("Write failed: %v", err) + } + _, err = io.ReadFull(client, buf[:]) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + }) + if mallocs > 0 { + t.Fatalf("Got %v allocs, want 0", mallocs) + } +} + +func TestTCPStress(t *testing.T) { + const conns = 2 + const msgLen = 512 + msgs := int(1e4) + if testing.Short() { + msgs = 1e2 + } + + sendMsg := func(c Conn, buf []byte) bool { + n, err := c.Write(buf) + if n != len(buf) || err != nil { + t.Logf("Write failed: %v", err) + return false + } + return true + } + recvMsg := func(c Conn, buf []byte) bool { + for read := 0; read != len(buf); { + n, err := c.Read(buf) + read += n + if err != nil { + t.Logf("Read failed: %v", err) + return false + } + } + return true + } + + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer ln.Close() + // Acceptor. + go func() { + for { + c, err := ln.Accept() + if err != nil { + break + } + // Server connection. + go func(c Conn) { + defer c.Close() + var buf [msgLen]byte + for m := 0; m < msgs; m++ { + if !recvMsg(c, buf[:]) || !sendMsg(c, buf[:]) { + break + } + } + }(c) + } + }() + done := make(chan bool) + for i := 0; i < conns; i++ { + // Client connection. + go func() { + defer func() { + done <- true + }() + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Logf("Dial failed: %v", err) + return + } + defer c.Close() + var buf [msgLen]byte + for m := 0; m < msgs; m++ { + if !sendMsg(c, buf[:]) || !recvMsg(c, buf[:]) { + break + } + } + }() + } + for i := 0; i < conns; i++ { + <-done + } +} diff --git a/src/net/tcpsock.go b/src/net/tcpsock.go new file mode 100644 index 000000000..f3dfbd23d --- /dev/null +++ b/src/net/tcpsock.go @@ -0,0 +1,54 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// TCPAddr represents the address of a TCP end point. +type TCPAddr struct { + IP IP + Port int + Zone string // IPv6 scoped addressing zone +} + +// Network returns the address's network name, "tcp". +func (a *TCPAddr) Network() string { return "tcp" } + +func (a *TCPAddr) String() string { + if a == nil { + return "<nil>" + } + ip := ipEmptyString(a.IP) + if a.Zone != "" { + return JoinHostPort(ip+"%"+a.Zone, itoa(a.Port)) + } + return JoinHostPort(ip, itoa(a.Port)) +} + +func (a *TCPAddr) toAddr() Addr { + if a == nil { + return nil + } + return a +} + +// ResolveTCPAddr parses addr as a TCP address of the form "host:port" +// or "[ipv6-host%zone]:port" and resolves a pair of domain name and +// port name on the network net, which must be "tcp", "tcp4" or +// "tcp6". A literal address or host name for IPv6 must be enclosed +// in square brackets, as in "[::1]:80", "[ipv6-host]:http" or +// "[ipv6-host%zone]:80". +func ResolveTCPAddr(net, addr string) (*TCPAddr, error) { + switch net { + case "tcp", "tcp4", "tcp6": + case "": // a hint wildcard for Go 1.0 undocumented behavior + net = "tcp" + default: + return nil, UnknownNetworkError(net) + } + a, err := resolveInternetAddr(net, addr, noDeadline) + if err != nil { + return nil, err + } + return a.toAddr().(*TCPAddr), nil +} diff --git a/src/net/tcpsock_plan9.go b/src/net/tcpsock_plan9.go new file mode 100644 index 000000000..52019d7b4 --- /dev/null +++ b/src/net/tcpsock_plan9.go @@ -0,0 +1,198 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "io" + "os" + "syscall" + "time" +) + +// TCPConn is an implementation of the Conn interface for TCP network +// connections. +type TCPConn struct { + conn +} + +func newTCPConn(fd *netFD) *TCPConn { + return &TCPConn{conn{fd}} +} + +// ReadFrom implements the io.ReaderFrom ReadFrom method. +func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { + return genericReadFrom(c, r) +} + +// CloseRead shuts down the reading side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.closeRead() +} + +// CloseWrite shuts down the writing side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.closeWrite() +} + +// SetLinger sets the behavior of Close on a connection which still +// has data waiting to be sent or to be acknowledged. +// +// If sec < 0 (the default), the operating system finishes sending the +// data in the background. +// +// If sec == 0, the operating system discards any unsent or +// unacknowledged data. +// +// If sec > 0, the data is sent in the background as with sec < 0. On +// some operating systems after sec seconds have elapsed any remaining +// unsent data may be discarded. +func (c *TCPConn) SetLinger(sec int) error { + return syscall.EPLAN9 +} + +// SetKeepAlive sets whether the operating system should send +// keepalive messages on the connection. +func (c *TCPConn) SetKeepAlive(keepalive bool) error { + if !c.ok() { + return syscall.EPLAN9 + } + return setKeepAlive(c.fd, keepalive) +} + +// SetKeepAlivePeriod sets period between keep alives. +func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { + if !c.ok() { + return syscall.EPLAN9 + } + return setKeepAlivePeriod(c.fd, d) +} + +// SetNoDelay controls whether the operating system should delay +// packet transmission in hopes of sending fewer packets (Nagle's +// algorithm). The default is true (no delay), meaning that data is +// sent as soon as possible after a Write. +func (c *TCPConn) SetNoDelay(noDelay bool) error { + return syscall.EPLAN9 +} + +// DialTCP connects to the remote address raddr on the network net, +// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + return dialTCP(net, laddr, raddr, noDeadline) +} + +func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time) (*TCPConn, error) { + if !deadline.IsZero() { + panic("net.dialTCP: deadline not implemented on Plan 9") + } + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{"dial", net, raddr, UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{"dial", net, nil, errMissingAddress} + } + fd, err := dialPlan9(net, laddr, raddr) + if err != nil { + return nil, err + } + return newTCPConn(fd), nil +} + +// TCPListener is a TCP network listener. Clients should typically +// use variables of type Listener instead of assuming TCP. +type TCPListener struct { + fd *netFD +} + +// AcceptTCP accepts the next incoming call and returns the new +// connection. +func (l *TCPListener) AcceptTCP() (*TCPConn, error) { + if l == nil || l.fd == nil || l.fd.ctl == nil { + return nil, syscall.EINVAL + } + fd, err := l.fd.acceptPlan9() + if err != nil { + return nil, err + } + return newTCPConn(fd), nil +} + +// Accept implements the Accept method in the Listener interface; it +// waits for the next call and returns a generic Conn. +func (l *TCPListener) Accept() (Conn, error) { + if l == nil || l.fd == nil || l.fd.ctl == nil { + return nil, syscall.EINVAL + } + c, err := l.AcceptTCP() + if err != nil { + return nil, err + } + return c, nil +} + +// Close stops listening on the TCP address. +// Already Accepted connections are not closed. +func (l *TCPListener) Close() error { + if l == nil || l.fd == nil || l.fd.ctl == nil { + return syscall.EINVAL + } + if _, err := l.fd.ctl.WriteString("hangup"); err != nil { + l.fd.ctl.Close() + return &OpError{"close", l.fd.ctl.Name(), l.fd.laddr, err} + } + return l.fd.ctl.Close() +} + +// Addr returns the listener's network address, a *TCPAddr. +func (l *TCPListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *TCPListener) SetDeadline(t time.Time) error { + if l == nil || l.fd == nil || l.fd.ctl == nil { + return syscall.EINVAL + } + return l.fd.setDeadline(t) +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *TCPListener) File() (f *os.File, err error) { return l.dup() } + +// ListenTCP announces on the TCP address laddr and returns a TCP +// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a +// port of 0, ListenTCP will choose an available port. The caller can +// use the Addr method of TCPListener to retrieve the chosen address. +func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{"listen", net, laddr, UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &TCPAddr{} + } + fd, err := listenPlan9(net, laddr) + if err != nil { + return nil, err + } + return &TCPListener{fd}, nil +} diff --git a/src/net/tcpsock_posix.go b/src/net/tcpsock_posix.go new file mode 100644 index 000000000..dd78aefa7 --- /dev/null +++ b/src/net/tcpsock_posix.go @@ -0,0 +1,299 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "io" + "os" + "syscall" + "time" +) + +// BUG(rsc): On OpenBSD, listening on the "tcp" network does not listen for +// both IPv4 and IPv6 connections. This is due to the fact that IPv4 traffic +// will not be routed to an IPv6 socket - two separate sockets are required +// if both AFs are to be supported. See inet6(4) on OpenBSD for details. + +func sockaddrToTCP(sa syscall.Sockaddr) Addr { + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + return &TCPAddr{IP: sa.Addr[0:], Port: sa.Port} + case *syscall.SockaddrInet6: + return &TCPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + } + return nil +} + +func (a *TCPAddr) family() int { + if a == nil || len(a.IP) <= IPv4len { + return syscall.AF_INET + } + if a.IP.To4() != nil { + return syscall.AF_INET + } + return syscall.AF_INET6 +} + +func (a *TCPAddr) isWildcard() bool { + if a == nil || a.IP == nil { + return true + } + return a.IP.IsUnspecified() +} + +func (a *TCPAddr) sockaddr(family int) (syscall.Sockaddr, error) { + if a == nil { + return nil, nil + } + return ipToSockaddr(family, a.IP, a.Port, a.Zone) +} + +// TCPConn is an implementation of the Conn interface for TCP network +// connections. +type TCPConn struct { + conn +} + +func newTCPConn(fd *netFD) *TCPConn { + c := &TCPConn{conn{fd}} + c.SetNoDelay(true) + return c +} + +// ReadFrom implements the io.ReaderFrom ReadFrom method. +func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { + if n, err, handled := sendFile(c.fd, r); handled { + return n, err + } + return genericReadFrom(c, r) +} + +// CloseRead shuts down the reading side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.closeRead() +} + +// CloseWrite shuts down the writing side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.closeWrite() +} + +// SetLinger sets the behavior of Close on a connection which still +// has data waiting to be sent or to be acknowledged. +// +// If sec < 0 (the default), the operating system finishes sending the +// data in the background. +// +// If sec == 0, the operating system discards any unsent or +// unacknowledged data. +// +// If sec > 0, the data is sent in the background as with sec < 0. On +// some operating systems after sec seconds have elapsed any remaining +// unsent data may be discarded. +func (c *TCPConn) SetLinger(sec int) error { + if !c.ok() { + return syscall.EINVAL + } + return setLinger(c.fd, sec) +} + +// SetKeepAlive sets whether the operating system should send +// keepalive messages on the connection. +func (c *TCPConn) SetKeepAlive(keepalive bool) error { + if !c.ok() { + return syscall.EINVAL + } + return setKeepAlive(c.fd, keepalive) +} + +// SetKeepAlivePeriod sets period between keep alives. +func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { + if !c.ok() { + return syscall.EINVAL + } + return setKeepAlivePeriod(c.fd, d) +} + +// SetNoDelay controls whether the operating system should delay +// packet transmission in hopes of sending fewer packets (Nagle's +// algorithm). The default is true (no delay), meaning that data is +// sent as soon as possible after a Write. +func (c *TCPConn) SetNoDelay(noDelay bool) error { + if !c.ok() { + return syscall.EINVAL + } + return setNoDelay(c.fd, noDelay) +} + +// DialTCP connects to the remote address raddr on the network net, +// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{Op: "dial", Net: net, Addr: raddr, Err: UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: net, Addr: nil, Err: errMissingAddress} + } + return dialTCP(net, laddr, raddr, noDeadline) +} + +func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time) (*TCPConn, error) { + fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_STREAM, 0, "dial") + + // TCP has a rarely used mechanism called a 'simultaneous connection' in + // which Dial("tcp", addr1, addr2) run on the machine at addr1 can + // connect to a simultaneous Dial("tcp", addr2, addr1) run on the machine + // at addr2, without either machine executing Listen. If laddr == nil, + // it means we want the kernel to pick an appropriate originating local + // address. Some Linux kernels cycle blindly through a fixed range of + // local ports, regardless of destination port. If a kernel happens to + // pick local port 50001 as the source for a Dial("tcp", "", "localhost:50001"), + // then the Dial will succeed, having simultaneously connected to itself. + // This can only happen when we are letting the kernel pick a port (laddr == nil) + // and when there is no listener for the destination address. + // It's hard to argue this is anything other than a kernel bug. If we + // see this happen, rather than expose the buggy effect to users, we + // close the fd and try again. If it happens twice more, we relent and + // use the result. See also: + // http://golang.org/issue/2690 + // http://stackoverflow.com/questions/4949858/ + // + // The opposite can also happen: if we ask the kernel to pick an appropriate + // originating local address, sometimes it picks one that is already in use. + // So if the error is EADDRNOTAVAIL, we have to try again too, just for + // a different reason. + // + // The kernel socket code is no doubt enjoying watching us squirm. + for i := 0; i < 2 && (laddr == nil || laddr.Port == 0) && (selfConnect(fd, err) || spuriousENOTAVAIL(err)); i++ { + if err == nil { + fd.Close() + } + fd, err = internetSocket(net, laddr, raddr, deadline, syscall.SOCK_STREAM, 0, "dial") + } + + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Addr: raddr, Err: err} + } + return newTCPConn(fd), nil +} + +func selfConnect(fd *netFD, err error) bool { + // If the connect failed, we clearly didn't connect to ourselves. + if err != nil { + return false + } + + // The socket constructor can return an fd with raddr nil under certain + // unknown conditions. The errors in the calls there to Getpeername + // are discarded, but we can't catch the problem there because those + // calls are sometimes legally erroneous with a "socket not connected". + // Since this code (selfConnect) is already trying to work around + // a problem, we make sure if this happens we recognize trouble and + // ask the DialTCP routine to try again. + // TODO: try to understand what's really going on. + if fd.laddr == nil || fd.raddr == nil { + return true + } + l := fd.laddr.(*TCPAddr) + r := fd.raddr.(*TCPAddr) + return l.Port == r.Port && l.IP.Equal(r.IP) +} + +func spuriousENOTAVAIL(err error) bool { + e, ok := err.(*OpError) + return ok && e.Err == syscall.EADDRNOTAVAIL +} + +// TCPListener is a TCP network listener. Clients should typically +// use variables of type Listener instead of assuming TCP. +type TCPListener struct { + fd *netFD +} + +// AcceptTCP accepts the next incoming call and returns the new +// connection. +func (l *TCPListener) AcceptTCP() (*TCPConn, error) { + if l == nil || l.fd == nil { + return nil, syscall.EINVAL + } + fd, err := l.fd.accept() + if err != nil { + return nil, err + } + return newTCPConn(fd), nil +} + +// Accept implements the Accept method in the Listener interface; it +// waits for the next call and returns a generic Conn. +func (l *TCPListener) Accept() (Conn, error) { + c, err := l.AcceptTCP() + if err != nil { + return nil, err + } + return c, nil +} + +// Close stops listening on the TCP address. +// Already Accepted connections are not closed. +func (l *TCPListener) Close() error { + if l == nil || l.fd == nil { + return syscall.EINVAL + } + return l.fd.Close() +} + +// Addr returns the listener's network address, a *TCPAddr. +func (l *TCPListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *TCPListener) SetDeadline(t time.Time) error { + if l == nil || l.fd == nil { + return syscall.EINVAL + } + return l.fd.setDeadline(t) +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *TCPListener) File() (f *os.File, err error) { return l.fd.dup() } + +// ListenTCP announces on the TCP address laddr and returns a TCP +// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a +// port of 0, ListenTCP will choose an available port. The caller can +// use the Addr method of TCPListener to retrieve the chosen address. +func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &TCPAddr{} + } + fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_STREAM, 0, "listen") + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: err} + } + return &TCPListener{fd}, nil +} diff --git a/src/net/tcpsockopt_darwin.go b/src/net/tcpsockopt_darwin.go new file mode 100644 index 000000000..33140849c --- /dev/null +++ b/src/net/tcpsockopt_darwin.go @@ -0,0 +1,27 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TCP socket options for darwin + +package net + +import ( + "os" + "syscall" + "time" +) + +// Set keep alive period. +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + // The kernel expects seconds so round to next highest second. + d += (time.Second - time.Nanosecond) + secs := int(d.Seconds()) + + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPALIVE, secs)) +} diff --git a/src/net/tcpsockopt_dragonfly.go b/src/net/tcpsockopt_dragonfly.go new file mode 100644 index 000000000..d10a77773 --- /dev/null +++ b/src/net/tcpsockopt_dragonfly.go @@ -0,0 +1,29 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" + "time" +) + +// Set keep alive period. +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + // The kernel expects milliseconds so round to next highest millisecond. + d += (time.Millisecond - time.Nanosecond) + msecs := int(time.Duration(d.Nanoseconds()) / time.Millisecond) + + err := os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, msecs)) + if err != nil { + return err + } + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, msecs)) +} diff --git a/src/net/tcpsockopt_openbsd.go b/src/net/tcpsockopt_openbsd.go new file mode 100644 index 000000000..3480f932c --- /dev/null +++ b/src/net/tcpsockopt_openbsd.go @@ -0,0 +1,27 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TCP socket options for openbsd + +package net + +import ( + "os" + "syscall" + "time" +) + +// Set keep alive period. +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + // The kernel expects seconds so round to next highest second. + d += (time.Second - time.Nanosecond) + secs := int(d.Seconds()) + + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.SO_KEEPALIVE, secs)) +} diff --git a/src/net/tcpsockopt_plan9.go b/src/net/tcpsockopt_plan9.go new file mode 100644 index 000000000..0e7a6647c --- /dev/null +++ b/src/net/tcpsockopt_plan9.go @@ -0,0 +1,18 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TCP socket options for plan9 + +package net + +import ( + "time" +) + +// Set keep alive period. +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + cmd := "keepalive " + string(int64(d/time.Millisecond)) + _, e := fd.ctl.WriteAt([]byte(cmd), 0) + return e +} diff --git a/src/net/tcpsockopt_posix.go b/src/net/tcpsockopt_posix.go new file mode 100644 index 000000000..6484bad4b --- /dev/null +++ b/src/net/tcpsockopt_posix.go @@ -0,0 +1,20 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "os" + "syscall" +) + +func setNoDelay(fd *netFD, noDelay bool) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_NODELAY, boolint(noDelay))) +} diff --git a/src/net/tcpsockopt_solaris.go b/src/net/tcpsockopt_solaris.go new file mode 100644 index 000000000..eaab6b678 --- /dev/null +++ b/src/net/tcpsockopt_solaris.go @@ -0,0 +1,27 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TCP socket options for solaris + +package net + +import ( + "os" + "syscall" + "time" +) + +// Set keep alive period. +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + // The kernel expects seconds so round to next highest second. + d += (time.Second - time.Nanosecond) + secs := int(d.Seconds()) + + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.SO_KEEPALIVE, secs)) +} diff --git a/src/net/tcpsockopt_unix.go b/src/net/tcpsockopt_unix.go new file mode 100644 index 000000000..2693a541d --- /dev/null +++ b/src/net/tcpsockopt_unix.go @@ -0,0 +1,31 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build freebsd linux nacl netbsd + +package net + +import ( + "os" + "syscall" + "time" +) + +// Set keep alive period. +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + // The kernel expects seconds so round to next highest second. + d += (time.Second - time.Nanosecond) + secs := int(d.Seconds()) + + err := os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, secs)) + if err != nil { + return err + } + return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd.sysfd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, secs)) +} diff --git a/src/net/tcpsockopt_windows.go b/src/net/tcpsockopt_windows.go new file mode 100644 index 000000000..8ef140797 --- /dev/null +++ b/src/net/tcpsockopt_windows.go @@ -0,0 +1,34 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// TCP socket options for windows + +package net + +import ( + "os" + "syscall" + "time" + "unsafe" +) + +func setKeepAlivePeriod(fd *netFD, d time.Duration) error { + if err := fd.incref(); err != nil { + return err + } + defer fd.decref() + + // Windows expects milliseconds so round to next highest millisecond. + d += (time.Millisecond - time.Nanosecond) + millis := uint32(d / time.Millisecond) + ka := syscall.TCPKeepalive{ + OnOff: 1, + Time: millis, + Interval: millis, + } + ret := uint32(0) + size := uint32(unsafe.Sizeof(ka)) + err := syscall.WSAIoctl(fd.sysfd, syscall.SIO_KEEPALIVE_VALS, (*byte)(unsafe.Pointer(&ka)), size, nil, 0, &ret, nil, 0) + return os.NewSyscallError("WSAIoctl", err) +} diff --git a/src/net/testdata/domain-resolv.conf b/src/net/testdata/domain-resolv.conf new file mode 100644 index 000000000..ff269180f --- /dev/null +++ b/src/net/testdata/domain-resolv.conf @@ -0,0 +1,5 @@ +# /etc/resolv.conf + +search test invalid +domain localdomain +nameserver 8.8.8.8 diff --git a/src/net/testdata/empty-resolv.conf b/src/net/testdata/empty-resolv.conf new file mode 100644 index 000000000..c4b2b5765 --- /dev/null +++ b/src/net/testdata/empty-resolv.conf @@ -0,0 +1 @@ +# /etc/resolv.conf diff --git a/src/net/testdata/hosts b/src/net/testdata/hosts new file mode 100644 index 000000000..b60176389 --- /dev/null +++ b/src/net/testdata/hosts @@ -0,0 +1,12 @@ +255.255.255.255 broadcasthost +127.0.0.2 odin +127.0.0.3 odin # inline comment +::2 odin +127.1.1.1 thor +# aliases +127.1.1.2 ullr ullrhost +# Bogus entries that must be ignored. +123.123.123 loki +321.321.321.321 +# TODO(yvesj): Should we be able to parse this? From a Darwin system. +fe80::1%lo0 localhost diff --git a/src/net/testdata/hosts_singleline b/src/net/testdata/hosts_singleline new file mode 100644 index 000000000..5f5f74a3f --- /dev/null +++ b/src/net/testdata/hosts_singleline @@ -0,0 +1 @@ +127.0.0.2 odin
\ No newline at end of file diff --git a/src/net/testdata/igmp b/src/net/testdata/igmp new file mode 100644 index 000000000..5f380a2c7 --- /dev/null +++ b/src/net/testdata/igmp @@ -0,0 +1,24 @@ +Idx Device : Count Querier Group Users Timer Reporter +1 lo : 1 V3 + 010000E0 1 0:00000000 0 +2 eth0 : 2 V2 + FB0000E0 1 0:00000000 1 + 010000E0 1 0:00000000 0 +3 eth1 : 1 V3 + 010000E0 1 0:00000000 0 +4 eth2 : 1 V3 + 010000E0 1 0:00000000 0 +5 eth0.100 : 2 V3 + FB0000E0 1 0:00000000 0 + 010000E0 1 0:00000000 0 +6 eth0.101 : 2 V3 + FB0000E0 1 0:00000000 0 + 010000E0 1 0:00000000 0 +7 eth0.102 : 2 V3 + FB0000E0 1 0:00000000 0 + 010000E0 1 0:00000000 0 +8 eth0.103 : 2 V3 + FB0000E0 1 0:00000000 0 + 010000E0 1 0:00000000 0 +9 device1tap2: 1 V3 + 010000E0 1 0:00000000 0 diff --git a/src/net/testdata/igmp6 b/src/net/testdata/igmp6 new file mode 100644 index 000000000..6cd5a2d4d --- /dev/null +++ b/src/net/testdata/igmp6 @@ -0,0 +1,18 @@ +1 lo ff020000000000000000000000000001 1 0000000C 0 +2 eth0 ff0200000000000000000001ffac891e 1 00000006 0 +2 eth0 ff020000000000000000000000000001 1 0000000C 0 +3 eth1 ff0200000000000000000001ffac8928 2 00000006 0 +3 eth1 ff020000000000000000000000000001 1 0000000C 0 +4 eth2 ff0200000000000000000001ffac8932 2 00000006 0 +4 eth2 ff020000000000000000000000000001 1 0000000C 0 +5 eth0.100 ff0200000000000000000001ffac891e 1 00000004 0 +5 eth0.100 ff020000000000000000000000000001 1 0000000C 0 +6 pan0 ff020000000000000000000000000001 1 0000000C 0 +7 eth0.101 ff0200000000000000000001ffac891e 1 00000004 0 +7 eth0.101 ff020000000000000000000000000001 1 0000000C 0 +8 eth0.102 ff0200000000000000000001ffac891e 1 00000004 0 +8 eth0.102 ff020000000000000000000000000001 1 0000000C 0 +9 eth0.103 ff0200000000000000000001ffac891e 1 00000004 0 +9 eth0.103 ff020000000000000000000000000001 1 0000000C 0 +10 device1tap2 ff0200000000000000000001ff4cc3a3 1 00000004 0 +10 device1tap2 ff020000000000000000000000000001 1 0000000C 0 diff --git a/src/net/testdata/resolv.conf b/src/net/testdata/resolv.conf new file mode 100644 index 000000000..04e87eed0 --- /dev/null +++ b/src/net/testdata/resolv.conf @@ -0,0 +1,8 @@ +# /etc/resolv.conf + +domain localdomain +nameserver 8.8.8.8 +nameserver 2001:4860:4860::8888 +nameserver fe80::1%lo0 +options ndots:5 timeout:10 attempts:3 rotate +options attempts 3 diff --git a/src/net/testdata/search-resolv.conf b/src/net/testdata/search-resolv.conf new file mode 100644 index 000000000..1c846bfaf --- /dev/null +++ b/src/net/testdata/search-resolv.conf @@ -0,0 +1,5 @@ +# /etc/resolv.conf + +domain localdomain +search test invalid +nameserver 8.8.8.8 diff --git a/src/net/textproto/header.go b/src/net/textproto/header.go new file mode 100644 index 000000000..7fb32f804 --- /dev/null +++ b/src/net/textproto/header.go @@ -0,0 +1,43 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +// A MIMEHeader represents a MIME-style header mapping +// keys to sets of values. +type MIMEHeader map[string][]string + +// Add adds the key, value pair to the header. +// It appends to any existing values associated with key. +func (h MIMEHeader) Add(key, value string) { + key = CanonicalMIMEHeaderKey(key) + h[key] = append(h[key], value) +} + +// Set sets the header entries associated with key to +// the single element value. It replaces any existing +// values associated with key. +func (h MIMEHeader) Set(key, value string) { + h[CanonicalMIMEHeaderKey(key)] = []string{value} +} + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns "". +// Get is a convenience method. For more complex queries, +// access the map directly. +func (h MIMEHeader) Get(key string) string { + if h == nil { + return "" + } + v := h[CanonicalMIMEHeaderKey(key)] + if len(v) == 0 { + return "" + } + return v[0] +} + +// Del deletes the values associated with key. +func (h MIMEHeader) Del(key string) { + delete(h, CanonicalMIMEHeaderKey(key)) +} diff --git a/src/net/textproto/pipeline.go b/src/net/textproto/pipeline.go new file mode 100644 index 000000000..ca50eddac --- /dev/null +++ b/src/net/textproto/pipeline.go @@ -0,0 +1,117 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import ( + "sync" +) + +// A Pipeline manages a pipelined in-order request/response sequence. +// +// To use a Pipeline p to manage multiple clients on a connection, +// each client should run: +// +// id := p.Next() // take a number +// +// p.StartRequest(id) // wait for turn to send request +// «send request» +// p.EndRequest(id) // notify Pipeline that request is sent +// +// p.StartResponse(id) // wait for turn to read response +// «read response» +// p.EndResponse(id) // notify Pipeline that response is read +// +// A pipelined server can use the same calls to ensure that +// responses computed in parallel are written in the correct order. +type Pipeline struct { + mu sync.Mutex + id uint + request sequencer + response sequencer +} + +// Next returns the next id for a request/response pair. +func (p *Pipeline) Next() uint { + p.mu.Lock() + id := p.id + p.id++ + p.mu.Unlock() + return id +} + +// StartRequest blocks until it is time to send (or, if this is a server, receive) +// the request with the given id. +func (p *Pipeline) StartRequest(id uint) { + p.request.Start(id) +} + +// EndRequest notifies p that the request with the given id has been sent +// (or, if this is a server, received). +func (p *Pipeline) EndRequest(id uint) { + p.request.End(id) +} + +// StartResponse blocks until it is time to receive (or, if this is a server, send) +// the request with the given id. +func (p *Pipeline) StartResponse(id uint) { + p.response.Start(id) +} + +// EndResponse notifies p that the response with the given id has been received +// (or, if this is a server, sent). +func (p *Pipeline) EndResponse(id uint) { + p.response.End(id) +} + +// A sequencer schedules a sequence of numbered events that must +// happen in order, one after the other. The event numbering must start +// at 0 and increment without skipping. The event number wraps around +// safely as long as there are not 2^32 simultaneous events pending. +type sequencer struct { + mu sync.Mutex + id uint + wait map[uint]chan uint +} + +// Start waits until it is time for the event numbered id to begin. +// That is, except for the first event, it waits until End(id-1) has +// been called. +func (s *sequencer) Start(id uint) { + s.mu.Lock() + if s.id == id { + s.mu.Unlock() + return + } + c := make(chan uint) + if s.wait == nil { + s.wait = make(map[uint]chan uint) + } + s.wait[id] = c + s.mu.Unlock() + <-c +} + +// End notifies the sequencer that the event numbered id has completed, +// allowing it to schedule the event numbered id+1. It is a run-time error +// to call End with an id that is not the number of the active event. +func (s *sequencer) End(id uint) { + s.mu.Lock() + if s.id != id { + panic("out of sync") + } + id++ + s.id = id + if s.wait == nil { + s.wait = make(map[uint]chan uint) + } + c, ok := s.wait[id] + if ok { + delete(s.wait, id) + } + s.mu.Unlock() + if ok { + c <- 1 + } +} diff --git a/src/net/textproto/reader.go b/src/net/textproto/reader.go new file mode 100644 index 000000000..eea9207f2 --- /dev/null +++ b/src/net/textproto/reader.go @@ -0,0 +1,637 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import ( + "bufio" + "bytes" + "io" + "io/ioutil" + "strconv" + "strings" +) + +// BUG(rsc): To let callers manage exposure to denial of service +// attacks, Reader should allow them to set and reset a limit on +// the number of bytes read from the connection. + +// A Reader implements convenience methods for reading requests +// or responses from a text protocol network connection. +type Reader struct { + R *bufio.Reader + dot *dotReader + buf []byte // a re-usable buffer for readContinuedLineSlice +} + +// NewReader returns a new Reader reading from r. +func NewReader(r *bufio.Reader) *Reader { + return &Reader{R: r} +} + +// ReadLine reads a single line from r, +// eliding the final \n or \r\n from the returned string. +func (r *Reader) ReadLine() (string, error) { + line, err := r.readLineSlice() + return string(line), err +} + +// ReadLineBytes is like ReadLine but returns a []byte instead of a string. +func (r *Reader) ReadLineBytes() ([]byte, error) { + line, err := r.readLineSlice() + if line != nil { + buf := make([]byte, len(line)) + copy(buf, line) + line = buf + } + return line, err +} + +func (r *Reader) readLineSlice() ([]byte, error) { + r.closeDot() + var line []byte + for { + l, more, err := r.R.ReadLine() + if err != nil { + return nil, err + } + // Avoid the copy if the first call produced a full line. + if line == nil && !more { + return l, nil + } + line = append(line, l...) + if !more { + break + } + } + return line, nil +} + +// ReadContinuedLine reads a possibly continued line from r, +// eliding the final trailing ASCII white space. +// Lines after the first are considered continuations if they +// begin with a space or tab character. In the returned data, +// continuation lines are separated from the previous line +// only by a single space: the newline and leading white space +// are removed. +// +// For example, consider this input: +// +// Line 1 +// continued... +// Line 2 +// +// The first call to ReadContinuedLine will return "Line 1 continued..." +// and the second will return "Line 2". +// +// A line consisting of only white space is never continued. +// +func (r *Reader) ReadContinuedLine() (string, error) { + line, err := r.readContinuedLineSlice() + return string(line), err +} + +// trim returns s with leading and trailing spaces and tabs removed. +// It does not assume Unicode or UTF-8. +func trim(s []byte) []byte { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + n := len(s) + for n > i && (s[n-1] == ' ' || s[n-1] == '\t') { + n-- + } + return s[i:n] +} + +// ReadContinuedLineBytes is like ReadContinuedLine but +// returns a []byte instead of a string. +func (r *Reader) ReadContinuedLineBytes() ([]byte, error) { + line, err := r.readContinuedLineSlice() + if line != nil { + buf := make([]byte, len(line)) + copy(buf, line) + line = buf + } + return line, err +} + +func (r *Reader) readContinuedLineSlice() ([]byte, error) { + // Read the first line. + line, err := r.readLineSlice() + if err != nil { + return nil, err + } + if len(line) == 0 { // blank line - no continuation + return line, nil + } + + // Optimistically assume that we have started to buffer the next line + // and it starts with an ASCII letter (the next header key), so we can + // avoid copying that buffered data around in memory and skipping over + // non-existent whitespace. + if r.R.Buffered() > 1 { + peek, err := r.R.Peek(1) + if err == nil && isASCIILetter(peek[0]) { + return trim(line), nil + } + } + + // ReadByte or the next readLineSlice will flush the read buffer; + // copy the slice into buf. + r.buf = append(r.buf[:0], trim(line)...) + + // Read continuation lines. + for r.skipSpace() > 0 { + line, err := r.readLineSlice() + if err != nil { + break + } + r.buf = append(r.buf, ' ') + r.buf = append(r.buf, line...) + } + return r.buf, nil +} + +// skipSpace skips R over all spaces and returns the number of bytes skipped. +func (r *Reader) skipSpace() int { + n := 0 + for { + c, err := r.R.ReadByte() + if err != nil { + // Bufio will keep err until next read. + break + } + if c != ' ' && c != '\t' { + r.R.UnreadByte() + break + } + n++ + } + return n +} + +func (r *Reader) readCodeLine(expectCode int) (code int, continued bool, message string, err error) { + line, err := r.ReadLine() + if err != nil { + return + } + return parseCodeLine(line, expectCode) +} + +func parseCodeLine(line string, expectCode int) (code int, continued bool, message string, err error) { + if len(line) < 4 || line[3] != ' ' && line[3] != '-' { + err = ProtocolError("short response: " + line) + return + } + continued = line[3] == '-' + code, err = strconv.Atoi(line[0:3]) + if err != nil || code < 100 { + err = ProtocolError("invalid response code: " + line) + return + } + message = line[4:] + if 1 <= expectCode && expectCode < 10 && code/100 != expectCode || + 10 <= expectCode && expectCode < 100 && code/10 != expectCode || + 100 <= expectCode && expectCode < 1000 && code != expectCode { + err = &Error{code, message} + } + return +} + +// ReadCodeLine reads a response code line of the form +// code message +// where code is a three-digit status code and the message +// extends to the rest of the line. An example of such a line is: +// 220 plan9.bell-labs.com ESMTP +// +// If the prefix of the status does not match the digits in expectCode, +// ReadCodeLine returns with err set to &Error{code, message}. +// For example, if expectCode is 31, an error will be returned if +// the status is not in the range [310,319]. +// +// If the response is multi-line, ReadCodeLine returns an error. +// +// An expectCode <= 0 disables the check of the status code. +// +func (r *Reader) ReadCodeLine(expectCode int) (code int, message string, err error) { + code, continued, message, err := r.readCodeLine(expectCode) + if err == nil && continued { + err = ProtocolError("unexpected multi-line response: " + message) + } + return +} + +// ReadResponse reads a multi-line response of the form: +// +// code-message line 1 +// code-message line 2 +// ... +// code message line n +// +// where code is a three-digit status code. The first line starts with the +// code and a hyphen. The response is terminated by a line that starts +// with the same code followed by a space. Each line in message is +// separated by a newline (\n). +// +// See page 36 of RFC 959 (http://www.ietf.org/rfc/rfc959.txt) for +// details. +// +// If the prefix of the status does not match the digits in expectCode, +// ReadResponse returns with err set to &Error{code, message}. +// For example, if expectCode is 31, an error will be returned if +// the status is not in the range [310,319]. +// +// An expectCode <= 0 disables the check of the status code. +// +func (r *Reader) ReadResponse(expectCode int) (code int, message string, err error) { + code, continued, message, err := r.readCodeLine(expectCode) + for err == nil && continued { + line, err := r.ReadLine() + if err != nil { + return 0, "", err + } + + var code2 int + var moreMessage string + code2, continued, moreMessage, err = parseCodeLine(line, expectCode) + if err != nil || code2 != code { + message += "\n" + strings.TrimRight(line, "\r\n") + continued = true + continue + } + message += "\n" + moreMessage + } + return +} + +// DotReader returns a new Reader that satisfies Reads using the +// decoded text of a dot-encoded block read from r. +// The returned Reader is only valid until the next call +// to a method on r. +// +// Dot encoding is a common framing used for data blocks +// in text protocols such as SMTP. The data consists of a sequence +// of lines, each of which ends in "\r\n". The sequence itself +// ends at a line containing just a dot: ".\r\n". Lines beginning +// with a dot are escaped with an additional dot to avoid +// looking like the end of the sequence. +// +// The decoded form returned by the Reader's Read method +// rewrites the "\r\n" line endings into the simpler "\n", +// removes leading dot escapes if present, and stops with error io.EOF +// after consuming (and discarding) the end-of-sequence line. +func (r *Reader) DotReader() io.Reader { + r.closeDot() + r.dot = &dotReader{r: r} + return r.dot +} + +type dotReader struct { + r *Reader + state int +} + +// Read satisfies reads by decoding dot-encoded data read from d.r. +func (d *dotReader) Read(b []byte) (n int, err error) { + // Run data through a simple state machine to + // elide leading dots, rewrite trailing \r\n into \n, + // and detect ending .\r\n line. + const ( + stateBeginLine = iota // beginning of line; initial state; must be zero + stateDot // read . at beginning of line + stateDotCR // read .\r at beginning of line + stateCR // read \r (possibly at end of line) + stateData // reading data in middle of line + stateEOF // reached .\r\n end marker line + ) + br := d.r.R + for n < len(b) && d.state != stateEOF { + var c byte + c, err = br.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + switch d.state { + case stateBeginLine: + if c == '.' { + d.state = stateDot + continue + } + if c == '\r' { + d.state = stateCR + continue + } + d.state = stateData + + case stateDot: + if c == '\r' { + d.state = stateDotCR + continue + } + if c == '\n' { + d.state = stateEOF + continue + } + d.state = stateData + + case stateDotCR: + if c == '\n' { + d.state = stateEOF + continue + } + // Not part of .\r\n. + // Consume leading dot and emit saved \r. + br.UnreadByte() + c = '\r' + d.state = stateData + + case stateCR: + if c == '\n' { + d.state = stateBeginLine + break + } + // Not part of \r\n. Emit saved \r + br.UnreadByte() + c = '\r' + d.state = stateData + + case stateData: + if c == '\r' { + d.state = stateCR + continue + } + if c == '\n' { + d.state = stateBeginLine + } + } + b[n] = c + n++ + } + if err == nil && d.state == stateEOF { + err = io.EOF + } + if err != nil && d.r.dot == d { + d.r.dot = nil + } + return +} + +// closeDot drains the current DotReader if any, +// making sure that it reads until the ending dot line. +func (r *Reader) closeDot() { + if r.dot == nil { + return + } + buf := make([]byte, 128) + for r.dot != nil { + // When Read reaches EOF or an error, + // it will set r.dot == nil. + r.dot.Read(buf) + } +} + +// ReadDotBytes reads a dot-encoding and returns the decoded data. +// +// See the documentation for the DotReader method for details about dot-encoding. +func (r *Reader) ReadDotBytes() ([]byte, error) { + return ioutil.ReadAll(r.DotReader()) +} + +// ReadDotLines reads a dot-encoding and returns a slice +// containing the decoded lines, with the final \r\n or \n elided from each. +// +// See the documentation for the DotReader method for details about dot-encoding. +func (r *Reader) ReadDotLines() ([]string, error) { + // We could use ReadDotBytes and then Split it, + // but reading a line at a time avoids needing a + // large contiguous block of memory and is simpler. + var v []string + var err error + for { + var line string + line, err = r.ReadLine() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + + // Dot by itself marks end; otherwise cut one dot. + if len(line) > 0 && line[0] == '.' { + if len(line) == 1 { + break + } + line = line[1:] + } + v = append(v, line) + } + return v, err +} + +// ReadMIMEHeader reads a MIME-style header from r. +// The header is a sequence of possibly continued Key: Value lines +// ending in a blank line. +// The returned map m maps CanonicalMIMEHeaderKey(key) to a +// sequence of values in the same order encountered in the input. +// +// For example, consider this input: +// +// My-Key: Value 1 +// Long-Key: Even +// Longer Value +// My-Key: Value 2 +// +// Given that input, ReadMIMEHeader returns the map: +// +// map[string][]string{ +// "My-Key": {"Value 1", "Value 2"}, +// "Long-Key": {"Even Longer Value"}, +// } +// +func (r *Reader) ReadMIMEHeader() (MIMEHeader, error) { + // Avoid lots of small slice allocations later by allocating one + // large one ahead of time which we'll cut up into smaller + // slices. If this isn't big enough later, we allocate small ones. + var strs []string + hint := r.upcomingHeaderNewlines() + if hint > 0 { + strs = make([]string, hint) + } + + m := make(MIMEHeader, hint) + for { + kv, err := r.readContinuedLineSlice() + if len(kv) == 0 { + return m, err + } + + // Key ends at first colon; should not have spaces but + // they appear in the wild, violating specs, so we + // remove them if present. + i := bytes.IndexByte(kv, ':') + if i < 0 { + return m, ProtocolError("malformed MIME header line: " + string(kv)) + } + endKey := i + for endKey > 0 && kv[endKey-1] == ' ' { + endKey-- + } + key := canonicalMIMEHeaderKey(kv[:endKey]) + + // Skip initial spaces in value. + i++ // skip colon + for i < len(kv) && (kv[i] == ' ' || kv[i] == '\t') { + i++ + } + value := string(kv[i:]) + + vv := m[key] + if vv == nil && len(strs) > 0 { + // More than likely this will be a single-element key. + // Most headers aren't multi-valued. + // Set the capacity on strs[0] to 1, so any future append + // won't extend the slice into the other strings. + vv, strs = strs[:1:1], strs[1:] + vv[0] = value + m[key] = vv + } else { + m[key] = append(vv, value) + } + + if err != nil { + return m, err + } + } +} + +// upcomingHeaderNewlines returns an approximation of the number of newlines +// that will be in this header. If it gets confused, it returns 0. +func (r *Reader) upcomingHeaderNewlines() (n int) { + // Try to determine the 'hint' size. + r.R.Peek(1) // force a buffer load if empty + s := r.R.Buffered() + if s == 0 { + return + } + peek, _ := r.R.Peek(s) + for len(peek) > 0 { + i := bytes.IndexByte(peek, '\n') + if i < 3 { + // Not present (-1) or found within the next few bytes, + // implying we're at the end ("\r\n\r\n" or "\n\n") + return + } + n++ + peek = peek[i+1:] + } + return +} + +// CanonicalMIMEHeaderKey returns the canonical format of the +// MIME header key s. The canonicalization converts the first +// letter and any letter following a hyphen to upper case; +// the rest are converted to lowercase. For example, the +// canonical key for "accept-encoding" is "Accept-Encoding". +// MIME header keys are assumed to be ASCII only. +func CanonicalMIMEHeaderKey(s string) string { + // Quick check for canonical encoding. + upper := true + for i := 0; i < len(s); i++ { + c := s[i] + if upper && 'a' <= c && c <= 'z' { + return canonicalMIMEHeaderKey([]byte(s)) + } + if !upper && 'A' <= c && c <= 'Z' { + return canonicalMIMEHeaderKey([]byte(s)) + } + upper = c == '-' + } + return s +} + +const toLower = 'a' - 'A' + +// canonicalMIMEHeaderKey is like CanonicalMIMEHeaderKey but is +// allowed to mutate the provided byte slice before returning the +// string. +func canonicalMIMEHeaderKey(a []byte) string { + upper := true + for i, c := range a { + // Canonicalize: first letter upper case + // and upper case after each dash. + // (Host, User-Agent, If-Modified-Since). + // MIME headers are ASCII only, so no Unicode issues. + if c == ' ' { + c = '-' + } else if upper && 'a' <= c && c <= 'z' { + c -= toLower + } else if !upper && 'A' <= c && c <= 'Z' { + c += toLower + } + a[i] = c + upper = c == '-' // for next time + } + // The compiler recognizes m[string(byteSlice)] as a special + // case, so a copy of a's bytes into a new string does not + // happen in this map lookup: + if v := commonHeader[string(a)]; v != "" { + return v + } + return string(a) +} + +// commonHeader interns common header strings. +var commonHeader = make(map[string]string) + +func init() { + for _, v := range []string{ + "Accept", + "Accept-Charset", + "Accept-Encoding", + "Accept-Language", + "Accept-Ranges", + "Cache-Control", + "Cc", + "Connection", + "Content-Id", + "Content-Language", + "Content-Length", + "Content-Transfer-Encoding", + "Content-Type", + "Cookie", + "Date", + "Dkim-Signature", + "Etag", + "Expires", + "From", + "Host", + "If-Modified-Since", + "If-None-Match", + "In-Reply-To", + "Last-Modified", + "Location", + "Message-Id", + "Mime-Version", + "Pragma", + "Received", + "Return-Path", + "Server", + "Set-Cookie", + "Subject", + "To", + "User-Agent", + "Via", + "X-Forwarded-For", + "X-Imforwards", + "X-Powered-By", + } { + commonHeader[v] = v + } +} diff --git a/src/net/textproto/reader_test.go b/src/net/textproto/reader_test.go new file mode 100644 index 000000000..cbc0ed183 --- /dev/null +++ b/src/net/textproto/reader_test.go @@ -0,0 +1,337 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import ( + "bufio" + "bytes" + "io" + "reflect" + "strings" + "testing" +) + +type canonicalHeaderKeyTest struct { + in, out string +} + +var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ + {"a-b-c", "A-B-C"}, + {"a-1-c", "A-1-C"}, + {"User-Agent", "User-Agent"}, + {"uSER-aGENT", "User-Agent"}, + {"user-agent", "User-Agent"}, + {"USER-AGENT", "User-Agent"}, + {"üser-agenT", "üser-Agent"}, // non-ASCII unchanged + + // This caused a panic due to mishandling of a space: + {"C Ontent-Transfer-Encoding", "C-Ontent-Transfer-Encoding"}, + {"foo bar", "Foo-Bar"}, +} + +func TestCanonicalMIMEHeaderKey(t *testing.T) { + for _, tt := range canonicalHeaderKeyTests { + if s := CanonicalMIMEHeaderKey(tt.in); s != tt.out { + t.Errorf("CanonicalMIMEHeaderKey(%q) = %q, want %q", tt.in, s, tt.out) + } + } +} + +func reader(s string) *Reader { + return NewReader(bufio.NewReader(strings.NewReader(s))) +} + +func TestReadLine(t *testing.T) { + r := reader("line1\nline2\n") + s, err := r.ReadLine() + if s != "line1" || err != nil { + t.Fatalf("Line 1: %s, %v", s, err) + } + s, err = r.ReadLine() + if s != "line2" || err != nil { + t.Fatalf("Line 2: %s, %v", s, err) + } + s, err = r.ReadLine() + if s != "" || err != io.EOF { + t.Fatalf("EOF: %s, %v", s, err) + } +} + +func TestReadContinuedLine(t *testing.T) { + r := reader("line1\nline\n 2\nline3\n") + s, err := r.ReadContinuedLine() + if s != "line1" || err != nil { + t.Fatalf("Line 1: %s, %v", s, err) + } + s, err = r.ReadContinuedLine() + if s != "line 2" || err != nil { + t.Fatalf("Line 2: %s, %v", s, err) + } + s, err = r.ReadContinuedLine() + if s != "line3" || err != nil { + t.Fatalf("Line 3: %s, %v", s, err) + } + s, err = r.ReadContinuedLine() + if s != "" || err != io.EOF { + t.Fatalf("EOF: %s, %v", s, err) + } +} + +func TestReadCodeLine(t *testing.T) { + r := reader("123 hi\n234 bye\n345 no way\n") + code, msg, err := r.ReadCodeLine(0) + if code != 123 || msg != "hi" || err != nil { + t.Fatalf("Line 1: %d, %s, %v", code, msg, err) + } + code, msg, err = r.ReadCodeLine(23) + if code != 234 || msg != "bye" || err != nil { + t.Fatalf("Line 2: %d, %s, %v", code, msg, err) + } + code, msg, err = r.ReadCodeLine(346) + if code != 345 || msg != "no way" || err == nil { + t.Fatalf("Line 3: %d, %s, %v", code, msg, err) + } + if e, ok := err.(*Error); !ok || e.Code != code || e.Msg != msg { + t.Fatalf("Line 3: wrong error %v\n", err) + } + code, msg, err = r.ReadCodeLine(1) + if code != 0 || msg != "" || err != io.EOF { + t.Fatalf("EOF: %d, %s, %v", code, msg, err) + } +} + +func TestReadDotLines(t *testing.T) { + r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanother\n") + s, err := r.ReadDotLines() + want := []string{"dotlines", "foo", ".bar", "..baz", "quux", ""} + if !reflect.DeepEqual(s, want) || err != nil { + t.Fatalf("ReadDotLines: %v, %v", s, err) + } + + s, err = r.ReadDotLines() + want = []string{"another"} + if !reflect.DeepEqual(s, want) || err != io.ErrUnexpectedEOF { + t.Fatalf("ReadDotLines2: %v, %v", s, err) + } +} + +func TestReadDotBytes(t *testing.T) { + r := reader("dotlines\r\n.foo\r\n..bar\n...baz\nquux\r\n\r\n.\r\nanot.her\r\n") + b, err := r.ReadDotBytes() + want := []byte("dotlines\nfoo\n.bar\n..baz\nquux\n\n") + if !reflect.DeepEqual(b, want) || err != nil { + t.Fatalf("ReadDotBytes: %q, %v", b, err) + } + + b, err = r.ReadDotBytes() + want = []byte("anot.her\n") + if !reflect.DeepEqual(b, want) || err != io.ErrUnexpectedEOF { + t.Fatalf("ReadDotBytes2: %q, %v", b, err) + } +} + +func TestReadMIMEHeader(t *testing.T) { + r := reader("my-key: Value 1 \r\nLong-key: Even \n Longer Value\r\nmy-Key: Value 2\r\n\n") + m, err := r.ReadMIMEHeader() + want := MIMEHeader{ + "My-Key": {"Value 1", "Value 2"}, + "Long-Key": {"Even Longer Value"}, + } + if !reflect.DeepEqual(m, want) || err != nil { + t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want) + } +} + +func TestReadMIMEHeaderSingle(t *testing.T) { + r := reader("Foo: bar\n\n") + m, err := r.ReadMIMEHeader() + want := MIMEHeader{"Foo": {"bar"}} + if !reflect.DeepEqual(m, want) || err != nil { + t.Fatalf("ReadMIMEHeader: %v, %v; want %v", m, err, want) + } +} + +func TestLargeReadMIMEHeader(t *testing.T) { + data := make([]byte, 16*1024) + for i := 0; i < len(data); i++ { + data[i] = 'x' + } + sdata := string(data) + r := reader("Cookie: " + sdata + "\r\n\n") + m, err := r.ReadMIMEHeader() + if err != nil { + t.Fatalf("ReadMIMEHeader: %v", err) + } + cookie := m.Get("Cookie") + if cookie != sdata { + t.Fatalf("ReadMIMEHeader: %v bytes, want %v bytes", len(cookie), len(sdata)) + } +} + +// Test that we read slightly-bogus MIME headers seen in the wild, +// with spaces before colons, and spaces in keys. +func TestReadMIMEHeaderNonCompliant(t *testing.T) { + // Invalid HTTP response header as sent by an Axis security + // camera: (this is handled by IE, Firefox, Chrome, curl, etc.) + r := reader("Foo: bar\r\n" + + "Content-Language: en\r\n" + + "SID : 0\r\n" + + "Audio Mode : None\r\n" + + "Privilege : 127\r\n\r\n") + m, err := r.ReadMIMEHeader() + want := MIMEHeader{ + "Foo": {"bar"}, + "Content-Language": {"en"}, + "Sid": {"0"}, + "Audio-Mode": {"None"}, + "Privilege": {"127"}, + } + if !reflect.DeepEqual(m, want) || err != nil { + t.Fatalf("ReadMIMEHeader =\n%v, %v; want:\n%v", m, err, want) + } +} + +type readResponseTest struct { + in string + inCode int + wantCode int + wantMsg string +} + +var readResponseTests = []readResponseTest{ + {"230-Anonymous access granted, restrictions apply\n" + + "Read the file README.txt,\n" + + "230 please", + 23, + 230, + "Anonymous access granted, restrictions apply\nRead the file README.txt,\n please", + }, + + {"230 Anonymous access granted, restrictions apply\n", + 23, + 230, + "Anonymous access granted, restrictions apply", + }, + + {"400-A\n400-B\n400 C", + 4, + 400, + "A\nB\nC", + }, + + {"400-A\r\n400-B\r\n400 C\r\n", + 4, + 400, + "A\nB\nC", + }, +} + +// See http://www.ietf.org/rfc/rfc959.txt page 36. +func TestRFC959Lines(t *testing.T) { + for i, tt := range readResponseTests { + r := reader(tt.in + "\nFOLLOWING DATA") + code, msg, err := r.ReadResponse(tt.inCode) + if err != nil { + t.Errorf("#%d: ReadResponse: %v", i, err) + continue + } + if code != tt.wantCode { + t.Errorf("#%d: code=%d, want %d", i, code, tt.wantCode) + } + if msg != tt.wantMsg { + t.Errorf("#%d: msg=%q, want %q", i, msg, tt.wantMsg) + } + } +} + +func TestCommonHeaders(t *testing.T) { + for h := range commonHeader { + if h != CanonicalMIMEHeaderKey(h) { + t.Errorf("Non-canonical header %q in commonHeader", h) + } + } + b := []byte("content-Length") + want := "Content-Length" + n := testing.AllocsPerRun(200, func() { + if x := canonicalMIMEHeaderKey(b); x != want { + t.Fatalf("canonicalMIMEHeaderKey(%q) = %q; want %q", b, x, want) + } + }) + if n > 0 { + t.Errorf("canonicalMIMEHeaderKey allocs = %v; want 0", n) + } +} + +var clientHeaders = strings.Replace(`Host: golang.org +Connection: keep-alive +Cache-Control: max-age=0 +Accept: application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5 +User-Agent: Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.63 Safari/534.3 +Accept-Encoding: gzip,deflate,sdch +Accept-Language: en-US,en;q=0.8,fr-CH;q=0.6 +Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3 +COOKIE: __utma=000000000.0000000000.0000000000.0000000000.0000000000.00; __utmb=000000000.0.00.0000000000; __utmc=000000000; __utmz=000000000.0000000000.00.0.utmcsr=code.google.com|utmccn=(referral)|utmcmd=referral|utmcct=/p/go/issues/detail +Non-Interned: test + +`, "\n", "\r\n", -1) + +var serverHeaders = strings.Replace(`Content-Type: text/html; charset=utf-8 +Content-Encoding: gzip +Date: Thu, 27 Sep 2012 09:03:33 GMT +Server: Google Frontend +Cache-Control: private +Content-Length: 2298 +VIA: 1.1 proxy.example.com:80 (XXX/n.n.n-nnn) +Connection: Close +Non-Interned: test + +`, "\n", "\r\n", -1) + +func BenchmarkReadMIMEHeader(b *testing.B) { + b.ReportAllocs() + var buf bytes.Buffer + br := bufio.NewReader(&buf) + r := NewReader(br) + for i := 0; i < b.N; i++ { + var want int + var find string + if (i & 1) == 1 { + buf.WriteString(clientHeaders) + want = 10 + find = "Cookie" + } else { + buf.WriteString(serverHeaders) + want = 9 + find = "Via" + } + h, err := r.ReadMIMEHeader() + if err != nil { + b.Fatal(err) + } + if len(h) != want { + b.Fatalf("wrong number of headers: got %d, want %d", len(h), want) + } + if _, ok := h[find]; !ok { + b.Fatalf("did not find key %s", find) + } + } +} + +func BenchmarkUncommon(b *testing.B) { + b.ReportAllocs() + var buf bytes.Buffer + br := bufio.NewReader(&buf) + r := NewReader(br) + for i := 0; i < b.N; i++ { + buf.WriteString("uncommon-header-for-benchmark: foo\r\n\r\n") + h, err := r.ReadMIMEHeader() + if err != nil { + b.Fatal(err) + } + if _, ok := h["Uncommon-Header-For-Benchmark"]; !ok { + b.Fatal("Missing result header.") + } + } +} diff --git a/src/net/textproto/textproto.go b/src/net/textproto/textproto.go new file mode 100644 index 000000000..026eb026b --- /dev/null +++ b/src/net/textproto/textproto.go @@ -0,0 +1,154 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package textproto implements generic support for text-based request/response +// protocols in the style of HTTP, NNTP, and SMTP. +// +// The package provides: +// +// Error, which represents a numeric error response from +// a server. +// +// Pipeline, to manage pipelined requests and responses +// in a client. +// +// Reader, to read numeric response code lines, +// key: value headers, lines wrapped with leading spaces +// on continuation lines, and whole text blocks ending +// with a dot on a line by itself. +// +// Writer, to write dot-encoded text blocks. +// +// Conn, a convenient packaging of Reader, Writer, and Pipeline for use +// with a single network connection. +// +package textproto + +import ( + "bufio" + "fmt" + "io" + "net" +) + +// An Error represents a numeric error response from a server. +type Error struct { + Code int + Msg string +} + +func (e *Error) Error() string { + return fmt.Sprintf("%03d %s", e.Code, e.Msg) +} + +// A ProtocolError describes a protocol violation such +// as an invalid response or a hung-up connection. +type ProtocolError string + +func (p ProtocolError) Error() string { + return string(p) +} + +// A Conn represents a textual network protocol connection. +// It consists of a Reader and Writer to manage I/O +// and a Pipeline to sequence concurrent requests on the connection. +// These embedded types carry methods with them; +// see the documentation of those types for details. +type Conn struct { + Reader + Writer + Pipeline + conn io.ReadWriteCloser +} + +// NewConn returns a new Conn using conn for I/O. +func NewConn(conn io.ReadWriteCloser) *Conn { + return &Conn{ + Reader: Reader{R: bufio.NewReader(conn)}, + Writer: Writer{W: bufio.NewWriter(conn)}, + conn: conn, + } +} + +// Close closes the connection. +func (c *Conn) Close() error { + return c.conn.Close() +} + +// Dial connects to the given address on the given network using net.Dial +// and then returns a new Conn for the connection. +func Dial(network, addr string) (*Conn, error) { + c, err := net.Dial(network, addr) + if err != nil { + return nil, err + } + return NewConn(c), nil +} + +// Cmd is a convenience method that sends a command after +// waiting its turn in the pipeline. The command text is the +// result of formatting format with args and appending \r\n. +// Cmd returns the id of the command, for use with StartResponse and EndResponse. +// +// For example, a client might run a HELP command that returns a dot-body +// by using: +// +// id, err := c.Cmd("HELP") +// if err != nil { +// return nil, err +// } +// +// c.StartResponse(id) +// defer c.EndResponse(id) +// +// if _, _, err = c.ReadCodeLine(110); err != nil { +// return nil, err +// } +// text, err := c.ReadDotBytes() +// if err != nil { +// return nil, err +// } +// return c.ReadCodeLine(250) +// +func (c *Conn) Cmd(format string, args ...interface{}) (id uint, err error) { + id = c.Next() + c.StartRequest(id) + err = c.PrintfLine(format, args...) + c.EndRequest(id) + if err != nil { + return 0, err + } + return id, nil +} + +// TrimString returns s without leading and trailing ASCII space. +func TrimString(s string) string { + for len(s) > 0 && isASCIISpace(s[0]) { + s = s[1:] + } + for len(s) > 0 && isASCIISpace(s[len(s)-1]) { + s = s[:len(s)-1] + } + return s +} + +// TrimBytes returns b without leading and trailing ASCII space. +func TrimBytes(b []byte) []byte { + for len(b) > 0 && isASCIISpace(b[0]) { + b = b[1:] + } + for len(b) > 0 && isASCIISpace(b[len(b)-1]) { + b = b[:len(b)-1] + } + return b +} + +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +func isASCIILetter(b byte) bool { + b |= 0x20 // make lower case + return 'a' <= b && b <= 'z' +} diff --git a/src/net/textproto/writer.go b/src/net/textproto/writer.go new file mode 100644 index 000000000..03e2fd658 --- /dev/null +++ b/src/net/textproto/writer.go @@ -0,0 +1,118 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import ( + "bufio" + "fmt" + "io" +) + +// A Writer implements convenience methods for writing +// requests or responses to a text protocol network connection. +type Writer struct { + W *bufio.Writer + dot *dotWriter +} + +// NewWriter returns a new Writer writing to w. +func NewWriter(w *bufio.Writer) *Writer { + return &Writer{W: w} +} + +var crnl = []byte{'\r', '\n'} +var dotcrnl = []byte{'.', '\r', '\n'} + +// PrintfLine writes the formatted output followed by \r\n. +func (w *Writer) PrintfLine(format string, args ...interface{}) error { + w.closeDot() + fmt.Fprintf(w.W, format, args...) + w.W.Write(crnl) + return w.W.Flush() +} + +// DotWriter returns a writer that can be used to write a dot-encoding to w. +// It takes care of inserting leading dots when necessary, +// translating line-ending \n into \r\n, and adding the final .\r\n line +// when the DotWriter is closed. The caller should close the +// DotWriter before the next call to a method on w. +// +// See the documentation for Reader's DotReader method for details about dot-encoding. +func (w *Writer) DotWriter() io.WriteCloser { + w.closeDot() + w.dot = &dotWriter{w: w} + return w.dot +} + +func (w *Writer) closeDot() { + if w.dot != nil { + w.dot.Close() // sets w.dot = nil + } +} + +type dotWriter struct { + w *Writer + state int +} + +const ( + wstateBeginLine = iota // beginning of line; initial state; must be zero + wstateCR // wrote \r (possibly at end of line) + wstateData // writing data in middle of line +) + +func (d *dotWriter) Write(b []byte) (n int, err error) { + bw := d.w.W + for n < len(b) { + c := b[n] + switch d.state { + case wstateBeginLine: + d.state = wstateData + if c == '.' { + // escape leading dot + bw.WriteByte('.') + } + fallthrough + + case wstateData: + if c == '\r' { + d.state = wstateCR + } + if c == '\n' { + bw.WriteByte('\r') + d.state = wstateBeginLine + } + + case wstateCR: + d.state = wstateData + if c == '\n' { + d.state = wstateBeginLine + } + } + if err = bw.WriteByte(c); err != nil { + break + } + n++ + } + return +} + +func (d *dotWriter) Close() error { + if d.w.dot == d { + d.w.dot = nil + } + bw := d.w.W + switch d.state { + default: + bw.WriteByte('\r') + fallthrough + case wstateCR: + bw.WriteByte('\n') + fallthrough + case wstateBeginLine: + bw.Write(dotcrnl) + } + return bw.Flush() +} diff --git a/src/net/textproto/writer_test.go b/src/net/textproto/writer_test.go new file mode 100644 index 000000000..e03ab5e15 --- /dev/null +++ b/src/net/textproto/writer_test.go @@ -0,0 +1,35 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package textproto + +import ( + "bufio" + "bytes" + "testing" +) + +func TestPrintfLine(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(bufio.NewWriter(&buf)) + err := w.PrintfLine("foo %d", 123) + if s := buf.String(); s != "foo 123\r\n" || err != nil { + t.Fatalf("s=%q; err=%s", s, err) + } +} + +func TestDotWriter(t *testing.T) { + var buf bytes.Buffer + w := NewWriter(bufio.NewWriter(&buf)) + d := w.DotWriter() + n, err := d.Write([]byte("abc\n.def\n..ghi\n.jkl\n.")) + if n != 21 || err != nil { + t.Fatalf("Write: %d, %s", n, err) + } + d.Close() + want := "abc\r\n..def\r\n...ghi\r\n..jkl\r\n..\r\n.\r\n" + if s := buf.String(); s != want { + t.Fatalf("wrote %q", s) + } +} diff --git a/src/net/timeout_test.go b/src/net/timeout_test.go new file mode 100644 index 000000000..9ef0c4d15 --- /dev/null +++ b/src/net/timeout_test.go @@ -0,0 +1,747 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "fmt" + "io" + "io/ioutil" + "runtime" + "testing" + "time" +) + +func isTimeout(err error) bool { + e, ok := err.(Error) + return ok && e.Timeout() +} + +type copyRes struct { + n int64 + err error + d time.Duration +} + +func TestAcceptTimeout(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t).(*TCPListener) + defer ln.Close() + ln.SetDeadline(time.Now().Add(-1 * time.Second)) + if _, err := ln.Accept(); !isTimeout(err) { + t.Fatalf("Accept: expected err %v, got %v", errTimeout, err) + } + if _, err := ln.Accept(); !isTimeout(err) { + t.Fatalf("Accept: expected err %v, got %v", errTimeout, err) + } + ln.SetDeadline(time.Now().Add(100 * time.Millisecond)) + if _, err := ln.Accept(); !isTimeout(err) { + t.Fatalf("Accept: expected err %v, got %v", errTimeout, err) + } + if _, err := ln.Accept(); !isTimeout(err) { + t.Fatalf("Accept: expected err %v, got %v", errTimeout, err) + } + ln.SetDeadline(noDeadline) + errc := make(chan error) + go func() { + _, err := ln.Accept() + errc <- err + }() + time.Sleep(100 * time.Millisecond) + select { + case err := <-errc: + t.Fatalf("Expected Accept() to not return, but it returned with %v\n", err) + default: + } + ln.Close() + switch nerr := <-errc; err := nerr.(type) { + case *OpError: + if err.Err != errClosing { + t.Fatalf("Accept: expected err %v, got %v", errClosing, err) + } + default: + if err != errClosing { + t.Fatalf("Accept: expected err %v, got %v", errClosing, err) + } + } +} + +func TestReadTimeout(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t) + defer ln.Close() + c, err := DialTCP("tcp", nil, ln.Addr().(*TCPAddr)) + if err != nil { + t.Fatalf("Connect: %v", err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(time.Hour)) + c.SetReadDeadline(time.Now().Add(-1 * time.Second)) + buf := make([]byte, 1) + if _, err = c.Read(buf); !isTimeout(err) { + t.Fatalf("Read: expected err %v, got %v", errTimeout, err) + } + if _, err = c.Read(buf); !isTimeout(err) { + t.Fatalf("Read: expected err %v, got %v", errTimeout, err) + } + c.SetDeadline(time.Now().Add(100 * time.Millisecond)) + if _, err = c.Read(buf); !isTimeout(err) { + t.Fatalf("Read: expected err %v, got %v", errTimeout, err) + } + if _, err = c.Read(buf); !isTimeout(err) { + t.Fatalf("Read: expected err %v, got %v", errTimeout, err) + } + c.SetReadDeadline(noDeadline) + c.SetWriteDeadline(time.Now().Add(-1 * time.Second)) + errc := make(chan error) + go func() { + _, err := c.Read(buf) + errc <- err + }() + time.Sleep(100 * time.Millisecond) + select { + case err := <-errc: + t.Fatalf("Expected Read() to not return, but it returned with %v\n", err) + default: + } + c.Close() + switch nerr := <-errc; err := nerr.(type) { + case *OpError: + if err.Err != errClosing { + t.Fatalf("Read: expected err %v, got %v", errClosing, err) + } + default: + if err == io.EOF && runtime.GOOS == "nacl" { // close enough; golang.org/issue/8044 + break + } + if err != errClosing { + t.Fatalf("Read: expected err %v, got %v", errClosing, err) + } + } +} + +func TestWriteTimeout(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t) + defer ln.Close() + c, err := DialTCP("tcp", nil, ln.Addr().(*TCPAddr)) + if err != nil { + t.Fatalf("Connect: %v", err) + } + defer c.Close() + c.SetDeadline(time.Now().Add(time.Hour)) + c.SetWriteDeadline(time.Now().Add(-1 * time.Second)) + buf := make([]byte, 4096) + writeUntilTimeout := func() { + for { + _, err := c.Write(buf) + if err != nil { + if isTimeout(err) { + return + } + t.Fatalf("Write: expected err %v, got %v", errTimeout, err) + } + } + } + writeUntilTimeout() + c.SetDeadline(time.Now().Add(10 * time.Millisecond)) + writeUntilTimeout() + writeUntilTimeout() + c.SetWriteDeadline(noDeadline) + c.SetReadDeadline(time.Now().Add(-1 * time.Second)) + errc := make(chan error) + go func() { + for { + _, err := c.Write(buf) + if err != nil { + errc <- err + } + } + }() + time.Sleep(100 * time.Millisecond) + select { + case err := <-errc: + t.Fatalf("Expected Write() to not return, but it returned with %v\n", err) + default: + } + c.Close() + switch nerr := <-errc; err := nerr.(type) { + case *OpError: + if err.Err != errClosing { + t.Fatalf("Write: expected err %v, got %v", errClosing, err) + } + default: + if err != errClosing { + t.Fatalf("Write: expected err %v, got %v", errClosing, err) + } + } +} + +func testTimeout(t *testing.T, net, addr string, readFrom bool) { + c, err := Dial(net, addr) + if err != nil { + t.Errorf("Dial(%q, %q) failed: %v", net, addr, err) + return + } + defer c.Close() + what := "Read" + if readFrom { + what = "ReadFrom" + } + + errc := make(chan error, 1) + go func() { + t0 := time.Now() + c.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + var b [100]byte + var n int + var err error + if readFrom { + n, _, err = c.(PacketConn).ReadFrom(b[0:]) + } else { + n, err = c.Read(b[0:]) + } + t1 := time.Now() + if n != 0 || err == nil || !err.(Error).Timeout() { + errc <- fmt.Errorf("%s(%q, %q) did not return 0, timeout: %v, %v", what, net, addr, n, err) + return + } + if dt := t1.Sub(t0); dt < 50*time.Millisecond || !testing.Short() && dt > 250*time.Millisecond { + errc <- fmt.Errorf("%s(%q, %q) took %s, expected 0.1s", what, net, addr, dt) + return + } + errc <- nil + }() + select { + case err := <-errc: + if err != nil { + t.Error(err) + } + case <-time.After(1 * time.Second): + t.Errorf("%s(%q, %q) took over 1 second, expected 0.1s", what, net, addr) + } +} + +func TestTimeoutUDP(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + // set up a listener that won't talk back + listening := make(chan string) + done := make(chan int) + go runDatagramPacketConnServer(t, "udp", "127.0.0.1:0", listening, done) + addr := <-listening + + testTimeout(t, "udp", addr, false) + testTimeout(t, "udp", addr, true) + <-done +} + +func TestTimeoutTCP(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + // set up a listener that won't talk back + listening := make(chan string) + done := make(chan int) + go runStreamConnServer(t, "tcp", "127.0.0.1:0", listening, done) + addr := <-listening + + testTimeout(t, "tcp", addr, false) + <-done +} + +func TestDeadlineReset(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + tl := ln.(*TCPListener) + tl.SetDeadline(time.Now().Add(1 * time.Minute)) + tl.SetDeadline(noDeadline) // reset it + errc := make(chan error, 1) + go func() { + _, err := ln.Accept() + errc <- err + }() + select { + case <-time.After(50 * time.Millisecond): + // Pass. + case err := <-errc: + // Accept should never return; we never + // connected to it. + t.Errorf("unexpected return from Accept; err=%v", err) + } +} + +func TestTimeoutAccept(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + tl := ln.(*TCPListener) + tl.SetDeadline(time.Now().Add(100 * time.Millisecond)) + errc := make(chan error, 1) + go func() { + _, err := ln.Accept() + errc <- err + }() + select { + case <-time.After(1 * time.Second): + // Accept shouldn't block indefinitely + t.Errorf("Accept didn't return in an expected time") + case <-errc: + // Pass. + } +} + +func TestReadWriteDeadline(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + const ( + readTimeout = 50 * time.Millisecond + writeTimeout = 250 * time.Millisecond + ) + checkTimeout := func(command string, start time.Time, should time.Duration) { + is := time.Now().Sub(start) + d := is - should + if d < -30*time.Millisecond || !testing.Short() && 150*time.Millisecond < d { + t.Errorf("%s timeout test failed: is=%v should=%v\n", command, is, should) + } + } + + ln, err := Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("ListenTCP on :0: %v", err) + } + defer ln.Close() + + lnquit := make(chan bool) + + go func() { + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + defer c.Close() + lnquit <- true + }() + + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + + start := time.Now() + err = c.SetReadDeadline(start.Add(readTimeout)) + if err != nil { + t.Fatalf("SetReadDeadline: %v", err) + } + err = c.SetWriteDeadline(start.Add(writeTimeout)) + if err != nil { + t.Fatalf("SetWriteDeadline: %v", err) + } + + quit := make(chan bool) + + go func() { + var buf [10]byte + _, err := c.Read(buf[:]) + if err == nil { + t.Errorf("Read should not succeed") + } + checkTimeout("Read", start, readTimeout) + quit <- true + }() + + go func() { + var buf [10000]byte + for { + _, err := c.Write(buf[:]) + if err != nil { + break + } + } + checkTimeout("Write", start, writeTimeout) + quit <- true + }() + + <-quit + <-quit + <-lnquit +} + +type neverEnding byte + +func (b neverEnding) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = byte(b) + } + return len(p), nil +} + +func TestVariousDeadlines1Proc(t *testing.T) { + testVariousDeadlines(t, 1) +} + +func TestVariousDeadlines4Proc(t *testing.T) { + testVariousDeadlines(t, 4) +} + +func testVariousDeadlines(t *testing.T, maxProcs int) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(maxProcs)) + ln := newLocalListener(t) + defer ln.Close() + acceptc := make(chan error, 1) + + // The server, with no timeouts of its own, sending bytes to clients + // as fast as it can. + servec := make(chan copyRes) + go func() { + for { + c, err := ln.Accept() + if err != nil { + acceptc <- err + return + } + go func() { + t0 := time.Now() + n, err := io.Copy(c, neverEnding('a')) + d := time.Since(t0) + c.Close() + servec <- copyRes{n, err, d} + }() + } + }() + + for _, timeout := range []time.Duration{ + 1 * time.Nanosecond, + 2 * time.Nanosecond, + 5 * time.Nanosecond, + 50 * time.Nanosecond, + 100 * time.Nanosecond, + 200 * time.Nanosecond, + 500 * time.Nanosecond, + 750 * time.Nanosecond, + 1 * time.Microsecond, + 5 * time.Microsecond, + 25 * time.Microsecond, + 250 * time.Microsecond, + 500 * time.Microsecond, + 1 * time.Millisecond, + 5 * time.Millisecond, + 100 * time.Millisecond, + 250 * time.Millisecond, + 500 * time.Millisecond, + 1 * time.Second, + } { + numRuns := 3 + if testing.Short() { + numRuns = 1 + if timeout > 500*time.Microsecond { + continue + } + } + for run := 0; run < numRuns; run++ { + name := fmt.Sprintf("%v run %d/%d", timeout, run+1, numRuns) + t.Log(name) + + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + clientc := make(chan copyRes) + go func() { + t0 := time.Now() + c.SetDeadline(t0.Add(timeout)) + n, err := io.Copy(ioutil.Discard, c) + d := time.Since(t0) + c.Close() + clientc <- copyRes{n, err, d} + }() + + tooLong := 5 * time.Second + select { + case res := <-clientc: + if isTimeout(res.err) { + t.Logf("for %v, good client timeout after %v, reading %d bytes", name, res.d, res.n) + } else { + t.Fatalf("for %v: client Copy = %d, %v (want timeout)", name, res.n, res.err) + } + case <-time.After(tooLong): + t.Fatalf("for %v: timeout (%v) waiting for client to timeout (%v) reading", name, tooLong, timeout) + } + + select { + case res := <-servec: + t.Logf("for %v: server in %v wrote %d, %v", name, res.d, res.n, res.err) + case err := <-acceptc: + t.Fatalf("for %v: server Accept = %v", name, err) + case <-time.After(tooLong): + t.Fatalf("for %v, timeout waiting for server to finish writing", name) + } + } + } +} + +// TestReadDeadlineDataAvailable tests that read deadlines work, even +// if there's data ready to be read. +func TestReadDeadlineDataAvailable(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t) + defer ln.Close() + + servec := make(chan copyRes) + const msg = "data client shouldn't read, even though it'll be waiting" + go func() { + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + defer c.Close() + n, err := c.Write([]byte(msg)) + servec <- copyRes{n: int64(n), err: err} + }() + + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + if res := <-servec; res.err != nil || res.n != int64(len(msg)) { + t.Fatalf("unexpected server Write: n=%d, err=%v; want n=%d, err=nil", res.n, res.err, len(msg)) + } + c.SetReadDeadline(time.Now().Add(-5 * time.Second)) // in the psat. + buf := make([]byte, len(msg)/2) + n, err := c.Read(buf) + if n > 0 || !isTimeout(err) { + t.Fatalf("client read = %d (%q) err=%v; want 0, timeout", n, buf[:n], err) + } +} + +// TestWriteDeadlineBufferAvailable tests that write deadlines work, even +// if there's buffer space available to write. +func TestWriteDeadlineBufferAvailable(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t) + defer ln.Close() + + servec := make(chan copyRes) + go func() { + c, err := ln.Accept() + if err != nil { + t.Errorf("Accept: %v", err) + return + } + defer c.Close() + c.SetWriteDeadline(time.Now().Add(-5 * time.Second)) // in the past + n, err := c.Write([]byte{'x'}) + servec <- copyRes{n: int64(n), err: err} + }() + + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + res := <-servec + if res.n != 0 { + t.Errorf("Write = %d; want 0", res.n) + } + if !isTimeout(res.err) { + t.Errorf("Write error = %v; want timeout", res.err) + } +} + +// TestAcceptDeadlineConnectionAvailable tests that accept deadlines work, even +// if there's incoming connections available. +func TestAcceptDeadlineConnectionAvailable(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t).(*TCPListener) + defer ln.Close() + + go func() { + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Errorf("Dial: %v", err) + return + } + defer c.Close() + var buf [1]byte + c.Read(buf[:]) // block until the connection or listener is closed + }() + time.Sleep(10 * time.Millisecond) + ln.SetDeadline(time.Now().Add(-5 * time.Second)) // in the past + c, err := ln.Accept() + if err == nil { + defer c.Close() + } + if !isTimeout(err) { + t.Fatalf("Accept: got %v; want timeout", err) + } +} + +// TestConnectDeadlineInThePast tests that connect deadlines work, even +// if the connection can be established w/o blocking. +func TestConnectDeadlineInThePast(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t).(*TCPListener) + defer ln.Close() + + go func() { + c, err := ln.Accept() + if err == nil { + defer c.Close() + } + }() + time.Sleep(10 * time.Millisecond) + c, err := DialTimeout("tcp", ln.Addr().String(), -5*time.Second) // in the past + if err == nil { + defer c.Close() + } + if !isTimeout(err) { + t.Fatalf("DialTimeout: got %v; want timeout", err) + } +} + +// TestProlongTimeout tests concurrent deadline modification. +// Known to cause data races in the past. +func TestProlongTimeout(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + ln := newLocalListener(t) + defer ln.Close() + connected := make(chan bool) + go func() { + s, err := ln.Accept() + connected <- true + if err != nil { + t.Errorf("ln.Accept: %v", err) + return + } + defer s.Close() + s.SetDeadline(time.Now().Add(time.Hour)) + go func() { + var buf [4096]byte + for { + _, err := s.Write(buf[:]) + if err != nil { + break + } + s.SetDeadline(time.Now().Add(time.Hour)) + } + }() + buf := make([]byte, 1) + for { + _, err := s.Read(buf) + if err != nil { + break + } + s.SetDeadline(time.Now().Add(time.Hour)) + } + }() + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("DialTCP: %v", err) + } + defer c.Close() + <-connected + for i := 0; i < 1024; i++ { + var buf [1]byte + c.Write(buf[:]) + } +} + +func TestDeadlineRace(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + N := 1000 + if testing.Short() { + N = 50 + } + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(4)) + ln := newLocalListener(t) + defer ln.Close() + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + done := make(chan bool) + go func() { + t := time.NewTicker(2 * time.Microsecond).C + for i := 0; i < N; i++ { + if err := c.SetDeadline(time.Now().Add(2 * time.Microsecond)); err != nil { + break + } + <-t + } + done <- true + }() + var buf [1]byte + for i := 0; i < N; i++ { + c.Read(buf[:]) // ignore possible timeout errors + } + c.Close() + <-done +} diff --git a/src/net/udp_test.go b/src/net/udp_test.go new file mode 100644 index 000000000..e1778779c --- /dev/null +++ b/src/net/udp_test.go @@ -0,0 +1,257 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "reflect" + "runtime" + "strings" + "testing" +) + +func TestResolveUDPAddr(t *testing.T) { + for _, tt := range resolveTCPAddrTests { + net := strings.Replace(tt.net, "tcp", "udp", -1) + addr, err := ResolveUDPAddr(net, tt.litAddrOrName) + if err != tt.err { + t.Fatalf("ResolveUDPAddr(%q, %q) failed: %v", net, tt.litAddrOrName, err) + } + if !reflect.DeepEqual(addr, (*UDPAddr)(tt.addr)) { + t.Fatalf("ResolveUDPAddr(%q, %q) = %#v, want %#v", net, tt.litAddrOrName, addr, tt.addr) + } + if err == nil { + str := addr.String() + addr1, err := ResolveUDPAddr(net, str) + if err != nil { + t.Fatalf("ResolveUDPAddr(%q, %q) [from %q]: %v", net, str, tt.litAddrOrName, err) + } + if !reflect.DeepEqual(addr1, addr) { + t.Fatalf("ResolveUDPAddr(%q, %q) [from %q] = %#v, want %#v", net, str, tt.litAddrOrName, addr1, addr) + } + } + } +} + +func TestWriteToUDP(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + l, err := ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Listen failed: %v", err) + } + defer l.Close() + + testWriteToConn(t, l.LocalAddr().String()) + testWriteToPacketConn(t, l.LocalAddr().String()) +} + +func testWriteToConn(t *testing.T, raddr string) { + c, err := Dial("udp", raddr) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c.Close() + + ra, err := ResolveUDPAddr("udp", raddr) + if err != nil { + t.Fatalf("ResolveUDPAddr failed: %v", err) + } + + _, err = c.(*UDPConn).WriteToUDP([]byte("Connection-oriented mode socket"), ra) + if err == nil { + t.Fatal("WriteToUDP should fail") + } + if err != nil && err.(*OpError).Err != ErrWriteToConnected { + t.Fatalf("WriteToUDP should fail as ErrWriteToConnected: %v", err) + } + + _, err = c.(*UDPConn).WriteTo([]byte("Connection-oriented mode socket"), ra) + if err == nil { + t.Fatal("WriteTo should fail") + } + if err != nil && err.(*OpError).Err != ErrWriteToConnected { + t.Fatalf("WriteTo should fail as ErrWriteToConnected: %v", err) + } + + _, err = c.Write([]byte("Connection-oriented mode socket")) + if err != nil { + t.Fatalf("Write failed: %v", err) + } +} + +func testWriteToPacketConn(t *testing.T, raddr string) { + c, err := ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + defer c.Close() + + ra, err := ResolveUDPAddr("udp", raddr) + if err != nil { + t.Fatalf("ResolveUDPAddr failed: %v", err) + } + + _, err = c.(*UDPConn).WriteToUDP([]byte("Connection-less mode socket"), ra) + if err != nil { + t.Fatalf("WriteToUDP failed: %v", err) + } + + _, err = c.WriteTo([]byte("Connection-less mode socket"), ra) + if err != nil { + t.Fatalf("WriteTo failed: %v", err) + } + + _, err = c.(*UDPConn).Write([]byte("Connection-less mode socket")) + if err == nil { + t.Fatal("Write should fail") + } +} + +var udpConnLocalNameTests = []struct { + net string + laddr *UDPAddr +}{ + {"udp4", &UDPAddr{IP: IPv4(127, 0, 0, 1)}}, + {"udp4", &UDPAddr{}}, + {"udp4", nil}, +} + +func TestUDPConnLocalName(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + for _, tt := range udpConnLocalNameTests { + c, err := ListenUDP(tt.net, tt.laddr) + if err != nil { + t.Fatalf("ListenUDP failed: %v", err) + } + defer c.Close() + la := c.LocalAddr() + if a, ok := la.(*UDPAddr); !ok || a.Port == 0 { + t.Fatalf("got %v; expected a proper address with non-zero port number", la) + } + } +} + +func TestUDPConnLocalAndRemoteNames(t *testing.T) { + for _, laddr := range []string{"", "127.0.0.1:0"} { + c1, err := ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatalf("ListenUDP failed: %v", err) + } + defer c1.Close() + + var la *UDPAddr + if laddr != "" { + var err error + if la, err = ResolveUDPAddr("udp", laddr); err != nil { + t.Fatalf("ResolveUDPAddr failed: %v", err) + } + } + c2, err := DialUDP("udp", la, c1.LocalAddr().(*UDPAddr)) + if err != nil { + t.Fatalf("DialUDP failed: %v", err) + } + defer c2.Close() + + var connAddrs = [4]struct { + got Addr + ok bool + }{ + {c1.LocalAddr(), true}, + {c1.(*UDPConn).RemoteAddr(), false}, + {c2.LocalAddr(), true}, + {c2.RemoteAddr(), true}, + } + for _, ca := range connAddrs { + if a, ok := ca.got.(*UDPAddr); ok != ca.ok || ok && a.Port == 0 { + t.Fatalf("got %v; expected a proper address with non-zero port number", ca.got) + } + } + } +} + +func TestIPv6LinkLocalUnicastUDP(t *testing.T) { + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + if !supportsIPv6 { + t.Skip("ipv6 is not supported") + } + ifi := loopbackInterface() + if ifi == nil { + t.Skip("loopback interface not found") + } + laddr := ipv6LinkLocalUnicastAddr(ifi) + if laddr == "" { + t.Skip("ipv6 unicast address on loopback not found") + } + + type test struct { + net, addr string + nameLookup bool + } + var tests = []test{ + {"udp", "[" + laddr + "%" + ifi.Name + "]:0", false}, + {"udp6", "[" + laddr + "%" + ifi.Name + "]:0", false}, + } + // The first udp test fails on DragonFly - see issue 7473. + if runtime.GOOS == "dragonfly" { + tests = tests[1:] + } + switch runtime.GOOS { + case "darwin", "dragonfly", "freebsd", "openbsd", "netbsd": + tests = append(tests, []test{ + {"udp", "[localhost%" + ifi.Name + "]:0", true}, + {"udp6", "[localhost%" + ifi.Name + "]:0", true}, + }...) + case "linux": + tests = append(tests, []test{ + {"udp", "[ip6-localhost%" + ifi.Name + "]:0", true}, + {"udp6", "[ip6-localhost%" + ifi.Name + "]:0", true}, + }...) + } + for _, tt := range tests { + c1, err := ListenPacket(tt.net, tt.addr) + if err != nil { + // It might return "LookupHost returned no + // suitable address" error on some platforms. + t.Logf("ListenPacket failed: %v", err) + continue + } + defer c1.Close() + if la, ok := c1.LocalAddr().(*UDPAddr); !ok || !tt.nameLookup && la.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", la) + } + + c2, err := Dial(tt.net, c1.LocalAddr().String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c2.Close() + if la, ok := c2.LocalAddr().(*UDPAddr); !ok || !tt.nameLookup && la.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", la) + } + if ra, ok := c2.RemoteAddr().(*UDPAddr); !ok || !tt.nameLookup && ra.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", ra) + } + + if _, err := c2.Write([]byte("UDP OVER IPV6 LINKLOCAL TEST")); err != nil { + t.Fatalf("Conn.Write failed: %v", err) + } + b := make([]byte, 32) + if _, from, err := c1.ReadFrom(b); err != nil { + t.Fatalf("PacketConn.ReadFrom failed: %v", err) + } else { + if ra, ok := from.(*UDPAddr); !ok || !tt.nameLookup && ra.Zone == "" { + t.Fatalf("got %v; expected a proper address with zone identifier", ra) + } + } + } +} diff --git a/src/net/udpsock.go b/src/net/udpsock.go new file mode 100644 index 000000000..4c99ae4af --- /dev/null +++ b/src/net/udpsock.go @@ -0,0 +1,54 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// UDPAddr represents the address of a UDP end point. +type UDPAddr struct { + IP IP + Port int + Zone string // IPv6 scoped addressing zone +} + +// Network returns the address's network name, "udp". +func (a *UDPAddr) Network() string { return "udp" } + +func (a *UDPAddr) String() string { + if a == nil { + return "<nil>" + } + ip := ipEmptyString(a.IP) + if a.Zone != "" { + return JoinHostPort(ip+"%"+a.Zone, itoa(a.Port)) + } + return JoinHostPort(ip, itoa(a.Port)) +} + +func (a *UDPAddr) toAddr() Addr { + if a == nil { + return nil + } + return a +} + +// ResolveUDPAddr parses addr as a UDP address of the form "host:port" +// or "[ipv6-host%zone]:port" and resolves a pair of domain name and +// port name on the network net, which must be "udp", "udp4" or +// "udp6". A literal address or host name for IPv6 must be enclosed +// in square brackets, as in "[::1]:80", "[ipv6-host]:http" or +// "[ipv6-host%zone]:80". +func ResolveUDPAddr(net, addr string) (*UDPAddr, error) { + switch net { + case "udp", "udp4", "udp6": + case "": // a hint wildcard for Go 1.0 undocumented behavior + net = "udp" + default: + return nil, UnknownNetworkError(net) + } + a, err := resolveInternetAddr(net, addr, noDeadline) + if err != nil { + return nil, err + } + return a.toAddr().(*UDPAddr), nil +} diff --git a/src/net/udpsock_plan9.go b/src/net/udpsock_plan9.go new file mode 100644 index 000000000..510ac5e4a --- /dev/null +++ b/src/net/udpsock_plan9.go @@ -0,0 +1,203 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "os" + "syscall" + "time" +) + +// UDPConn is the implementation of the Conn and PacketConn interfaces +// for UDP network connections. +type UDPConn struct { + conn +} + +func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } + +// ReadFromUDP reads a UDP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { + if !c.ok() || c.fd.data == nil { + return 0, nil, syscall.EINVAL + } + buf := make([]byte, udpHeaderSize+len(b)) + m, err := c.fd.data.Read(buf) + if err != nil { + return + } + if m < udpHeaderSize { + return 0, nil, errors.New("short read reading UDP header") + } + buf = buf[:m] + + h, buf := unmarshalUDPHeader(buf) + n = copy(b, buf) + return n, &UDPAddr{IP: h.raddr, Port: int(h.rport)}, nil +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + return c.ReadFromUDP(b) +} + +// ReadMsgUDP reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number +// of bytes copied into b, the number of bytes copied into oob, the +// flags that were set on the packet and the source address of the +// packet. +func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 +} + +// WriteToUDP writes a UDP packet to addr via c, copying the payload +// from b. +// +// WriteToUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { + if !c.ok() || c.fd.data == nil { + return 0, syscall.EINVAL + } + if addr == nil { + return 0, &OpError{Op: "write", Net: c.fd.dir, Addr: nil, Err: errMissingAddress} + } + h := new(udpHeader) + h.raddr = addr.IP.To16() + h.laddr = c.fd.laddr.(*UDPAddr).IP.To16() + h.ifcaddr = IPv6zero // ignored (receive only) + h.rport = uint16(addr.Port) + h.lport = uint16(c.fd.laddr.(*UDPAddr).Port) + + buf := make([]byte, udpHeaderSize+len(b)) + i := copy(buf, h.Bytes()) + copy(buf[i:], b) + return c.fd.data.Write(buf) +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UDPAddr) + if !ok { + return 0, &OpError{"write", c.fd.dir, addr, syscall.EINVAL} + } + return c.WriteToUDP(b, a) +} + +// WriteMsgUDP writes a packet to addr via c, copying the payload from +// b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. +func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 +} + +// DialUDP connects to the remote address raddr on the network net, +// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + return dialUDP(net, laddr, raddr, noDeadline) +} + +func dialUDP(net string, laddr, raddr *UDPAddr, deadline time.Time) (*UDPConn, error) { + if !deadline.IsZero() { + panic("net.dialUDP: deadline not implemented on Plan 9") + } + switch net { + case "udp", "udp4", "udp6": + default: + return nil, UnknownNetworkError(net) + } + if raddr == nil { + return nil, &OpError{"dial", net, nil, errMissingAddress} + } + fd, err := dialPlan9(net, laddr, raddr) + if err != nil { + return nil, err + } + return newUDPConn(fd), nil +} + +const udpHeaderSize = 16*3 + 2*2 + +type udpHeader struct { + raddr, laddr, ifcaddr IP + rport, lport uint16 +} + +func (h *udpHeader) Bytes() []byte { + b := make([]byte, udpHeaderSize) + i := 0 + i += copy(b[i:i+16], h.raddr) + i += copy(b[i:i+16], h.laddr) + i += copy(b[i:i+16], h.ifcaddr) + b[i], b[i+1], i = byte(h.rport>>8), byte(h.rport), i+2 + b[i], b[i+1], i = byte(h.lport>>8), byte(h.lport), i+2 + return b +} + +func unmarshalUDPHeader(b []byte) (*udpHeader, []byte) { + h := new(udpHeader) + h.raddr, b = IP(b[:16]), b[16:] + h.laddr, b = IP(b[:16]), b[16:] + h.ifcaddr, b = IP(b[:16]), b[16:] + h.rport, b = uint16(b[0])<<8|uint16(b[1]), b[2:] + h.lport, b = uint16(b[0])<<8|uint16(b[1]), b[2:] + return h, b +} + +// ListenUDP listens for incoming UDP packets addressed to the local +// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has +// a port of 0, ListenUDP will choose an available port. +// The LocalAddr method of the returned UDPConn can be used to +// discover the port. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send UDP packets with per-packet +// addressing. +func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, UnknownNetworkError(net) + } + if laddr == nil { + laddr = &UDPAddr{} + } + l, err := listenPlan9(net, laddr) + if err != nil { + return nil, err + } + _, err = l.ctl.WriteString("headers") + if err != nil { + return nil, err + } + l.data, err = os.OpenFile(l.dir+"/data", os.O_RDWR, 0) + if err != nil { + return nil, err + } + fd, err := l.netFD() + return newUDPConn(fd), err +} + +// ListenMulticastUDP listens for incoming multicast UDP packets +// addressed to the group address gaddr on ifi, which specifies the +// interface to join. ListenMulticastUDP uses default multicast +// interface if ifi is nil. +func ListenMulticastUDP(net string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + return nil, syscall.EPLAN9 +} diff --git a/src/net/udpsock_posix.go b/src/net/udpsock_posix.go new file mode 100644 index 000000000..a0533366a --- /dev/null +++ b/src/net/udpsock_posix.go @@ -0,0 +1,268 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "syscall" + "time" +) + +func sockaddrToUDP(sa syscall.Sockaddr) Addr { + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + return &UDPAddr{IP: sa.Addr[0:], Port: sa.Port} + case *syscall.SockaddrInet6: + return &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + } + return nil +} + +func (a *UDPAddr) family() int { + if a == nil || len(a.IP) <= IPv4len { + return syscall.AF_INET + } + if a.IP.To4() != nil { + return syscall.AF_INET + } + return syscall.AF_INET6 +} + +func (a *UDPAddr) isWildcard() bool { + if a == nil || a.IP == nil { + return true + } + return a.IP.IsUnspecified() +} + +func (a *UDPAddr) sockaddr(family int) (syscall.Sockaddr, error) { + if a == nil { + return nil, nil + } + return ipToSockaddr(family, a.IP, a.Port, a.Zone) +} + +// UDPConn is the implementation of the Conn and PacketConn interfaces +// for UDP network connections. +type UDPConn struct { + conn +} + +func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } + +// ReadFromUDP reads a UDP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, sa, err := c.fd.readFrom(b) + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port} + case *syscall.SockaddrInet6: + addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + } + return +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.ReadFromUDP(b) + return n, addr.toAddr(), err +} + +// ReadMsgUDP reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number +// of bytes copied into b, the number of bytes copied into oob, the +// flags that were set on the packet and the source address of the +// packet. +func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + var sa syscall.Sockaddr + n, oobn, flags, sa, err = c.fd.readMsg(b, oob) + switch sa := sa.(type) { + case *syscall.SockaddrInet4: + addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port} + case *syscall.SockaddrInet6: + addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} + } + return +} + +// WriteToUDP writes a UDP packet to addr via c, copying the payload +// from b. +// +// WriteToUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + if c.fd.isConnected { + return 0, &OpError{"write", c.fd.net, addr, ErrWriteToConnected} + } + if addr == nil { + return 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress} + } + sa, err := addr.sockaddr(c.fd.family) + if err != nil { + return 0, &OpError{"write", c.fd.net, addr, err} + } + return c.fd.writeTo(b, sa) +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UDPAddr) + if !ok { + return 0, &OpError{"write", c.fd.net, addr, syscall.EINVAL} + } + return c.WriteToUDP(b, a) +} + +// WriteMsgUDP writes a packet to addr via c, copying the payload from +// b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. +func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + if c.fd.isConnected { + return 0, 0, &OpError{"write", c.fd.net, addr, ErrWriteToConnected} + } + if addr == nil { + return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress} + } + sa, err := addr.sockaddr(c.fd.family) + if err != nil { + return 0, 0, &OpError{"write", c.fd.net, addr, err} + } + return c.fd.writeMsg(b, oob, sa) +} + +// DialUDP connects to the remote address raddr on the network net, +// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "dial", Net: net, Addr: raddr, Err: UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: net, Addr: nil, Err: errMissingAddress} + } + return dialUDP(net, laddr, raddr, noDeadline) +} + +func dialUDP(net string, laddr, raddr *UDPAddr, deadline time.Time) (*UDPConn, error) { + fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_DGRAM, 0, "dial") + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Addr: raddr, Err: err} + } + return newUDPConn(fd), nil +} + +// ListenUDP listens for incoming UDP packets addressed to the local +// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has +// a port of 0, ListenUDP will choose an available port. +// The LocalAddr method of the returned UDPConn can be used to +// discover the port. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send UDP packets with per-packet +// addressing. +func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &UDPAddr{} + } + fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_DGRAM, 0, "listen") + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: err} + } + return newUDPConn(fd), nil +} + +// ListenMulticastUDP listens for incoming multicast UDP packets +// addressed to the group address gaddr on ifi, which specifies the +// interface to join. ListenMulticastUDP uses default multicast +// interface if ifi is nil. +func ListenMulticastUDP(net string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "listen", Net: net, Addr: gaddr, Err: UnknownNetworkError(net)} + } + if gaddr == nil || gaddr.IP == nil { + return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: errMissingAddress} + } + fd, err := internetSocket(net, gaddr, nil, noDeadline, syscall.SOCK_DGRAM, 0, "listen") + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: gaddr, Err: err} + } + c := newUDPConn(fd) + if ip4 := gaddr.IP.To4(); ip4 != nil { + if err := listenIPv4MulticastUDP(c, ifi, ip4); err != nil { + c.Close() + return nil, &OpError{Op: "listen", Net: net, Addr: &IPAddr{IP: ip4}, Err: err} + } + } else { + if err := listenIPv6MulticastUDP(c, ifi, gaddr.IP); err != nil { + c.Close() + return nil, &OpError{Op: "listen", Net: net, Addr: &IPAddr{IP: gaddr.IP}, Err: err} + } + } + return c, nil +} + +func listenIPv4MulticastUDP(c *UDPConn, ifi *Interface, ip IP) error { + if ifi != nil { + if err := setIPv4MulticastInterface(c.fd, ifi); err != nil { + return err + } + } + if err := setIPv4MulticastLoopback(c.fd, false); err != nil { + return err + } + if err := joinIPv4Group(c.fd, ifi, ip); err != nil { + return err + } + return nil +} + +func listenIPv6MulticastUDP(c *UDPConn, ifi *Interface, ip IP) error { + if ifi != nil { + if err := setIPv6MulticastInterface(c.fd, ifi); err != nil { + return err + } + } + if err := setIPv6MulticastLoopback(c.fd, false); err != nil { + return err + } + if err := joinIPv6Group(c.fd, ifi, ip); err != nil { + return err + } + return nil +} diff --git a/src/net/unicast_posix_test.go b/src/net/unicast_posix_test.go new file mode 100644 index 000000000..452ac9254 --- /dev/null +++ b/src/net/unicast_posix_test.go @@ -0,0 +1,466 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9 + +package net + +import ( + "runtime" + "syscall" + "testing" +) + +var listenerTests = []struct { + net string + laddr string + ipv6 bool // test with underlying AF_INET6 socket + wildcard bool // test with wildcard address +}{ + {net: "tcp", laddr: "", wildcard: true}, + {net: "tcp", laddr: "0.0.0.0", wildcard: true}, + {net: "tcp", laddr: "[::ffff:0.0.0.0]", wildcard: true}, + {net: "tcp", laddr: "[::]", ipv6: true, wildcard: true}, + + {net: "tcp", laddr: "127.0.0.1"}, + {net: "tcp", laddr: "[::ffff:127.0.0.1]"}, + {net: "tcp", laddr: "[::1]", ipv6: true}, + + {net: "tcp4", laddr: "", wildcard: true}, + {net: "tcp4", laddr: "0.0.0.0", wildcard: true}, + {net: "tcp4", laddr: "[::ffff:0.0.0.0]", wildcard: true}, + + {net: "tcp4", laddr: "127.0.0.1"}, + {net: "tcp4", laddr: "[::ffff:127.0.0.1]"}, + + {net: "tcp6", laddr: "", ipv6: true, wildcard: true}, + {net: "tcp6", laddr: "[::]", ipv6: true, wildcard: true}, + + {net: "tcp6", laddr: "[::1]", ipv6: true}, +} + +// TestTCPListener tests both single and double listen to a test +// listener with same address family, same listening address and +// same port. +func TestTCPListener(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + for _, tt := range listenerTests { + if tt.wildcard && (testing.Short() || !*testExternal) { + continue + } + if tt.ipv6 && !supportsIPv6 { + continue + } + l1, port := usableListenPort(t, tt.net, tt.laddr) + checkFirstListener(t, tt.net, tt.laddr+":"+port, l1) + l2, err := Listen(tt.net, tt.laddr+":"+port) + checkSecondListener(t, tt.net, tt.laddr+":"+port, err, l2) + l1.Close() + } +} + +// TestUDPListener tests both single and double listen to a test +// listener with same address family, same listening address and +// same port. +func TestUDPListener(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + toudpnet := func(net string) string { + switch net { + case "tcp": + return "udp" + case "tcp4": + return "udp4" + case "tcp6": + return "udp6" + } + return "<nil>" + } + + for _, tt := range listenerTests { + if tt.wildcard && (testing.Short() || !*testExternal) { + continue + } + if tt.ipv6 && !supportsIPv6 { + continue + } + tt.net = toudpnet(tt.net) + l1, port := usableListenPacketPort(t, tt.net, tt.laddr) + checkFirstListener(t, tt.net, tt.laddr+":"+port, l1) + l2, err := ListenPacket(tt.net, tt.laddr+":"+port) + checkSecondListener(t, tt.net, tt.laddr+":"+port, err, l2) + l1.Close() + } +} + +var dualStackListenerTests = []struct { + net1 string // first listener + laddr1 string + net2 string // second listener + laddr2 string + wildcard bool // test with wildcard address + xerr error // expected error value, nil or other +}{ + // Test cases and expected results for the attemping 2nd listen on the same port + // 1st listen 2nd listen darwin freebsd linux openbsd + // ------------------------------------------------------------------------------------ + // "tcp" "" "tcp" "" - - - - + // "tcp" "" "tcp" "0.0.0.0" - - - - + // "tcp" "0.0.0.0" "tcp" "" - - - - + // ------------------------------------------------------------------------------------ + // "tcp" "" "tcp" "[::]" - - - ok + // "tcp" "[::]" "tcp" "" - - - ok + // "tcp" "0.0.0.0" "tcp" "[::]" - - - ok + // "tcp" "[::]" "tcp" "0.0.0.0" - - - ok + // "tcp" "[::ffff:0.0.0.0]" "tcp" "[::]" - - - ok + // "tcp" "[::]" "tcp" "[::ffff:0.0.0.0]" - - - ok + // ------------------------------------------------------------------------------------ + // "tcp4" "" "tcp6" "" ok ok ok ok + // "tcp6" "" "tcp4" "" ok ok ok ok + // "tcp4" "0.0.0.0" "tcp6" "[::]" ok ok ok ok + // "tcp6" "[::]" "tcp4" "0.0.0.0" ok ok ok ok + // ------------------------------------------------------------------------------------ + // "tcp" "127.0.0.1" "tcp" "[::1]" ok ok ok ok + // "tcp" "[::1]" "tcp" "127.0.0.1" ok ok ok ok + // "tcp4" "127.0.0.1" "tcp6" "[::1]" ok ok ok ok + // "tcp6" "[::1]" "tcp4" "127.0.0.1" ok ok ok ok + // + // Platform default configurations: + // darwin, kernel version 11.3.0 + // net.inet6.ip6.v6only=0 (overridable by sysctl or IPV6_V6ONLY option) + // freebsd, kernel version 8.2 + // net.inet6.ip6.v6only=1 (overridable by sysctl or IPV6_V6ONLY option) + // linux, kernel version 3.0.0 + // net.ipv6.bindv6only=0 (overridable by sysctl or IPV6_V6ONLY option) + // openbsd, kernel version 5.0 + // net.inet6.ip6.v6only=1 (overriding is prohibited) + + {net1: "tcp", laddr1: "", net2: "tcp", laddr2: "", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "", net2: "tcp", laddr2: "0.0.0.0", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "0.0.0.0", net2: "tcp", laddr2: "", wildcard: true, xerr: syscall.EADDRINUSE}, + + {net1: "tcp", laddr1: "", net2: "tcp", laddr2: "[::]", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "[::]", net2: "tcp", laddr2: "", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "0.0.0.0", net2: "tcp", laddr2: "[::]", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "[::]", net2: "tcp", laddr2: "0.0.0.0", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "[::ffff:0.0.0.0]", net2: "tcp", laddr2: "[::]", wildcard: true, xerr: syscall.EADDRINUSE}, + {net1: "tcp", laddr1: "[::]", net2: "tcp", laddr2: "[::ffff:0.0.0.0]", wildcard: true, xerr: syscall.EADDRINUSE}, + + {net1: "tcp4", laddr1: "", net2: "tcp6", laddr2: "", wildcard: true}, + {net1: "tcp6", laddr1: "", net2: "tcp4", laddr2: "", wildcard: true}, + {net1: "tcp4", laddr1: "0.0.0.0", net2: "tcp6", laddr2: "[::]", wildcard: true}, + {net1: "tcp6", laddr1: "[::]", net2: "tcp4", laddr2: "0.0.0.0", wildcard: true}, + + {net1: "tcp", laddr1: "127.0.0.1", net2: "tcp", laddr2: "[::1]"}, + {net1: "tcp", laddr1: "[::1]", net2: "tcp", laddr2: "127.0.0.1"}, + {net1: "tcp4", laddr1: "127.0.0.1", net2: "tcp6", laddr2: "[::1]"}, + {net1: "tcp6", laddr1: "[::1]", net2: "tcp4", laddr2: "127.0.0.1"}, +} + +// TestDualStackTCPListener tests both single and double listen +// to a test listener with various address families, different +// listening address and same port. +func TestDualStackTCPListener(t *testing.T) { + if testing.Short() { + t.Skip("skipping in -short mode, see issue 5001") + } + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + if !supportsIPv6 { + t.Skip("ipv6 is not supported") + } + + for _, tt := range dualStackListenerTests { + if tt.wildcard && !*testExternal { + continue + } + switch runtime.GOOS { + case "openbsd": + if tt.wildcard && differentWildcardAddr(tt.laddr1, tt.laddr2) { + tt.xerr = nil + } + } + l1, port := usableListenPort(t, tt.net1, tt.laddr1) + laddr := tt.laddr1 + ":" + port + checkFirstListener(t, tt.net1, laddr, l1) + laddr = tt.laddr2 + ":" + port + l2, err := Listen(tt.net2, laddr) + checkDualStackSecondListener(t, tt.net2, laddr, tt.xerr, err, l2) + l1.Close() + } +} + +// TestDualStackUDPListener tests both single and double listen +// to a test listener with various address families, differnet +// listening address and same port. +func TestDualStackUDPListener(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + if !supportsIPv6 { + t.Skip("ipv6 is not supported") + } + + toudpnet := func(net string) string { + switch net { + case "tcp": + return "udp" + case "tcp4": + return "udp4" + case "tcp6": + return "udp6" + } + return "<nil>" + } + + for _, tt := range dualStackListenerTests { + if tt.wildcard && (testing.Short() || !*testExternal) { + continue + } + tt.net1 = toudpnet(tt.net1) + tt.net2 = toudpnet(tt.net2) + switch runtime.GOOS { + case "openbsd": + if tt.wildcard && differentWildcardAddr(tt.laddr1, tt.laddr2) { + tt.xerr = nil + } + } + l1, port := usableListenPacketPort(t, tt.net1, tt.laddr1) + laddr := tt.laddr1 + ":" + port + checkFirstListener(t, tt.net1, laddr, l1) + laddr = tt.laddr2 + ":" + port + l2, err := ListenPacket(tt.net2, laddr) + checkDualStackSecondListener(t, tt.net2, laddr, tt.xerr, err, l2) + l1.Close() + } +} + +func usableListenPort(t *testing.T, net, laddr string) (l Listener, port string) { + var nladdr string + var err error + switch net { + default: + panic("usableListenPort net=" + net) + case "tcp", "tcp4", "tcp6": + l, err = Listen(net, laddr+":0") + if err != nil { + t.Fatalf("Probe Listen(%q, %q) failed: %v", net, laddr, err) + } + nladdr = l.(*TCPListener).Addr().String() + } + _, port, err = SplitHostPort(nladdr) + if err != nil { + t.Fatalf("SplitHostPort failed: %v", err) + } + return l, port +} + +func usableListenPacketPort(t *testing.T, net, laddr string) (l PacketConn, port string) { + var nladdr string + var err error + switch net { + default: + panic("usableListenPacketPort net=" + net) + case "udp", "udp4", "udp6": + l, err = ListenPacket(net, laddr+":0") + if err != nil { + t.Fatalf("Probe ListenPacket(%q, %q) failed: %v", net, laddr, err) + } + nladdr = l.(*UDPConn).LocalAddr().String() + } + _, port, err = SplitHostPort(nladdr) + if err != nil { + t.Fatalf("SplitHostPort failed: %v", err) + } + return l, port +} + +func differentWildcardAddr(i, j string) bool { + if (i == "" || i == "0.0.0.0" || i == "::ffff:0.0.0.0") && (j == "" || j == "0.0.0.0" || j == "::ffff:0.0.0.0") { + return false + } + if i == "[::]" && j == "[::]" { + return false + } + return true +} + +func checkFirstListener(t *testing.T, net, laddr string, l interface{}) { + switch net { + case "tcp": + fd := l.(*TCPListener).fd + checkDualStackAddrFamily(t, net, laddr, fd) + case "tcp4": + fd := l.(*TCPListener).fd + if fd.family != syscall.AF_INET { + t.Fatalf("First Listen(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, syscall.AF_INET) + } + case "tcp6": + fd := l.(*TCPListener).fd + if fd.family != syscall.AF_INET6 { + t.Fatalf("First Listen(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, syscall.AF_INET6) + } + case "udp": + fd := l.(*UDPConn).fd + checkDualStackAddrFamily(t, net, laddr, fd) + case "udp4": + fd := l.(*UDPConn).fd + if fd.family != syscall.AF_INET { + t.Fatalf("First ListenPacket(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, syscall.AF_INET) + } + case "udp6": + fd := l.(*UDPConn).fd + if fd.family != syscall.AF_INET6 { + t.Fatalf("First ListenPacket(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, syscall.AF_INET6) + } + default: + t.Fatalf("Unexpected network: %q", net) + } +} + +func checkSecondListener(t *testing.T, net, laddr string, err error, l interface{}) { + switch net { + case "tcp", "tcp4", "tcp6": + if err == nil { + l.(*TCPListener).Close() + t.Fatalf("Second Listen(%q, %q) should fail", net, laddr) + } + case "udp", "udp4", "udp6": + if err == nil { + l.(*UDPConn).Close() + t.Fatalf("Second ListenPacket(%q, %q) should fail", net, laddr) + } + default: + t.Fatalf("Unexpected network: %q", net) + } +} + +func checkDualStackSecondListener(t *testing.T, net, laddr string, xerr, err error, l interface{}) { + switch net { + case "tcp", "tcp4", "tcp6": + if xerr == nil && err != nil || xerr != nil && err == nil { + t.Fatalf("Second Listen(%q, %q) returns %v, expected %v", net, laddr, err, xerr) + } + if err == nil { + l.(*TCPListener).Close() + } + case "udp", "udp4", "udp6": + if xerr == nil && err != nil || xerr != nil && err == nil { + t.Fatalf("Second ListenPacket(%q, %q) returns %v, expected %v", net, laddr, err, xerr) + } + if err == nil { + l.(*UDPConn).Close() + } + default: + t.Fatalf("Unexpected network: %q", net) + } +} + +func checkDualStackAddrFamily(t *testing.T, net, laddr string, fd *netFD) { + switch a := fd.laddr.(type) { + case *TCPAddr: + // If a node under test supports both IPv6 capability + // and IPv6 IPv4-mapping capability, we can assume + // that the node listens on a wildcard address with an + // AF_INET6 socket. + if supportsIPv4map && fd.laddr.(*TCPAddr).isWildcard() { + if fd.family != syscall.AF_INET6 { + t.Fatalf("Listen(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, syscall.AF_INET6) + } + } else { + if fd.family != a.family() { + t.Fatalf("Listen(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, a.family()) + } + } + case *UDPAddr: + // If a node under test supports both IPv6 capability + // and IPv6 IPv4-mapping capability, we can assume + // that the node listens on a wildcard address with an + // AF_INET6 socket. + if supportsIPv4map && fd.laddr.(*UDPAddr).isWildcard() { + if fd.family != syscall.AF_INET6 { + t.Fatalf("ListenPacket(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, syscall.AF_INET6) + } + } else { + if fd.family != a.family() { + t.Fatalf("ListenPacket(%q, %q) returns address family %v, expected %v", net, laddr, fd.family, a.family()) + } + } + default: + t.Fatalf("Unexpected protocol address type: %T", a) + } +} + +var prohibitionaryDialArgTests = []struct { + net string + addr string +}{ + {"tcp6", "127.0.0.1"}, + {"tcp6", "[::ffff:127.0.0.1]"}, +} + +func TestProhibitionaryDialArgs(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + // This test requires both IPv6 and IPv6 IPv4-mapping functionality. + if !supportsIPv4map || testing.Short() || !*testExternal { + return + } + + l, port := usableListenPort(t, "tcp", "[::]") + defer l.Close() + + for _, tt := range prohibitionaryDialArgTests { + c, err := Dial(tt.net, tt.addr+":"+port) + if err == nil { + c.Close() + t.Fatalf("Dial(%q, %q) should fail", tt.net, tt.addr) + } + } +} + +func TestWildWildcardListener(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("skipping test on %q", runtime.GOOS) + } + + if testing.Short() || !*testExternal { + t.Skip("skipping test to avoid external network") + } + + defer func() { + if p := recover(); p != nil { + t.Fatalf("Listen, ListenPacket or protocol-specific Listen panicked: %v", p) + } + }() + + if ln, err := Listen("tcp", ""); err == nil { + ln.Close() + } + if ln, err := ListenPacket("udp", ""); err == nil { + ln.Close() + } + if ln, err := ListenTCP("tcp", nil); err == nil { + ln.Close() + } + if ln, err := ListenUDP("udp", nil); err == nil { + ln.Close() + } + if ln, err := ListenIP("ip:icmp", nil); err == nil { + ln.Close() + } +} diff --git a/src/net/unix_test.go b/src/net/unix_test.go new file mode 100644 index 000000000..1cdff3908 --- /dev/null +++ b/src/net/unix_test.go @@ -0,0 +1,333 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !nacl,!plan9,!windows + +package net + +import ( + "bytes" + "os" + "reflect" + "runtime" + "syscall" + "testing" + "time" +) + +func TestReadUnixgramWithUnnamedSocket(t *testing.T) { + addr := testUnixAddr() + la, err := ResolveUnixAddr("unixgram", addr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c, err := ListenUnixgram("unixgram", la) + if err != nil { + t.Fatalf("ListenUnixgram failed: %v", err) + } + defer func() { + c.Close() + os.Remove(addr) + }() + + off := make(chan bool) + data := [5]byte{1, 2, 3, 4, 5} + go func() { + defer func() { off <- true }() + s, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_DGRAM, 0) + if err != nil { + t.Errorf("syscall.Socket failed: %v", err) + return + } + defer syscall.Close(s) + rsa := &syscall.SockaddrUnix{Name: addr} + if err := syscall.Sendto(s, data[:], 0, rsa); err != nil { + t.Errorf("syscall.Sendto failed: %v", err) + return + } + }() + + <-off + b := make([]byte, 64) + c.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + n, from, err := c.ReadFrom(b) + if err != nil { + t.Fatalf("UnixConn.ReadFrom failed: %v", err) + } + if from != nil { + t.Fatalf("neighbor address is %v", from) + } + if !bytes.Equal(b[:n], data[:]) { + t.Fatalf("got %v, want %v", b[:n], data[:]) + } +} + +func TestReadUnixgramWithZeroBytesBuffer(t *testing.T) { + // issue 4352: Recvfrom failed with "address family not + // supported by protocol family" if zero-length buffer provided + + addr := testUnixAddr() + la, err := ResolveUnixAddr("unixgram", addr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c, err := ListenUnixgram("unixgram", la) + if err != nil { + t.Fatalf("ListenUnixgram failed: %v", err) + } + defer func() { + c.Close() + os.Remove(addr) + }() + + off := make(chan bool) + go func() { + defer func() { off <- true }() + c, err := DialUnix("unixgram", nil, la) + if err != nil { + t.Errorf("DialUnix failed: %v", err) + return + } + defer c.Close() + if _, err := c.Write([]byte{1, 2, 3, 4, 5}); err != nil { + t.Errorf("UnixConn.Write failed: %v", err) + return + } + }() + + <-off + c.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + _, from, err := c.ReadFrom(nil) + if err != nil { + t.Fatalf("UnixConn.ReadFrom failed: %v", err) + } + if from != nil { + t.Fatalf("neighbor address is %v", from) + } +} + +func TestUnixgramAutobind(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("skipping: autobind is linux only") + } + + laddr := &UnixAddr{Name: "", Net: "unixgram"} + c1, err := ListenUnixgram("unixgram", laddr) + if err != nil { + t.Fatalf("ListenUnixgram failed: %v", err) + } + defer c1.Close() + + // retrieve the autobind address + autoAddr := c1.LocalAddr().(*UnixAddr) + if len(autoAddr.Name) <= 1 { + t.Fatalf("invalid autobind address: %v", autoAddr) + } + if autoAddr.Name[0] != '@' { + t.Fatalf("invalid autobind address: %v", autoAddr) + } + + c2, err := DialUnix("unixgram", nil, autoAddr) + if err != nil { + t.Fatalf("DialUnix failed: %v", err) + } + defer c2.Close() + + if !reflect.DeepEqual(c1.LocalAddr(), c2.RemoteAddr()) { + t.Fatalf("expected autobind address %v, got %v", c1.LocalAddr(), c2.RemoteAddr()) + } +} + +func TestUnixAutobindClose(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("skipping: autobind is linux only") + } + laddr := &UnixAddr{Name: "", Net: "unix"} + ln, err := ListenUnix("unix", laddr) + if err != nil { + t.Fatalf("ListenUnix failed: %v", err) + } + ln.Close() +} + +func TestUnixgramWrite(t *testing.T) { + addr := testUnixAddr() + laddr, err := ResolveUnixAddr("unixgram", addr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c, err := ListenPacket("unixgram", addr) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + defer os.Remove(addr) + defer c.Close() + + testUnixgramWriteConn(t, laddr) + testUnixgramWritePacketConn(t, laddr) +} + +func testUnixgramWriteConn(t *testing.T, raddr *UnixAddr) { + c, err := Dial("unixgram", raddr.String()) + if err != nil { + t.Fatalf("Dial failed: %v", err) + } + defer c.Close() + + if _, err := c.(*UnixConn).WriteToUnix([]byte("Connection-oriented mode socket"), raddr); err == nil { + t.Fatal("WriteToUnix should fail") + } else if err.(*OpError).Err != ErrWriteToConnected { + t.Fatalf("WriteToUnix should fail as ErrWriteToConnected: %v", err) + } + if _, err = c.(*UnixConn).WriteTo([]byte("Connection-oriented mode socket"), raddr); err == nil { + t.Fatal("WriteTo should fail") + } else if err.(*OpError).Err != ErrWriteToConnected { + t.Fatalf("WriteTo should fail as ErrWriteToConnected: %v", err) + } + if _, _, err = c.(*UnixConn).WriteMsgUnix([]byte("Connection-oriented mode socket"), nil, raddr); err == nil { + t.Fatal("WriteTo should fail") + } else if err.(*OpError).Err != ErrWriteToConnected { + t.Fatalf("WriteMsgUnix should fail as ErrWriteToConnected: %v", err) + } + if _, err := c.Write([]byte("Connection-oriented mode socket")); err != nil { + t.Fatalf("Write failed: %v", err) + } +} + +func testUnixgramWritePacketConn(t *testing.T, raddr *UnixAddr) { + addr := testUnixAddr() + c, err := ListenPacket("unixgram", addr) + if err != nil { + t.Fatalf("ListenPacket failed: %v", err) + } + defer os.Remove(addr) + defer c.Close() + + if _, err := c.(*UnixConn).WriteToUnix([]byte("Connectionless mode socket"), raddr); err != nil { + t.Fatalf("WriteToUnix failed: %v", err) + } + if _, err := c.WriteTo([]byte("Connectionless mode socket"), raddr); err != nil { + t.Fatalf("WriteTo failed: %v", err) + } + if _, _, err := c.(*UnixConn).WriteMsgUnix([]byte("Connectionless mode socket"), nil, raddr); err != nil { + t.Fatalf("WriteMsgUnix failed: %v", err) + } + if _, err := c.(*UnixConn).Write([]byte("Connectionless mode socket")); err == nil { + t.Fatal("Write should fail") + } +} + +func TestUnixConnLocalAndRemoteNames(t *testing.T) { + for _, laddr := range []string{"", testUnixAddr()} { + laddr := laddr + taddr := testUnixAddr() + ta, err := ResolveUnixAddr("unix", taddr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + ln, err := ListenUnix("unix", ta) + if err != nil { + t.Fatalf("ListenUnix failed: %v", err) + } + defer func() { + ln.Close() + os.Remove(taddr) + }() + + done := make(chan int) + go transponder(t, ln, done) + + la, err := ResolveUnixAddr("unix", laddr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c, err := DialUnix("unix", la, ta) + if err != nil { + t.Fatalf("DialUnix failed: %v", err) + } + defer func() { + c.Close() + if la != nil { + defer os.Remove(laddr) + } + }() + if _, err := c.Write([]byte("UNIXCONN LOCAL AND REMOTE NAME TEST")); err != nil { + t.Fatalf("UnixConn.Write failed: %v", err) + } + + switch runtime.GOOS { + case "android", "linux": + if laddr == "" { + laddr = "@" // autobind feature + } + } + var connAddrs = [3]struct{ got, want Addr }{ + {ln.Addr(), ta}, + {c.LocalAddr(), &UnixAddr{Name: laddr, Net: "unix"}}, + {c.RemoteAddr(), ta}, + } + for _, ca := range connAddrs { + if !reflect.DeepEqual(ca.got, ca.want) { + t.Fatalf("got %#v, expected %#v", ca.got, ca.want) + } + } + + <-done + } +} + +func TestUnixgramConnLocalAndRemoteNames(t *testing.T) { + for _, laddr := range []string{"", testUnixAddr()} { + laddr := laddr + taddr := testUnixAddr() + ta, err := ResolveUnixAddr("unixgram", taddr) + if err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + c1, err := ListenUnixgram("unixgram", ta) + if err != nil { + t.Fatalf("ListenUnixgram failed: %v", err) + } + defer func() { + c1.Close() + os.Remove(taddr) + }() + + var la *UnixAddr + if laddr != "" { + if la, err = ResolveUnixAddr("unixgram", laddr); err != nil { + t.Fatalf("ResolveUnixAddr failed: %v", err) + } + } + c2, err := DialUnix("unixgram", la, ta) + if err != nil { + t.Fatalf("DialUnix failed: %v", err) + } + defer func() { + c2.Close() + if la != nil { + defer os.Remove(laddr) + } + }() + + switch runtime.GOOS { + case "android", "linux": + if laddr == "" { + laddr = "@" // autobind feature + } + } + + var connAddrs = [4]struct{ got, want Addr }{ + {c1.LocalAddr(), ta}, + {c1.RemoteAddr(), nil}, + {c2.LocalAddr(), &UnixAddr{Name: laddr, Net: "unixgram"}}, + {c2.RemoteAddr(), ta}, + } + for _, ca := range connAddrs { + if !reflect.DeepEqual(ca.got, ca.want) { + t.Fatalf("got %#v, expected %#v", ca.got, ca.want) + } + } + } +} diff --git a/src/net/unixsock.go b/src/net/unixsock.go new file mode 100644 index 000000000..85955845b --- /dev/null +++ b/src/net/unixsock.go @@ -0,0 +1,43 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// UnixAddr represents the address of a Unix domain socket end point. +type UnixAddr struct { + Name string + Net string +} + +// Network returns the address's network name, "unix", "unixgram" or +// "unixpacket". +func (a *UnixAddr) Network() string { + return a.Net +} + +func (a *UnixAddr) String() string { + if a == nil { + return "<nil>" + } + return a.Name +} + +func (a *UnixAddr) toAddr() Addr { + if a == nil { + return nil + } + return a +} + +// ResolveUnixAddr parses addr as a Unix domain socket address. +// The string net gives the network name, "unix", "unixgram" or +// "unixpacket". +func ResolveUnixAddr(net, addr string) (*UnixAddr, error) { + switch net { + case "unix", "unixgram", "unixpacket": + return &UnixAddr{Name: addr, Net: net}, nil + default: + return nil, UnknownNetworkError(net) + } +} diff --git a/src/net/unixsock_plan9.go b/src/net/unixsock_plan9.go new file mode 100644 index 000000000..c60c1d83b --- /dev/null +++ b/src/net/unixsock_plan9.go @@ -0,0 +1,143 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "syscall" + "time" +) + +// UnixConn is an implementation of the Conn interface for connections +// to Unix domain sockets. +type UnixConn struct { + conn +} + +// ReadFromUnix reads a packet from c, copying the payload into b. It +// returns the number of bytes copied into b and the source address of +// the packet. +// +// ReadFromUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { + return 0, nil, syscall.EPLAN9 +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { + return 0, nil, syscall.EPLAN9 +} + +// ReadMsgUnix reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet, and the source address of the packet. +func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 +} + +// WriteToUnix writes a packet to addr via c, copying the payload from b. +// +// WriteToUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { + return 0, syscall.EPLAN9 +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { + return 0, syscall.EPLAN9 +} + +// WriteMsgUnix writes a packet to addr via c, copying the payload +// from b and the associated out-of-band data from oob. It returns +// the number of payload and out-of-band bytes written. +func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 +} + +// CloseRead shuts down the reading side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseRead() error { + return syscall.EPLAN9 +} + +// CloseWrite shuts down the writing side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseWrite() error { + return syscall.EPLAN9 +} + +// DialUnix connects to the remote address raddr on the network net, +// which must be "unix", "unixgram" or "unixpacket". If laddr is not +// nil, it is used as the local address for the connection. +func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { + return dialUnix(net, laddr, raddr, noDeadline) +} + +func dialUnix(net string, laddr, raddr *UnixAddr, deadline time.Time) (*UnixConn, error) { + return nil, syscall.EPLAN9 +} + +// UnixListener is a Unix domain socket listener. Clients should +// typically use variables of type Listener instead of assuming Unix +// domain sockets. +type UnixListener struct{} + +// ListenUnix announces on the Unix domain socket laddr and returns a +// Unix listener. The network net must be "unix" or "unixpacket". +func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { + return nil, syscall.EPLAN9 +} + +// AcceptUnix accepts the next incoming call and returns the new +// connection. +func (l *UnixListener) AcceptUnix() (*UnixConn, error) { + return nil, syscall.EPLAN9 +} + +// Accept implements the Accept method in the Listener interface; it +// waits for the next call and returns a generic Conn. +func (l *UnixListener) Accept() (Conn, error) { + return nil, syscall.EPLAN9 +} + +// Close stops listening on the Unix address. Already accepted +// connections are not closed. +func (l *UnixListener) Close() error { + return syscall.EPLAN9 +} + +// Addr returns the listener's network address. +func (l *UnixListener) Addr() Addr { return nil } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *UnixListener) SetDeadline(t time.Time) error { + return syscall.EPLAN9 +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *UnixListener) File() (*os.File, error) { + return nil, syscall.EPLAN9 +} + +// ListenUnixgram listens for incoming Unix datagram packets addressed +// to the local address laddr. The network net must be "unixgram". +// The returned connection's ReadFrom and WriteTo methods can be used +// to receive and send packets with per-packet addressing. +func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { + return nil, syscall.EPLAN9 +} diff --git a/src/net/unixsock_posix.go b/src/net/unixsock_posix.go new file mode 100644 index 000000000..3c2e78bdc --- /dev/null +++ b/src/net/unixsock_posix.go @@ -0,0 +1,362 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows + +package net + +import ( + "errors" + "os" + "syscall" + "time" +) + +func unixSocket(net string, laddr, raddr sockaddr, mode string, deadline time.Time) (*netFD, error) { + var sotype int + switch net { + case "unix": + sotype = syscall.SOCK_STREAM + case "unixgram": + sotype = syscall.SOCK_DGRAM + case "unixpacket": + sotype = syscall.SOCK_SEQPACKET + default: + return nil, UnknownNetworkError(net) + } + + switch mode { + case "dial": + if laddr != nil && laddr.isWildcard() { + laddr = nil + } + if raddr != nil && raddr.isWildcard() { + raddr = nil + } + if raddr == nil && (sotype != syscall.SOCK_DGRAM || laddr == nil) { + return nil, errMissingAddress + } + case "listen": + default: + return nil, errors.New("unknown mode: " + mode) + } + + fd, err := socket(net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr, deadline) + if err != nil { + return nil, err + } + return fd, nil +} + +func sockaddrToUnix(sa syscall.Sockaddr) Addr { + if s, ok := sa.(*syscall.SockaddrUnix); ok { + return &UnixAddr{Name: s.Name, Net: "unix"} + } + return nil +} + +func sockaddrToUnixgram(sa syscall.Sockaddr) Addr { + if s, ok := sa.(*syscall.SockaddrUnix); ok { + return &UnixAddr{Name: s.Name, Net: "unixgram"} + } + return nil +} + +func sockaddrToUnixpacket(sa syscall.Sockaddr) Addr { + if s, ok := sa.(*syscall.SockaddrUnix); ok { + return &UnixAddr{Name: s.Name, Net: "unixpacket"} + } + return nil +} + +func sotypeToNet(sotype int) string { + switch sotype { + case syscall.SOCK_STREAM: + return "unix" + case syscall.SOCK_DGRAM: + return "unixgram" + case syscall.SOCK_SEQPACKET: + return "unixpacket" + default: + panic("sotypeToNet unknown socket type") + } +} + +func (a *UnixAddr) family() int { + return syscall.AF_UNIX +} + +func (a *UnixAddr) isWildcard() bool { + return a == nil || a.Name == "" +} + +func (a *UnixAddr) sockaddr(family int) (syscall.Sockaddr, error) { + if a == nil { + return nil, nil + } + return &syscall.SockaddrUnix{Name: a.Name}, nil +} + +// UnixConn is an implementation of the Conn interface for connections +// to Unix domain sockets. +type UnixConn struct { + conn +} + +func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } + +// ReadFromUnix reads a packet from c, copying the payload into b. It +// returns the number of bytes copied into b and the source address of +// the packet. +// +// ReadFromUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UnixConn) ReadFromUnix(b []byte) (n int, addr *UnixAddr, err error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, sa, err := c.fd.readFrom(b) + switch sa := sa.(type) { + case *syscall.SockaddrUnix: + if sa.Name != "" { + addr = &UnixAddr{Name: sa.Name, Net: sotypeToNet(c.fd.sotype)} + } + } + return +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.ReadFromUnix(b) + return n, addr.toAddr(), err +} + +// ReadMsgUnix reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet, and the source address of the packet. +func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, sa, err := c.fd.readMsg(b, oob) + switch sa := sa.(type) { + case *syscall.SockaddrUnix: + if sa.Name != "" { + addr = &UnixAddr{Name: sa.Name, Net: sotypeToNet(c.fd.sotype)} + } + } + return +} + +// WriteToUnix writes a packet to addr via c, copying the payload from b. +// +// WriteToUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (n int, err error) { + if !c.ok() { + return 0, syscall.EINVAL + } + if c.fd.isConnected { + return 0, &OpError{Op: "write", Net: c.fd.net, Addr: addr, Err: ErrWriteToConnected} + } + if addr == nil { + return 0, &OpError{Op: "write", Net: c.fd.net, Addr: nil, Err: errMissingAddress} + } + if addr.Net != sotypeToNet(c.fd.sotype) { + return 0, syscall.EAFNOSUPPORT + } + sa := &syscall.SockaddrUnix{Name: addr.Name} + return c.fd.writeTo(b, sa) +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UnixConn) WriteTo(b []byte, addr Addr) (n int, err error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UnixAddr) + if !ok { + return 0, &OpError{"write", c.fd.net, addr, syscall.EINVAL} + } + return c.WriteToUnix(b, a) +} + +// WriteMsgUnix writes a packet to addr via c, copying the payload +// from b and the associated out-of-band data from oob. It returns +// the number of payload and out-of-band bytes written. +func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + if c.fd.sotype == syscall.SOCK_DGRAM && c.fd.isConnected { + return 0, 0, &OpError{Op: "write", Net: c.fd.net, Addr: addr, Err: ErrWriteToConnected} + } + if addr != nil { + if addr.Net != sotypeToNet(c.fd.sotype) { + return 0, 0, syscall.EAFNOSUPPORT + } + sa := &syscall.SockaddrUnix{Name: addr.Name} + return c.fd.writeMsg(b, oob, sa) + } + return c.fd.writeMsg(b, oob, nil) +} + +// CloseRead shuts down the reading side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.closeRead() +} + +// CloseWrite shuts down the writing side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + return c.fd.closeWrite() +} + +// DialUnix connects to the remote address raddr on the network net, +// which must be "unix", "unixgram" or "unixpacket". If laddr is not +// nil, it is used as the local address for the connection. +func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { + switch net { + case "unix", "unixgram", "unixpacket": + default: + return nil, &OpError{Op: "dial", Net: net, Addr: raddr, Err: UnknownNetworkError(net)} + } + return dialUnix(net, laddr, raddr, noDeadline) +} + +func dialUnix(net string, laddr, raddr *UnixAddr, deadline time.Time) (*UnixConn, error) { + fd, err := unixSocket(net, laddr, raddr, "dial", deadline) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Addr: raddr, Err: err} + } + return newUnixConn(fd), nil +} + +// UnixListener is a Unix domain socket listener. Clients should +// typically use variables of type Listener instead of assuming Unix +// domain sockets. +type UnixListener struct { + fd *netFD + path string +} + +// ListenUnix announces on the Unix domain socket laddr and returns a +// Unix listener. The network net must be "unix" or "unixpacket". +func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { + switch net { + case "unix", "unixpacket": + default: + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: UnknownNetworkError(net)} + } + if laddr == nil { + return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: errMissingAddress} + } + fd, err := unixSocket(net, laddr, nil, "listen", noDeadline) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: err} + } + return &UnixListener{fd, fd.laddr.String()}, nil +} + +// AcceptUnix accepts the next incoming call and returns the new +// connection. +func (l *UnixListener) AcceptUnix() (*UnixConn, error) { + if l == nil || l.fd == nil { + return nil, syscall.EINVAL + } + fd, err := l.fd.accept() + if err != nil { + return nil, err + } + c := newUnixConn(fd) + return c, nil +} + +// Accept implements the Accept method in the Listener interface; it +// waits for the next call and returns a generic Conn. +func (l *UnixListener) Accept() (c Conn, err error) { + c1, err := l.AcceptUnix() + if err != nil { + return nil, err + } + return c1, nil +} + +// Close stops listening on the Unix address. Already accepted +// connections are not closed. +func (l *UnixListener) Close() error { + if l == nil || l.fd == nil { + return syscall.EINVAL + } + + // The operating system doesn't clean up + // the file that announcing created, so + // we have to clean it up ourselves. + // There's a race here--we can't know for + // sure whether someone else has come along + // and replaced our socket name already-- + // but this sequence (remove then close) + // is at least compatible with the auto-remove + // sequence in ListenUnix. It's only non-Go + // programs that can mess us up. + if l.path[0] != '@' { + syscall.Unlink(l.path) + } + return l.fd.Close() +} + +// Addr returns the listener's network address. +func (l *UnixListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *UnixListener) SetDeadline(t time.Time) (err error) { + if l == nil || l.fd == nil { + return syscall.EINVAL + } + return l.fd.setDeadline(t) +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *UnixListener) File() (f *os.File, err error) { return l.fd.dup() } + +// ListenUnixgram listens for incoming Unix datagram packets addressed +// to the local address laddr. The network net must be "unixgram". +// The returned connection's ReadFrom and WriteTo methods can be used +// to receive and send packets with per-packet addressing. +func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { + switch net { + case "unixgram": + default: + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: UnknownNetworkError(net)} + } + if laddr == nil { + return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: errMissingAddress} + } + fd, err := unixSocket(net, laddr, nil, "listen", noDeadline) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Addr: laddr, Err: err} + } + return newUnixConn(fd), nil +} diff --git a/src/net/url/example_test.go b/src/net/url/example_test.go new file mode 100644 index 000000000..56c5dc696 --- /dev/null +++ b/src/net/url/example_test.go @@ -0,0 +1,41 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package url_test + +import ( + "fmt" + "log" + "net/url" +) + +func ExampleValues() { + v := url.Values{} + v.Set("name", "Ava") + v.Add("friend", "Jess") + v.Add("friend", "Sarah") + v.Add("friend", "Zoe") + // v.Encode() == "name=Ava&friend=Jess&friend=Sarah&friend=Zoe" + fmt.Println(v.Get("name")) + fmt.Println(v.Get("friend")) + fmt.Println(v["friend"]) + // Output: + // Ava + // Jess + // [Jess Sarah Zoe] +} + +func ExampleURL() { + u, err := url.Parse("http://bing.com/search?q=dotnet") + if err != nil { + log.Fatal(err) + } + u.Scheme = "https" + u.Host = "google.com" + q := u.Query() + q.Set("q", "golang") + u.RawQuery = q.Encode() + fmt.Println(u) + // Output: https://google.com/search?q=golang +} diff --git a/src/net/url/url.go b/src/net/url/url.go new file mode 100644 index 000000000..0b32cd7c8 --- /dev/null +++ b/src/net/url/url.go @@ -0,0 +1,701 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package url parses URLs and implements query escaping. +// See RFC 3986. +package url + +import ( + "bytes" + "errors" + "sort" + "strconv" + "strings" +) + +// Error reports an error and the operation and URL that caused it. +type Error struct { + Op string + URL string + Err error +} + +func (e *Error) Error() string { return e.Op + " " + e.URL + ": " + e.Err.Error() } + +func ishex(c byte) bool { + switch { + case '0' <= c && c <= '9': + return true + case 'a' <= c && c <= 'f': + return true + case 'A' <= c && c <= 'F': + return true + } + return false +} + +func unhex(c byte) byte { + switch { + case '0' <= c && c <= '9': + return c - '0' + case 'a' <= c && c <= 'f': + return c - 'a' + 10 + case 'A' <= c && c <= 'F': + return c - 'A' + 10 + } + return 0 +} + +type encoding int + +const ( + encodePath encoding = 1 + iota + encodeUserPassword + encodeQueryComponent + encodeFragment +) + +type EscapeError string + +func (e EscapeError) Error() string { + return "invalid URL escape " + strconv.Quote(string(e)) +} + +// Return true if the specified character should be escaped when +// appearing in a URL string, according to RFC 3986. +func shouldEscape(c byte, mode encoding) bool { + // §2.3 Unreserved characters (alphanum) + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { + return false + } + + switch c { + case '-', '_', '.', '~': // §2.3 Unreserved characters (mark) + return false + + case '$', '&', '+', ',', '/', ':', ';', '=', '?', '@': // §2.2 Reserved characters (reserved) + // Different sections of the URL allow a few of + // the reserved characters to appear unescaped. + switch mode { + case encodePath: // §3.3 + // The RFC allows : @ & = + $ but saves / ; , for assigning + // meaning to individual path segments. This package + // only manipulates the path as a whole, so we allow those + // last two as well. That leaves only ? to escape. + return c == '?' + + case encodeUserPassword: // §3.2.1 + // The RFC allows ';', ':', '&', '=', '+', '$', and ',' in + // userinfo, so we must escape only '@', '/', and '?'. + // The parsing of userinfo treats ':' as special so we must escape + // that too. + return c == '@' || c == '/' || c == '?' || c == ':' + + case encodeQueryComponent: // §3.4 + // The RFC reserves (so we must escape) everything. + return true + + case encodeFragment: // §4.1 + // The RFC text is silent but the grammar allows + // everything, so escape nothing. + return false + } + } + + // Everything else must be escaped. + return true +} + +// QueryUnescape does the inverse transformation of QueryEscape, converting +// %AB into the byte 0xAB and '+' into ' ' (space). It returns an error if +// any % is not followed by two hexadecimal digits. +func QueryUnescape(s string) (string, error) { + return unescape(s, encodeQueryComponent) +} + +// unescape unescapes a string; the mode specifies +// which section of the URL string is being unescaped. +func unescape(s string, mode encoding) (string, error) { + // Count %, check that they're well-formed. + n := 0 + hasPlus := false + for i := 0; i < len(s); { + switch s[i] { + case '%': + n++ + if i+2 >= len(s) || !ishex(s[i+1]) || !ishex(s[i+2]) { + s = s[i:] + if len(s) > 3 { + s = s[0:3] + } + return "", EscapeError(s) + } + i += 3 + case '+': + hasPlus = mode == encodeQueryComponent + i++ + default: + i++ + } + } + + if n == 0 && !hasPlus { + return s, nil + } + + t := make([]byte, len(s)-2*n) + j := 0 + for i := 0; i < len(s); { + switch s[i] { + case '%': + t[j] = unhex(s[i+1])<<4 | unhex(s[i+2]) + j++ + i += 3 + case '+': + if mode == encodeQueryComponent { + t[j] = ' ' + } else { + t[j] = '+' + } + j++ + i++ + default: + t[j] = s[i] + j++ + i++ + } + } + return string(t), nil +} + +// QueryEscape escapes the string so it can be safely placed +// inside a URL query. +func QueryEscape(s string) string { + return escape(s, encodeQueryComponent) +} + +func escape(s string, mode encoding) string { + spaceCount, hexCount := 0, 0 + for i := 0; i < len(s); i++ { + c := s[i] + if shouldEscape(c, mode) { + if c == ' ' && mode == encodeQueryComponent { + spaceCount++ + } else { + hexCount++ + } + } + } + + if spaceCount == 0 && hexCount == 0 { + return s + } + + t := make([]byte, len(s)+2*hexCount) + j := 0 + for i := 0; i < len(s); i++ { + switch c := s[i]; { + case c == ' ' && mode == encodeQueryComponent: + t[j] = '+' + j++ + case shouldEscape(c, mode): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} + +// A URL represents a parsed URL (technically, a URI reference). +// The general form represented is: +// +// scheme://[userinfo@]host/path[?query][#fragment] +// +// URLs that do not start with a slash after the scheme are interpreted as: +// +// scheme:opaque[?query][#fragment] +// +// Note that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/. +// A consequence is that it is impossible to tell which slashes in the Path were +// slashes in the raw URL and which were %2f. This distinction is rarely important, +// but when it is a client must use other routines to parse the raw URL or construct +// the parsed URL. For example, an HTTP server can consult req.RequestURI, and +// an HTTP client can use URL{Host: "example.com", Opaque: "//example.com/Go%2f"} +// instead of URL{Host: "example.com", Path: "/Go/"}. +type URL struct { + Scheme string + Opaque string // encoded opaque data + User *Userinfo // username and password information + Host string // host or host:port + Path string + RawQuery string // encoded query values, without '?' + Fragment string // fragment for references, without '#' +} + +// User returns a Userinfo containing the provided username +// and no password set. +func User(username string) *Userinfo { + return &Userinfo{username, "", false} +} + +// UserPassword returns a Userinfo containing the provided username +// and password. +// This functionality should only be used with legacy web sites. +// RFC 2396 warns that interpreting Userinfo this way +// ``is NOT RECOMMENDED, because the passing of authentication +// information in clear text (such as URI) has proven to be a +// security risk in almost every case where it has been used.'' +func UserPassword(username, password string) *Userinfo { + return &Userinfo{username, password, true} +} + +// The Userinfo type is an immutable encapsulation of username and +// password details for a URL. An existing Userinfo value is guaranteed +// to have a username set (potentially empty, as allowed by RFC 2396), +// and optionally a password. +type Userinfo struct { + username string + password string + passwordSet bool +} + +// Username returns the username. +func (u *Userinfo) Username() string { + return u.username +} + +// Password returns the password in case it is set, and whether it is set. +func (u *Userinfo) Password() (string, bool) { + if u.passwordSet { + return u.password, true + } + return "", false +} + +// String returns the encoded userinfo information in the standard form +// of "username[:password]". +func (u *Userinfo) String() string { + s := escape(u.username, encodeUserPassword) + if u.passwordSet { + s += ":" + escape(u.password, encodeUserPassword) + } + return s +} + +// Maybe rawurl is of the form scheme:path. +// (Scheme must be [a-zA-Z][a-zA-Z0-9+-.]*) +// If so, return scheme, path; else return "", rawurl. +func getscheme(rawurl string) (scheme, path string, err error) { + for i := 0; i < len(rawurl); i++ { + c := rawurl[i] + switch { + case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z': + // do nothing + case '0' <= c && c <= '9' || c == '+' || c == '-' || c == '.': + if i == 0 { + return "", rawurl, nil + } + case c == ':': + if i == 0 { + return "", "", errors.New("missing protocol scheme") + } + return rawurl[0:i], rawurl[i+1:], nil + default: + // we have encountered an invalid character, + // so there is no valid scheme + return "", rawurl, nil + } + } + return "", rawurl, nil +} + +// Maybe s is of the form t c u. +// If so, return t, c u (or t, u if cutc == true). +// If not, return s, "". +func split(s string, c string, cutc bool) (string, string) { + i := strings.Index(s, c) + if i < 0 { + return s, "" + } + if cutc { + return s[0:i], s[i+len(c):] + } + return s[0:i], s[i:] +} + +// Parse parses rawurl into a URL structure. +// The rawurl may be relative or absolute. +func Parse(rawurl string) (url *URL, err error) { + // Cut off #frag + u, frag := split(rawurl, "#", true) + if url, err = parse(u, false); err != nil { + return nil, err + } + if frag == "" { + return url, nil + } + if url.Fragment, err = unescape(frag, encodeFragment); err != nil { + return nil, &Error{"parse", rawurl, err} + } + return url, nil +} + +// ParseRequestURI parses rawurl into a URL structure. It assumes that +// rawurl was received in an HTTP request, so the rawurl is interpreted +// only as an absolute URI or an absolute path. +// The string rawurl is assumed not to have a #fragment suffix. +// (Web browsers strip #fragment before sending the URL to a web server.) +func ParseRequestURI(rawurl string) (url *URL, err error) { + return parse(rawurl, true) +} + +// parse parses a URL from a string in one of two contexts. If +// viaRequest is true, the URL is assumed to have arrived via an HTTP request, +// in which case only absolute URLs or path-absolute relative URLs are allowed. +// If viaRequest is false, all forms of relative URLs are allowed. +func parse(rawurl string, viaRequest bool) (url *URL, err error) { + var rest string + + if rawurl == "" && viaRequest { + err = errors.New("empty url") + goto Error + } + url = new(URL) + + if rawurl == "*" { + url.Path = "*" + return + } + + // Split off possible leading "http:", "mailto:", etc. + // Cannot contain escaped characters. + if url.Scheme, rest, err = getscheme(rawurl); err != nil { + goto Error + } + url.Scheme = strings.ToLower(url.Scheme) + + rest, url.RawQuery = split(rest, "?", true) + + if !strings.HasPrefix(rest, "/") { + if url.Scheme != "" { + // We consider rootless paths per RFC 3986 as opaque. + url.Opaque = rest + return url, nil + } + if viaRequest { + err = errors.New("invalid URI for request") + goto Error + } + } + + if (url.Scheme != "" || !viaRequest && !strings.HasPrefix(rest, "///")) && strings.HasPrefix(rest, "//") { + var authority string + authority, rest = split(rest[2:], "/", false) + url.User, url.Host, err = parseAuthority(authority) + if err != nil { + goto Error + } + if strings.Contains(url.Host, "%") { + err = errors.New("hexadecimal escape in host") + goto Error + } + } + if url.Path, err = unescape(rest, encodePath); err != nil { + goto Error + } + return url, nil + +Error: + return nil, &Error{"parse", rawurl, err} +} + +func parseAuthority(authority string) (user *Userinfo, host string, err error) { + i := strings.LastIndex(authority, "@") + if i < 0 { + host = authority + return + } + userinfo, host := authority[:i], authority[i+1:] + if strings.Index(userinfo, ":") < 0 { + if userinfo, err = unescape(userinfo, encodeUserPassword); err != nil { + return + } + user = User(userinfo) + } else { + username, password := split(userinfo, ":", true) + if username, err = unescape(username, encodeUserPassword); err != nil { + return + } + if password, err = unescape(password, encodeUserPassword); err != nil { + return + } + user = UserPassword(username, password) + } + return +} + +// String reassembles the URL into a valid URL string. +func (u *URL) String() string { + var buf bytes.Buffer + if u.Scheme != "" { + buf.WriteString(u.Scheme) + buf.WriteByte(':') + } + if u.Opaque != "" { + buf.WriteString(u.Opaque) + } else { + if u.Scheme != "" || u.Host != "" || u.User != nil { + buf.WriteString("//") + if ui := u.User; ui != nil { + buf.WriteString(ui.String()) + buf.WriteByte('@') + } + if h := u.Host; h != "" { + buf.WriteString(h) + } + } + if u.Path != "" && u.Path[0] != '/' && u.Host != "" { + buf.WriteByte('/') + } + buf.WriteString(escape(u.Path, encodePath)) + } + if u.RawQuery != "" { + buf.WriteByte('?') + buf.WriteString(u.RawQuery) + } + if u.Fragment != "" { + buf.WriteByte('#') + buf.WriteString(escape(u.Fragment, encodeFragment)) + } + return buf.String() +} + +// Values maps a string key to a list of values. +// It is typically used for query parameters and form values. +// Unlike in the http.Header map, the keys in a Values map +// are case-sensitive. +type Values map[string][]string + +// Get gets the first value associated with the given key. +// If there are no values associated with the key, Get returns +// the empty string. To access multiple values, use the map +// directly. +func (v Values) Get(key string) string { + if v == nil { + return "" + } + vs, ok := v[key] + if !ok || len(vs) == 0 { + return "" + } + return vs[0] +} + +// Set sets the key to value. It replaces any existing +// values. +func (v Values) Set(key, value string) { + v[key] = []string{value} +} + +// Add adds the value to key. It appends to any existing +// values associated with key. +func (v Values) Add(key, value string) { + v[key] = append(v[key], value) +} + +// Del deletes the values associated with key. +func (v Values) Del(key string) { + delete(v, key) +} + +// ParseQuery parses the URL-encoded query string and returns +// a map listing the values specified for each key. +// ParseQuery always returns a non-nil map containing all the +// valid query parameters found; err describes the first decoding error +// encountered, if any. +func ParseQuery(query string) (m Values, err error) { + m = make(Values) + err = parseQuery(m, query) + return +} + +func parseQuery(m Values, query string) (err error) { + for query != "" { + key := query + if i := strings.IndexAny(key, "&;"); i >= 0 { + key, query = key[:i], key[i+1:] + } else { + query = "" + } + if key == "" { + continue + } + value := "" + if i := strings.Index(key, "="); i >= 0 { + key, value = key[:i], key[i+1:] + } + key, err1 := QueryUnescape(key) + if err1 != nil { + if err == nil { + err = err1 + } + continue + } + value, err1 = QueryUnescape(value) + if err1 != nil { + if err == nil { + err = err1 + } + continue + } + m[key] = append(m[key], value) + } + return err +} + +// Encode encodes the values into ``URL encoded'' form +// ("bar=baz&foo=quux") sorted by key. +func (v Values) Encode() string { + if v == nil { + return "" + } + var buf bytes.Buffer + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := v[k] + prefix := QueryEscape(k) + "=" + for _, v := range vs { + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(prefix) + buf.WriteString(QueryEscape(v)) + } + } + return buf.String() +} + +// resolvePath applies special path segments from refs and applies +// them to base, per RFC 3986. +func resolvePath(base, ref string) string { + var full string + if ref == "" { + full = base + } else if ref[0] != '/' { + i := strings.LastIndex(base, "/") + full = base[:i+1] + ref + } else { + full = ref + } + if full == "" { + return "" + } + var dst []string + src := strings.Split(full, "/") + for _, elem := range src { + switch elem { + case ".": + // drop + case "..": + if len(dst) > 0 { + dst = dst[:len(dst)-1] + } + default: + dst = append(dst, elem) + } + } + if last := src[len(src)-1]; last == "." || last == ".." { + // Add final slash to the joined path. + dst = append(dst, "") + } + return "/" + strings.TrimLeft(strings.Join(dst, "/"), "/") +} + +// IsAbs returns true if the URL is absolute. +func (u *URL) IsAbs() bool { + return u.Scheme != "" +} + +// Parse parses a URL in the context of the receiver. The provided URL +// may be relative or absolute. Parse returns nil, err on parse +// failure, otherwise its return value is the same as ResolveReference. +func (u *URL) Parse(ref string) (*URL, error) { + refurl, err := Parse(ref) + if err != nil { + return nil, err + } + return u.ResolveReference(refurl), nil +} + +// ResolveReference resolves a URI reference to an absolute URI from +// an absolute base URI, per RFC 3986 Section 5.2. The URI reference +// may be relative or absolute. ResolveReference always returns a new +// URL instance, even if the returned URL is identical to either the +// base or reference. If ref is an absolute URL, then ResolveReference +// ignores base and returns a copy of ref. +func (u *URL) ResolveReference(ref *URL) *URL { + url := *ref + if ref.Scheme == "" { + url.Scheme = u.Scheme + } + if ref.Scheme != "" || ref.Host != "" || ref.User != nil { + // The "absoluteURI" or "net_path" cases. + url.Path = resolvePath(ref.Path, "") + return &url + } + if ref.Opaque != "" { + url.User = nil + url.Host = "" + url.Path = "" + return &url + } + if ref.Path == "" { + if ref.RawQuery == "" { + url.RawQuery = u.RawQuery + if ref.Fragment == "" { + url.Fragment = u.Fragment + } + } + } + // The "abs_path" or "rel_path" cases. + url.Host = u.Host + url.User = u.User + url.Path = resolvePath(u.Path, ref.Path) + return &url +} + +// Query parses RawQuery and returns the corresponding values. +func (u *URL) Query() Values { + v, _ := ParseQuery(u.RawQuery) + return v +} + +// RequestURI returns the encoded path?query or opaque?query +// string that would be used in an HTTP request for u. +func (u *URL) RequestURI() string { + result := u.Opaque + if result == "" { + result = escape(u.Path, encodePath) + if result == "" { + result = "/" + } + } else { + if strings.HasPrefix(result, "//") { + result = u.Scheme + ":" + result + } + } + if u.RawQuery != "" { + result += "?" + u.RawQuery + } + return result +} diff --git a/src/net/url/url_test.go b/src/net/url/url_test.go new file mode 100644 index 000000000..d8b19d805 --- /dev/null +++ b/src/net/url/url_test.go @@ -0,0 +1,961 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package url + +import ( + "fmt" + "reflect" + "strings" + "testing" +) + +type URLTest struct { + in string + out *URL + roundtrip string // expected result of reserializing the URL; empty means same as "in". +} + +var urltests = []URLTest{ + // no path + { + "http://www.google.com", + &URL{ + Scheme: "http", + Host: "www.google.com", + }, + "", + }, + // path + { + "http://www.google.com/", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + }, + "", + }, + // path with hex escaping + { + "http://www.google.com/file%20one%26two", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/file one&two", + }, + "http://www.google.com/file%20one&two", + }, + // user + { + "ftp://webmaster@www.google.com/", + &URL{ + Scheme: "ftp", + User: User("webmaster"), + Host: "www.google.com", + Path: "/", + }, + "", + }, + // escape sequence in username + { + "ftp://john%20doe@www.google.com/", + &URL{ + Scheme: "ftp", + User: User("john doe"), + Host: "www.google.com", + Path: "/", + }, + "ftp://john%20doe@www.google.com/", + }, + // query + { + "http://www.google.com/?q=go+language", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + RawQuery: "q=go+language", + }, + "", + }, + // query with hex escaping: NOT parsed + { + "http://www.google.com/?q=go%20language", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + RawQuery: "q=go%20language", + }, + "", + }, + // %20 outside query + { + "http://www.google.com/a%20b?q=c+d", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/a b", + RawQuery: "q=c+d", + }, + "", + }, + // path without leading /, so no parsing + { + "http:www.google.com/?q=go+language", + &URL{ + Scheme: "http", + Opaque: "www.google.com/", + RawQuery: "q=go+language", + }, + "http:www.google.com/?q=go+language", + }, + // path without leading /, so no parsing + { + "http:%2f%2fwww.google.com/?q=go+language", + &URL{ + Scheme: "http", + Opaque: "%2f%2fwww.google.com/", + RawQuery: "q=go+language", + }, + "http:%2f%2fwww.google.com/?q=go+language", + }, + // non-authority with path + { + "mailto:/webmaster@golang.org", + &URL{ + Scheme: "mailto", + Path: "/webmaster@golang.org", + }, + "mailto:///webmaster@golang.org", // unfortunate compromise + }, + // non-authority + { + "mailto:webmaster@golang.org", + &URL{ + Scheme: "mailto", + Opaque: "webmaster@golang.org", + }, + "", + }, + // unescaped :// in query should not create a scheme + { + "/foo?query=http://bad", + &URL{ + Path: "/foo", + RawQuery: "query=http://bad", + }, + "", + }, + // leading // without scheme should create an authority + { + "//foo", + &URL{ + Host: "foo", + }, + "", + }, + // leading // without scheme, with userinfo, path, and query + { + "//user@foo/path?a=b", + &URL{ + User: User("user"), + Host: "foo", + Path: "/path", + RawQuery: "a=b", + }, + "", + }, + // Three leading slashes isn't an authority, but doesn't return an error. + // (We can't return an error, as this code is also used via + // ServeHTTP -> ReadRequest -> Parse, which is arguably a + // different URL parsing context, but currently shares the + // same codepath) + { + "///threeslashes", + &URL{ + Path: "///threeslashes", + }, + "", + }, + { + "http://user:password@google.com", + &URL{ + Scheme: "http", + User: UserPassword("user", "password"), + Host: "google.com", + }, + "http://user:password@google.com", + }, + // unescaped @ in username should not confuse host + { + "http://j@ne:password@google.com", + &URL{ + Scheme: "http", + User: UserPassword("j@ne", "password"), + Host: "google.com", + }, + "http://j%40ne:password@google.com", + }, + // unescaped @ in password should not confuse host + { + "http://jane:p@ssword@google.com", + &URL{ + Scheme: "http", + User: UserPassword("jane", "p@ssword"), + Host: "google.com", + }, + "http://jane:p%40ssword@google.com", + }, + { + "http://j@ne:password@google.com/p@th?q=@go", + &URL{ + Scheme: "http", + User: UserPassword("j@ne", "password"), + Host: "google.com", + Path: "/p@th", + RawQuery: "q=@go", + }, + "http://j%40ne:password@google.com/p@th?q=@go", + }, + { + "http://www.google.com/?q=go+language#foo", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + RawQuery: "q=go+language", + Fragment: "foo", + }, + "", + }, + { + "http://www.google.com/?q=go+language#foo%26bar", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + RawQuery: "q=go+language", + Fragment: "foo&bar", + }, + "http://www.google.com/?q=go+language#foo&bar", + }, + { + "file:///home/adg/rabbits", + &URL{ + Scheme: "file", + Host: "", + Path: "/home/adg/rabbits", + }, + "file:///home/adg/rabbits", + }, + // "Windows" paths are no exception to the rule. + // See golang.org/issue/6027, especially comment #9. + { + "file:///C:/FooBar/Baz.txt", + &URL{ + Scheme: "file", + Host: "", + Path: "/C:/FooBar/Baz.txt", + }, + "file:///C:/FooBar/Baz.txt", + }, + // case-insensitive scheme + { + "MaIlTo:webmaster@golang.org", + &URL{ + Scheme: "mailto", + Opaque: "webmaster@golang.org", + }, + "mailto:webmaster@golang.org", + }, + // Relative path + { + "a/b/c", + &URL{ + Path: "a/b/c", + }, + "a/b/c", + }, + // escaped '?' in username and password + { + "http://%3Fam:pa%3Fsword@google.com", + &URL{ + Scheme: "http", + User: UserPassword("?am", "pa?sword"), + Host: "google.com", + }, + "", + }, +} + +// more useful string for debugging than fmt's struct printer +func ufmt(u *URL) string { + var user, pass interface{} + if u.User != nil { + user = u.User.Username() + if p, ok := u.User.Password(); ok { + pass = p + } + } + return fmt.Sprintf("opaque=%q, scheme=%q, user=%#v, pass=%#v, host=%q, path=%q, rawq=%q, frag=%q", + u.Opaque, u.Scheme, user, pass, u.Host, u.Path, u.RawQuery, u.Fragment) +} + +func DoTest(t *testing.T, parse func(string) (*URL, error), name string, tests []URLTest) { + for _, tt := range tests { + u, err := parse(tt.in) + if err != nil { + t.Errorf("%s(%q) returned error %s", name, tt.in, err) + continue + } + if !reflect.DeepEqual(u, tt.out) { + t.Errorf("%s(%q):\n\thave %v\n\twant %v\n", + name, tt.in, ufmt(u), ufmt(tt.out)) + } + } +} + +func BenchmarkString(b *testing.B) { + b.StopTimer() + b.ReportAllocs() + for _, tt := range urltests { + u, err := Parse(tt.in) + if err != nil { + b.Errorf("Parse(%q) returned error %s", tt.in, err) + continue + } + if tt.roundtrip == "" { + continue + } + b.StartTimer() + var g string + for i := 0; i < b.N; i++ { + g = u.String() + } + b.StopTimer() + if w := tt.roundtrip; g != w { + b.Errorf("Parse(%q).String() == %q, want %q", tt.in, g, w) + } + } +} + +func TestParse(t *testing.T) { + DoTest(t, Parse, "Parse", urltests) +} + +const pathThatLooksSchemeRelative = "//not.a.user@not.a.host/just/a/path" + +var parseRequestURLTests = []struct { + url string + expectedValid bool +}{ + {"http://foo.com", true}, + {"http://foo.com/", true}, + {"http://foo.com/path", true}, + {"/", true}, + {pathThatLooksSchemeRelative, true}, + {"//not.a.user@%66%6f%6f.com/just/a/path/also", true}, + {"foo.html", false}, + {"../dir/", false}, + {"*", true}, +} + +func TestParseRequestURI(t *testing.T) { + for _, test := range parseRequestURLTests { + _, err := ParseRequestURI(test.url) + valid := err == nil + if valid != test.expectedValid { + t.Errorf("Expected valid=%v for %q; got %v", test.expectedValid, test.url, valid) + } + } + + url, err := ParseRequestURI(pathThatLooksSchemeRelative) + if err != nil { + t.Fatalf("Unexpected error %v", err) + } + if url.Path != pathThatLooksSchemeRelative { + t.Errorf("Expected path %q; got %q", pathThatLooksSchemeRelative, url.Path) + } +} + +func DoTestString(t *testing.T, parse func(string) (*URL, error), name string, tests []URLTest) { + for _, tt := range tests { + u, err := parse(tt.in) + if err != nil { + t.Errorf("%s(%q) returned error %s", name, tt.in, err) + continue + } + expected := tt.in + if len(tt.roundtrip) > 0 { + expected = tt.roundtrip + } + s := u.String() + if s != expected { + t.Errorf("%s(%q).String() == %q (expected %q)", name, tt.in, s, expected) + } + } +} + +func TestURLString(t *testing.T) { + DoTestString(t, Parse, "Parse", urltests) + + // no leading slash on path should prepend + // slash on String() call + noslash := URLTest{ + "http://www.google.com/search", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "search", + }, + "", + } + s := noslash.out.String() + if s != noslash.in { + t.Errorf("Expected %s; go %s", noslash.in, s) + } +} + +type EscapeTest struct { + in string + out string + err error +} + +var unescapeTests = []EscapeTest{ + { + "", + "", + nil, + }, + { + "abc", + "abc", + nil, + }, + { + "1%41", + "1A", + nil, + }, + { + "1%41%42%43", + "1ABC", + nil, + }, + { + "%4a", + "J", + nil, + }, + { + "%6F", + "o", + nil, + }, + { + "%", // not enough characters after % + "", + EscapeError("%"), + }, + { + "%a", // not enough characters after % + "", + EscapeError("%a"), + }, + { + "%1", // not enough characters after % + "", + EscapeError("%1"), + }, + { + "123%45%6", // not enough characters after % + "", + EscapeError("%6"), + }, + { + "%zzzzz", // invalid hex digits + "", + EscapeError("%zz"), + }, +} + +func TestUnescape(t *testing.T) { + for _, tt := range unescapeTests { + actual, err := QueryUnescape(tt.in) + if actual != tt.out || (err != nil) != (tt.err != nil) { + t.Errorf("QueryUnescape(%q) = %q, %s; want %q, %s", tt.in, actual, err, tt.out, tt.err) + } + } +} + +var escapeTests = []EscapeTest{ + { + "", + "", + nil, + }, + { + "abc", + "abc", + nil, + }, + { + "one two", + "one+two", + nil, + }, + { + "10%", + "10%25", + nil, + }, + { + " ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;", + "+%3F%26%3D%23%2B%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%E2%98%BA%09%3A%2F%40%24%27%28%29%2A%2C%3B", + nil, + }, +} + +func TestEscape(t *testing.T) { + for _, tt := range escapeTests { + actual := QueryEscape(tt.in) + if tt.out != actual { + t.Errorf("QueryEscape(%q) = %q, want %q", tt.in, actual, tt.out) + } + + // for bonus points, verify that escape:unescape is an identity. + roundtrip, err := QueryUnescape(actual) + if roundtrip != tt.in || err != nil { + t.Errorf("QueryUnescape(%q) = %q, %s; want %q, %s", actual, roundtrip, err, tt.in, "[no error]") + } + } +} + +//var userinfoTests = []UserinfoTest{ +// {"user", "password", "user:password"}, +// {"foo:bar", "~!@#$%^&*()_+{}|[]\\-=`:;'\"<>?,./", +// "foo%3Abar:~!%40%23$%25%5E&*()_+%7B%7D%7C%5B%5D%5C-=%60%3A;'%22%3C%3E?,.%2F"}, +//} + +type EncodeQueryTest struct { + m Values + expected string +} + +var encodeQueryTests = []EncodeQueryTest{ + {nil, ""}, + {Values{"q": {"puppies"}, "oe": {"utf8"}}, "oe=utf8&q=puppies"}, + {Values{"q": {"dogs", "&", "7"}}, "q=dogs&q=%26&q=7"}, + {Values{ + "a": {"a1", "a2", "a3"}, + "b": {"b1", "b2", "b3"}, + "c": {"c1", "c2", "c3"}, + }, "a=a1&a=a2&a=a3&b=b1&b=b2&b=b3&c=c1&c=c2&c=c3"}, +} + +func TestEncodeQuery(t *testing.T) { + for _, tt := range encodeQueryTests { + if q := tt.m.Encode(); q != tt.expected { + t.Errorf(`EncodeQuery(%+v) = %q, want %q`, tt.m, q, tt.expected) + } + } +} + +var resolvePathTests = []struct { + base, ref, expected string +}{ + {"a/b", ".", "/a/"}, + {"a/b", "c", "/a/c"}, + {"a/b", "..", "/"}, + {"a/", "..", "/"}, + {"a/", "../..", "/"}, + {"a/b/c", "..", "/a/"}, + {"a/b/c", "../d", "/a/d"}, + {"a/b/c", ".././d", "/a/d"}, + {"a/b", "./..", "/"}, + {"a/./b", ".", "/a/"}, + {"a/../", ".", "/"}, + {"a/.././b", "c", "/c"}, +} + +func TestResolvePath(t *testing.T) { + for _, test := range resolvePathTests { + got := resolvePath(test.base, test.ref) + if got != test.expected { + t.Errorf("For %q + %q got %q; expected %q", test.base, test.ref, got, test.expected) + } + } +} + +var resolveReferenceTests = []struct { + base, rel, expected string +}{ + // Absolute URL references + {"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"}, + {"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/?a=b"}, + {"http://foo.com/bar", "mailto:foo@example.com", "mailto:foo@example.com"}, + + // Path-absolute references + {"http://foo.com/bar", "/baz", "http://foo.com/baz"}, + {"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"}, + {"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz?c=d"}, + + // Scheme-relative + {"https://foo.com/bar?a=b", "//bar.com/quux", "https://bar.com/quux"}, + + // Path-relative references: + + // ... current directory + {"http://foo.com", ".", "http://foo.com/"}, + {"http://foo.com/bar", ".", "http://foo.com/"}, + {"http://foo.com/bar/", ".", "http://foo.com/bar/"}, + + // ... going down + {"http://foo.com", "bar", "http://foo.com/bar"}, + {"http://foo.com/", "bar", "http://foo.com/bar"}, + {"http://foo.com/bar/baz", "quux", "http://foo.com/bar/quux"}, + + // ... going up + {"http://foo.com/bar/baz", "../quux", "http://foo.com/quux"}, + {"http://foo.com/bar/baz", "../../../../../quux", "http://foo.com/quux"}, + {"http://foo.com/bar", "..", "http://foo.com/"}, + {"http://foo.com/bar/baz", "./..", "http://foo.com/"}, + // ".." in the middle (issue 3560) + {"http://foo.com/bar/baz", "quux/dotdot/../tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/../tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/.././tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/./../tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/././../../tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/./.././../tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/dotdot/dotdot/./../../.././././tail", "http://foo.com/bar/quux/tail"}, + {"http://foo.com/bar/baz", "quux/./dotdot/../dotdot/../dot/./tail/..", "http://foo.com/bar/quux/dot/"}, + + // Remove any dot-segments prior to forming the target URI. + // http://tools.ietf.org/html/rfc3986#section-5.2.4 + {"http://foo.com/dot/./dotdot/../foo/bar", "../baz", "http://foo.com/dot/baz"}, + + // Triple dot isn't special + {"http://foo.com/bar", "...", "http://foo.com/..."}, + + // Fragment + {"http://foo.com/bar", ".#frag", "http://foo.com/#frag"}, + + // RFC 3986: Normal Examples + // http://tools.ietf.org/html/rfc3986#section-5.4.1 + {"http://a/b/c/d;p?q", "g:h", "g:h"}, + {"http://a/b/c/d;p?q", "g", "http://a/b/c/g"}, + {"http://a/b/c/d;p?q", "./g", "http://a/b/c/g"}, + {"http://a/b/c/d;p?q", "g/", "http://a/b/c/g/"}, + {"http://a/b/c/d;p?q", "/g", "http://a/g"}, + {"http://a/b/c/d;p?q", "//g", "http://g"}, + {"http://a/b/c/d;p?q", "?y", "http://a/b/c/d;p?y"}, + {"http://a/b/c/d;p?q", "g?y", "http://a/b/c/g?y"}, + {"http://a/b/c/d;p?q", "#s", "http://a/b/c/d;p?q#s"}, + {"http://a/b/c/d;p?q", "g#s", "http://a/b/c/g#s"}, + {"http://a/b/c/d;p?q", "g?y#s", "http://a/b/c/g?y#s"}, + {"http://a/b/c/d;p?q", ";x", "http://a/b/c/;x"}, + {"http://a/b/c/d;p?q", "g;x", "http://a/b/c/g;x"}, + {"http://a/b/c/d;p?q", "g;x?y#s", "http://a/b/c/g;x?y#s"}, + {"http://a/b/c/d;p?q", "", "http://a/b/c/d;p?q"}, + {"http://a/b/c/d;p?q", ".", "http://a/b/c/"}, + {"http://a/b/c/d;p?q", "./", "http://a/b/c/"}, + {"http://a/b/c/d;p?q", "..", "http://a/b/"}, + {"http://a/b/c/d;p?q", "../", "http://a/b/"}, + {"http://a/b/c/d;p?q", "../g", "http://a/b/g"}, + {"http://a/b/c/d;p?q", "../..", "http://a/"}, + {"http://a/b/c/d;p?q", "../../", "http://a/"}, + {"http://a/b/c/d;p?q", "../../g", "http://a/g"}, + + // RFC 3986: Abnormal Examples + // http://tools.ietf.org/html/rfc3986#section-5.4.2 + {"http://a/b/c/d;p?q", "../../../g", "http://a/g"}, + {"http://a/b/c/d;p?q", "../../../../g", "http://a/g"}, + {"http://a/b/c/d;p?q", "/./g", "http://a/g"}, + {"http://a/b/c/d;p?q", "/../g", "http://a/g"}, + {"http://a/b/c/d;p?q", "g.", "http://a/b/c/g."}, + {"http://a/b/c/d;p?q", ".g", "http://a/b/c/.g"}, + {"http://a/b/c/d;p?q", "g..", "http://a/b/c/g.."}, + {"http://a/b/c/d;p?q", "..g", "http://a/b/c/..g"}, + {"http://a/b/c/d;p?q", "./../g", "http://a/b/g"}, + {"http://a/b/c/d;p?q", "./g/.", "http://a/b/c/g/"}, + {"http://a/b/c/d;p?q", "g/./h", "http://a/b/c/g/h"}, + {"http://a/b/c/d;p?q", "g/../h", "http://a/b/c/h"}, + {"http://a/b/c/d;p?q", "g;x=1/./y", "http://a/b/c/g;x=1/y"}, + {"http://a/b/c/d;p?q", "g;x=1/../y", "http://a/b/c/y"}, + {"http://a/b/c/d;p?q", "g?y/./x", "http://a/b/c/g?y/./x"}, + {"http://a/b/c/d;p?q", "g?y/../x", "http://a/b/c/g?y/../x"}, + {"http://a/b/c/d;p?q", "g#s/./x", "http://a/b/c/g#s/./x"}, + {"http://a/b/c/d;p?q", "g#s/../x", "http://a/b/c/g#s/../x"}, + + // Extras. + {"https://a/b/c/d;p?q", "//g?q", "https://g?q"}, + {"https://a/b/c/d;p?q", "//g#s", "https://g#s"}, + {"https://a/b/c/d;p?q", "//g/d/e/f?y#s", "https://g/d/e/f?y#s"}, + {"https://a/b/c/d;p#s", "?y", "https://a/b/c/d;p?y"}, + {"https://a/b/c/d;p?q#s", "?y", "https://a/b/c/d;p?y"}, +} + +func TestResolveReference(t *testing.T) { + mustParse := func(url string) *URL { + u, err := Parse(url) + if err != nil { + t.Fatalf("Expected URL to parse: %q, got error: %v", url, err) + } + return u + } + opaque := &URL{Scheme: "scheme", Opaque: "opaque"} + for _, test := range resolveReferenceTests { + base := mustParse(test.base) + rel := mustParse(test.rel) + url := base.ResolveReference(rel) + if url.String() != test.expected { + t.Errorf("URL(%q).ResolveReference(%q) == %q, got %q", test.base, test.rel, test.expected, url.String()) + } + // Ensure that new instances are returned. + if base == url { + t.Errorf("Expected URL.ResolveReference to return new URL instance.") + } + // Test the convenience wrapper too. + url, err := base.Parse(test.rel) + if err != nil { + t.Errorf("URL(%q).Parse(%q) failed: %v", test.base, test.rel, err) + } else if url.String() != test.expected { + t.Errorf("URL(%q).Parse(%q) == %q, got %q", test.base, test.rel, test.expected, url.String()) + } else if base == url { + // Ensure that new instances are returned for the wrapper too. + t.Errorf("Expected URL.Parse to return new URL instance.") + } + // Ensure Opaque resets the URL. + url = base.ResolveReference(opaque) + if *url != *opaque { + t.Errorf("ResolveReference failed to resolve opaque URL: want %#v, got %#v", url, opaque) + } + // Test the convenience wrapper with an opaque URL too. + url, err = base.Parse("scheme:opaque") + if err != nil { + t.Errorf(`URL(%q).Parse("scheme:opaque") failed: %v`, test.base, err) + } else if *url != *opaque { + t.Errorf("Parse failed to resolve opaque URL: want %#v, got %#v", url, opaque) + } else if base == url { + // Ensure that new instances are returned, again. + t.Errorf("Expected URL.Parse to return new URL instance.") + } + } +} + +func TestQueryValues(t *testing.T) { + u, _ := Parse("http://x.com?foo=bar&bar=1&bar=2") + v := u.Query() + if len(v) != 2 { + t.Errorf("got %d keys in Query values, want 2", len(v)) + } + if g, e := v.Get("foo"), "bar"; g != e { + t.Errorf("Get(foo) = %q, want %q", g, e) + } + // Case sensitive: + if g, e := v.Get("Foo"), ""; g != e { + t.Errorf("Get(Foo) = %q, want %q", g, e) + } + if g, e := v.Get("bar"), "1"; g != e { + t.Errorf("Get(bar) = %q, want %q", g, e) + } + if g, e := v.Get("baz"), ""; g != e { + t.Errorf("Get(baz) = %q, want %q", g, e) + } + v.Del("bar") + if g, e := v.Get("bar"), ""; g != e { + t.Errorf("second Get(bar) = %q, want %q", g, e) + } +} + +type parseTest struct { + query string + out Values +} + +var parseTests = []parseTest{ + { + query: "a=1&b=2", + out: Values{"a": []string{"1"}, "b": []string{"2"}}, + }, + { + query: "a=1&a=2&a=banana", + out: Values{"a": []string{"1", "2", "banana"}}, + }, + { + query: "ascii=%3Ckey%3A+0x90%3E", + out: Values{"ascii": []string{"<key: 0x90>"}}, + }, + { + query: "a=1;b=2", + out: Values{"a": []string{"1"}, "b": []string{"2"}}, + }, + { + query: "a=1&a=2;a=banana", + out: Values{"a": []string{"1", "2", "banana"}}, + }, +} + +func TestParseQuery(t *testing.T) { + for i, test := range parseTests { + form, err := ParseQuery(test.query) + if err != nil { + t.Errorf("test %d: Unexpected error: %v", i, err) + continue + } + if len(form) != len(test.out) { + t.Errorf("test %d: len(form) = %d, want %d", i, len(form), len(test.out)) + } + for k, evs := range test.out { + vs, ok := form[k] + if !ok { + t.Errorf("test %d: Missing key %q", i, k) + continue + } + if len(vs) != len(evs) { + t.Errorf("test %d: len(form[%q]) = %d, want %d", i, k, len(vs), len(evs)) + continue + } + for j, ev := range evs { + if v := vs[j]; v != ev { + t.Errorf("test %d: form[%q][%d] = %q, want %q", i, k, j, v, ev) + } + } + } + } +} + +type RequestURITest struct { + url *URL + out string +} + +var requritests = []RequestURITest{ + { + &URL{ + Scheme: "http", + Host: "example.com", + Path: "", + }, + "/", + }, + { + &URL{ + Scheme: "http", + Host: "example.com", + Path: "/a b", + }, + "/a%20b", + }, + // golang.org/issue/4860 variant 1 + { + &URL{ + Scheme: "http", + Host: "example.com", + Opaque: "/%2F/%2F/", + }, + "/%2F/%2F/", + }, + // golang.org/issue/4860 variant 2 + { + &URL{ + Scheme: "http", + Host: "example.com", + Opaque: "//other.example.com/%2F/%2F/", + }, + "http://other.example.com/%2F/%2F/", + }, + { + &URL{ + Scheme: "http", + Host: "example.com", + Path: "/a b", + RawQuery: "q=go+language", + }, + "/a%20b?q=go+language", + }, + { + &URL{ + Scheme: "myschema", + Opaque: "opaque", + }, + "opaque", + }, + { + &URL{ + Scheme: "myschema", + Opaque: "opaque", + RawQuery: "q=go+language", + }, + "opaque?q=go+language", + }, +} + +func TestRequestURI(t *testing.T) { + for _, tt := range requritests { + s := tt.url.RequestURI() + if s != tt.out { + t.Errorf("%#v.RequestURI() == %q (expected %q)", tt.url, s, tt.out) + } + } +} + +func TestParseFailure(t *testing.T) { + // Test that the first parse error is returned. + const url = "%gh&%ij" + _, err := ParseQuery(url) + errStr := fmt.Sprint(err) + if !strings.Contains(errStr, "%gh") { + t.Errorf(`ParseQuery(%q) returned error %q, want something containing %q"`, url, errStr, "%gh") + } +} + +type shouldEscapeTest struct { + in byte + mode encoding + escape bool +} + +var shouldEscapeTests = []shouldEscapeTest{ + // Unreserved characters (§2.3) + {'a', encodePath, false}, + {'a', encodeUserPassword, false}, + {'a', encodeQueryComponent, false}, + {'a', encodeFragment, false}, + {'z', encodePath, false}, + {'A', encodePath, false}, + {'Z', encodePath, false}, + {'0', encodePath, false}, + {'9', encodePath, false}, + {'-', encodePath, false}, + {'-', encodeUserPassword, false}, + {'-', encodeQueryComponent, false}, + {'-', encodeFragment, false}, + {'.', encodePath, false}, + {'_', encodePath, false}, + {'~', encodePath, false}, + + // User information (§3.2.1) + {':', encodeUserPassword, true}, + {'/', encodeUserPassword, true}, + {'?', encodeUserPassword, true}, + {'@', encodeUserPassword, true}, + {'$', encodeUserPassword, false}, + {'&', encodeUserPassword, false}, + {'+', encodeUserPassword, false}, + {',', encodeUserPassword, false}, + {';', encodeUserPassword, false}, + {'=', encodeUserPassword, false}, +} + +func TestShouldEscape(t *testing.T) { + for _, tt := range shouldEscapeTests { + if shouldEscape(tt.in, tt.mode) != tt.escape { + t.Errorf("shouldEscape(%q, %v) returned %v; expected %v", tt.in, tt.mode, !tt.escape, tt.escape) + } + } +} diff --git a/src/net/z_last_test.go b/src/net/z_last_test.go new file mode 100644 index 000000000..4f6a54a56 --- /dev/null +++ b/src/net/z_last_test.go @@ -0,0 +1,37 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "flag" + "fmt" + "testing" +) + +var testDNSFlood = flag.Bool("dnsflood", false, "whether to test dns query flooding") + +func TestDNSThreadLimit(t *testing.T) { + if !*testDNSFlood { + t.Skip("test disabled; use -dnsflood to enable") + } + + const N = 10000 + c := make(chan int, N) + for i := 0; i < N; i++ { + go func(i int) { + LookupIP(fmt.Sprintf("%d.net-test.golang.org", i)) + c <- 1 + }(i) + } + // Don't bother waiting for the stragglers; stop at 0.9 N. + for i := 0; i < N*9/10; i++ { + if i%100 == 0 { + //println("TestDNSThreadLimit:", i) + } + <-c + } + + // If we're still here, it worked. +} |