diff options
Diffstat (limited to 'libgo/go/net/http/transfer.go')
-rw-r--r-- | libgo/go/net/http/transfer.go | 256 |
1 files changed, 192 insertions, 64 deletions
diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index 6e59af8f6f..4f47637aa7 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -17,6 +17,9 @@ import ( "strconv" "strings" "sync" + "time" + + "golang_org/x/net/lex/httplex" ) // ErrLineTooLong is returned when reading request or response bodies @@ -31,6 +34,23 @@ func (r errorReader) Read(p []byte) (n int, err error) { return 0, r.err } +type byteReader struct { + b byte + done bool +} + +func (br *byteReader) Read(p []byte) (n int, err error) { + if br.done { + return 0, io.EOF + } + if len(p) == 0 { + return 0, nil + } + br.done = true + p[0] = br.b + return 1, io.EOF +} + // transferWriter inspects the fields of a user-supplied Request or Response, // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. @@ -44,6 +64,9 @@ type transferWriter struct { TransferEncoding []string Trailer Header IsResponse bool + + FlushHeaders bool // flush headers to network before body + ByteReadCh chan readResult // non-nil if probeRequestBody called } func newTransferWriter(r interface{}) (t *transferWriter, err error) { @@ -57,37 +80,15 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) } t.Method = valueOrDefault(rr.Method, "GET") - t.Body = rr.Body - t.BodyCloser = rr.Body - t.ContentLength = rr.ContentLength t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer - atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 { - if t.ContentLength == 0 { - // Test to see if it's actually zero or just unset. - var buf [1]byte - n, rerr := io.ReadFull(t.Body, buf[:]) - if rerr != nil && rerr != io.EOF { - t.ContentLength = -1 - t.Body = errorReader{rerr} - } else if n == 1 { - // Oh, guess there is data in this Body Reader after all. - // The ContentLength field just wasn't set. - // Stich the Body back together again, re-attaching our - // consumed byte. - t.ContentLength = -1 - t.Body = io.MultiReader(bytes.NewReader(buf[:]), t.Body) - } else { - // Body is actually empty. - t.Body = nil - t.BodyCloser = nil - } - } - if t.ContentLength < 0 { - t.TransferEncoding = []string{"chunked"} - } + atLeastHTTP11 = rr.protoAtLeastOutgoing(1, 1) + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = rr.outgoingLength() + if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 && t.shouldSendChunkedRequestBody() { + t.TransferEncoding = []string{"chunked"} } case *Response: t.IsResponse = true @@ -101,7 +102,7 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - t.ResponseToHEAD = noBodyExpected(t.Method) + t.ResponseToHEAD = noResponseBodyExpected(t.Method) } // Sanitize Body,ContentLength,TransferEncoding @@ -129,7 +130,100 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { return t, nil } -func noBodyExpected(requestMethod string) bool { +// shouldSendChunkedRequestBody reports whether we should try to send a +// chunked request body to the server. In particular, the case we really +// want to prevent is sending a GET or other typically-bodyless request to a +// server with a chunked body when the body has zero bytes, since GETs with +// bodies (while acceptable according to specs), even zero-byte chunked +// bodies, are approximately never seen in the wild and confuse most +// servers. See Issue 18257, as one example. +// +// The only reason we'd send such a request is if the user set the Body to a +// non-nil value (say, ioutil.NopCloser(bytes.NewReader(nil))) and didn't +// set ContentLength, or NewRequest set it to -1 (unknown), so then we assume +// there's bytes to send. +// +// This code tries to read a byte from the Request.Body in such cases to see +// whether the body actually has content (super rare) or is actually just +// a non-nil content-less ReadCloser (the more common case). In that more +// common case, we act as if their Body were nil instead, and don't send +// a body. +func (t *transferWriter) shouldSendChunkedRequestBody() bool { + // Note that t.ContentLength is the corrected content length + // from rr.outgoingLength, so 0 actually means zero, not unknown. + if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them + return false + } + if requestMethodUsuallyLacksBody(t.Method) { + // Only probe the Request.Body for GET/HEAD/DELETE/etc + // requests, because it's only those types of requests + // that confuse servers. + t.probeRequestBody() // adjusts t.Body, t.ContentLength + return t.Body != nil + } + // For all other request types (PUT, POST, PATCH, or anything + // made-up we've never heard of), assume it's normal and the server + // can deal with a chunked request body. Maybe we'll adjust this + // later. + return true +} + +// probeRequestBody reads a byte from t.Body to see whether it's empty +// (returns io.EOF right away). +// +// But because we've had problems with this blocking users in the past +// (issue 17480) when the body is a pipe (perhaps waiting on the response +// headers before the pipe is fed data), we need to be careful and bound how +// long we wait for it. This delay will only affect users if all the following +// are true: +// * the request body blocks +// * the content length is not set (or set to -1) +// * the method doesn't usually have a body (GET, HEAD, DELETE, ...) +// * there is no transfer-encoding=chunked already set. +// In other words, this delay will not normally affect anybody, and there +// are workarounds if it does. +func (t *transferWriter) probeRequestBody() { + t.ByteReadCh = make(chan readResult, 1) + go func(body io.Reader) { + var buf [1]byte + var rres readResult + rres.n, rres.err = body.Read(buf[:]) + if rres.n == 1 { + rres.b = buf[0] + } + t.ByteReadCh <- rres + }(t.Body) + timer := time.NewTimer(200 * time.Millisecond) + select { + case rres := <-t.ByteReadCh: + timer.Stop() + if rres.n == 0 && rres.err == io.EOF { + // It was empty. + t.Body = nil + t.ContentLength = 0 + } else if rres.n == 1 { + if rres.err != nil { + t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) + } else { + t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) + } + } else if rres.err != nil { + t.Body = errorReader{rres.err} + } + case <-timer.C: + // Too slow. Don't wait. Read it later, and keep + // assuming that this is ContentLength == -1 + // (unknown), which means we'll send a + // "Transfer-Encoding: chunked" header. + t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) + // Request that Request.Write flush the headers to the + // network before writing the body, since our body may not + // become readable until it's seen the response headers. + t.FlushHeaders = true + } +} + +func noResponseBodyExpected(requestMethod string) bool { return requestMethod == "HEAD" } @@ -212,7 +306,7 @@ func (t *transferWriter) WriteBody(w io.Writer) error { if t.Body != nil { if chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { - w = &internal.FlushAfterChunkWriter{bw} + w = &internal.FlushAfterChunkWriter{Writer: bw} } cw := internal.NewChunkedWriter(w) _, err = io.Copy(cw, t.Body) @@ -233,7 +327,9 @@ func (t *transferWriter) WriteBody(w io.Writer) error { if err != nil { return err } - if err = t.BodyCloser.Close(); err != nil { + } + if t.BodyCloser != nil { + if err := t.BodyCloser.Close(); err != nil { return err } } @@ -276,7 +372,7 @@ func (t *transferReader) protoAtLeast(m, n int) bool { } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC2616, section 4.4. +// permits a body. See RFC 2616, section 4.4. func bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: @@ -368,7 +464,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { // If there is no Content-Length or chunked Transfer-Encoding on a *Response // and the status is not 1xx, 204 or 304, then the body is unbounded. - // See RFC2616, section 4.4. + // See RFC 2616, section 4.4. switch msg.(type) { case *Response: if realLength == -1 && @@ -379,17 +475,17 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { } } - // Prepare body reader. ContentLength < 0 means chunked encoding + // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { case chunked(t.TransferEncoding): - if noBodyExpected(t.RequestMethod) { - t.Body = eofReader + if noResponseBodyExpected(t.RequestMethod) { + t.Body = NoBody } else { t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} } case realLength == 0: - t.Body = eofReader + t.Body = NoBody case realLength > 0: t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} default: @@ -399,7 +495,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { t.Body = &body{src: r, closing: t.Close} } else { // Persistent connection (i.e. HTTP/1.1) - t.Body = eofReader + t.Body = NoBody } } @@ -491,10 +587,31 @@ func (t *transferReader) fixTransferEncoding() error { // function is not a method, because ultimately it should be shared by // ReadResponse and ReadRequest. func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { - contentLens := header["Content-Length"] isRequest := !isResponse + contentLens := header["Content-Length"] + + // Hardening against HTTP request smuggling + if len(contentLens) > 1 { + // Per RFC 7230 Section 3.3.2, prevent multiple + // Content-Length headers if they differ in value. + // If there are dups of the value, remove the dups. + // See Issue 16490. + first := strings.TrimSpace(contentLens[0]) + for _, ct := range contentLens[1:] { + if first != strings.TrimSpace(ct) { + return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) + } + } + + // deduplicate Content-Length + header.Del("Content-Length") + header.Add("Content-Length", first) + + contentLens = header["Content-Length"] + } + // Logic based on response type or status - if noBodyExpected(requestMethod) { + if noResponseBodyExpected(requestMethod) { // For HTTP requests, as part of hardening against request // smuggling (RFC 7230), don't allow a Content-Length header for // methods which don't permit bodies. As an exception, allow @@ -512,11 +629,6 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, return 0, nil } - if len(contentLens) > 1 { - // harden against HTTP request smuggling. See RFC 7230. - return 0, errors.New("http: message cannot contain multiple Content-Length headers") - } - // Logic based on Transfer-Encoding if chunked(te) { return -1, nil @@ -537,7 +649,7 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, header.Del("Content-Length") } - if !isResponse { + if isRequest { // RFC 2616 neither explicitly permits nor forbids an // entity-body on a GET request so we permit one if // declared, but we default to 0 here (not -1 below) @@ -558,21 +670,19 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { if major < 1 { return true - } else if major == 1 && minor == 0 { - vv := header["Connection"] - if headerValuesContainsToken(vv, "close") || !headerValuesContainsToken(vv, "keep-alive") { - return true - } - return false - } else { - if headerValuesContainsToken(header["Connection"], "close") { - if removeCloseHeader { - header.Del("Connection") - } - return true - } } - return false + + conv := header["Connection"] + hasClose := httplex.HeaderValuesContainsToken(conv, "close") + if major == 1 && minor == 0 { + return hasClose || !httplex.HeaderValuesContainsToken(conv, "keep-alive") + } + + if hasClose && removeCloseHeader { + header.Del("Connection") + } + + return hasClose } // Parse the trailer header @@ -729,11 +839,11 @@ func (b *body) readTrailer() error { } // Make sure there's a header terminator coming up, to prevent - // a DoS with an unbounded size Trailer. It's not easy to + // a DoS with an unbounded size Trailer. It's not easy to // slip in a LimitReader here, as textproto.NewReader requires - // a concrete *bufio.Reader. Also, we can't get all the way + // a concrete *bufio.Reader. Also, we can't get all the way // back up to our conn's LimitedReader that *might* be backing - // this bufio.Reader. Instead, a hack: we iteratively Peek up + // this bufio.Reader. Instead, a hack: we iteratively Peek up // to the bufio.Reader's max size, looking for a double CRLF. // This limits the trailer to the underlying buffer size, typically 4kB. if !seeUpcomingDoubleCRLF(b.r) { @@ -864,3 +974,21 @@ func parseContentLength(cl string) (int64, error) { return n, nil } + +// finishAsyncByteRead finishes reading the 1-byte sniff +// from the ContentLength==0, Body!=nil case. +type finishAsyncByteRead struct { + tw *transferWriter +} + +func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + rres := <-fr.tw.ByteReadCh + n, err = rres.n, rres.err + if n == 1 { + p[0] = rres.b + } + return +} |