Source file src/net/http/transfer.go

Documentation: net/http

		 1  // Copyright 2009 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  package http
		 6  
		 7  import (
		 8  	"bufio"
		 9  	"bytes"
		10  	"errors"
		11  	"fmt"
		12  	"io"
		13  	"net/http/httptrace"
		14  	"net/http/internal"
		15  	"net/http/internal/ascii"
		16  	"net/textproto"
		17  	"reflect"
		18  	"sort"
		19  	"strconv"
		20  	"strings"
		21  	"sync"
		22  	"time"
		23  
		24  	"golang.org/x/net/http/httpguts"
		25  )
		26  
		27  // ErrLineTooLong is returned when reading request or response bodies
		28  // with malformed chunked encoding.
		29  var ErrLineTooLong = internal.ErrLineTooLong
		30  
		31  type errorReader struct {
		32  	err error
		33  }
		34  
		35  func (r errorReader) Read(p []byte) (n int, err error) {
		36  	return 0, r.err
		37  }
		38  
		39  type byteReader struct {
		40  	b		byte
		41  	done bool
		42  }
		43  
		44  func (br *byteReader) Read(p []byte) (n int, err error) {
		45  	if br.done {
		46  		return 0, io.EOF
		47  	}
		48  	if len(p) == 0 {
		49  		return 0, nil
		50  	}
		51  	br.done = true
		52  	p[0] = br.b
		53  	return 1, io.EOF
		54  }
		55  
		56  // transferWriter inspects the fields of a user-supplied Request or Response,
		57  // sanitizes them without changing the user object and provides methods for
		58  // writing the respective header, body and trailer in wire format.
		59  type transferWriter struct {
		60  	Method					 string
		61  	Body						 io.Reader
		62  	BodyCloser			 io.Closer
		63  	ResponseToHEAD	 bool
		64  	ContentLength		int64 // -1 means unknown, 0 means exactly none
		65  	Close						bool
		66  	TransferEncoding []string
		67  	Header					 Header
		68  	Trailer					Header
		69  	IsResponse			 bool
		70  	bodyReadError		error // any non-EOF error from reading Body
		71  
		72  	FlushHeaders bool						// flush headers to network before body
		73  	ByteReadCh	 chan readResult // non-nil if probeRequestBody called
		74  }
		75  
		76  func newTransferWriter(r interface{}) (t *transferWriter, err error) {
		77  	t = &transferWriter{}
		78  
		79  	// Extract relevant fields
		80  	atLeastHTTP11 := false
		81  	switch rr := r.(type) {
		82  	case *Request:
		83  		if rr.ContentLength != 0 && rr.Body == nil {
		84  			return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength)
		85  		}
		86  		t.Method = valueOrDefault(rr.Method, "GET")
		87  		t.Close = rr.Close
		88  		t.TransferEncoding = rr.TransferEncoding
		89  		t.Header = rr.Header
		90  		t.Trailer = rr.Trailer
		91  		t.Body = rr.Body
		92  		t.BodyCloser = rr.Body
		93  		t.ContentLength = rr.outgoingLength()
		94  		if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && t.shouldSendChunkedRequestBody() {
		95  			t.TransferEncoding = []string{"chunked"}
		96  		}
		97  		// If there's a body, conservatively flush the headers
		98  		// to any bufio.Writer we're writing to, just in case
		99  		// the server needs the headers early, before we copy
	 100  		// the body and possibly block. We make an exception
	 101  		// for the common standard library in-memory types,
	 102  		// though, to avoid unnecessary TCP packets on the
	 103  		// wire. (Issue 22088.)
	 104  		if t.ContentLength != 0 && !isKnownInMemoryReader(t.Body) {
	 105  			t.FlushHeaders = true
	 106  		}
	 107  
	 108  		atLeastHTTP11 = true // Transport requests are always 1.1 or 2.0
	 109  	case *Response:
	 110  		t.IsResponse = true
	 111  		if rr.Request != nil {
	 112  			t.Method = rr.Request.Method
	 113  		}
	 114  		t.Body = rr.Body
	 115  		t.BodyCloser = rr.Body
	 116  		t.ContentLength = rr.ContentLength
	 117  		t.Close = rr.Close
	 118  		t.TransferEncoding = rr.TransferEncoding
	 119  		t.Header = rr.Header
	 120  		t.Trailer = rr.Trailer
	 121  		atLeastHTTP11 = rr.ProtoAtLeast(1, 1)
	 122  		t.ResponseToHEAD = noResponseBodyExpected(t.Method)
	 123  	}
	 124  
	 125  	// Sanitize Body,ContentLength,TransferEncoding
	 126  	if t.ResponseToHEAD {
	 127  		t.Body = nil
	 128  		if chunked(t.TransferEncoding) {
	 129  			t.ContentLength = -1
	 130  		}
	 131  	} else {
	 132  		if !atLeastHTTP11 || t.Body == nil {
	 133  			t.TransferEncoding = nil
	 134  		}
	 135  		if chunked(t.TransferEncoding) {
	 136  			t.ContentLength = -1
	 137  		} else if t.Body == nil { // no chunking, no body
	 138  			t.ContentLength = 0
	 139  		}
	 140  	}
	 141  
	 142  	// Sanitize Trailer
	 143  	if !chunked(t.TransferEncoding) {
	 144  		t.Trailer = nil
	 145  	}
	 146  
	 147  	return t, nil
	 148  }
	 149  
	 150  // shouldSendChunkedRequestBody reports whether we should try to send a
	 151  // chunked request body to the server. In particular, the case we really
	 152  // want to prevent is sending a GET or other typically-bodyless request to a
	 153  // server with a chunked body when the body has zero bytes, since GETs with
	 154  // bodies (while acceptable according to specs), even zero-byte chunked
	 155  // bodies, are approximately never seen in the wild and confuse most
	 156  // servers. See Issue 18257, as one example.
	 157  //
	 158  // The only reason we'd send such a request is if the user set the Body to a
	 159  // non-nil value (say, io.NopCloser(bytes.NewReader(nil))) and didn't
	 160  // set ContentLength, or NewRequest set it to -1 (unknown), so then we assume
	 161  // there's bytes to send.
	 162  //
	 163  // This code tries to read a byte from the Request.Body in such cases to see
	 164  // whether the body actually has content (super rare) or is actually just
	 165  // a non-nil content-less ReadCloser (the more common case). In that more
	 166  // common case, we act as if their Body were nil instead, and don't send
	 167  // a body.
	 168  func (t *transferWriter) shouldSendChunkedRequestBody() bool {
	 169  	// Note that t.ContentLength is the corrected content length
	 170  	// from rr.outgoingLength, so 0 actually means zero, not unknown.
	 171  	if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them
	 172  		return false
	 173  	}
	 174  	if t.Method == "CONNECT" {
	 175  		return false
	 176  	}
	 177  	if requestMethodUsuallyLacksBody(t.Method) {
	 178  		// Only probe the Request.Body for GET/HEAD/DELETE/etc
	 179  		// requests, because it's only those types of requests
	 180  		// that confuse servers.
	 181  		t.probeRequestBody() // adjusts t.Body, t.ContentLength
	 182  		return t.Body != nil
	 183  	}
	 184  	// For all other request types (PUT, POST, PATCH, or anything
	 185  	// made-up we've never heard of), assume it's normal and the server
	 186  	// can deal with a chunked request body. Maybe we'll adjust this
	 187  	// later.
	 188  	return true
	 189  }
	 190  
	 191  // probeRequestBody reads a byte from t.Body to see whether it's empty
	 192  // (returns io.EOF right away).
	 193  //
	 194  // But because we've had problems with this blocking users in the past
	 195  // (issue 17480) when the body is a pipe (perhaps waiting on the response
	 196  // headers before the pipe is fed data), we need to be careful and bound how
	 197  // long we wait for it. This delay will only affect users if all the following
	 198  // are true:
	 199  //	 * the request body blocks
	 200  //	 * the content length is not set (or set to -1)
	 201  //	 * the method doesn't usually have a body (GET, HEAD, DELETE, ...)
	 202  //	 * there is no transfer-encoding=chunked already set.
	 203  // In other words, this delay will not normally affect anybody, and there
	 204  // are workarounds if it does.
	 205  func (t *transferWriter) probeRequestBody() {
	 206  	t.ByteReadCh = make(chan readResult, 1)
	 207  	go func(body io.Reader) {
	 208  		var buf [1]byte
	 209  		var rres readResult
	 210  		rres.n, rres.err = body.Read(buf[:])
	 211  		if rres.n == 1 {
	 212  			rres.b = buf[0]
	 213  		}
	 214  		t.ByteReadCh <- rres
	 215  	}(t.Body)
	 216  	timer := time.NewTimer(200 * time.Millisecond)
	 217  	select {
	 218  	case rres := <-t.ByteReadCh:
	 219  		timer.Stop()
	 220  		if rres.n == 0 && rres.err == io.EOF {
	 221  			// It was empty.
	 222  			t.Body = nil
	 223  			t.ContentLength = 0
	 224  		} else if rres.n == 1 {
	 225  			if rres.err != nil {
	 226  				t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err})
	 227  			} else {
	 228  				t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body)
	 229  			}
	 230  		} else if rres.err != nil {
	 231  			t.Body = errorReader{rres.err}
	 232  		}
	 233  	case <-timer.C:
	 234  		// Too slow. Don't wait. Read it later, and keep
	 235  		// assuming that this is ContentLength == -1
	 236  		// (unknown), which means we'll send a
	 237  		// "Transfer-Encoding: chunked" header.
	 238  		t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body)
	 239  		// Request that Request.Write flush the headers to the
	 240  		// network before writing the body, since our body may not
	 241  		// become readable until it's seen the response headers.
	 242  		t.FlushHeaders = true
	 243  	}
	 244  }
	 245  
	 246  func noResponseBodyExpected(requestMethod string) bool {
	 247  	return requestMethod == "HEAD"
	 248  }
	 249  
	 250  func (t *transferWriter) shouldSendContentLength() bool {
	 251  	if chunked(t.TransferEncoding) {
	 252  		return false
	 253  	}
	 254  	if t.ContentLength > 0 {
	 255  		return true
	 256  	}
	 257  	if t.ContentLength < 0 {
	 258  		return false
	 259  	}
	 260  	// Many servers expect a Content-Length for these methods
	 261  	if t.Method == "POST" || t.Method == "PUT" || t.Method == "PATCH" {
	 262  		return true
	 263  	}
	 264  	if t.ContentLength == 0 && isIdentity(t.TransferEncoding) {
	 265  		if t.Method == "GET" || t.Method == "HEAD" {
	 266  			return false
	 267  		}
	 268  		return true
	 269  	}
	 270  
	 271  	return false
	 272  }
	 273  
	 274  func (t *transferWriter) writeHeader(w io.Writer, trace *httptrace.ClientTrace) error {
	 275  	if t.Close && !hasToken(t.Header.get("Connection"), "close") {
	 276  		if _, err := io.WriteString(w, "Connection: close\r\n"); err != nil {
	 277  			return err
	 278  		}
	 279  		if trace != nil && trace.WroteHeaderField != nil {
	 280  			trace.WroteHeaderField("Connection", []string{"close"})
	 281  		}
	 282  	}
	 283  
	 284  	// Write Content-Length and/or Transfer-Encoding whose values are a
	 285  	// function of the sanitized field triple (Body, ContentLength,
	 286  	// TransferEncoding)
	 287  	if t.shouldSendContentLength() {
	 288  		if _, err := io.WriteString(w, "Content-Length: "); err != nil {
	 289  			return err
	 290  		}
	 291  		if _, err := io.WriteString(w, strconv.FormatInt(t.ContentLength, 10)+"\r\n"); err != nil {
	 292  			return err
	 293  		}
	 294  		if trace != nil && trace.WroteHeaderField != nil {
	 295  			trace.WroteHeaderField("Content-Length", []string{strconv.FormatInt(t.ContentLength, 10)})
	 296  		}
	 297  	} else if chunked(t.TransferEncoding) {
	 298  		if _, err := io.WriteString(w, "Transfer-Encoding: chunked\r\n"); err != nil {
	 299  			return err
	 300  		}
	 301  		if trace != nil && trace.WroteHeaderField != nil {
	 302  			trace.WroteHeaderField("Transfer-Encoding", []string{"chunked"})
	 303  		}
	 304  	}
	 305  
	 306  	// Write Trailer header
	 307  	if t.Trailer != nil {
	 308  		keys := make([]string, 0, len(t.Trailer))
	 309  		for k := range t.Trailer {
	 310  			k = CanonicalHeaderKey(k)
	 311  			switch k {
	 312  			case "Transfer-Encoding", "Trailer", "Content-Length":
	 313  				return badStringError("invalid Trailer key", k)
	 314  			}
	 315  			keys = append(keys, k)
	 316  		}
	 317  		if len(keys) > 0 {
	 318  			sort.Strings(keys)
	 319  			// TODO: could do better allocation-wise here, but trailers are rare,
	 320  			// so being lazy for now.
	 321  			if _, err := io.WriteString(w, "Trailer: "+strings.Join(keys, ",")+"\r\n"); err != nil {
	 322  				return err
	 323  			}
	 324  			if trace != nil && trace.WroteHeaderField != nil {
	 325  				trace.WroteHeaderField("Trailer", keys)
	 326  			}
	 327  		}
	 328  	}
	 329  
	 330  	return nil
	 331  }
	 332  
	 333  // always closes t.BodyCloser
	 334  func (t *transferWriter) writeBody(w io.Writer) (err error) {
	 335  	var ncopy int64
	 336  	closed := false
	 337  	defer func() {
	 338  		if closed || t.BodyCloser == nil {
	 339  			return
	 340  		}
	 341  		if closeErr := t.BodyCloser.Close(); closeErr != nil && err == nil {
	 342  			err = closeErr
	 343  		}
	 344  	}()
	 345  
	 346  	// Write body. We "unwrap" the body first if it was wrapped in a
	 347  	// nopCloser or readTrackingBody. This is to ensure that we can take advantage of
	 348  	// OS-level optimizations in the event that the body is an
	 349  	// *os.File.
	 350  	if t.Body != nil {
	 351  		var body = t.unwrapBody()
	 352  		if chunked(t.TransferEncoding) {
	 353  			if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse {
	 354  				w = &internal.FlushAfterChunkWriter{Writer: bw}
	 355  			}
	 356  			cw := internal.NewChunkedWriter(w)
	 357  			_, err = t.doBodyCopy(cw, body)
	 358  			if err == nil {
	 359  				err = cw.Close()
	 360  			}
	 361  		} else if t.ContentLength == -1 {
	 362  			dst := w
	 363  			if t.Method == "CONNECT" {
	 364  				dst = bufioFlushWriter{dst}
	 365  			}
	 366  			ncopy, err = t.doBodyCopy(dst, body)
	 367  		} else {
	 368  			ncopy, err = t.doBodyCopy(w, io.LimitReader(body, t.ContentLength))
	 369  			if err != nil {
	 370  				return err
	 371  			}
	 372  			var nextra int64
	 373  			nextra, err = t.doBodyCopy(io.Discard, body)
	 374  			ncopy += nextra
	 375  		}
	 376  		if err != nil {
	 377  			return err
	 378  		}
	 379  	}
	 380  	if t.BodyCloser != nil {
	 381  		closed = true
	 382  		if err := t.BodyCloser.Close(); err != nil {
	 383  			return err
	 384  		}
	 385  	}
	 386  
	 387  	if !t.ResponseToHEAD && t.ContentLength != -1 && t.ContentLength != ncopy {
	 388  		return fmt.Errorf("http: ContentLength=%d with Body length %d",
	 389  			t.ContentLength, ncopy)
	 390  	}
	 391  
	 392  	if chunked(t.TransferEncoding) {
	 393  		// Write Trailer header
	 394  		if t.Trailer != nil {
	 395  			if err := t.Trailer.Write(w); err != nil {
	 396  				return err
	 397  			}
	 398  		}
	 399  		// Last chunk, empty trailer
	 400  		_, err = io.WriteString(w, "\r\n")
	 401  	}
	 402  	return err
	 403  }
	 404  
	 405  // doBodyCopy wraps a copy operation, with any resulting error also
	 406  // being saved in bodyReadError.
	 407  //
	 408  // This function is only intended for use in writeBody.
	 409  func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) {
	 410  	n, err = io.Copy(dst, src)
	 411  	if err != nil && err != io.EOF {
	 412  		t.bodyReadError = err
	 413  	}
	 414  	return
	 415  }
	 416  
	 417  // unwrapBodyReader unwraps the body's inner reader if it's a
	 418  // nopCloser. This is to ensure that body writes sourced from local
	 419  // files (*os.File types) are properly optimized.
	 420  //
	 421  // This function is only intended for use in writeBody.
	 422  func (t *transferWriter) unwrapBody() io.Reader {
	 423  	if reflect.TypeOf(t.Body) == nopCloserType {
	 424  		return reflect.ValueOf(t.Body).Field(0).Interface().(io.Reader)
	 425  	}
	 426  	if r, ok := t.Body.(*readTrackingBody); ok {
	 427  		r.didRead = true
	 428  		return r.ReadCloser
	 429  	}
	 430  	return t.Body
	 431  }
	 432  
	 433  type transferReader struct {
	 434  	// Input
	 435  	Header				Header
	 436  	StatusCode		int
	 437  	RequestMethod string
	 438  	ProtoMajor		int
	 439  	ProtoMinor		int
	 440  	// Output
	 441  	Body					io.ReadCloser
	 442  	ContentLength int64
	 443  	Chunked			 bool
	 444  	Close				 bool
	 445  	Trailer			 Header
	 446  }
	 447  
	 448  func (t *transferReader) protoAtLeast(m, n int) bool {
	 449  	return t.ProtoMajor > m || (t.ProtoMajor == m && t.ProtoMinor >= n)
	 450  }
	 451  
	 452  // bodyAllowedForStatus reports whether a given response status code
	 453  // permits a body. See RFC 7230, section 3.3.
	 454  func bodyAllowedForStatus(status int) bool {
	 455  	switch {
	 456  	case status >= 100 && status <= 199:
	 457  		return false
	 458  	case status == 204:
	 459  		return false
	 460  	case status == 304:
	 461  		return false
	 462  	}
	 463  	return true
	 464  }
	 465  
	 466  var (
	 467  	suppressedHeaders304		= []string{"Content-Type", "Content-Length", "Transfer-Encoding"}
	 468  	suppressedHeadersNoBody = []string{"Content-Length", "Transfer-Encoding"}
	 469  )
	 470  
	 471  func suppressedHeaders(status int) []string {
	 472  	switch {
	 473  	case status == 304:
	 474  		// RFC 7232 section 4.1
	 475  		return suppressedHeaders304
	 476  	case !bodyAllowedForStatus(status):
	 477  		return suppressedHeadersNoBody
	 478  	}
	 479  	return nil
	 480  }
	 481  
	 482  // msg is *Request or *Response.
	 483  func readTransfer(msg interface{}, r *bufio.Reader) (err error) {
	 484  	t := &transferReader{RequestMethod: "GET"}
	 485  
	 486  	// Unify input
	 487  	isResponse := false
	 488  	switch rr := msg.(type) {
	 489  	case *Response:
	 490  		t.Header = rr.Header
	 491  		t.StatusCode = rr.StatusCode
	 492  		t.ProtoMajor = rr.ProtoMajor
	 493  		t.ProtoMinor = rr.ProtoMinor
	 494  		t.Close = shouldClose(t.ProtoMajor, t.ProtoMinor, t.Header, true)
	 495  		isResponse = true
	 496  		if rr.Request != nil {
	 497  			t.RequestMethod = rr.Request.Method
	 498  		}
	 499  	case *Request:
	 500  		t.Header = rr.Header
	 501  		t.RequestMethod = rr.Method
	 502  		t.ProtoMajor = rr.ProtoMajor
	 503  		t.ProtoMinor = rr.ProtoMinor
	 504  		// Transfer semantics for Requests are exactly like those for
	 505  		// Responses with status code 200, responding to a GET method
	 506  		t.StatusCode = 200
	 507  		t.Close = rr.Close
	 508  	default:
	 509  		panic("unexpected type")
	 510  	}
	 511  
	 512  	// Default to HTTP/1.1
	 513  	if t.ProtoMajor == 0 && t.ProtoMinor == 0 {
	 514  		t.ProtoMajor, t.ProtoMinor = 1, 1
	 515  	}
	 516  
	 517  	// Transfer-Encoding: chunked, and overriding Content-Length.
	 518  	if err := t.parseTransferEncoding(); err != nil {
	 519  		return err
	 520  	}
	 521  
	 522  	realLength, err := fixLength(isResponse, t.StatusCode, t.RequestMethod, t.Header, t.Chunked)
	 523  	if err != nil {
	 524  		return err
	 525  	}
	 526  	if isResponse && t.RequestMethod == "HEAD" {
	 527  		if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil {
	 528  			return err
	 529  		} else {
	 530  			t.ContentLength = n
	 531  		}
	 532  	} else {
	 533  		t.ContentLength = realLength
	 534  	}
	 535  
	 536  	// Trailer
	 537  	t.Trailer, err = fixTrailer(t.Header, t.Chunked)
	 538  	if err != nil {
	 539  		return err
	 540  	}
	 541  
	 542  	// If there is no Content-Length or chunked Transfer-Encoding on a *Response
	 543  	// and the status is not 1xx, 204 or 304, then the body is unbounded.
	 544  	// See RFC 7230, section 3.3.
	 545  	switch msg.(type) {
	 546  	case *Response:
	 547  		if realLength == -1 && !t.Chunked && bodyAllowedForStatus(t.StatusCode) {
	 548  			// Unbounded body.
	 549  			t.Close = true
	 550  		}
	 551  	}
	 552  
	 553  	// Prepare body reader. ContentLength < 0 means chunked encoding
	 554  	// or close connection when finished, since multipart is not supported yet
	 555  	switch {
	 556  	case t.Chunked:
	 557  		if noResponseBodyExpected(t.RequestMethod) || !bodyAllowedForStatus(t.StatusCode) {
	 558  			t.Body = NoBody
	 559  		} else {
	 560  			t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close}
	 561  		}
	 562  	case realLength == 0:
	 563  		t.Body = NoBody
	 564  	case realLength > 0:
	 565  		t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close}
	 566  	default:
	 567  		// realLength < 0, i.e. "Content-Length" not mentioned in header
	 568  		if t.Close {
	 569  			// Close semantics (i.e. HTTP/1.0)
	 570  			t.Body = &body{src: r, closing: t.Close}
	 571  		} else {
	 572  			// Persistent connection (i.e. HTTP/1.1)
	 573  			t.Body = NoBody
	 574  		}
	 575  	}
	 576  
	 577  	// Unify output
	 578  	switch rr := msg.(type) {
	 579  	case *Request:
	 580  		rr.Body = t.Body
	 581  		rr.ContentLength = t.ContentLength
	 582  		if t.Chunked {
	 583  			rr.TransferEncoding = []string{"chunked"}
	 584  		}
	 585  		rr.Close = t.Close
	 586  		rr.Trailer = t.Trailer
	 587  	case *Response:
	 588  		rr.Body = t.Body
	 589  		rr.ContentLength = t.ContentLength
	 590  		if t.Chunked {
	 591  			rr.TransferEncoding = []string{"chunked"}
	 592  		}
	 593  		rr.Close = t.Close
	 594  		rr.Trailer = t.Trailer
	 595  	}
	 596  
	 597  	return nil
	 598  }
	 599  
	 600  // Checks whether chunked is part of the encodings stack
	 601  func chunked(te []string) bool { return len(te) > 0 && te[0] == "chunked" }
	 602  
	 603  // Checks whether the encoding is explicitly "identity".
	 604  func isIdentity(te []string) bool { return len(te) == 1 && te[0] == "identity" }
	 605  
	 606  // unsupportedTEError reports unsupported transfer-encodings.
	 607  type unsupportedTEError struct {
	 608  	err string
	 609  }
	 610  
	 611  func (uste *unsupportedTEError) Error() string {
	 612  	return uste.err
	 613  }
	 614  
	 615  // isUnsupportedTEError checks if the error is of type
	 616  // unsupportedTEError. It is usually invoked with a non-nil err.
	 617  func isUnsupportedTEError(err error) bool {
	 618  	_, ok := err.(*unsupportedTEError)
	 619  	return ok
	 620  }
	 621  
	 622  // parseTransferEncoding sets t.Chunked based on the Transfer-Encoding header.
	 623  func (t *transferReader) parseTransferEncoding() error {
	 624  	raw, present := t.Header["Transfer-Encoding"]
	 625  	if !present {
	 626  		return nil
	 627  	}
	 628  	delete(t.Header, "Transfer-Encoding")
	 629  
	 630  	// Issue 12785; ignore Transfer-Encoding on HTTP/1.0 requests.
	 631  	if !t.protoAtLeast(1, 1) {
	 632  		return nil
	 633  	}
	 634  
	 635  	// Like nginx, we only support a single Transfer-Encoding header field, and
	 636  	// only if set to "chunked". This is one of the most security sensitive
	 637  	// surfaces in HTTP/1.1 due to the risk of request smuggling, so we keep it
	 638  	// strict and simple.
	 639  	if len(raw) != 1 {
	 640  		return &unsupportedTEError{fmt.Sprintf("too many transfer encodings: %q", raw)}
	 641  	}
	 642  	if !ascii.EqualFold(raw[0], "chunked") {
	 643  		return &unsupportedTEError{fmt.Sprintf("unsupported transfer encoding: %q", raw[0])}
	 644  	}
	 645  
	 646  	// RFC 7230 3.3.2 says "A sender MUST NOT send a Content-Length header field
	 647  	// in any message that contains a Transfer-Encoding header field."
	 648  	//
	 649  	// but also: "If a message is received with both a Transfer-Encoding and a
	 650  	// Content-Length header field, the Transfer-Encoding overrides the
	 651  	// Content-Length. Such a message might indicate an attempt to perform
	 652  	// request smuggling (Section 9.5) or response splitting (Section 9.4) and
	 653  	// ought to be handled as an error. A sender MUST remove the received
	 654  	// Content-Length field prior to forwarding such a message downstream."
	 655  	//
	 656  	// Reportedly, these appear in the wild.
	 657  	delete(t.Header, "Content-Length")
	 658  
	 659  	t.Chunked = true
	 660  	return nil
	 661  }
	 662  
	 663  // Determine the expected body length, using RFC 7230 Section 3.3. This
	 664  // function is not a method, because ultimately it should be shared by
	 665  // ReadResponse and ReadRequest.
	 666  func fixLength(isResponse bool, status int, requestMethod string, header Header, chunked bool) (int64, error) {
	 667  	isRequest := !isResponse
	 668  	contentLens := header["Content-Length"]
	 669  
	 670  	// Hardening against HTTP request smuggling
	 671  	if len(contentLens) > 1 {
	 672  		// Per RFC 7230 Section 3.3.2, prevent multiple
	 673  		// Content-Length headers if they differ in value.
	 674  		// If there are dups of the value, remove the dups.
	 675  		// See Issue 16490.
	 676  		first := textproto.TrimString(contentLens[0])
	 677  		for _, ct := range contentLens[1:] {
	 678  			if first != textproto.TrimString(ct) {
	 679  				return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens)
	 680  			}
	 681  		}
	 682  
	 683  		// deduplicate Content-Length
	 684  		header.Del("Content-Length")
	 685  		header.Add("Content-Length", first)
	 686  
	 687  		contentLens = header["Content-Length"]
	 688  	}
	 689  
	 690  	// Logic based on response type or status
	 691  	if noResponseBodyExpected(requestMethod) {
	 692  		// For HTTP requests, as part of hardening against request
	 693  		// smuggling (RFC 7230), don't allow a Content-Length header for
	 694  		// methods which don't permit bodies. As an exception, allow
	 695  		// exactly one Content-Length header if its value is "0".
	 696  		if isRequest && len(contentLens) > 0 && !(len(contentLens) == 1 && contentLens[0] == "0") {
	 697  			return 0, fmt.Errorf("http: method cannot contain a Content-Length; got %q", contentLens)
	 698  		}
	 699  		return 0, nil
	 700  	}
	 701  	if status/100 == 1 {
	 702  		return 0, nil
	 703  	}
	 704  	switch status {
	 705  	case 204, 304:
	 706  		return 0, nil
	 707  	}
	 708  
	 709  	// Logic based on Transfer-Encoding
	 710  	if chunked {
	 711  		return -1, nil
	 712  	}
	 713  
	 714  	// Logic based on Content-Length
	 715  	var cl string
	 716  	if len(contentLens) == 1 {
	 717  		cl = textproto.TrimString(contentLens[0])
	 718  	}
	 719  	if cl != "" {
	 720  		n, err := parseContentLength(cl)
	 721  		if err != nil {
	 722  			return -1, err
	 723  		}
	 724  		return n, nil
	 725  	}
	 726  	header.Del("Content-Length")
	 727  
	 728  	if isRequest {
	 729  		// RFC 7230 neither explicitly permits nor forbids an
	 730  		// entity-body on a GET request so we permit one if
	 731  		// declared, but we default to 0 here (not -1 below)
	 732  		// if there's no mention of a body.
	 733  		// Likewise, all other request methods are assumed to have
	 734  		// no body if neither Transfer-Encoding chunked nor a
	 735  		// Content-Length are set.
	 736  		return 0, nil
	 737  	}
	 738  
	 739  	// Body-EOF logic based on other methods (like closing, or chunked coding)
	 740  	return -1, nil
	 741  }
	 742  
	 743  // Determine whether to hang up after sending a request and body, or
	 744  // receiving a response and body
	 745  // 'header' is the request headers
	 746  func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool {
	 747  	if major < 1 {
	 748  		return true
	 749  	}
	 750  
	 751  	conv := header["Connection"]
	 752  	hasClose := httpguts.HeaderValuesContainsToken(conv, "close")
	 753  	if major == 1 && minor == 0 {
	 754  		return hasClose || !httpguts.HeaderValuesContainsToken(conv, "keep-alive")
	 755  	}
	 756  
	 757  	if hasClose && removeCloseHeader {
	 758  		header.Del("Connection")
	 759  	}
	 760  
	 761  	return hasClose
	 762  }
	 763  
	 764  // Parse the trailer header
	 765  func fixTrailer(header Header, chunked bool) (Header, error) {
	 766  	vv, ok := header["Trailer"]
	 767  	if !ok {
	 768  		return nil, nil
	 769  	}
	 770  	if !chunked {
	 771  		// Trailer and no chunking:
	 772  		// this is an invalid use case for trailer header.
	 773  		// Nevertheless, no error will be returned and we
	 774  		// let users decide if this is a valid HTTP message.
	 775  		// The Trailer header will be kept in Response.Header
	 776  		// but not populate Response.Trailer.
	 777  		// See issue #27197.
	 778  		return nil, nil
	 779  	}
	 780  	header.Del("Trailer")
	 781  
	 782  	trailer := make(Header)
	 783  	var err error
	 784  	for _, v := range vv {
	 785  		foreachHeaderElement(v, func(key string) {
	 786  			key = CanonicalHeaderKey(key)
	 787  			switch key {
	 788  			case "Transfer-Encoding", "Trailer", "Content-Length":
	 789  				if err == nil {
	 790  					err = badStringError("bad trailer key", key)
	 791  					return
	 792  				}
	 793  			}
	 794  			trailer[key] = nil
	 795  		})
	 796  	}
	 797  	if err != nil {
	 798  		return nil, err
	 799  	}
	 800  	if len(trailer) == 0 {
	 801  		return nil, nil
	 802  	}
	 803  	return trailer, nil
	 804  }
	 805  
	 806  // body turns a Reader into a ReadCloser.
	 807  // Close ensures that the body has been fully read
	 808  // and then reads the trailer if necessary.
	 809  type body struct {
	 810  	src					io.Reader
	 811  	hdr					interface{}	 // non-nil (Response or Request) value means read trailer
	 812  	r						*bufio.Reader // underlying wire-format reader for the trailer
	 813  	closing			bool					// is the connection to be closed after reading body?
	 814  	doEarlyClose bool					// whether Close should stop early
	 815  
	 816  	mu				 sync.Mutex // guards following, and calls to Read and Close
	 817  	sawEOF		 bool
	 818  	closed		 bool
	 819  	earlyClose bool	 // Close called and we didn't read to the end of src
	 820  	onHitEOF	 func() // if non-nil, func to call when EOF is Read
	 821  }
	 822  
	 823  // ErrBodyReadAfterClose is returned when reading a Request or Response
	 824  // Body after the body has been closed. This typically happens when the body is
	 825  // read after an HTTP Handler calls WriteHeader or Write on its
	 826  // ResponseWriter.
	 827  var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body")
	 828  
	 829  func (b *body) Read(p []byte) (n int, err error) {
	 830  	b.mu.Lock()
	 831  	defer b.mu.Unlock()
	 832  	if b.closed {
	 833  		return 0, ErrBodyReadAfterClose
	 834  	}
	 835  	return b.readLocked(p)
	 836  }
	 837  
	 838  // Must hold b.mu.
	 839  func (b *body) readLocked(p []byte) (n int, err error) {
	 840  	if b.sawEOF {
	 841  		return 0, io.EOF
	 842  	}
	 843  	n, err = b.src.Read(p)
	 844  
	 845  	if err == io.EOF {
	 846  		b.sawEOF = true
	 847  		// Chunked case. Read the trailer.
	 848  		if b.hdr != nil {
	 849  			if e := b.readTrailer(); e != nil {
	 850  				err = e
	 851  				// Something went wrong in the trailer, we must not allow any
	 852  				// further reads of any kind to succeed from body, nor any
	 853  				// subsequent requests on the server connection. See
	 854  				// golang.org/issue/12027
	 855  				b.sawEOF = false
	 856  				b.closed = true
	 857  			}
	 858  			b.hdr = nil
	 859  		} else {
	 860  			// If the server declared the Content-Length, our body is a LimitedReader
	 861  			// and we need to check whether this EOF arrived early.
	 862  			if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > 0 {
	 863  				err = io.ErrUnexpectedEOF
	 864  			}
	 865  		}
	 866  	}
	 867  
	 868  	// If we can return an EOF here along with the read data, do
	 869  	// so. This is optional per the io.Reader contract, but doing
	 870  	// so helps the HTTP transport code recycle its connection
	 871  	// earlier (since it will see this EOF itself), even if the
	 872  	// client doesn't do future reads or Close.
	 873  	if err == nil && n > 0 {
	 874  		if lr, ok := b.src.(*io.LimitedReader); ok && lr.N == 0 {
	 875  			err = io.EOF
	 876  			b.sawEOF = true
	 877  		}
	 878  	}
	 879  
	 880  	if b.sawEOF && b.onHitEOF != nil {
	 881  		b.onHitEOF()
	 882  	}
	 883  
	 884  	return n, err
	 885  }
	 886  
	 887  var (
	 888  	singleCRLF = []byte("\r\n")
	 889  	doubleCRLF = []byte("\r\n\r\n")
	 890  )
	 891  
	 892  func seeUpcomingDoubleCRLF(r *bufio.Reader) bool {
	 893  	for peekSize := 4; ; peekSize++ {
	 894  		// This loop stops when Peek returns an error,
	 895  		// which it does when r's buffer has been filled.
	 896  		buf, err := r.Peek(peekSize)
	 897  		if bytes.HasSuffix(buf, doubleCRLF) {
	 898  			return true
	 899  		}
	 900  		if err != nil {
	 901  			break
	 902  		}
	 903  	}
	 904  	return false
	 905  }
	 906  
	 907  var errTrailerEOF = errors.New("http: unexpected EOF reading trailer")
	 908  
	 909  func (b *body) readTrailer() error {
	 910  	// The common case, since nobody uses trailers.
	 911  	buf, err := b.r.Peek(2)
	 912  	if bytes.Equal(buf, singleCRLF) {
	 913  		b.r.Discard(2)
	 914  		return nil
	 915  	}
	 916  	if len(buf) < 2 {
	 917  		return errTrailerEOF
	 918  	}
	 919  	if err != nil {
	 920  		return err
	 921  	}
	 922  
	 923  	// Make sure there's a header terminator coming up, to prevent
	 924  	// a DoS with an unbounded size Trailer. It's not easy to
	 925  	// slip in a LimitReader here, as textproto.NewReader requires
	 926  	// a concrete *bufio.Reader. Also, we can't get all the way
	 927  	// back up to our conn's LimitedReader that *might* be backing
	 928  	// this bufio.Reader. Instead, a hack: we iteratively Peek up
	 929  	// to the bufio.Reader's max size, looking for a double CRLF.
	 930  	// This limits the trailer to the underlying buffer size, typically 4kB.
	 931  	if !seeUpcomingDoubleCRLF(b.r) {
	 932  		return errors.New("http: suspiciously long trailer after chunked body")
	 933  	}
	 934  
	 935  	hdr, err := textproto.NewReader(b.r).ReadMIMEHeader()
	 936  	if err != nil {
	 937  		if err == io.EOF {
	 938  			return errTrailerEOF
	 939  		}
	 940  		return err
	 941  	}
	 942  	switch rr := b.hdr.(type) {
	 943  	case *Request:
	 944  		mergeSetHeader(&rr.Trailer, Header(hdr))
	 945  	case *Response:
	 946  		mergeSetHeader(&rr.Trailer, Header(hdr))
	 947  	}
	 948  	return nil
	 949  }
	 950  
	 951  func mergeSetHeader(dst *Header, src Header) {
	 952  	if *dst == nil {
	 953  		*dst = src
	 954  		return
	 955  	}
	 956  	for k, vv := range src {
	 957  		(*dst)[k] = vv
	 958  	}
	 959  }
	 960  
	 961  // unreadDataSizeLocked returns the number of bytes of unread input.
	 962  // It returns -1 if unknown.
	 963  // b.mu must be held.
	 964  func (b *body) unreadDataSizeLocked() int64 {
	 965  	if lr, ok := b.src.(*io.LimitedReader); ok {
	 966  		return lr.N
	 967  	}
	 968  	return -1
	 969  }
	 970  
	 971  func (b *body) Close() error {
	 972  	b.mu.Lock()
	 973  	defer b.mu.Unlock()
	 974  	if b.closed {
	 975  		return nil
	 976  	}
	 977  	var err error
	 978  	switch {
	 979  	case b.sawEOF:
	 980  		// Already saw EOF, so no need going to look for it.
	 981  	case b.hdr == nil && b.closing:
	 982  		// no trailer and closing the connection next.
	 983  		// no point in reading to EOF.
	 984  	case b.doEarlyClose:
	 985  		// Read up to maxPostHandlerReadBytes bytes of the body, looking
	 986  		// for EOF (and trailers), so we can re-use this connection.
	 987  		if lr, ok := b.src.(*io.LimitedReader); ok && lr.N > maxPostHandlerReadBytes {
	 988  			// There was a declared Content-Length, and we have more bytes remaining
	 989  			// than our maxPostHandlerReadBytes tolerance. So, give up.
	 990  			b.earlyClose = true
	 991  		} else {
	 992  			var n int64
	 993  			// Consume the body, or, which will also lead to us reading
	 994  			// the trailer headers after the body, if present.
	 995  			n, err = io.CopyN(io.Discard, bodyLocked{b}, maxPostHandlerReadBytes)
	 996  			if err == io.EOF {
	 997  				err = nil
	 998  			}
	 999  			if n == maxPostHandlerReadBytes {
	1000  				b.earlyClose = true
	1001  			}
	1002  		}
	1003  	default:
	1004  		// Fully consume the body, which will also lead to us reading
	1005  		// the trailer headers after the body, if present.
	1006  		_, err = io.Copy(io.Discard, bodyLocked{b})
	1007  	}
	1008  	b.closed = true
	1009  	return err
	1010  }
	1011  
	1012  func (b *body) didEarlyClose() bool {
	1013  	b.mu.Lock()
	1014  	defer b.mu.Unlock()
	1015  	return b.earlyClose
	1016  }
	1017  
	1018  // bodyRemains reports whether future Read calls might
	1019  // yield data.
	1020  func (b *body) bodyRemains() bool {
	1021  	b.mu.Lock()
	1022  	defer b.mu.Unlock()
	1023  	return !b.sawEOF
	1024  }
	1025  
	1026  func (b *body) registerOnHitEOF(fn func()) {
	1027  	b.mu.Lock()
	1028  	defer b.mu.Unlock()
	1029  	b.onHitEOF = fn
	1030  }
	1031  
	1032  // bodyLocked is a io.Reader reading from a *body when its mutex is
	1033  // already held.
	1034  type bodyLocked struct {
	1035  	b *body
	1036  }
	1037  
	1038  func (bl bodyLocked) Read(p []byte) (n int, err error) {
	1039  	if bl.b.closed {
	1040  		return 0, ErrBodyReadAfterClose
	1041  	}
	1042  	return bl.b.readLocked(p)
	1043  }
	1044  
	1045  // parseContentLength trims whitespace from s and returns -1 if no value
	1046  // is set, or the value if it's >= 0.
	1047  func parseContentLength(cl string) (int64, error) {
	1048  	cl = textproto.TrimString(cl)
	1049  	if cl == "" {
	1050  		return -1, nil
	1051  	}
	1052  	n, err := strconv.ParseUint(cl, 10, 63)
	1053  	if err != nil {
	1054  		return 0, badStringError("bad Content-Length", cl)
	1055  	}
	1056  	return int64(n), nil
	1057  
	1058  }
	1059  
	1060  // finishAsyncByteRead finishes reading the 1-byte sniff
	1061  // from the ContentLength==0, Body!=nil case.
	1062  type finishAsyncByteRead struct {
	1063  	tw *transferWriter
	1064  }
	1065  
	1066  func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) {
	1067  	if len(p) == 0 {
	1068  		return
	1069  	}
	1070  	rres := <-fr.tw.ByteReadCh
	1071  	n, err = rres.n, rres.err
	1072  	if n == 1 {
	1073  		p[0] = rres.b
	1074  	}
	1075  	return
	1076  }
	1077  
	1078  var nopCloserType = reflect.TypeOf(io.NopCloser(nil))
	1079  
	1080  // isKnownInMemoryReader reports whether r is a type known to not
	1081  // block on Read. Its caller uses this as an optional optimization to
	1082  // send fewer TCP packets.
	1083  func isKnownInMemoryReader(r io.Reader) bool {
	1084  	switch r.(type) {
	1085  	case *bytes.Reader, *bytes.Buffer, *strings.Reader:
	1086  		return true
	1087  	}
	1088  	if reflect.TypeOf(r) == nopCloserType {
	1089  		return isKnownInMemoryReader(reflect.ValueOf(r).Field(0).Interface().(io.Reader))
	1090  	}
	1091  	if r, ok := r.(*readTrackingBody); ok {
	1092  		return isKnownInMemoryReader(r.ReadCloser)
	1093  	}
	1094  	return false
	1095  }
	1096  
	1097  // bufioFlushWriter is an io.Writer wrapper that flushes all writes
	1098  // on its wrapped writer if it's a *bufio.Writer.
	1099  type bufioFlushWriter struct{ w io.Writer }
	1100  
	1101  func (fw bufioFlushWriter) Write(p []byte) (n int, err error) {
	1102  	n, err = fw.w.Write(p)
	1103  	if bw, ok := fw.w.(*bufio.Writer); n > 0 && ok {
	1104  		ferr := bw.Flush()
	1105  		if ferr != nil && err == nil {
	1106  			err = ferr
	1107  		}
	1108  	}
	1109  	return
	1110  }
	1111  

View as plain text