1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Package io provides basic interfaces to I/O primitives. 6 // Its primary job is to wrap existing implementations of such primitives, 7 // such as those in package os, into shared public interfaces that 8 // abstract the functionality, plus some other related primitives. 9 // 10 // Because these interfaces and primitives wrap lower-level operations with 11 // various implementations, unless otherwise informed clients should not 12 // assume they are safe for parallel execution. 13 package io 14 15 import ( 16 "errors" 17 "sync" 18 ) 19 20 // Seek whence values. 21 const ( 22 SeekStart = 0 // seek relative to the origin of the file 23 SeekCurrent = 1 // seek relative to the current offset 24 SeekEnd = 2 // seek relative to the end 25 ) 26 27 // ErrShortWrite means that a write accepted fewer bytes than requested 28 // but failed to return an explicit error. 29 var ErrShortWrite = errors.New("short write") 30 31 // errInvalidWrite means that a write returned an impossible count. 32 var errInvalidWrite = errors.New("invalid write result") 33 34 // ErrShortBuffer means that a read required a longer buffer than was provided. 35 var ErrShortBuffer = errors.New("short buffer") 36 37 // EOF is the error returned by Read when no more input is available. 38 // (Read must return EOF itself, not an error wrapping EOF, 39 // because callers will test for EOF using ==.) 40 // Functions should return EOF only to signal a graceful end of input. 41 // If the EOF occurs unexpectedly in a structured data stream, 42 // the appropriate error is either ErrUnexpectedEOF or some other error 43 // giving more detail. 44 var EOF = errors.New("EOF") 45 46 // ErrUnexpectedEOF means that EOF was encountered in the 47 // middle of reading a fixed-size block or data structure. 48 var ErrUnexpectedEOF = errors.New("unexpected EOF") 49 50 // ErrNoProgress is returned by some clients of an Reader when 51 // many calls to Read have failed to return any data or error, 52 // usually the sign of a broken Reader implementation. 53 var ErrNoProgress = errors.New("multiple Read calls return no data or error") 54 55 // Reader is the interface that wraps the basic Read method. 56 // 57 // Read reads up to len(p) bytes into p. It returns the number of bytes 58 // read (0 <= n <= len(p)) and any error encountered. Even if Read 59 // returns n < len(p), it may use all of p as scratch space during the call. 60 // If some data is available but not len(p) bytes, Read conventionally 61 // returns what is available instead of waiting for more. 62 // 63 // When Read encounters an error or end-of-file condition after 64 // successfully reading n > 0 bytes, it returns the number of 65 // bytes read. It may return the (non-nil) error from the same call 66 // or return the error (and n == 0) from a subsequent call. 67 // An instance of this general case is that a Reader returning 68 // a non-zero number of bytes at the end of the input stream may 69 // return either err == EOF or err == nil. The next Read should 70 // return 0, EOF. 71 // 72 // Callers should always process the n > 0 bytes returned before 73 // considering the error err. Doing so correctly handles I/O errors 74 // that happen after reading some bytes and also both of the 75 // allowed EOF behaviors. 76 // 77 // Implementations of Read are discouraged from returning a 78 // zero byte count with a nil error, except when len(p) == 0. 79 // Callers should treat a return of 0 and nil as indicating that 80 // nothing happened; in particular it does not indicate EOF. 81 // 82 // Implementations must not retain p. 83 type Reader interface { 84 Read(p []byte) (n int, err error) 85 } 86 87 // Writer is the interface that wraps the basic Write method. 88 // 89 // Write writes len(p) bytes from p to the underlying data stream. 90 // It returns the number of bytes written from p (0 <= n <= len(p)) 91 // and any error encountered that caused the write to stop early. 92 // Write must return a non-nil error if it returns n < len(p). 93 // Write must not modify the slice data, even temporarily. 94 // 95 // Implementations must not retain p. 96 type Writer interface { 97 Write(p []byte) (n int, err error) 98 } 99 100 // Closer is the interface that wraps the basic Close method. 101 // 102 // The behavior of Close after the first call is undefined. 103 // Specific implementations may document their own behavior. 104 type Closer interface { 105 Close() error 106 } 107 108 // Seeker is the interface that wraps the basic Seek method. 109 // 110 // Seek sets the offset for the next Read or Write to offset, 111 // interpreted according to whence: 112 // SeekStart means relative to the start of the file, 113 // SeekCurrent means relative to the current offset, and 114 // SeekEnd means relative to the end. 115 // Seek returns the new offset relative to the start of the 116 // file and an error, if any. 117 // 118 // Seeking to an offset before the start of the file is an error. 119 // Seeking to any positive offset is legal, but the behavior of subsequent 120 // I/O operations on the underlying object is implementation-dependent. 121 type Seeker interface { 122 Seek(offset int64, whence int) (int64, error) 123 } 124 125 // ReadWriter is the interface that groups the basic Read and Write methods. 126 type ReadWriter interface { 127 Reader 128 Writer 129 } 130 131 // ReadCloser is the interface that groups the basic Read and Close methods. 132 type ReadCloser interface { 133 Reader 134 Closer 135 } 136 137 // WriteCloser is the interface that groups the basic Write and Close methods. 138 type WriteCloser interface { 139 Writer 140 Closer 141 } 142 143 // ReadWriteCloser is the interface that groups the basic Read, Write and Close methods. 144 type ReadWriteCloser interface { 145 Reader 146 Writer 147 Closer 148 } 149 150 // ReadSeeker is the interface that groups the basic Read and Seek methods. 151 type ReadSeeker interface { 152 Reader 153 Seeker 154 } 155 156 // ReadSeekCloser is the interface that groups the basic Read, Seek and Close 157 // methods. 158 type ReadSeekCloser interface { 159 Reader 160 Seeker 161 Closer 162 } 163 164 // WriteSeeker is the interface that groups the basic Write and Seek methods. 165 type WriteSeeker interface { 166 Writer 167 Seeker 168 } 169 170 // ReadWriteSeeker is the interface that groups the basic Read, Write and Seek methods. 171 type ReadWriteSeeker interface { 172 Reader 173 Writer 174 Seeker 175 } 176 177 // ReaderFrom is the interface that wraps the ReadFrom method. 178 // 179 // ReadFrom reads data from r until EOF or error. 180 // The return value n is the number of bytes read. 181 // Any error except EOF encountered during the read is also returned. 182 // 183 // The Copy function uses ReaderFrom if available. 184 type ReaderFrom interface { 185 ReadFrom(r Reader) (n int64, err error) 186 } 187 188 // WriterTo is the interface that wraps the WriteTo method. 189 // 190 // WriteTo writes data to w until there's no more data to write or 191 // when an error occurs. The return value n is the number of bytes 192 // written. Any error encountered during the write is also returned. 193 // 194 // The Copy function uses WriterTo if available. 195 type WriterTo interface { 196 WriteTo(w Writer) (n int64, err error) 197 } 198 199 // ReaderAt is the interface that wraps the basic ReadAt method. 200 // 201 // ReadAt reads len(p) bytes into p starting at offset off in the 202 // underlying input source. It returns the number of bytes 203 // read (0 <= n <= len(p)) and any error encountered. 204 // 205 // When ReadAt returns n < len(p), it returns a non-nil error 206 // explaining why more bytes were not returned. In this respect, 207 // ReadAt is stricter than Read. 208 // 209 // Even if ReadAt returns n < len(p), it may use all of p as scratch 210 // space during the call. If some data is available but not len(p) bytes, 211 // ReadAt blocks until either all the data is available or an error occurs. 212 // In this respect ReadAt is different from Read. 213 // 214 // If the n = len(p) bytes returned by ReadAt are at the end of the 215 // input source, ReadAt may return either err == EOF or err == nil. 216 // 217 // If ReadAt is reading from an input source with a seek offset, 218 // ReadAt should not affect nor be affected by the underlying 219 // seek offset. 220 // 221 // Clients of ReadAt can execute parallel ReadAt calls on the 222 // same input source. 223 // 224 // Implementations must not retain p. 225 type ReaderAt interface { 226 ReadAt(p []byte, off int64) (n int, err error) 227 } 228 229 // WriterAt is the interface that wraps the basic WriteAt method. 230 // 231 // WriteAt writes len(p) bytes from p to the underlying data stream 232 // at offset off. It returns the number of bytes written from p (0 <= n <= len(p)) 233 // and any error encountered that caused the write to stop early. 234 // WriteAt must return a non-nil error if it returns n < len(p). 235 // 236 // If WriteAt is writing to a destination with a seek offset, 237 // WriteAt should not affect nor be affected by the underlying 238 // seek offset. 239 // 240 // Clients of WriteAt can execute parallel WriteAt calls on the same 241 // destination if the ranges do not overlap. 242 // 243 // Implementations must not retain p. 244 type WriterAt interface { 245 WriteAt(p []byte, off int64) (n int, err error) 246 } 247 248 // ByteReader is the interface that wraps the ReadByte method. 249 // 250 // ReadByte reads and returns the next byte from the input or 251 // any error encountered. If ReadByte returns an error, no input 252 // byte was consumed, and the returned byte value is undefined. 253 // 254 // ReadByte provides an efficient interface for byte-at-time 255 // processing. A Reader that does not implement ByteReader 256 // can be wrapped using bufio.NewReader to add this method. 257 type ByteReader interface { 258 ReadByte() (byte, error) 259 } 260 261 // ByteScanner is the interface that adds the UnreadByte method to the 262 // basic ReadByte method. 263 // 264 // UnreadByte causes the next call to ReadByte to return the same byte 265 // as the previous call to ReadByte. 266 // It may be an error to call UnreadByte twice without an intervening 267 // call to ReadByte. 268 type ByteScanner interface { 269 ByteReader 270 UnreadByte() error 271 } 272 273 // ByteWriter is the interface that wraps the WriteByte method. 274 type ByteWriter interface { 275 WriteByte(c byte) error 276 } 277 278 // RuneReader is the interface that wraps the ReadRune method. 279 // 280 // ReadRune reads a single UTF-8 encoded Unicode character 281 // and returns the rune and its size in bytes. If no character is 282 // available, err will be set. 283 type RuneReader interface { 284 ReadRune() (r rune, size int, err error) 285 } 286 287 // RuneScanner is the interface that adds the UnreadRune method to the 288 // basic ReadRune method. 289 // 290 // UnreadRune causes the next call to ReadRune to return the same rune 291 // as the previous call to ReadRune. 292 // It may be an error to call UnreadRune twice without an intervening 293 // call to ReadRune. 294 type RuneScanner interface { 295 RuneReader 296 UnreadRune() error 297 } 298 299 // StringWriter is the interface that wraps the WriteString method. 300 type StringWriter interface { 301 WriteString(s string) (n int, err error) 302 } 303 304 // WriteString writes the contents of the string s to w, which accepts a slice of bytes. 305 // If w implements StringWriter, its WriteString method is invoked directly. 306 // Otherwise, w.Write is called exactly once. 307 func WriteString(w Writer, s string) (n int, err error) { 308 if sw, ok := w.(StringWriter); ok { 309 return sw.WriteString(s) 310 } 311 return w.Write([]byte(s)) 312 } 313 314 // ReadAtLeast reads from r into buf until it has read at least min bytes. 315 // It returns the number of bytes copied and an error if fewer bytes were read. 316 // The error is EOF only if no bytes were read. 317 // If an EOF happens after reading fewer than min bytes, 318 // ReadAtLeast returns ErrUnexpectedEOF. 319 // If min is greater than the length of buf, ReadAtLeast returns ErrShortBuffer. 320 // On return, n >= min if and only if err == nil. 321 // If r returns an error having read at least min bytes, the error is dropped. 322 func ReadAtLeast(r Reader, buf []byte, min int) (n int, err error) { 323 if len(buf) < min { 324 return 0, ErrShortBuffer 325 } 326 for n < min && err == nil { 327 var nn int 328 nn, err = r.Read(buf[n:]) 329 n += nn 330 } 331 if n >= min { 332 err = nil 333 } else if n > 0 && err == EOF { 334 err = ErrUnexpectedEOF 335 } 336 return 337 } 338 339 // ReadFull reads exactly len(buf) bytes from r into buf. 340 // It returns the number of bytes copied and an error if fewer bytes were read. 341 // The error is EOF only if no bytes were read. 342 // If an EOF happens after reading some but not all the bytes, 343 // ReadFull returns ErrUnexpectedEOF. 344 // On return, n == len(buf) if and only if err == nil. 345 // If r returns an error having read at least len(buf) bytes, the error is dropped. 346 func ReadFull(r Reader, buf []byte) (n int, err error) { 347 return ReadAtLeast(r, buf, len(buf)) 348 } 349 350 // CopyN copies n bytes (or until an error) from src to dst. 351 // It returns the number of bytes copied and the earliest 352 // error encountered while copying. 353 // On return, written == n if and only if err == nil. 354 // 355 // If dst implements the ReaderFrom interface, 356 // the copy is implemented using it. 357 func CopyN(dst Writer, src Reader, n int64) (written int64, err error) { 358 written, err = Copy(dst, LimitReader(src, n)) 359 if written == n { 360 return n, nil 361 } 362 if written < n && err == nil { 363 // src stopped early; must have been EOF. 364 err = EOF 365 } 366 return 367 } 368 369 // Copy copies from src to dst until either EOF is reached 370 // on src or an error occurs. It returns the number of bytes 371 // copied and the first error encountered while copying, if any. 372 // 373 // A successful Copy returns err == nil, not err == EOF. 374 // Because Copy is defined to read from src until EOF, it does 375 // not treat an EOF from Read as an error to be reported. 376 // 377 // If src implements the WriterTo interface, 378 // the copy is implemented by calling src.WriteTo(dst). 379 // Otherwise, if dst implements the ReaderFrom interface, 380 // the copy is implemented by calling dst.ReadFrom(src). 381 func Copy(dst Writer, src Reader) (written int64, err error) { 382 return copyBuffer(dst, src, nil) 383 } 384 385 // CopyBuffer is identical to Copy except that it stages through the 386 // provided buffer (if one is required) rather than allocating a 387 // temporary one. If buf is nil, one is allocated; otherwise if it has 388 // zero length, CopyBuffer panics. 389 // 390 // If either src implements WriterTo or dst implements ReaderFrom, 391 // buf will not be used to perform the copy. 392 func CopyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 393 if buf != nil && len(buf) == 0 { 394 panic("empty buffer in CopyBuffer") 395 } 396 return copyBuffer(dst, src, buf) 397 } 398 399 // copyBuffer is the actual implementation of Copy and CopyBuffer. 400 // if buf is nil, one is allocated. 401 func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) { 402 // If the reader has a WriteTo method, use it to do the copy. 403 // Avoids an allocation and a copy. 404 if wt, ok := src.(WriterTo); ok { 405 return wt.WriteTo(dst) 406 } 407 // Similarly, if the writer has a ReadFrom method, use it to do the copy. 408 if rt, ok := dst.(ReaderFrom); ok { 409 return rt.ReadFrom(src) 410 } 411 if buf == nil { 412 size := 32 * 1024 413 if l, ok := src.(*LimitedReader); ok && int64(size) > l.N { 414 if l.N < 1 { 415 size = 1 416 } else { 417 size = int(l.N) 418 } 419 } 420 buf = make([]byte, size) 421 } 422 for { 423 nr, er := src.Read(buf) 424 if nr > 0 { 425 nw, ew := dst.Write(buf[0:nr]) 426 if nw < 0 || nr < nw { 427 nw = 0 428 if ew == nil { 429 ew = errInvalidWrite 430 } 431 } 432 written += int64(nw) 433 if ew != nil { 434 err = ew 435 break 436 } 437 if nr != nw { 438 err = ErrShortWrite 439 break 440 } 441 } 442 if er != nil { 443 if er != EOF { 444 err = er 445 } 446 break 447 } 448 } 449 return written, err 450 } 451 452 // LimitReader returns a Reader that reads from r 453 // but stops with EOF after n bytes. 454 // The underlying implementation is a *LimitedReader. 455 func LimitReader(r Reader, n int64) Reader { return &LimitedReader{r, n} } 456 457 // A LimitedReader reads from R but limits the amount of 458 // data returned to just N bytes. Each call to Read 459 // updates N to reflect the new amount remaining. 460 // Read returns EOF when N <= 0 or when the underlying R returns EOF. 461 type LimitedReader struct { 462 R Reader // underlying reader 463 N int64 // max bytes remaining 464 } 465 466 func (l *LimitedReader) Read(p []byte) (n int, err error) { 467 if l.N <= 0 { 468 return 0, EOF 469 } 470 if int64(len(p)) > l.N { 471 p = p[0:l.N] 472 } 473 n, err = l.R.Read(p) 474 l.N -= int64(n) 475 return 476 } 477 478 // NewSectionReader returns a SectionReader that reads from r 479 // starting at offset off and stops with EOF after n bytes. 480 func NewSectionReader(r ReaderAt, off int64, n int64) *SectionReader { 481 return &SectionReader{r, off, off, off + n} 482 } 483 484 // SectionReader implements Read, Seek, and ReadAt on a section 485 // of an underlying ReaderAt. 486 type SectionReader struct { 487 r ReaderAt 488 base int64 489 off int64 490 limit int64 491 } 492 493 func (s *SectionReader) Read(p []byte) (n int, err error) { 494 if s.off >= s.limit { 495 return 0, EOF 496 } 497 if max := s.limit - s.off; int64(len(p)) > max { 498 p = p[0:max] 499 } 500 n, err = s.r.ReadAt(p, s.off) 501 s.off += int64(n) 502 return 503 } 504 505 var errWhence = errors.New("Seek: invalid whence") 506 var errOffset = errors.New("Seek: invalid offset") 507 508 func (s *SectionReader) Seek(offset int64, whence int) (int64, error) { 509 switch whence { 510 default: 511 return 0, errWhence 512 case SeekStart: 513 offset += s.base 514 case SeekCurrent: 515 offset += s.off 516 case SeekEnd: 517 offset += s.limit 518 } 519 if offset < s.base { 520 return 0, errOffset 521 } 522 s.off = offset 523 return offset - s.base, nil 524 } 525 526 func (s *SectionReader) ReadAt(p []byte, off int64) (n int, err error) { 527 if off < 0 || off >= s.limit-s.base { 528 return 0, EOF 529 } 530 off += s.base 531 if max := s.limit - off; int64(len(p)) > max { 532 p = p[0:max] 533 n, err = s.r.ReadAt(p, off) 534 if err == nil { 535 err = EOF 536 } 537 return n, err 538 } 539 return s.r.ReadAt(p, off) 540 } 541 542 // Size returns the size of the section in bytes. 543 func (s *SectionReader) Size() int64 { return s.limit - s.base } 544 545 // TeeReader returns a Reader that writes to w what it reads from r. 546 // All reads from r performed through it are matched with 547 // corresponding writes to w. There is no internal buffering - 548 // the write must complete before the read completes. 549 // Any error encountered while writing is reported as a read error. 550 func TeeReader(r Reader, w Writer) Reader { 551 return &teeReader{r, w} 552 } 553 554 type teeReader struct { 555 r Reader 556 w Writer 557 } 558 559 func (t *teeReader) Read(p []byte) (n int, err error) { 560 n, err = t.r.Read(p) 561 if n > 0 { 562 if n, err := t.w.Write(p[:n]); err != nil { 563 return n, err 564 } 565 } 566 return 567 } 568 569 // Discard is a Writer on which all Write calls succeed 570 // without doing anything. 571 var Discard Writer = discard{} 572 573 type discard struct{} 574 575 // discard implements ReaderFrom as an optimization so Copy to 576 // io.Discard can avoid doing unnecessary work. 577 var _ ReaderFrom = discard{} 578 579 func (discard) Write(p []byte) (int, error) { 580 return len(p), nil 581 } 582 583 func (discard) WriteString(s string) (int, error) { 584 return len(s), nil 585 } 586 587 var blackHolePool = sync.Pool{ 588 New: func() interface{} { 589 b := make([]byte, 8192) 590 return &b 591 }, 592 } 593 594 func (discard) ReadFrom(r Reader) (n int64, err error) { 595 bufp := blackHolePool.Get().(*[]byte) 596 readSize := 0 597 for { 598 readSize, err = r.Read(*bufp) 599 n += int64(readSize) 600 if err != nil { 601 blackHolePool.Put(bufp) 602 if err == EOF { 603 return n, nil 604 } 605 return 606 } 607 } 608 } 609 610 // NopCloser returns a ReadCloser with a no-op Close method wrapping 611 // the provided Reader r. 612 func NopCloser(r Reader) ReadCloser { 613 return nopCloser{r} 614 } 615 616 type nopCloser struct { 617 Reader 618 } 619 620 func (nopCloser) Close() error { return nil } 621 622 // ReadAll reads from r until an error or EOF and returns the data it read. 623 // A successful call returns err == nil, not err == EOF. Because ReadAll is 624 // defined to read from src until EOF, it does not treat an EOF from Read 625 // as an error to be reported. 626 func ReadAll(r Reader) ([]byte, error) { 627 b := make([]byte, 0, 512) 628 for { 629 if len(b) == cap(b) { 630 // Add more capacity (let append pick how much). 631 b = append(b, 0)[:len(b)] 632 } 633 n, err := r.Read(b[len(b):cap(b)]) 634 b = b[:len(b)+n] 635 if err != nil { 636 if err == EOF { 637 err = nil 638 } 639 return b, err 640 } 641 } 642 } 643