bufio.go 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699
  1. // Copyright 2009 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer
  5. // object, creating another object (Reader or Writer) that also implements
  6. // the interface but provides buffering and some help for textual I/O.
  7. package bufio
  8. import (
  9. "bytes"
  10. "errors"
  11. "io"
  12. "unicode/utf8"
  13. )
  14. const (
  15. defaultBufSize = 4096
  16. )
  17. var (
  18. ErrInvalidUnreadByte = errors.New("bufio: invalid use of UnreadByte")
  19. ErrInvalidUnreadRune = errors.New("bufio: invalid use of UnreadRune")
  20. ErrBufferFull = errors.New("bufio: buffer full")
  21. ErrNegativeCount = errors.New("bufio: negative count")
  22. )
  23. // Buffered input.
  24. // Reader implements buffering for an io.Reader object.
  25. type Reader struct {
  26. buf []byte
  27. rd io.Reader // reader provided by the client
  28. r, w int // buf read and write positions
  29. err error
  30. lastByte int
  31. lastRuneSize int
  32. }
  33. const minReadBufferSize = 16
  34. const maxConsecutiveEmptyReads = 100
  35. // NewReaderSize returns a new Reader whose buffer has at least the specified
  36. // size. If the argument io.Reader is already a Reader with large enough
  37. // size, it returns the underlying Reader.
  38. func NewReaderSize(rd io.Reader, size int) *Reader {
  39. // Is it already a Reader?
  40. b, ok := rd.(*Reader)
  41. if ok && len(b.buf) >= size {
  42. return b
  43. }
  44. if size < minReadBufferSize {
  45. size = minReadBufferSize
  46. }
  47. r := new(Reader)
  48. r.reset(make([]byte, size), rd)
  49. return r
  50. }
  51. // NewReader returns a new Reader whose buffer has the default size.
  52. func NewReader(rd io.Reader) *Reader {
  53. return NewReaderSize(rd, defaultBufSize)
  54. }
  55. // Reset discards any buffered data, resets all state, and switches
  56. // the buffered reader to read from r.
  57. func (b *Reader) Reset(r io.Reader) {
  58. b.reset(b.buf, r)
  59. }
  60. func (b *Reader) reset(buf []byte, r io.Reader) {
  61. *b = Reader{
  62. buf: buf,
  63. rd: r,
  64. lastByte: -1,
  65. lastRuneSize: -1,
  66. }
  67. }
  68. var errNegativeRead = errors.New("bufio: reader returned negative count from Read")
  69. // fill reads a new chunk into the buffer.
  70. func (b *Reader) fill() {
  71. // Slide existing data to beginning.
  72. if b.r > 0 {
  73. copy(b.buf, b.buf[b.r:b.w])
  74. b.w -= b.r
  75. b.r = 0
  76. }
  77. if b.w >= len(b.buf) {
  78. panic("bufio: tried to fill full buffer")
  79. }
  80. // Read new data: try a limited number of times.
  81. for i := maxConsecutiveEmptyReads; i > 0; i-- {
  82. n, err := b.rd.Read(b.buf[b.w:])
  83. if n < 0 {
  84. panic(errNegativeRead)
  85. }
  86. b.w += n
  87. if err != nil {
  88. b.err = err
  89. return
  90. }
  91. if n > 0 {
  92. return
  93. }
  94. }
  95. b.err = io.ErrNoProgress
  96. }
  97. func (b *Reader) readErr() error {
  98. err := b.err
  99. b.err = nil
  100. return err
  101. }
  102. // Peek returns the next n bytes without advancing the reader. The bytes stop
  103. // being valid at the next read call. If Peek returns fewer than n bytes, it
  104. // also returns an error explaining why the read is short. The error is
  105. // ErrBufferFull if n is larger than b's buffer size.
  106. func (b *Reader) Peek(n int) ([]byte, error) {
  107. if n < 0 {
  108. return nil, ErrNegativeCount
  109. }
  110. if n > len(b.buf) {
  111. return nil, ErrBufferFull
  112. }
  113. // 0 <= n <= len(b.buf)
  114. for b.w-b.r < n && b.err == nil {
  115. b.fill() // b.w-b.r < len(b.buf) => buffer is not full
  116. }
  117. var err error
  118. if avail := b.w - b.r; avail < n {
  119. // not enough data in buffer
  120. n = avail
  121. err = b.readErr()
  122. if err == nil {
  123. err = ErrBufferFull
  124. }
  125. }
  126. return b.buf[b.r : b.r+n], err
  127. }
  128. // Read reads data into p.
  129. // It returns the number of bytes read into p.
  130. // It calls Read at most once on the underlying Reader,
  131. // hence n may be less than len(p).
  132. // At EOF, the count will be zero and err will be io.EOF.
  133. func (b *Reader) Read(p []byte) (n int, err error) {
  134. n = len(p)
  135. if n == 0 {
  136. return 0, b.readErr()
  137. }
  138. if b.r == b.w {
  139. if b.err != nil {
  140. return 0, b.readErr()
  141. }
  142. if len(p) >= len(b.buf) {
  143. // Large read, empty buffer.
  144. // Read directly into p to avoid copy.
  145. n, b.err = b.rd.Read(p)
  146. if n < 0 {
  147. panic(errNegativeRead)
  148. }
  149. if n > 0 {
  150. b.lastByte = int(p[n-1])
  151. b.lastRuneSize = -1
  152. }
  153. return n, b.readErr()
  154. }
  155. b.fill() // buffer is empty
  156. if b.r == b.w {
  157. return 0, b.readErr()
  158. }
  159. }
  160. // copy as much as we can
  161. n = copy(p, b.buf[b.r:b.w])
  162. b.r += n
  163. b.lastByte = int(b.buf[b.r-1])
  164. b.lastRuneSize = -1
  165. return n, nil
  166. }
  167. // ReadByte reads and returns a single byte.
  168. // If no byte is available, returns an error.
  169. func (b *Reader) ReadByte() (c byte, err error) {
  170. b.lastRuneSize = -1
  171. for b.r == b.w {
  172. if b.err != nil {
  173. return 0, b.readErr()
  174. }
  175. b.fill() // buffer is empty
  176. }
  177. c = b.buf[b.r]
  178. b.r++
  179. b.lastByte = int(c)
  180. return c, nil
  181. }
  182. // UnreadByte unreads the last byte. Only the most recently read byte can be unread.
  183. func (b *Reader) UnreadByte() error {
  184. if b.lastByte < 0 || b.r == 0 && b.w > 0 {
  185. return ErrInvalidUnreadByte
  186. }
  187. // b.r > 0 || b.w == 0
  188. if b.r > 0 {
  189. b.r--
  190. } else {
  191. // b.r == 0 && b.w == 0
  192. b.w = 1
  193. }
  194. b.buf[b.r] = byte(b.lastByte)
  195. b.lastByte = -1
  196. b.lastRuneSize = -1
  197. return nil
  198. }
  199. // ReadRune reads a single UTF-8 encoded Unicode character and returns the
  200. // rune and its size in bytes. If the encoded rune is invalid, it consumes one byte
  201. // and returns unicode.ReplacementChar (U+FFFD) with a size of 1.
  202. func (b *Reader) ReadRune() (r rune, size int, err error) {
  203. for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil && b.w-b.r < len(b.buf) {
  204. b.fill() // b.w-b.r < len(buf) => buffer is not full
  205. }
  206. b.lastRuneSize = -1
  207. if b.r == b.w {
  208. return 0, 0, b.readErr()
  209. }
  210. r, size = rune(b.buf[b.r]), 1
  211. if r >= 0x80 {
  212. r, size = utf8.DecodeRune(b.buf[b.r:b.w])
  213. }
  214. b.r += size
  215. b.lastByte = int(b.buf[b.r-1])
  216. b.lastRuneSize = size
  217. return r, size, nil
  218. }
  219. // UnreadRune unreads the last rune. If the most recent read operation on
  220. // the buffer was not a ReadRune, UnreadRune returns an error. (In this
  221. // regard it is stricter than UnreadByte, which will unread the last byte
  222. // from any read operation.)
  223. func (b *Reader) UnreadRune() error {
  224. if b.lastRuneSize < 0 || b.r < b.lastRuneSize {
  225. return ErrInvalidUnreadRune
  226. }
  227. b.r -= b.lastRuneSize
  228. b.lastByte = -1
  229. b.lastRuneSize = -1
  230. return nil
  231. }
  232. // Buffered returns the number of bytes that can be read from the current buffer.
  233. func (b *Reader) Buffered() int { return b.w - b.r }
  234. // ReadSlice reads until the first occurrence of delim in the input,
  235. // returning a slice pointing at the bytes in the buffer.
  236. // The bytes stop being valid at the next read.
  237. // If ReadSlice encounters an error before finding a delimiter,
  238. // it returns all the data in the buffer and the error itself (often io.EOF).
  239. // ReadSlice fails with error ErrBufferFull if the buffer fills without a delim.
  240. // Because the data returned from ReadSlice will be overwritten
  241. // by the next I/O operation, most clients should use
  242. // ReadBytes or ReadString instead.
  243. // ReadSlice returns err != nil if and only if line does not end in delim.
  244. func (b *Reader) ReadSlice(delim byte) (line []byte, err error) {
  245. for {
  246. // Search buffer.
  247. if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 {
  248. line = b.buf[b.r : b.r+i+1]
  249. b.r += i + 1
  250. break
  251. }
  252. // Pending error?
  253. if b.err != nil {
  254. line = b.buf[b.r:b.w]
  255. b.r = b.w
  256. err = b.readErr()
  257. break
  258. }
  259. // Buffer full?
  260. if b.Buffered() >= len(b.buf) {
  261. b.r = b.w
  262. line = b.buf
  263. err = ErrBufferFull
  264. break
  265. }
  266. b.fill() // buffer is not full
  267. }
  268. // Handle last byte, if any.
  269. if i := len(line) - 1; i >= 0 {
  270. b.lastByte = int(line[i])
  271. b.lastRuneSize = -1
  272. }
  273. return
  274. }
  275. // ReadLine is a low-level line-reading primitive. Most callers should use
  276. // ReadBytes('\n') or ReadString('\n') instead or use a Scanner.
  277. //
  278. // ReadLine tries to return a single line, not including the end-of-line bytes.
  279. // If the line was too long for the buffer then isPrefix is set and the
  280. // beginning of the line is returned. The rest of the line will be returned
  281. // from future calls. isPrefix will be false when returning the last fragment
  282. // of the line. The returned buffer is only valid until the next call to
  283. // ReadLine. ReadLine either returns a non-nil line or it returns an error,
  284. // never both.
  285. //
  286. // The text returned from ReadLine does not include the line end ("\r\n" or "\n").
  287. // No indication or error is given if the input ends without a final line end.
  288. // Calling UnreadByte after ReadLine will always unread the last byte read
  289. // (possibly a character belonging to the line end) even if that byte is not
  290. // part of the line returned by ReadLine.
  291. func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) {
  292. line, err = b.ReadSlice('\n')
  293. if err == ErrBufferFull {
  294. // Handle the case where "\r\n" straddles the buffer.
  295. if len(line) > 0 && line[len(line)-1] == '\r' {
  296. // Put the '\r' back on buf and drop it from line.
  297. // Let the next call to ReadLine check for "\r\n".
  298. if b.r == 0 {
  299. // should be unreachable
  300. panic("bufio: tried to rewind past start of buffer")
  301. }
  302. b.r--
  303. line = line[:len(line)-1]
  304. }
  305. return line, true, nil
  306. }
  307. if len(line) == 0 {
  308. if err != nil {
  309. line = nil
  310. }
  311. return
  312. }
  313. err = nil
  314. if line[len(line)-1] == '\n' {
  315. drop := 1
  316. if len(line) > 1 && line[len(line)-2] == '\r' {
  317. drop = 2
  318. }
  319. line = line[:len(line)-drop]
  320. }
  321. return
  322. }
  323. // ReadBytes reads until the first occurrence of delim in the input,
  324. // returning a slice containing the data up to and including the delimiter.
  325. // If ReadBytes encounters an error before finding a delimiter,
  326. // it returns the data read before the error and the error itself (often io.EOF).
  327. // ReadBytes returns err != nil if and only if the returned data does not end in
  328. // delim.
  329. // For simple uses, a Scanner may be more convenient.
  330. func (b *Reader) ReadBytes(delim byte) (line []byte, err error) {
  331. // Use ReadSlice to look for array,
  332. // accumulating full buffers.
  333. var frag []byte
  334. var full [][]byte
  335. err = nil
  336. for {
  337. var e error
  338. frag, e = b.ReadSlice(delim)
  339. if e == nil { // got final fragment
  340. break
  341. }
  342. if e != ErrBufferFull { // unexpected error
  343. err = e
  344. break
  345. }
  346. // Make a copy of the buffer.
  347. buf := make([]byte, len(frag))
  348. copy(buf, frag)
  349. full = append(full, buf)
  350. }
  351. // Allocate new buffer to hold the full pieces and the fragment.
  352. n := 0
  353. for i := range full {
  354. n += len(full[i])
  355. }
  356. n += len(frag)
  357. // Copy full pieces and fragment in.
  358. buf := make([]byte, n)
  359. n = 0
  360. for i := range full {
  361. n += copy(buf[n:], full[i])
  362. }
  363. copy(buf[n:], frag)
  364. return buf, err
  365. }
  366. // ReadString reads until the first occurrence of delim in the input,
  367. // returning a string containing the data up to and including the delimiter.
  368. // If ReadString encounters an error before finding a delimiter,
  369. // it returns the data read before the error and the error itself (often io.EOF).
  370. // ReadString returns err != nil if and only if the returned data does not end in
  371. // delim.
  372. // For simple uses, a Scanner may be more convenient.
  373. func (b *Reader) ReadString(delim byte) (line string, err error) {
  374. bytes, err := b.ReadBytes(delim)
  375. line = string(bytes)
  376. return line, err
  377. }
  378. // WriteTo implements io.WriterTo.
  379. func (b *Reader) WriteTo(w io.Writer) (n int64, err error) {
  380. n, err = b.writeBuf(w)
  381. if err != nil {
  382. return
  383. }
  384. if r, ok := b.rd.(io.WriterTo); ok {
  385. m, err := r.WriteTo(w)
  386. n += m
  387. return n, err
  388. }
  389. if w, ok := w.(io.ReaderFrom); ok {
  390. m, err := w.ReadFrom(b.rd)
  391. n += m
  392. return n, err
  393. }
  394. if b.w-b.r < len(b.buf) {
  395. b.fill() // buffer not full
  396. }
  397. for b.r < b.w {
  398. // b.r < b.w => buffer is not empty
  399. m, err := b.writeBuf(w)
  400. n += m
  401. if err != nil {
  402. return n, err
  403. }
  404. b.fill() // buffer is empty
  405. }
  406. if b.err == io.EOF {
  407. b.err = nil
  408. }
  409. return n, b.readErr()
  410. }
  411. var errNegativeWrite = errors.New("bufio: writer returned negative count from Write")
  412. // writeBuf writes the Reader's buffer to the writer.
  413. func (b *Reader) writeBuf(w io.Writer) (int64, error) {
  414. n, err := w.Write(b.buf[b.r:b.w])
  415. if n < 0 {
  416. panic(errNegativeWrite)
  417. }
  418. b.r += n
  419. return int64(n), err
  420. }
  421. // buffered output
  422. // Writer implements buffering for an io.Writer object.
  423. // If an error occurs writing to a Writer, no more data will be
  424. // accepted and all subsequent writes will return the error.
  425. // After all data has been written, the client should call the
  426. // Flush method to guarantee all data has been forwarded to
  427. // the underlying io.Writer.
  428. type Writer struct {
  429. err error
  430. buf []byte
  431. n int
  432. wr io.Writer
  433. }
  434. // NewWriterSize returns a new Writer whose buffer has at least the specified
  435. // size. If the argument io.Writer is already a Writer with large enough
  436. // size, it returns the underlying Writer.
  437. func NewWriterSize(w io.Writer, size int) *Writer {
  438. // Is it already a Writer?
  439. b, ok := w.(*Writer)
  440. if ok && len(b.buf) >= size {
  441. return b
  442. }
  443. if size <= 0 {
  444. size = defaultBufSize
  445. }
  446. return &Writer{
  447. buf: make([]byte, size),
  448. wr: w,
  449. }
  450. }
  451. // NewWriter returns a new Writer whose buffer has the default size.
  452. func NewWriter(w io.Writer) *Writer {
  453. return NewWriterSize(w, defaultBufSize)
  454. }
  455. // Reset discards any unflushed buffered data, clears any error, and
  456. // resets b to write its output to w.
  457. func (b *Writer) Reset(w io.Writer) {
  458. b.err = nil
  459. b.n = 0
  460. b.wr = w
  461. }
  462. // Flush writes any buffered data to the underlying io.Writer.
  463. func (b *Writer) Flush() error {
  464. err := b.flush()
  465. return err
  466. }
  467. func (b *Writer) flush() error {
  468. if b.err != nil {
  469. return b.err
  470. }
  471. if b.n == 0 {
  472. return nil
  473. }
  474. n, err := b.wr.Write(b.buf[0:b.n])
  475. if n < b.n && err == nil {
  476. err = io.ErrShortWrite
  477. }
  478. if err != nil {
  479. if n > 0 && n < b.n {
  480. copy(b.buf[0:b.n-n], b.buf[n:b.n])
  481. }
  482. b.n -= n
  483. b.err = err
  484. return err
  485. }
  486. b.n = 0
  487. return nil
  488. }
  489. // Available returns how many bytes are unused in the buffer.
  490. func (b *Writer) Available() int { return len(b.buf) - b.n }
  491. // Buffered returns the number of bytes that have been written into the current buffer.
  492. func (b *Writer) Buffered() int { return b.n }
  493. // Write writes the contents of p into the buffer.
  494. // It returns the number of bytes written.
  495. // If nn < len(p), it also returns an error explaining
  496. // why the write is short.
  497. func (b *Writer) Write(p []byte) (nn int, err error) {
  498. for len(p) > b.Available() && b.err == nil {
  499. var n int
  500. if b.Buffered() == 0 {
  501. // Large write, empty buffer.
  502. // Write directly from p to avoid copy.
  503. n, b.err = b.wr.Write(p)
  504. } else {
  505. n = copy(b.buf[b.n:], p)
  506. b.n += n
  507. b.flush()
  508. }
  509. nn += n
  510. p = p[n:]
  511. }
  512. if b.err != nil {
  513. return nn, b.err
  514. }
  515. n := copy(b.buf[b.n:], p)
  516. b.n += n
  517. nn += n
  518. return nn, nil
  519. }
  520. // WriteByte writes a single byte.
  521. func (b *Writer) WriteByte(c byte) error {
  522. if b.err != nil {
  523. return b.err
  524. }
  525. if b.Available() <= 0 && b.flush() != nil {
  526. return b.err
  527. }
  528. b.buf[b.n] = c
  529. b.n++
  530. return nil
  531. }
  532. // WriteRune writes a single Unicode code point, returning
  533. // the number of bytes written and any error.
  534. func (b *Writer) WriteRune(r rune) (size int, err error) {
  535. if r < utf8.RuneSelf {
  536. err = b.WriteByte(byte(r))
  537. if err != nil {
  538. return 0, err
  539. }
  540. return 1, nil
  541. }
  542. if b.err != nil {
  543. return 0, b.err
  544. }
  545. n := b.Available()
  546. if n < utf8.UTFMax {
  547. if b.flush(); b.err != nil {
  548. return 0, b.err
  549. }
  550. n = b.Available()
  551. if n < utf8.UTFMax {
  552. // Can only happen if buffer is silly small.
  553. return b.WriteString(string(r))
  554. }
  555. }
  556. size = utf8.EncodeRune(b.buf[b.n:], r)
  557. b.n += size
  558. return size, nil
  559. }
  560. // WriteString writes a string.
  561. // It returns the number of bytes written.
  562. // If the count is less than len(s), it also returns an error explaining
  563. // why the write is short.
  564. func (b *Writer) WriteString(s string) (int, error) {
  565. nn := 0
  566. for len(s) > b.Available() && b.err == nil {
  567. n := copy(b.buf[b.n:], s)
  568. b.n += n
  569. nn += n
  570. s = s[n:]
  571. b.flush()
  572. }
  573. if b.err != nil {
  574. return nn, b.err
  575. }
  576. n := copy(b.buf[b.n:], s)
  577. b.n += n
  578. nn += n
  579. return nn, nil
  580. }
  581. // ReadFrom implements io.ReaderFrom.
  582. func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) {
  583. if b.Buffered() == 0 {
  584. if w, ok := b.wr.(io.ReaderFrom); ok {
  585. return w.ReadFrom(r)
  586. }
  587. }
  588. var m int
  589. for {
  590. if b.Available() == 0 {
  591. if err1 := b.flush(); err1 != nil {
  592. return n, err1
  593. }
  594. }
  595. nr := 0
  596. for nr < maxConsecutiveEmptyReads {
  597. m, err = r.Read(b.buf[b.n:])
  598. if m != 0 || err != nil {
  599. break
  600. }
  601. nr++
  602. }
  603. if nr == maxConsecutiveEmptyReads {
  604. return n, io.ErrNoProgress
  605. }
  606. b.n += m
  607. n += int64(m)
  608. if err != nil {
  609. break
  610. }
  611. }
  612. if err == io.EOF {
  613. // If we filled the buffer exactly, flush pre-emptively.
  614. if b.Available() == 0 {
  615. err = b.flush()
  616. } else {
  617. err = nil
  618. }
  619. }
  620. return n, err
  621. }
  622. // buffered input and output
  623. // ReadWriter stores pointers to a Reader and a Writer.
  624. // It implements io.ReadWriter.
  625. type ReadWriter struct {
  626. *Reader
  627. *Writer
  628. }
  629. // NewReadWriter allocates a new ReadWriter that dispatches to r and w.
  630. func NewReadWriter(r *Reader, w *Writer) *ReadWriter {
  631. return &ReadWriter{r, w}
  632. }