golang reader 代码
文件路径：/src/encoding/csv/reader.go
/ Copyright 2011 The Go Authors. All rights reserved.
/ Use of this source code is governed by a BSD-style
/ license that can be found in the LICENSE file.
/ Package csv reads and writes comma-separated values (CSV) files.
/ There are many kinds of CSV files; this package supports the format
/ described in RFC 4180.
/
/ A csv file contains zero or more records of one or more fields per record.
/ Each record is separated by the newline character. The final record may
/ optionally be followed by a newline character.
/
/ field1,field2,field3
/
/ White space is considered part of a field.
/
/ Carriage returns before newline characters are silently removed.
/
/ Blank lines are ignored. A line with only whitespace characters (excluding
/ the ending newline character) is not considered a blank line.
/
/ Fields which start and stop with the quote character " are called
/ quoted-fields. The beginning and ending quote are not part of the
/ field.
/
/ The source:
/
/ normal string,"quoted-field"
/
/ results in the fields
/
/ {`normal string`, `quoted-field`}
/
/ Within a quoted-field a quote character followed by a second quote
/ character is considered a single quote.
/
/ "the ""word"" is true","a ""quoted-field"""
/
/ results in
/
/ {`the "word" is true`, `a "quoted-field"`}
/
/ Newlines and commas may be included in a quoted-field
/
/ "Multi-line
/ field","comma is ,"
/
/ results in
/
/ {`Multi-line
/ field`, `comma is ,`}
package csv
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"unicode"
"unicode/utf8"
)
/ A ParseError is returned for parsing errors.
/ Line numbers are 1-indexed and columns are 0-indexed.
type ParseError struct {
StartLine int / Line where the record starts
Line int / Line where the error occurred
Column int / Column (1-based byte index) where the error occurred
Err error / The actual error
}
func (e *ParseError) Error() string {
if e.Err == ErrFieldCount {
return fmt.Sprintf("record on line %d: %v", e.Line, e.Err)
}
if e.StartLine != e.Line {
return fmt.Sprintf("record on line %d; parse error on line %d, column %d: %v", e.StartLine, e.Line, e.Column, e.Err)
}
return fmt.Sprintf("parse error on line %d, column %d: %v", e.Line, e.Column, e.Err)
}
func (e *ParseError) Unwrap() error { return e.Err }
/ These are the errors that can be returned in ParseError.Err.
var (
ErrTrailingComma = errors.New("extra delimiter at end of line") / Deprecated: No longer used.
ErrBareQuote = errors.New("bare \" in non-quoted-field")
ErrQuote = errors.New("extraneous or missing \" in quoted-field")
ErrFieldCount = errors.New("wrong number of fields")
)
var errInvalidDelim = errors.New("csv: invalid field or comment delimiter")
func validDelim(r rune) bool {
return r != 0 && r != '"' && r != '\r' && r != '\n' && utf8.ValidRune(r) && r != utf8.RuneError
}
/ A Reader reads records from a CSV-encoded file.
/
/ As returned by NewReader, a Reader expects input conforming to RFC 4180.
/ The exported fields can be changed to customize the details before the
/ first call to Read or ReadAll.
/
/ The Reader converts all \r\n sequences in its input to plain \n,
/ including in multiline field values, so that the returned data does
/ not depend on which line-ending convention an input file uses.
type Reader struct {
/ Comma is the field delimiter.
/ It is set to comma (',') by NewReader.
/ Comma must be a valid rune and must not be \r, \n,
/ or the Unicode replacement character (0xFFFD).
Comma rune
/ Comment, if not 0, is the comment character. Lines beginning with the
/ Comment character without preceding whitespace are ignored.
/ With leading whitespace the Comment character becomes part of the
/ field, even if TrimLeadingSpace is true.
/ Comment must be a valid rune and must not be \r, \n,
/ or the Unicode replacement character (0xFFFD).
/ It must also not be equal to Comma.
Comment rune
/ FieldsPerRecord is the number of expected fields per record.
/ If FieldsPerRecord is positive, Read requires each record to
/ have the given number of fields. If FieldsPerRecord is 0, Read sets it to
/ the number of fields in the first record, so that future records must
/ have the same field count. If FieldsPerRecord is negative, no check is
/ made and records may have a variable number of fields.
FieldsPerRecord int
/ If LazyQuotes is true, a quote may appear in an unquoted field and a
/ non-doubled quote may appear in a quoted field.
LazyQuotes bool
/ If TrimLeadingSpace is true, leading white space in a field is ignored.
/ This is done even if the field delimiter, Comma, is white space.
TrimLeadingSpace bool
/ ReuseRecord controls whether calls to Read may return a slice sharing
/ the backing array of the previous call's returned slice for performance.
/ By default, each call to Read returns newly allocated memory owned by the caller.
ReuseRecord bool
TrailingComma bool / Deprecated: No longer used.
r *bufio.Reader
/ numLine is the current line being read in the CSV file.
numLine int
/ offset is the input stream byte offset of the current reader position.
offset int64
/ rawBuffer is a line buffer only used by the readLine method.
rawBuffer []byte
/ recordBuffer holds the unescaped fields, one after another.
/ The fields can be accessed by using the indexes in fieldIndexes.
/ E.g., For the row `a,"b","c""d",e`, recordBuffer will contain `abc"de`
/ and fieldIndexes will contain the indexes [1, 2, 5, 6].
recordBuffer []byte
/ fieldIndexes is an index of fields inside recordBuffer.
/ The i'th field ends at offset fieldIndexes[i] in recordBuffer.
fieldIndexes []int
/ fieldPositions is an index of field positions for the
/ last record returned by Read.
fieldPositions []position
/ lastRecord is a record cache and only used when ReuseRecord == true.
lastRecord []string
}
/ NewReader returns a new Reader that reads from r.
func NewReader(r io.Reader) *Reader {
return &Reader{
Comma: ',',
r: bufio.NewReader(r),
}
}
/ Read reads one record (a slice of fields) from r.
/ If the record has an unexpected number of fields,
/ Read returns the record along with the error ErrFieldCount.
/ Except for that case, Read always returns either a non-nil
/ record or a non-nil error, but not both.
/ If there is no data left to be read, Read returns nil, io.EOF.
/ If ReuseRecord is true, the returned slice may be shared
/ between multiple calls to Read.
func (r *Reader) Read() (record []string, err error) {
if r.ReuseRecord {
record, err = r.readRecord(r.lastRecord)
r.lastRecord = record
} else {
record, err = r.readRecord(nil)
}
return record, err
}
/ FieldPos returns the line and column corresponding to
/ the start of the field with the given index in the slice most recently
/ returned by Read. Numbering of lines and columns starts at 1;
/ columns are counted in bytes, not runes.
/
/ If this is called with an out-of-bounds index, it panics.
func (r *Reader) FieldPos(field int) (line, column int) {
if field < 0 || field >= len(r.fieldPositions) {
panic("out of range index passed to FieldPos")
}
p := &r.fieldPositions[field]
return p.line, p.col
}
/ InputOffset returns the input stream byte offset of the current reader
/ position. The offset gives the location of the end of the most recently
/ read row and the beginning of the next row.
func (r *Reader) InputOffset() int64 {
return r.offset
}
/ pos holds the position of a field in the current line.
type position struct {
line, col int
}
/ ReadAll reads all the remaining records from r.
/ Each record is a slice of fields.
/ A successful call returns err == nil, not err == io.EOF. Because ReadAll is
/ defined to read until EOF, it does not treat end of file as an error to be
/ reported.
func (r *Reader) ReadAll() (records [][]string, err error) {
for {
record, err := r.readRecord(nil)
if err == io.EOF {
return records, nil
}
if err != nil {
return nil, err
}
records = append(records, record)
}
}
/ readLine reads the next line (with the trailing endline).
/ If EOF is hit without a trailing endline, it will be omitted.
/ If some bytes were read, then the error is never io.EOF.
/ The result is only valid until the next call to readLine.
func (r *Reader) readLine() ([]byte, error) {
line, err := r.r.ReadSlice('\n')
if err == bufio.ErrBufferFull {
r.rawBuffer = append(r.rawBuffer[:0], line...)
for err == bufio.ErrBufferFull {
line, err = r.r.ReadSlice('\n')
r.rawBuffer = append(r.rawBuffer, line...)
}
line = r.rawBuffer
}
readSize := len(line)
if readSize > 0 && err == io.EOF {
err = nil
/ For backwards compatibility, drop trailing \r before EOF.
if line[readSize-1] == '\r' {
line = line[:readSize-1]
}
}
r.numLine++
r.offset += int64(readSize)
/ Normalize \r\n to \n on all input lines.
if n := len(line); n >= 2 && line[n-2] == '\r' && line[n-1] == '\n' {
line[n-2] = '\n'
line = line[:n-1]
}
return line, err
}
/ lengthNL reports the number of bytes for the trailing \n.
func lengthNL(b []byte) int {
if len(b) > 0 && b[len(b)-1] == '\n' {
return 1
}
return 0
}
/ nextRune returns the next rune in b or utf8.RuneError.
func nextRune(b []byte) rune {
r, _ := utf8.DecodeRune(b)
return r
}
func (r *Reader) readRecord(dst []string) ([]string, error) {
if r.Comma == r.Comment || !validDelim(r.Comma) || (r.Comment != 0 && !validDelim(r.Comment)) {
return nil, errInvalidDelim
}
/ Read line (automatically skipping past empty lines and any comments).
var line []byte
var errRead error
for errRead == nil {
line, errRead = r.readLine()
if r.Comment != 0 && nextRune(line) == r.Comment {
line = nil
continue / Skip comment lines
}
if errRead == nil && len(line) == lengthNL(line) {
line = nil
continue / Skip empty lines
}
break
}
if errRead == io.EOF {
return nil, errRead
}
/ Parse each field in the record.
var err error
const quoteLen = len(`"`)
commaLen := utf8.RuneLen(r.Comma)
recLine := r.numLine / Starting line for record
r.recordBuffer = r.recordBuffer[:0]
r.fieldIndexes = r.fieldIndexes[:0]
r.fieldPositions = r.fieldPositions[:0]
pos := position{line: r.numLine, col: 1}
parseField:
for {
if r.TrimLeadingSpace {
i := bytes.IndexFunc(line, func(r rune) bool {
return !unicode.IsSpace(r)
})
if i < 0 {
i = len(line)
pos.col -= lengthNL(line)
}
line = line[i:]
pos.col += i
}
if len(line) == 0 || line[0] != '"' {
/ Non-quoted string field
i := bytes.IndexRune(line, r.Comma)
field := line
if i >= 0 {
field = field[:i]
} else {
field = field[:len(field)-lengthNL(field)]
}
/ Check to make sure a quote does not appear in field.
if !r.LazyQuotes {
if j := bytes.IndexByte(field, '"'); j >= 0 {
col := pos.col + j
err = &ParseError{StartLine: recLine, Line: r.numLine, Column: col, Err: ErrBareQuote}
break parseField
}
}
r.recordBuffer = append(r.recordBuffer, field...)
r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
r.fieldPositions = append(r.fieldPositions, pos)
if i >= 0 {
line = line[i+commaLen:]
pos.col += i + commaLen
continue parseField
}
break parseField
} else {
/ Quoted string field
fieldPos := pos
line = line[quoteLen:]
pos.col += quoteLen
for {
i := bytes.IndexByte(line, '"')
if i >= 0 {
/ Hit next quote.
r.recordBuffer = append(r.recordBuffer, line[:i]...)
line = line[i+quoteLen:]
pos.col += i + quoteLen
switch rn := nextRune(line); {
case rn == '"':
/ `""` sequence (append quote).
r.recordBuffer = append(r.recordBuffer, '"')
line = line[quoteLen:]
pos.col += quoteLen
case rn == r.Comma:
/ `",` sequence (end of field).
line = line[commaLen:]
pos.col += commaLen
r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
r.fieldPositions = append(r.fieldPositions, fieldPos)
continue parseField
case lengthNL(line) == len(line):
/ `"\n` sequence (end of line).
r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
r.fieldPositions = append(r.fieldPositions, fieldPos)
break parseField
case r.LazyQuotes:
/ `"` sequence (bare quote).
r.recordBuffer = append(r.recordBuffer, '"')
default:
/ `"*` sequence (invalid non-escaped quote).
err = &ParseError{StartLine: recLine, Line: r.numLine, Column: pos.col - quoteLen, Err: ErrQuote}
break parseField
}
} else if len(line) > 0 {
/ Hit end of line (copy all data so far).
r.recordBuffer = append(r.recordBuffer, line...)
if errRead != nil {
break parseField
}
pos.col += len(line)
line, errRead = r.readLine()
if len(line) > 0 {
pos.line++
pos.col = 1
}
if errRead == io.EOF {
errRead = nil
}
} else {
/ Abrupt end of file (EOF or error).
if !r.LazyQuotes && errRead == nil {
err = &ParseError{StartLine: recLine, Line: pos.line, Column: pos.col, Err: ErrQuote}
break parseField
}
r.fieldIndexes = append(r.fieldIndexes, len(r.recordBuffer))
r.fieldPositions = append(r.fieldPositions, fieldPos)
break parseField
}
}
}
}
if err == nil {
err = errRead
}
/ Create a single string and create slices out of it.
/ This pins the memory of the fields together, but allocates once.
str := string(r.recordBuffer) / Convert to string once to batch allocations
dst = dst[:0]
if cap(dst) < len(r.fieldIndexes) {
dst = make([]string, len(r.fieldIndexes))
}
dst = dst[:len(r.fieldIndexes)]
var preIdx int
for i, idx := range r.fieldIndexes {
dst[i] = str[preIdx:idx]
preIdx = idx
}
/ Check or update the expected fields per record.
if r.FieldsPerRecord > 0 {
if len(dst) != r.FieldsPerRecord && err == nil {
err = &ParseError{
StartLine: recLine,
Line: recLine,
Column: 1,
Err: ErrFieldCount,
}
}
} else if r.FieldsPerRecord == 0 {
r.FieldsPerRecord = len(dst)
}
return dst, err
}