func (s *ss) Width() (wid int, ok bool) { if s.maxWid == hugeWid { return 0, false } return s.maxWid, true } func (s *ss) getRune() (r rune) { r, _, err := s.ReadRune() if err != nil { if err == io.EOF { return eof } s.error(err) } return } func (s *ss) mustReadRune() (r rune) { r = s.getRune() if r == eof { s.error(io.ErrUnexpectedEOF) } return } func (s *ss) UnreadRune() error { s.rs.UnreadRune() s.atEOF = false s.count-- return nil } func (s *ss) Token(skipSpace bool, f func(rune) bool) (tok []byte, err error) { defer func() { if e := recover(); e != nil { if se, ok := e.(scanError); ok { err = se.err } else { panic(e) } } }() if f == nil { f = notSpace } s.buf = s.buf[:0] tok = s.token(skipSpace, f) return } func isSpace(r rune) bool { if r >= 1<<16 { return false } rx := uint16(r) for _, rng := range space { if rx < rng[0] { return false } if rx <= rng[1] { return true } } return false } func notSpace(r rune) bool { return !isSpace(r) } func (s *ss) SkipSpace() { s.skipSpace(false) } type readRune struct { reader io.Reader buf [utf8.UTFMax]byte pending int pendBuf [utf8.UTFMax]byte peekRune rune } func (r *readRune) readByte() (b byte, err error) { if r.pending > 0 { b = r.pendBuf[0] copy(r.pendBuf[0:], r.pendBuf[1:]) r.pending-- return } n, err := io.ReadFull(r.reader, r.pendBuf[:1]) if n != 1 { return 0, err } return r.pendBuf[0], err } func (r *readRune) ReadRune() (rr rune, size int, err error) { if r.peekRune >= 0 { rr = r.peekRune r.peekRune = ^r.peekRune size = utf8.RuneLen(rr) return } r.buf[0], err = r.readByte() if err != nil { return } if r.buf[0] < utf8.RuneSelf { rr = rune(r.buf[0]) size = 1 r.peekRune = ^rr return } var n int for n = 1; !utf8.FullRune(r.buf[:n]); n++ { r.buf[n], err = r.readByte() if err != nil { if err == io.EOF { err = nil break } return } } rr, size = utf8.DecodeRune(r.buf[:n]) if size < n { // an error, save the bytes for the next read copy(r.pendBuf[r.pending:], r.buf[size:n]) r.pending += n - size } // Flip the bits of the rune so it's available to UnreadRune. r.peekRune = ^rr return } // free saves used ss structs in ssFree; avoid an allocation per invocation. func (s *ss) free(old ssave) { // If it was used recursively, just restore the old state. if old.validSave { s.ssave = old return } // Don't hold on to ss structs with large buffers. if cap(s.buf) > 1024 { return } s.buf = s.buf[:0] s.rs = nil ssFree.Put(s) }