/**
* Copyright 2014 Paul Querna
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
/* Portions of this file are on derived from yajl: <https://github.com/lloyd/yajl> */
/*
* Copyright (c) 2007-2014, Lloyd Hilaiel <me@lloyd.io>
*
* Permission to use, copy, modify, and/or distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
package v1
import (
"errors"
"fmt"
"io"
)
type FFParseState int
const (
FFParse_map_start FFParseState = iota
FFParse_want_key
FFParse_want_colon
FFParse_want_value
FFParse_after_value
)
type FFTok int
const (
FFTok_init FFTok = iota
FFTok_bool FFTok = iota
FFTok_colon FFTok = iota
FFTok_comma FFTok = iota
FFTok_eof FFTok = iota
FFTok_error FFTok = iota
FFTok_left_brace FFTok = iota
FFTok_left_bracket FFTok = iota
FFTok_null FFTok = iota
FFTok_right_brace FFTok = iota
FFTok_right_bracket FFTok = iota
/* we differentiate between integers and doubles to allow the
* parser to interpret the number without re-scanning */
FFTok_integer FFTok = iota
FFTok_double FFTok = iota
FFTok_string FFTok = iota
/* comment tokens are not currently returned to the parser, ever */
FFTok_comment FFTok = iota
)
type FFErr int
const (
FFErr_e_ok FFErr = iota
FFErr_io FFErr = iota
FFErr_string_invalid_utf8 FFErr = iota
FFErr_string_invalid_escaped_char FFErr = iota
FFErr_string_invalid_json_char FFErr = iota
FFErr_string_invalid_hex_char FFErr = iota
FFErr_invalid_char FFErr = iota
FFErr_invalid_string FFErr = iota
FFErr_missing_integer_after_decimal FFErr = iota
FFErr_missing_integer_after_exponent FFErr = iota
FFErr_missing_integer_after_minus FFErr = iota
FFErr_unallowed_comment FFErr = iota
FFErr_incomplete_comment FFErr = iota
FFErr_unexpected_token_type FFErr = iota // TODO: improve this error
)
type FFLexer struct {
reader *ffReader
Output DecodingBuffer
Token FFTok
Error FFErr
BigError error
// TODO: convert all of this to an interface
lastCurrentChar int
captureAll bool
buf Buffer
}
func NewFFLexer(input []byte) *FFLexer {
fl := &FFLexer{
Token: FFTok_init,
Error: FFErr_e_ok,
reader: newffReader(input),
Output: &Buffer{},
}
// TODO: guess size?
//fl.Output.Grow(64)
return fl
}
type LexerError struct {
offset int
line int
char int
err error
}
// Reset the Lexer and add new input.
func (ffl *FFLexer) Reset(input []byte) {
ffl.Token = FFTok_init
ffl.Error = FFErr_e_ok
ffl.BigError = nil
ffl.reader.Reset(input)
ffl.lastCurrentChar = 0
ffl.Output.Reset()
}
func (le *LexerError) Error() string {
return fmt.Sprintf(`ffjson error: (%T)%s offset=%d line=%d char=%d`,
le.err, le.err.Error(),
le.offset, le.line, le.char)
}
func (ffl *FFLexer) WrapErr(err error) error {
line, char := ffl.reader.PosWithLine()
// TOOD: calcualte lines/characters based on offset
return &LexerError{
offset: ffl.reader.Pos(),
line: line,
char: char,
err: err,
}
}
func (ffl *FFLexer) scanReadByte() (byte, error) {
var c byte
var err error
if ffl.captureAll {
c, err = ffl.reader.ReadByte()
} else {
c, err = ffl.reader.ReadByteNoWS()
}
if err != nil {
ffl.Error = FFErr_io
ffl.BigError = err
return 0, err
}
return c, nil
}
func (ffl *FFLexer) readByte() (byte, error) {
c, err := ffl.reader.ReadByte()
if err != nil {
ffl.Error = FFErr_io
ffl.BigError = err
return 0, err
}
return c, nil
}
func (ffl *FFLexer) unreadByte() {
ffl.reader.UnreadByte()
}
func (ffl *FFLexer) wantBytes(want []byte, iftrue FFTok) FFTok {
startPos := ffl.reader.Pos()
for _, b := range want {
c, err := ffl.readByte()
if err != nil {
return FFTok_error
}
if c != b {
ffl.unreadByte()
// fmt.Printf("wanted bytes: %s\n", string(want))
// TODO(pquerna): thsi is a bad error message
ffl.Error = FFErr_invalid_string
return FFTok_error
}
}
endPos := ffl.reader.Pos()
ffl.Output.Write(ffl.reader.Slice(startPos, endPos))
return iftrue
}
func (ffl *FFLexer) lexComment() FFTok {
c, err := ffl.readByte()
if err != nil {
return FFTok_error
}
if c == '/' {
// a // comment, scan until line ends.
for {
c, err := ffl.readByte()
if err != nil {
return FFTok_error
}
if c == '\n' {
return FFTok_comment
}
}
} else if c == '*' {
// a /* */ comment, scan */
for {
c, err := ffl.readByte()
if err != nil {
return FFTok_error
}
if c == '*' {
c, err := ffl.readByte()
if err != nil {
return FFTok_error
}
if c == '/' {
return FFTok_comment
}
ffl.Error = FFErr_incomplete_comment
return FFTok_error
}
}
} else {
ffl.Error = FFErr_incomplete_comment
return FFTok_error
}
}
func (ffl *FFLexer) lexString() FFTok {
if ffl.captureAll {
ffl.buf.Reset()
err := ffl.reader.SliceString(&ffl.buf)
if err != nil {
ffl.BigError = err
return FFTok_error
}
WriteJson(ffl.Output, ffl.buf.Bytes())
return FFTok_string
} else {
err := ffl.reader.SliceString(ffl.Output)
if err != nil {
ffl.BigError = err
return FFTok_error
}
return FFTok_string
}
}
func (ffl *FFLexer) lexNumber() FFTok {
var numRead int = 0
tok := FFTok_integer
startPos := ffl.reader.Pos()
c, err := ffl.readByte()
if err != nil {
return FFTok_error
}
/* optional leading minus */
if c == '-' {
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
}
/* a single zero, or a series of integers */
if c == '0' {
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
} else if c >= '1' && c <= '9' {
for c >= '0' && c <= '9' {
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
}
} else {
ffl.unreadByte()
ffl.Error = FFErr_missing_integer_after_minus
return FFTok_error
}
if c == '.' {
numRead = 0
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
for c >= '0' && c <= '9' {
numRead++
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
}
if numRead == 0 {
ffl.unreadByte()
ffl.Error = FFErr_missing_integer_after_decimal
return FFTok_error
}
tok = FFTok_double
}
/* optional exponent (indicates this is floating point) */
if c == 'e' || c == 'E' {
numRead = 0
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
/* optional sign */
if c == '+' || c == '-' {
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
}
for c >= '0' && c <= '9' {
numRead++
c, err = ffl.readByte()
if err != nil {
return FFTok_error
}
}
if numRead == 0 {
ffl.Error = FFErr_missing_integer_after_exponent
return FFTok_error
}
tok = FFTok_double
}
ffl.unreadByte()
endPos := ffl.reader.Pos()
ffl.Output.Write(ffl.reader.Slice(startPos, endPos))
return tok
}
var true_bytes = []byte{'r', 'u', 'e'}
var false_bytes = []byte{'a', 'l', 's', 'e'}
var null_bytes = []byte{'u', 'l', 'l'}
func (ffl *FFLexer) Scan() FFTok {
tok := FFTok_error
if ffl.captureAll == false {
ffl.Output.Reset()
}
ffl.Token = FFTok_init
for {
c, err := ffl.scanReadByte()
if err != nil {
if err == io.EOF {
return FFTok_eof
} else {
return FFTok_error
}
}
switch c {
case '{':
tok = FFTok_left_bracket
if ffl.captureAll {
ffl.Output.WriteByte('{')
}
goto lexed
case '}':
tok = FFTok_right_bracket
if ffl.captureAll {
ffl.Output.WriteByte('}')
}
goto lexed
case '[':
tok = FFTok_left_brace
if ffl.captureAll {
ffl.Output.WriteByte('[')
}
goto lexed
case ']':
tok = FFTok_right_brace
if ffl.captureAll {
ffl.Output.WriteByte(']')
}
goto lexed
case ',':
tok = FFTok_comma
if ffl.captureAll {
ffl.Output.WriteByte(',')
}
goto lexed
case ':':
tok = FFTok_colon
if ffl.captureAll {
ffl.Output.WriteByte(':')
}
goto lexed
case '\t', '\n', '\v', '\f', '\r', ' ':
if ffl.captureAll {
ffl.Output.WriteByte(c)
}
break
case 't':
ffl.Output.WriteByte('t')
tok = ffl.wantBytes(true_bytes, FFTok_bool)
goto lexed
case 'f':
ffl.Output.WriteByte('f')
tok = ffl.wantBytes(false_bytes, FFTok_bool)
goto lexed
case 'n':
ffl.Output.WriteByte('n')
tok = ffl.wantBytes(null_bytes, FFTok_null)
goto lexed
case '"':
tok = ffl.lexString()
goto lexed
case '-', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
ffl.unreadByte()
tok = ffl.lexNumber()
goto lexed
case '/':
tok = ffl.lexComment()
goto lexed
default:
tok = FFTok_error
ffl.Error = FFErr_invalid_char
}
}
lexed:
ffl.Token = tok
return tok
}
func (ffl *FFLexer) scanField(start FFTok, capture bool) ([]byte, error) {
switch start {
case FFTok_left_brace,
FFTok_left_bracket:
{
end := FFTok_right_brace
if start == FFTok_left_bracket {
end = FFTok_right_bracket
if capture {
ffl.Output.WriteByte('{')
}
} else {
if capture {
ffl.Output.WriteByte('[')
}
}
depth := 1
if capture {
ffl.captureAll = true
}
// TODO: work.
scanloop:
for {
tok := ffl.Scan()
//fmt.Printf("capture-token: %v end: %v depth: %v\n", tok, end, depth)
switch tok {
case FFTok_eof:
return nil, errors.New("ffjson: unexpected EOF")
case FFTok_error:
if ffl.BigError != nil {
return nil, ffl.BigError
}
return nil, ffl.Error.ToError()
case end:
depth--
if depth == 0 {
break scanloop
}
case start:
depth++
}
}
if capture {
ffl.captureAll = false
}
if capture {
return ffl.Output.Bytes(), nil
} else {
return nil, nil
}
}
case FFTok_bool,
FFTok_integer,
FFTok_null,
FFTok_double:
// simple value, return it.
if capture {
return ffl.Output.Bytes(), nil
} else {
return nil, nil
}
case FFTok_string:
//TODO(pquerna): so, other users expect this to be a quoted string :(
if capture {
ffl.buf.Reset()
WriteJson(&ffl.buf, ffl.Output.Bytes())
return ffl.buf.Bytes(), nil
} else {
return nil, nil
}
}
return nil, fmt.Errorf("ffjson: invalid capture type: %v", start)
}
// Captures an entire field value, including recursive objects,
// and converts them to a []byte suitable to pass to a sub-object's
// UnmarshalJSON
func (ffl *FFLexer) CaptureField(start FFTok) ([]byte, error) {
return ffl.scanField(start, true)
}
func (ffl *FFLexer) SkipField(start FFTok) error {
_, err := ffl.scanField(start, false)
return err
}
// TODO(pquerna): return line number and offset.
func (err FFErr) ToError() error {
switch err {
case FFErr_e_ok:
return nil
case FFErr_io:
return errors.New("ffjson: IO error")
case FFErr_string_invalid_utf8:
return errors.New("ffjson: string with invalid UTF-8 sequence")
case FFErr_string_invalid_escaped_char:
return errors.New("ffjson: string with invalid escaped character")
case FFErr_string_invalid_json_char:
return errors.New("ffjson: string with invalid JSON character")
case FFErr_string_invalid_hex_char:
return errors.New("ffjson: string with invalid hex character")
case FFErr_invalid_char:
return errors.New("ffjson: invalid character")
case FFErr_invalid_string:
return errors.New("ffjson: invalid string")
case FFErr_missing_integer_after_decimal:
return errors.New("ffjson: missing integer after decimal")
case FFErr_missing_integer_after_exponent:
return errors.New("ffjson: missing integer after exponent")
case FFErr_missing_integer_after_minus:
return errors.New("ffjson: missing integer after minus")
case FFErr_unallowed_comment:
return errors.New("ffjson: unallowed comment")
case FFErr_incomplete_comment:
return errors.New("ffjson: incomplete comment")
case FFErr_unexpected_token_type:
return errors.New("ffjson: unexpected token sequence")
}
panic(fmt.Sprintf("unknown error type: %v ", err))
}
func (state FFParseState) String() string {
switch state {
case FFParse_map_start:
return "map:start"
case FFParse_want_key:
return "want_key"
case FFParse_want_colon:
return "want_colon"
case FFParse_want_value:
return "want_value"
case FFParse_after_value:
return "after_value"
}
panic(fmt.Sprintf("unknown parse state: %d", int(state)))
}
func (tok FFTok) String() string {
switch tok {
case FFTok_init:
return "tok:init"
case FFTok_bool:
return "tok:bool"
case FFTok_colon:
return "tok:colon"
case FFTok_comma:
return "tok:comma"
case FFTok_eof:
return "tok:eof"
case FFTok_error:
return "tok:error"
case FFTok_left_brace:
return "tok:left_brace"
case FFTok_left_bracket:
return "tok:left_bracket"
case FFTok_null:
return "tok:null"
case FFTok_right_brace:
return "tok:right_brace"
case FFTok_right_bracket:
return "tok:right_bracket"
case FFTok_integer:
return "tok:integer"
case FFTok_double:
return "tok:double"
case FFTok_string:
return "tok:string"
case FFTok_comment:
return "comment"
}
panic(fmt.Sprintf("unknown token: %d", int(tok)))
}
/* a lookup table which lets us quickly determine three things:
* cVEC - valid escaped control char
* note. the solidus '/' may be escaped or not.
* cIJC - invalid json char
* cVHC - valid hex char
* cNFP - needs further processing (from a string scanning perspective)
* cNUC - needs utf8 checking when enabled (from a string scanning perspective)
*/
const (
cVEC int8 = 0x01
cIJC int8 = 0x02
cVHC int8 = 0x04
cNFP int8 = 0x08
cNUC int8 = 0x10
)
var byteLookupTable [256]int8 = [256]int8{
cIJC, /* 0 */
cIJC, /* 1 */
cIJC, /* 2 */
cIJC, /* 3 */
cIJC, /* 4 */
cIJC, /* 5 */
cIJC, /* 6 */
cIJC, /* 7 */
cIJC, /* 8 */
cIJC, /* 9 */
cIJC, /* 10 */
cIJC, /* 11 */
cIJC, /* 12 */
cIJC, /* 13 */
cIJC, /* 14 */
cIJC, /* 15 */
cIJC, /* 16 */
cIJC, /* 17 */
cIJC, /* 18 */
cIJC, /* 19 */
cIJC, /* 20 */
cIJC, /* 21 */
cIJC, /* 22 */
cIJC, /* 23 */
cIJC, /* 24 */
cIJC, /* 25 */
cIJC, /* 26 */
cIJC, /* 27 */
cIJC, /* 28 */
cIJC, /* 29 */
cIJC, /* 30 */
cIJC, /* 31 */
0, /* 32 */
0, /* 33 */
cVEC | cIJC | cNFP, /* 34 */
0, /* 35 */
0, /* 36 */
0, /* 37 */
0, /* 38 */
0, /* 39 */
0, /* 40 */
0, /* 41 */
0, /* 42 */
0, /* 43 */
0, /* 44 */
0, /* 45 */
0, /* 46 */
cVEC, /* 47 */
cVHC, /* 48 */
cVHC, /* 49 */
cVHC, /* 50 */
cVHC, /* 51 */
cVHC, /* 52 */
cVHC, /* 53 */
cVHC, /* 54 */
cVHC, /* 55 */
cVHC, /* 56 */
cVHC, /* 57 */
0, /* 58 */
0, /* 59 */
0, /* 60 */
0, /* 61 */
0, /* 62 */
0, /* 63 */
0, /* 64 */
cVHC, /* 65 */
cVHC, /* 66 */
cVHC, /* 67 */
cVHC, /* 68 */
cVHC, /* 69 */
cVHC, /* 70 */
0, /* 71 */
0, /* 72 */
0, /* 73 */
0, /* 74 */
0, /* 75 */
0, /* 76 */
0, /* 77 */
0, /* 78 */
0, /* 79 */
0, /* 80 */
0, /* 81 */
0, /* 82 */
0, /* 83 */
0, /* 84 */
0, /* 85 */
0, /* 86 */
0, /* 87 */
0, /* 88 */
0, /* 89 */
0, /* 90 */
0, /* 91 */
cVEC | cIJC | cNFP, /* 92 */
0, /* 93 */
0, /* 94 */
0, /* 95 */
0, /* 96 */
cVHC, /* 97 */
cVEC | cVHC, /* 98 */
cVHC, /* 99 */
cVHC, /* 100 */
cVHC, /* 101 */
cVEC | cVHC, /* 102 */
0, /* 103 */
0, /* 104 */
0, /* 105 */
0, /* 106 */
0, /* 107 */
0, /* 108 */
0, /* 109 */
cVEC, /* 110 */
0, /* 111 */
0, /* 112 */
0, /* 113 */
cVEC, /* 114 */
0, /* 115 */
cVEC, /* 116 */
0, /* 117 */
0, /* 118 */
0, /* 119 */
0, /* 120 */
0, /* 121 */
0, /* 122 */
0, /* 123 */
0, /* 124 */
0, /* 125 */
0, /* 126 */
0, /* 127 */
cNUC, /* 128 */
cNUC, /* 129 */
cNUC, /* 130 */
cNUC, /* 131 */
cNUC, /* 132 */
cNUC, /* 133 */
cNUC, /* 134 */
cNUC, /* 135 */
cNUC, /* 136 */
cNUC, /* 137 */
cNUC, /* 138 */
cNUC, /* 139 */
cNUC, /* 140 */
cNUC, /* 141 */
cNUC, /* 142 */
cNUC, /* 143 */
cNUC, /* 144 */
cNUC, /* 145 */
cNUC, /* 146 */
cNUC, /* 147 */
cNUC, /* 148 */
cNUC, /* 149 */
cNUC, /* 150 */
cNUC, /* 151 */
cNUC, /* 152 */
cNUC, /* 153 */
cNUC, /* 154 */
cNUC, /* 155 */
cNUC, /* 156 */
cNUC, /* 157 */
cNUC, /* 158 */
cNUC, /* 159 */
cNUC, /* 160 */
cNUC, /* 161 */
cNUC, /* 162 */
cNUC, /* 163 */
cNUC, /* 164 */
cNUC, /* 165 */
cNUC, /* 166 */
cNUC, /* 167 */
cNUC, /* 168 */
cNUC, /* 169 */
cNUC, /* 170 */
cNUC, /* 171 */
cNUC, /* 172 */
cNUC, /* 173 */
cNUC, /* 174 */
cNUC, /* 175 */
cNUC, /* 176 */
cNUC, /* 177 */
cNUC, /* 178 */
cNUC, /* 179 */
cNUC, /* 180 */
cNUC, /* 181 */
cNUC, /* 182 */
cNUC, /* 183 */
cNUC, /* 184 */
cNUC, /* 185 */
cNUC, /* 186 */
cNUC, /* 187 */
cNUC, /* 188 */
cNUC, /* 189 */
cNUC, /* 190 */
cNUC, /* 191 */
cNUC, /* 192 */
cNUC, /* 193 */
cNUC, /* 194 */
cNUC, /* 195 */
cNUC, /* 196 */
cNUC, /* 197 */
cNUC, /* 198 */
cNUC, /* 199 */
cNUC, /* 200 */
cNUC, /* 201 */
cNUC, /* 202 */
cNUC, /* 203 */
cNUC, /* 204 */
cNUC, /* 205 */
cNUC, /* 206 */
cNUC, /* 207 */
cNUC, /* 208 */
cNUC, /* 209 */
cNUC, /* 210 */
cNUC, /* 211 */
cNUC, /* 212 */
cNUC, /* 213 */
cNUC, /* 214 */
cNUC, /* 215 */
cNUC, /* 216 */
cNUC, /* 217 */
cNUC, /* 218 */
cNUC, /* 219 */
cNUC, /* 220 */
cNUC, /* 221 */
cNUC, /* 222 */
cNUC, /* 223 */
cNUC, /* 224 */
cNUC, /* 225 */
cNUC, /* 226 */
cNUC, /* 227 */
cNUC, /* 228 */
cNUC, /* 229 */
cNUC, /* 230 */
cNUC, /* 231 */
cNUC, /* 232 */
cNUC, /* 233 */
cNUC, /* 234 */
cNUC, /* 235 */
cNUC, /* 236 */
cNUC, /* 237 */
cNUC, /* 238 */
cNUC, /* 239 */
cNUC, /* 240 */
cNUC, /* 241 */
cNUC, /* 242 */
cNUC, /* 243 */
cNUC, /* 244 */
cNUC, /* 245 */
cNUC, /* 246 */
cNUC, /* 247 */
cNUC, /* 248 */
cNUC, /* 249 */
cNUC, /* 250 */
cNUC, /* 251 */
cNUC, /* 252 */
cNUC, /* 253 */
cNUC, /* 254 */
cNUC, /* 255 */
}