decode.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708
  1. package msgpack
  2. import (
  3. "bufio"
  4. "bytes"
  5. "errors"
  6. "fmt"
  7. "io"
  8. "reflect"
  9. "sync"
  10. "time"
  11. "github.com/vmihailenco/msgpack/v5/msgpcode"
  12. )
  13. const (
  14. bytesAllocLimit = 1 << 20 // 1mb
  15. sliceAllocLimit = 1e6 // 1m elements
  16. maxMapSize = 1e6 // 1m elements
  17. )
  18. const (
  19. looseInterfaceDecodingFlag uint32 = 1 << iota
  20. disallowUnknownFieldsFlag
  21. usePreallocateValues
  22. disableAllocLimitFlag
  23. )
  24. type bufReader interface {
  25. io.Reader
  26. io.ByteScanner
  27. }
  28. //------------------------------------------------------------------------------
  29. var decPool = sync.Pool{
  30. New: func() interface{} {
  31. return NewDecoder(nil)
  32. },
  33. }
  34. func GetDecoder() *Decoder {
  35. return decPool.Get().(*Decoder)
  36. }
  37. func PutDecoder(dec *Decoder) {
  38. dec.r = nil
  39. dec.s = nil
  40. decPool.Put(dec)
  41. }
  42. //------------------------------------------------------------------------------
  43. // Unmarshal decodes the MessagePack-encoded data and stores the result
  44. // in the value pointed to by v.
  45. func Unmarshal(data []byte, v interface{}) error {
  46. dec := GetDecoder()
  47. dec.UsePreallocateValues(true)
  48. dec.Reset(bytes.NewReader(data))
  49. err := dec.Decode(v)
  50. PutDecoder(dec)
  51. return err
  52. }
  53. // A Decoder reads and decodes MessagePack values from an input stream.
  54. type Decoder struct {
  55. r io.Reader
  56. s io.ByteScanner
  57. mapDecoder func(*Decoder) (interface{}, error)
  58. structTag string
  59. buf []byte
  60. rec []byte
  61. dict []string
  62. flags uint32
  63. }
  64. // NewDecoder returns a new decoder that reads from r.
  65. //
  66. // The decoder introduces its own buffering and may read data from r
  67. // beyond the requested msgpack values. Buffering can be disabled
  68. // by passing a reader that implements io.ByteScanner interface.
  69. func NewDecoder(r io.Reader) *Decoder {
  70. d := new(Decoder)
  71. d.Reset(r)
  72. return d
  73. }
  74. // Reset discards any buffered data, resets all state, and switches the buffered
  75. // reader to read from r.
  76. func (d *Decoder) Reset(r io.Reader) {
  77. d.ResetDict(r, nil)
  78. }
  79. // ResetDict is like Reset, but also resets the dict.
  80. func (d *Decoder) ResetDict(r io.Reader, dict []string) {
  81. d.ResetReader(r)
  82. d.flags = 0
  83. d.structTag = ""
  84. d.dict = dict
  85. }
  86. func (d *Decoder) WithDict(dict []string, fn func(*Decoder) error) error {
  87. oldDict := d.dict
  88. d.dict = dict
  89. err := fn(d)
  90. d.dict = oldDict
  91. return err
  92. }
  93. func (d *Decoder) ResetReader(r io.Reader) {
  94. d.mapDecoder = nil
  95. d.dict = nil
  96. if br, ok := r.(bufReader); ok {
  97. d.r = br
  98. d.s = br
  99. } else if r == nil {
  100. d.r = nil
  101. d.s = nil
  102. } else {
  103. br := bufio.NewReader(r)
  104. d.r = br
  105. d.s = br
  106. }
  107. }
  108. func (d *Decoder) SetMapDecoder(fn func(*Decoder) (interface{}, error)) {
  109. d.mapDecoder = fn
  110. }
  111. // UseLooseInterfaceDecoding causes decoder to use DecodeInterfaceLoose
  112. // to decode msgpack value into Go interface{}.
  113. func (d *Decoder) UseLooseInterfaceDecoding(on bool) {
  114. if on {
  115. d.flags |= looseInterfaceDecodingFlag
  116. } else {
  117. d.flags &= ^looseInterfaceDecodingFlag
  118. }
  119. }
  120. // SetCustomStructTag causes the decoder to use the supplied tag as a fallback option
  121. // if there is no msgpack tag.
  122. func (d *Decoder) SetCustomStructTag(tag string) {
  123. d.structTag = tag
  124. }
  125. // DisallowUnknownFields causes the Decoder to return an error when the destination
  126. // is a struct and the input contains object keys which do not match any
  127. // non-ignored, exported fields in the destination.
  128. func (d *Decoder) DisallowUnknownFields(on bool) {
  129. if on {
  130. d.flags |= disallowUnknownFieldsFlag
  131. } else {
  132. d.flags &= ^disallowUnknownFieldsFlag
  133. }
  134. }
  135. // UseInternedStrings enables support for decoding interned strings.
  136. func (d *Decoder) UseInternedStrings(on bool) {
  137. if on {
  138. d.flags |= useInternedStringsFlag
  139. } else {
  140. d.flags &= ^useInternedStringsFlag
  141. }
  142. }
  143. // UsePreallocateValues enables preallocating values in chunks
  144. func (d *Decoder) UsePreallocateValues(on bool) {
  145. if on {
  146. d.flags |= usePreallocateValues
  147. } else {
  148. d.flags &= ^usePreallocateValues
  149. }
  150. }
  151. // DisableAllocLimit enables fully allocating slices/maps when the size is known
  152. func (d *Decoder) DisableAllocLimit(on bool) {
  153. if on {
  154. d.flags |= disableAllocLimitFlag
  155. } else {
  156. d.flags &= ^disableAllocLimitFlag
  157. }
  158. }
  159. // Buffered returns a reader of the data remaining in the Decoder's buffer.
  160. // The reader is valid until the next call to Decode.
  161. func (d *Decoder) Buffered() io.Reader {
  162. return d.r
  163. }
  164. //nolint:gocyclo
  165. func (d *Decoder) Decode(v interface{}) error {
  166. var err error
  167. switch v := v.(type) {
  168. case *string:
  169. if v != nil {
  170. *v, err = d.DecodeString()
  171. return err
  172. }
  173. case *[]byte:
  174. if v != nil {
  175. return d.decodeBytesPtr(v)
  176. }
  177. case *int:
  178. if v != nil {
  179. *v, err = d.DecodeInt()
  180. return err
  181. }
  182. case *int8:
  183. if v != nil {
  184. *v, err = d.DecodeInt8()
  185. return err
  186. }
  187. case *int16:
  188. if v != nil {
  189. *v, err = d.DecodeInt16()
  190. return err
  191. }
  192. case *int32:
  193. if v != nil {
  194. *v, err = d.DecodeInt32()
  195. return err
  196. }
  197. case *int64:
  198. if v != nil {
  199. *v, err = d.DecodeInt64()
  200. return err
  201. }
  202. case *uint:
  203. if v != nil {
  204. *v, err = d.DecodeUint()
  205. return err
  206. }
  207. case *uint8:
  208. if v != nil {
  209. *v, err = d.DecodeUint8()
  210. return err
  211. }
  212. case *uint16:
  213. if v != nil {
  214. *v, err = d.DecodeUint16()
  215. return err
  216. }
  217. case *uint32:
  218. if v != nil {
  219. *v, err = d.DecodeUint32()
  220. return err
  221. }
  222. case *uint64:
  223. if v != nil {
  224. *v, err = d.DecodeUint64()
  225. return err
  226. }
  227. case *bool:
  228. if v != nil {
  229. *v, err = d.DecodeBool()
  230. return err
  231. }
  232. case *float32:
  233. if v != nil {
  234. *v, err = d.DecodeFloat32()
  235. return err
  236. }
  237. case *float64:
  238. if v != nil {
  239. *v, err = d.DecodeFloat64()
  240. return err
  241. }
  242. case *[]string:
  243. return d.decodeStringSlicePtr(v)
  244. case *map[string]string:
  245. return d.decodeMapStringStringPtr(v)
  246. case *map[string]interface{}:
  247. return d.decodeMapStringInterfacePtr(v)
  248. case *time.Duration:
  249. if v != nil {
  250. vv, err := d.DecodeInt64()
  251. *v = time.Duration(vv)
  252. return err
  253. }
  254. case *time.Time:
  255. if v != nil {
  256. *v, err = d.DecodeTime()
  257. return err
  258. }
  259. }
  260. vv := reflect.ValueOf(v)
  261. if !vv.IsValid() {
  262. return errors.New("msgpack: Decode(nil)")
  263. }
  264. if vv.Kind() != reflect.Ptr {
  265. return fmt.Errorf("msgpack: Decode(non-pointer %T)", v)
  266. }
  267. if vv.IsNil() {
  268. return fmt.Errorf("msgpack: Decode(non-settable %T)", v)
  269. }
  270. vv = vv.Elem()
  271. if vv.Kind() == reflect.Interface {
  272. if !vv.IsNil() {
  273. vv = vv.Elem()
  274. if vv.Kind() != reflect.Ptr {
  275. return fmt.Errorf("msgpack: Decode(non-pointer %s)", vv.Type().String())
  276. }
  277. }
  278. }
  279. return d.DecodeValue(vv)
  280. }
  281. func (d *Decoder) DecodeMulti(v ...interface{}) error {
  282. for _, vv := range v {
  283. if err := d.Decode(vv); err != nil {
  284. return err
  285. }
  286. }
  287. return nil
  288. }
  289. func (d *Decoder) decodeInterfaceCond() (interface{}, error) {
  290. if d.flags&looseInterfaceDecodingFlag != 0 {
  291. return d.DecodeInterfaceLoose()
  292. }
  293. return d.DecodeInterface()
  294. }
  295. func (d *Decoder) DecodeValue(v reflect.Value) error {
  296. decode := getDecoder(v.Type())
  297. return decode(d, v)
  298. }
  299. func (d *Decoder) DecodeNil() error {
  300. c, err := d.readCode()
  301. if err != nil {
  302. return err
  303. }
  304. if c != msgpcode.Nil {
  305. return fmt.Errorf("msgpack: invalid code=%x decoding nil", c)
  306. }
  307. return nil
  308. }
  309. func (d *Decoder) decodeNilValue(v reflect.Value) error {
  310. err := d.DecodeNil()
  311. if v.IsNil() {
  312. return err
  313. }
  314. if v.Kind() == reflect.Ptr {
  315. v = v.Elem()
  316. }
  317. v.Set(reflect.Zero(v.Type()))
  318. return err
  319. }
  320. func (d *Decoder) DecodeBool() (bool, error) {
  321. c, err := d.readCode()
  322. if err != nil {
  323. return false, err
  324. }
  325. return d.bool(c)
  326. }
  327. func (d *Decoder) bool(c byte) (bool, error) {
  328. if c == msgpcode.Nil {
  329. return false, nil
  330. }
  331. if c == msgpcode.False {
  332. return false, nil
  333. }
  334. if c == msgpcode.True {
  335. return true, nil
  336. }
  337. return false, fmt.Errorf("msgpack: invalid code=%x decoding bool", c)
  338. }
  339. func (d *Decoder) DecodeDuration() (time.Duration, error) {
  340. n, err := d.DecodeInt64()
  341. if err != nil {
  342. return 0, err
  343. }
  344. return time.Duration(n), nil
  345. }
  346. // DecodeInterface decodes value into interface. It returns following types:
  347. // - nil,
  348. // - bool,
  349. // - int8, int16, int32, int64,
  350. // - uint8, uint16, uint32, uint64,
  351. // - float32 and float64,
  352. // - string,
  353. // - []byte,
  354. // - slices of any of the above,
  355. // - maps of any of the above.
  356. //
  357. // DecodeInterface should be used only when you don't know the type of value
  358. // you are decoding. For example, if you are decoding number it is better to use
  359. // DecodeInt64 for negative numbers and DecodeUint64 for positive numbers.
  360. func (d *Decoder) DecodeInterface() (interface{}, error) {
  361. c, err := d.readCode()
  362. if err != nil {
  363. return nil, err
  364. }
  365. if msgpcode.IsFixedNum(c) {
  366. return int8(c), nil
  367. }
  368. if msgpcode.IsFixedMap(c) {
  369. err = d.s.UnreadByte()
  370. if err != nil {
  371. return nil, err
  372. }
  373. return d.decodeMapDefault()
  374. }
  375. if msgpcode.IsFixedArray(c) {
  376. return d.decodeSlice(c)
  377. }
  378. if msgpcode.IsFixedString(c) {
  379. return d.string(c)
  380. }
  381. switch c {
  382. case msgpcode.Nil:
  383. return nil, nil
  384. case msgpcode.False, msgpcode.True:
  385. return d.bool(c)
  386. case msgpcode.Float:
  387. return d.float32(c)
  388. case msgpcode.Double:
  389. return d.float64(c)
  390. case msgpcode.Uint8:
  391. return d.uint8()
  392. case msgpcode.Uint16:
  393. return d.uint16()
  394. case msgpcode.Uint32:
  395. return d.uint32()
  396. case msgpcode.Uint64:
  397. return d.uint64()
  398. case msgpcode.Int8:
  399. return d.int8()
  400. case msgpcode.Int16:
  401. return d.int16()
  402. case msgpcode.Int32:
  403. return d.int32()
  404. case msgpcode.Int64:
  405. return d.int64()
  406. case msgpcode.Bin8, msgpcode.Bin16, msgpcode.Bin32:
  407. return d.bytes(c, nil)
  408. case msgpcode.Str8, msgpcode.Str16, msgpcode.Str32:
  409. return d.string(c)
  410. case msgpcode.Array16, msgpcode.Array32:
  411. return d.decodeSlice(c)
  412. case msgpcode.Map16, msgpcode.Map32:
  413. err = d.s.UnreadByte()
  414. if err != nil {
  415. return nil, err
  416. }
  417. return d.decodeMapDefault()
  418. case msgpcode.FixExt1, msgpcode.FixExt2, msgpcode.FixExt4, msgpcode.FixExt8, msgpcode.FixExt16,
  419. msgpcode.Ext8, msgpcode.Ext16, msgpcode.Ext32:
  420. return d.decodeInterfaceExt(c)
  421. }
  422. return 0, fmt.Errorf("msgpack: unknown code %x decoding interface{}", c)
  423. }
  424. // DecodeInterfaceLoose is like DecodeInterface except that:
  425. // - int8, int16, and int32 are converted to int64,
  426. // - uint8, uint16, and uint32 are converted to uint64,
  427. // - float32 is converted to float64.
  428. // - []byte is converted to string.
  429. func (d *Decoder) DecodeInterfaceLoose() (interface{}, error) {
  430. c, err := d.readCode()
  431. if err != nil {
  432. return nil, err
  433. }
  434. if msgpcode.IsFixedNum(c) {
  435. return int64(int8(c)), nil
  436. }
  437. if msgpcode.IsFixedMap(c) {
  438. err = d.s.UnreadByte()
  439. if err != nil {
  440. return nil, err
  441. }
  442. return d.decodeMapDefault()
  443. }
  444. if msgpcode.IsFixedArray(c) {
  445. return d.decodeSlice(c)
  446. }
  447. if msgpcode.IsFixedString(c) {
  448. return d.string(c)
  449. }
  450. switch c {
  451. case msgpcode.Nil:
  452. return nil, nil
  453. case msgpcode.False, msgpcode.True:
  454. return d.bool(c)
  455. case msgpcode.Float, msgpcode.Double:
  456. return d.float64(c)
  457. case msgpcode.Uint8, msgpcode.Uint16, msgpcode.Uint32, msgpcode.Uint64:
  458. return d.uint(c)
  459. case msgpcode.Int8, msgpcode.Int16, msgpcode.Int32, msgpcode.Int64:
  460. return d.int(c)
  461. case msgpcode.Str8, msgpcode.Str16, msgpcode.Str32,
  462. msgpcode.Bin8, msgpcode.Bin16, msgpcode.Bin32:
  463. return d.string(c)
  464. case msgpcode.Array16, msgpcode.Array32:
  465. return d.decodeSlice(c)
  466. case msgpcode.Map16, msgpcode.Map32:
  467. err = d.s.UnreadByte()
  468. if err != nil {
  469. return nil, err
  470. }
  471. return d.decodeMapDefault()
  472. case msgpcode.FixExt1, msgpcode.FixExt2, msgpcode.FixExt4, msgpcode.FixExt8, msgpcode.FixExt16,
  473. msgpcode.Ext8, msgpcode.Ext16, msgpcode.Ext32:
  474. return d.decodeInterfaceExt(c)
  475. }
  476. return 0, fmt.Errorf("msgpack: unknown code %x decoding interface{}", c)
  477. }
  478. // Skip skips next value.
  479. func (d *Decoder) Skip() error {
  480. c, err := d.readCode()
  481. if err != nil {
  482. return err
  483. }
  484. if msgpcode.IsFixedNum(c) {
  485. return nil
  486. }
  487. if msgpcode.IsFixedMap(c) {
  488. return d.skipMap(c)
  489. }
  490. if msgpcode.IsFixedArray(c) {
  491. return d.skipSlice(c)
  492. }
  493. if msgpcode.IsFixedString(c) {
  494. return d.skipBytes(c)
  495. }
  496. switch c {
  497. case msgpcode.Nil, msgpcode.False, msgpcode.True:
  498. return nil
  499. case msgpcode.Uint8, msgpcode.Int8:
  500. return d.skipN(1)
  501. case msgpcode.Uint16, msgpcode.Int16:
  502. return d.skipN(2)
  503. case msgpcode.Uint32, msgpcode.Int32, msgpcode.Float:
  504. return d.skipN(4)
  505. case msgpcode.Uint64, msgpcode.Int64, msgpcode.Double:
  506. return d.skipN(8)
  507. case msgpcode.Bin8, msgpcode.Bin16, msgpcode.Bin32:
  508. return d.skipBytes(c)
  509. case msgpcode.Str8, msgpcode.Str16, msgpcode.Str32:
  510. return d.skipBytes(c)
  511. case msgpcode.Array16, msgpcode.Array32:
  512. return d.skipSlice(c)
  513. case msgpcode.Map16, msgpcode.Map32:
  514. return d.skipMap(c)
  515. case msgpcode.FixExt1, msgpcode.FixExt2, msgpcode.FixExt4, msgpcode.FixExt8, msgpcode.FixExt16,
  516. msgpcode.Ext8, msgpcode.Ext16, msgpcode.Ext32:
  517. return d.skipExt(c)
  518. }
  519. return fmt.Errorf("msgpack: unknown code %x", c)
  520. }
  521. func (d *Decoder) DecodeRaw() (RawMessage, error) {
  522. d.rec = make([]byte, 0)
  523. if err := d.Skip(); err != nil {
  524. return nil, err
  525. }
  526. msg := RawMessage(d.rec)
  527. d.rec = nil
  528. return msg, nil
  529. }
  530. // PeekCode returns the next MessagePack code without advancing the reader.
  531. // Subpackage msgpack/codes defines the list of available msgpcode.
  532. func (d *Decoder) PeekCode() (byte, error) {
  533. c, err := d.s.ReadByte()
  534. if err != nil {
  535. return 0, err
  536. }
  537. return c, d.s.UnreadByte()
  538. }
  539. // ReadFull reads exactly len(buf) bytes into the buf.
  540. func (d *Decoder) ReadFull(buf []byte) error {
  541. _, err := readN(d.r, buf, len(buf))
  542. return err
  543. }
  544. func (d *Decoder) hasNilCode() bool {
  545. code, err := d.PeekCode()
  546. return err == nil && code == msgpcode.Nil
  547. }
  548. func (d *Decoder) readCode() (byte, error) {
  549. c, err := d.s.ReadByte()
  550. if err != nil {
  551. return 0, err
  552. }
  553. if d.rec != nil {
  554. d.rec = append(d.rec, c)
  555. }
  556. return c, nil
  557. }
  558. func (d *Decoder) readFull(b []byte) error {
  559. _, err := io.ReadFull(d.r, b)
  560. if err != nil {
  561. return err
  562. }
  563. if d.rec != nil {
  564. d.rec = append(d.rec, b...)
  565. }
  566. return nil
  567. }
  568. func (d *Decoder) readN(n int) ([]byte, error) {
  569. var err error
  570. if d.flags&disableAllocLimitFlag != 0 {
  571. d.buf, err = readN(d.r, d.buf, n)
  572. } else {
  573. d.buf, err = readNGrow(d.r, d.buf, n)
  574. }
  575. if err != nil {
  576. return nil, err
  577. }
  578. if d.rec != nil {
  579. // TODO: read directly into d.rec?
  580. d.rec = append(d.rec, d.buf...)
  581. }
  582. return d.buf, nil
  583. }
  584. func readN(r io.Reader, b []byte, n int) ([]byte, error) {
  585. if b == nil {
  586. if n == 0 {
  587. return make([]byte, 0), nil
  588. }
  589. b = make([]byte, 0, n)
  590. }
  591. if n > cap(b) {
  592. b = append(b, make([]byte, n-len(b))...)
  593. } else if n <= cap(b) {
  594. b = b[:n]
  595. }
  596. _, err := io.ReadFull(r, b)
  597. return b, err
  598. }
  599. func readNGrow(r io.Reader, b []byte, n int) ([]byte, error) {
  600. if b == nil {
  601. if n == 0 {
  602. return make([]byte, 0), nil
  603. }
  604. switch {
  605. case n < 64:
  606. b = make([]byte, 0, 64)
  607. case n <= bytesAllocLimit:
  608. b = make([]byte, 0, n)
  609. default:
  610. b = make([]byte, 0, bytesAllocLimit)
  611. }
  612. }
  613. if n <= cap(b) {
  614. b = b[:n]
  615. _, err := io.ReadFull(r, b)
  616. return b, err
  617. }
  618. b = b[:cap(b)]
  619. var pos int
  620. for {
  621. alloc := min(n-len(b), bytesAllocLimit)
  622. b = append(b, make([]byte, alloc)...)
  623. _, err := io.ReadFull(r, b[pos:])
  624. if err != nil {
  625. return b, err
  626. }
  627. if len(b) == n {
  628. break
  629. }
  630. pos = len(b)
  631. }
  632. return b, nil
  633. }
  634. func min(a, b int) int { //nolint:unparam
  635. if a <= b {
  636. return a
  637. }
  638. return b
  639. }