msgpack.unpacker source code

1 module msgpack.unpacker;
2 
3 import msgpack.common;
4 import msgpack.attribute;
5 import msgpack.exception;
6 
7 import std.array;
8 import std.exception;
9 import std.range;
10 import std.stdio;
11 import std.traits;
12 import std.typecons;
13 import std.typetuple;
14 import std.container;
15 
16 
17 // for unpack without calling constructor
18 private extern(C) Object _d_newclass(in ClassInfo);
19 
20 
21 /**
22  * This $(D Unpacker) is a $(D MessagePack) direct-conversion deserializer
23  *
24  * This implementation is suitable for fixed data.
25  *
26  * Example:
27  * -----
28  * // serializedData is [10, 0.1, false]
29  * auto unpacker = Unpacker(serializedData);
30  *
31  * uint   n;
32  * double d;
33  * bool   b;
34  *
35  * unpacker.unpackArray(n, d, b);
36  *
37  * // using Tuple
38  * Tuple!(uint, double, bool) record;
39  * unpacker.unpack(record);  // record is [10, 0.1, false]
40  * -----
41  *
42  * NOTE:
43  *  Unpacker becomes template struct if Phobos supports truly IO module.
44  */
45 struct Unpacker
46 {
47   private:
48     static @system
49     {
50         alias void delegate(ref Unpacker, void*) UnpackHandler;
51         UnpackHandler[TypeInfo] unpackHandlers;
52 
53         public void registerHandler(T, alias Handler)()
54         {
55             unpackHandlers[typeid(T)] = delegate(ref Unpacker unpacker, void* obj) {
56                 Handler(unpacker, *cast(T*)obj);
57             };
58         }
59 
60         public void register(T)()
61         {
62             unpackHandlers[typeid(T)] = delegate(ref Unpacker unpacker, void* obj) {
63                 unpacker.unpackObject(*cast(T*)obj);
64             };
65         }
66 
67     }
68 
69     enum Offset = 1;
70 
71     mixin InternalBuffer;
72 
73     bool withFieldName_;
74 
75 
76   public:
77     /**
78      * Constructs a $(D Unpacker).
79      *
80      * Params:
81      *  target     = byte buffer to deserialize
82      *  bufferSize = size limit of buffer size
83      */
84     this(in ubyte[] target, in size_t bufferSize = 8192, bool withFieldName = false)
85     {
86         initializeBuffer(target, bufferSize);
87         withFieldName_ = withFieldName;
88     }
89 
90 
91     /**
92      * Clears states for next deserialization.
93      */
94     @safe
95     nothrow void clear()
96     {
97         used_ = offset_ = parsed_ = 0;
98         hasRaw_ = false;
99     }
100 
101 
102     /**
103      * Deserializes $(D_PARAM T) object and assigns to $(D_PARAM value).
104      *
105      * If the argument is pointer, dereferences pointer and assigns deserialized value.
106      * -----
107      * int* a;
108      * unpacker.unpack(a)  // enforce throws Exception because a is null or
109      *                     // no throw if deserialized value is nil
110      *
111      * int b; a = &b;
112      * unpacker.unpack(b)  // b is deserialized value or
113      *                     // assigns null if deserialized value is nil
114      * -----
115      *
116      * Params:
117      *  value = the reference of value to assign.
118      *
119      * Returns:
120      *  self, i.e. for method chaining.
121      *
122      * Throws:
123      *  UnpackException when doesn't read from buffer or precision loss occurs and
124      *  MessagePackException when $(D_PARAM T) type doesn't match serialized type.
125      */
126     ref Unpacker unpack(T)(ref T value) if (is(Unqual!T == bool))
127     {
128         canRead(Offset, 0);
129         const header = read();
130 
131         switch (header) {
132         case Format.TRUE:
133             value = true;
134             break;
135         case Format.FALSE:
136             value = false;
137             break;
138         default:
139             rollback();
140         }
141 
142         return this;
143     }
144 
145 
146     /// ditto
147     ref Unpacker unpack(T)(ref T value) if (isUnsigned!T && !is(Unqual!T == enum))
148     {
149         canRead(Offset, 0);
150         const header = read();
151 
152         if (0x00 <= header && header <= 0x7f) {
153             value = header;
154         } else {
155             switch (header) {
156             case Format.UINT8:
157                 canRead(ubyte.sizeof);
158                 value = read();
159                 break;
160             case Format.UINT16:
161                 canRead(ushort.sizeof);
162                 auto us = load16To!ushort(read(ushort.sizeof));
163                 if (us > T.max)
164                     rollback(ushort.sizeof);
165                 value = cast(T)us;
166                 break;
167             case Format.UINT32:
168                 canRead(uint.sizeof);
169                 auto ui = load32To!uint(read(uint.sizeof));
170                 if (ui > T.max)
171                     rollback(uint.sizeof);
172                 value = cast(T)ui;
173                 break;
174             case Format.UINT64:
175                 canRead(ulong.sizeof);
176                 auto ul = load64To!ulong(read(ulong.sizeof));
177                 if (ul > T.max)
178                     rollback(ulong.sizeof);
179                 value = cast(T)ul;
180                 break;
181             default:
182                 rollback();
183             }
184         }
185 
186         return this;
187     }
188 
189 
190     /// ditto
191     ref Unpacker unpack(T)(ref T value) if (isSigned!T && isIntegral!T && !is(Unqual!T == enum))
192     {
193         canRead(Offset, 0);
194         const header = read();
195 
196         if (0x00 <= header && header <= 0x7f) {
197             value = cast(T)header;
198         } else if (0xe0 <= header && header <= 0xff) {
199             value = -(cast(T)-header);
200         } else {
201             switch (header) {
202             case Format.UINT8:
203                 canRead(ubyte.sizeof);
204                 auto ub = read();
205                 if (ub > T.max)
206                     rollback(ubyte.sizeof);
207                 value = cast(T)ub;
208                 break;
209             case Format.UINT16:
210                 canRead(ushort.sizeof);
211                 auto us = load16To!ushort(read(ushort.sizeof));
212                 if (us > T.max)
213                     rollback(ushort.sizeof);
214                 value = cast(T)us;
215                 break;
216             case Format.UINT32:
217                 canRead(uint.sizeof);
218                 auto ui = load32To!uint(read(uint.sizeof));
219                 if (ui > T.max)
220                     rollback(uint.sizeof);
221                 value = cast(T)ui;
222                 break;
223             case Format.UINT64:
224                 canRead(ulong.sizeof);
225                 auto ul = load64To!ulong(read(ulong.sizeof));
226                 if (ul > T.max)
227                     rollback(ulong.sizeof);
228                 value = cast(T)ul;
229                 break;
230             case Format.INT8:
231                 canRead(byte.sizeof);
232                 value = cast(byte)read();
233                 break;
234             case Format.INT16:
235                 canRead(short.sizeof);
236                 auto s = load16To!short(read(short.sizeof));
237                 if (s < T.min || T.max < s)
238                     rollback(short.sizeof);
239                 value = cast(T)s;
240                 break;
241             case Format.INT32:
242                 canRead(int.sizeof);
243                 auto i = load32To!int(read(int.sizeof));
244                 if (i < T.min || T.max < i)
245                     rollback(int.sizeof);
246                 value = cast(T)i;
247                 break;
248             case Format.INT64:
249                 canRead(long.sizeof);
250                 auto l = load64To!long(read(long.sizeof));
251                 if (l < T.min || T.max < l)
252                     rollback(long.sizeof);
253                 value = cast(T)l;
254                 break;
255             default:
256                 rollback();
257             }
258         }
259 
260         return this;
261     }
262 
263 
264     /// ditto
265     ref Unpacker unpack(T)(ref T value) if (isSomeChar!T && !is(Unqual!T == enum))
266     {
267         static if (is(Unqual!T == char)) {
268             ubyte tmp;
269         } else static if (is(Unqual!T == wchar)) {
270             ushort tmp;
271         } else static if (is(Unqual!T == dchar)) {
272             uint tmp;
273         }
274         unpack(tmp);
275         value = cast(T)(tmp);
276         return this;
277     }
278 
279 
280     /// ditto
281     ref Unpacker unpack(T)(ref T value) if (isFloatingPoint!T && !is(Unqual!T == enum))
282     {
283         canRead(Offset, 0);
284         const header = read();
285 
286         switch (header) {
287         case Format.FLOAT:
288             _f temp;
289 
290             canRead(uint.sizeof);
291             temp.i = load32To!uint(read(uint.sizeof));
292             value  = temp.f;
293             break;
294         case Format.DOUBLE:
295             // check precision loss
296             static if (is(Unqual!T == float))
297                 rollback();
298 
299             _d temp;
300 
301             canRead(ulong.sizeof);
302             temp.i = load64To!ulong(read(ulong.sizeof));
303             value  = temp.f;
304             break;
305         case Format.REAL:
306             static if (!EnableReal)
307             {
308                 rollback();
309             }
310             else
311             {
312                 // check precision loss
313                 static if (is(Unqual!T == float) || is(Unqual!T == double))
314                     rollback();
315 
316                 canRead(RealSize);
317 
318                 version (NonX86)
319                 {
320                     CustomFloat!80 temp;
321 
322                     const frac = load64To!ulong (read(ulong.sizeof));
323                     const exp  = load16To!ushort(read(ushort.sizeof));
324 
325                     temp.significand = frac;
326                     temp.exponent    = exp & 0x7fff;
327                     temp.sign        = exp & 0x8000 ? true : false;
328 
329                     // NOTE: temp.get!real is inf on non-x86 when deserialized value is larger than double.max.
330                     value = temp.get!real;
331                 }
332                 else
333                 {
334                     _r temp;
335 
336                     temp.fraction = load64To!(typeof(temp.fraction))(read(temp.fraction.sizeof));
337                     temp.exponent = load16To!(typeof(temp.exponent))(read(temp.exponent.sizeof));
338 
339                     value = temp.f;
340                 }
341             }
342 
343             break;
344         default:
345             rollback();
346         }
347 
348         return this;
349     }
350 
351 
352     /// ditto
353     ref Unpacker unpack(T)(ref T value) if (is(Unqual!T == enum))
354     {
355         OriginalType!T temp;
356 
357         unpack(temp);
358 
359         value = cast(T)temp;
360 
361         return this;
362     }
363 
364 
365     /// ditto
366     ref Unpacker unpack(T)(T value) if (isPointer!T)
367     {
368         static if (is(Unqual!T == void*)) {
369             enforce(value !is null,  "Can't deserialize void type");
370             unpackNil(value);
371         } else {
372             if (checkNil())
373                 unpackNil(value);
374             else
375                 enforce(value !is null, T.stringof ~ " is null pointer");
376 
377             unpack(mixin(AsteriskOf!T ~ "value"));
378         }
379 
380         return this;
381     }
382 
383 
384     /// ditto
385     ref Unpacker unpack(T)(ref T value) if (is(T == ExtValue))
386     {
387         canRead(Offset, 0);
388         const header = read();
389 
390         // Fixed
391         if (header >= Format.EXT && header <= Format.EXT + 4)
392         {
393             const length = 2^^(header - Format.EXT);
394             canRead(1 + length);
395 
396             value.type = read();
397             value.data = read(length);
398             return this;
399         }
400 
401         // Dynamic length
402         uint length;
403         switch (header) with (Format)
404         {
405             case EXT8:
406                 canRead(1);
407                 length = read();
408                 break;
409             case EXT16:
410                 canRead(2);
411                 length = load16To!ushort(read(2));
412                 break;
413             case EXT32:
414                 canRead(4);
415                 length = load32To!uint(read(4));
416                 break;
417             default:
418                 rollback();
419         }
420 
421         canRead(1 + length);
422         value.type = read();
423         value.data = read(length);
424 
425         return this;
426     }
427 
428 
429     /// ditto
430     ref Unpacker unpack(Types...)(ref Types objects) if (Types.length > 1)
431     {
432         foreach (i, T; Types)
433             unpack!(T)(objects[i]);
434 
435         return this;
436     }
437 
438 
439     /**
440      * Deserializes $(D_PARAM T) object and assigns to $(D_PARAM array).
441      *
442      * This is convenient method for array deserialization.
443      * Rollback will be completely successful if you deserialize raw type((u)byte[] or string types).
444      * But, Rollback will be one element(e.g. int) if you deserialize other types(e.g. int[], int[int])
445      *
446      * No assign if the length of deserialized object is 0.
447      *
448      * In a static array, this method checks the length. Do rollback and throw exception
449      * if length of $(D_PARAM array) is different from length of deserialized object.
450      *
451      * Params:
452      *  array = the reference of array to assign.
453      *
454      * Returns:
455      *  self, i.e. for method chaining.
456      *
457      * Throws:
458      *  UnpackException when doesn't read from buffer or precision loss occurs and
459      *  MessagePackException when $(D_PARAM T) type doesn't match serialized type.
460      */
461     ref Unpacker unpack(T)(ref T array) if ((isArray!T ||
462                                              isInstanceOf!(Array, T)) &&
463                                             !is(Unqual!T == enum))
464     {
465         alias typeof(T.init[0]) U;
466 
467         /*
468          * Deserializes type-information of raw type.
469          */
470         @safe
471         size_t beginRaw()
472         {
473             canRead(Offset, 0);
474             const  header = read();
475             size_t length;
476 
477             if (0xa0 <= header && header <= 0xbf) {
478                 length = header & 0x1f;
479             } else {
480                 switch (header) {
481                 case Format.BIN8, Format.STR8:
482                     canRead(ubyte.sizeof);
483                     length = read();
484                     break;
485                 case Format.BIN16, Format.RAW16:
486                     canRead(ushort.sizeof);
487                     length = load16To!size_t(read(ushort.sizeof));
488                     break;
489                 case Format.BIN32, Format.RAW32:
490                     canRead(uint.sizeof);
491                     length = load32To!size_t(read(uint.sizeof));
492                     break;
493                 case Format.NIL:
494                     break;
495                 default:
496                     rollback();
497                 }
498             }
499 
500             return length;
501         }
502 
503 
504         if (checkNil()) {
505             static if (isStaticArray!T) {
506                 onInvalidType();
507             } else {
508                 return unpackNil(array);
509             }
510         }
511 
512         // Raw bytes
513         static if (isByte!U || isSomeChar!U) {
514             auto length = beginRaw();
515             auto offset = calculateSize!(true)(length);
516             if (length == 0)
517                 return this;
518 
519             static if (isStaticArray!T) {
520                 if (length != array.length)
521                     rollback(offset);
522             }
523 
524             canRead(length, offset + Offset);
525             static if (isStaticArray!T) {
526                 array[] = (cast(U[])read(length))[0 .. T.length];
527             } else {
528                 array = cast(T)read(length);
529             }
530 
531             static if (isDynamicArray!T)
532                 hasRaw_ = true;
533         } else {
534             auto length = beginArray();
535             if (length == 0)
536                 return this;
537 
538             static if (isStaticArray!T) {
539                 if (length != array.length)
540                     rollback(calculateSize(length));
541             } else {
542                 array.length = length;
543             }
544 
545             foreach (i; 0..length)
546                 unpack(array[i]);
547         }
548 
549         return this;
550     }
551 
552 
553     /// ditto
554     ref Unpacker unpack(T)(ref T array) if (isAssociativeArray!T)
555     {
556         alias typeof(T.init.keys[0])   K;
557         alias typeof(T.init.values[0]) V;
558 
559         if (checkNil())
560             return unpackNil(array);
561 
562         auto length = beginMap();
563         if (length == 0)
564             return this;
565 
566         foreach (i; 0..length) {
567             K k; unpack(k);
568             V v; unpack(v);
569             array[k] = v;
570         }
571 
572         return this;
573     }
574 
575     /**
576      * Deserializes $(D_PARAM T) object and assigns to $(D_PARAM object).
577      *
578      * Calling $(D fromMsgpack) if $(D_KEYWORD class) and $(D_KEYWORD struct) implement $(D fromMsgpack) method. $(D fromMsgpack) signature is:
579      * -----
580      * void fromMsgpack(ref Unpacker unpacker)
581      * -----
582      * Assumes $(D std.typecons.Tuple) or simple struct if $(D_KEYWORD struct) doesn't implement $(D fromMsgpack).
583      * Checks length if $(D_PARAM T) is a $(D std.typecons.Tuple) or simple struct.
584      *
585      * Params:
586      *  object = the reference of object to assign.
587      *  args   = the arguments to class constructor(class only).
588      *           This is used at new statement if $(D_PARAM object) is $(D_KEYWORD null).
589      *
590      * Returns:
591      *  self, i.e. for method chaining.
592      */
593     ref Unpacker unpack(T, Args...)(ref T object, auto ref Args args) if (is(Unqual!T == class))
594     {
595         if (checkNil())
596             return unpackNil(object);
597 
598         if (object is null) {
599             static if (Args.length == 0) {
600                 static if (__traits(compiles, { new T(); }))
601                     object = new T();
602                 else
603                     object = cast(T)_d_newclass(T.classinfo);
604             } else static if (__traits(compiles, { new T(args); })) {
605                 object = new T(args);
606             } else {
607                 throw new MessagePackException("Don't know how to construct class type '" ~ Unqual!T.stringof ~ "' with argument types '" ~ Args.stringof ~ "'.");
608             }
609         }
610 
611         static if (hasMember!(T, "fromMsgpack"))
612         {
613             static if (__traits(compiles, { object.fromMsgpack(this, withFieldName_); })) {
614               object.fromMsgpack(this, withFieldName_);
615             } else static if (__traits(compiles, { object.fromMsgpack(this); })) { // backward compatible
616                 object.fromMsgpack(this);
617             } else {
618                 static assert(0, "Failed to invoke 'fromMsgpack' on type '" ~ Unqual!T.stringof ~ "'");
619             }
620         } else {
621             if (auto handler = object.classinfo in unpackHandlers) {
622                 (*handler)(this, cast(void*)&object);
623                 return this;
624             }
625             if (T.classinfo !is object.classinfo) {
626                 throw new MessagePackException("Can't unpack derived class through reference to base class.");
627             }
628 
629             unpackObject(object);
630         }
631 
632         return this;
633     }
634 
635 
636     /// ditto
637     ref Unpacker unpack(T)(ref T object) if (is(Unqual!T == struct) &&
638                                              !is(Unqual!T == ExtValue))
639     {
640         static if (hasMember!(T, "fromMsgpack"))
641         {
642             static if (__traits(compiles, { object.fromMsgpack(this); })) {
643                 object.fromMsgpack(this);
644             } else {
645                 static assert(0, "Failed to invoke 'fromMsgpack' on type '" ~ Unqual!T.stringof ~ "'");
646             }
647         } else {
648             if (auto handler = typeid(T) in unpackHandlers) {
649                 (*handler)(this, cast(void*)&object);
650                 return this;
651             }
652 
653             size_t length = withFieldName_ ? beginMap() : beginArray();
654             if (length == 0)
655                 return this;
656 
657             static if (isTuple!T) {
658                 if (length != T.Types.length)
659                     rollback(calculateSize(length));
660 
661                 foreach (i, Type; T.Types)
662                     unpack(object.field[i]);
663             } else {  // simple struct
664                 //if (length != object.tupleof.length)
665                 if (length != SerializingMemberNumbers!(T))
666                     rollback(calculateSize(length));
667 
668                 if (withFieldName_) {
669                     foreach (i, member; object.tupleof) {
670                         static if (isPackedField!(T.tupleof[i]))
671                         {
672                             string fieldName;
673                             unpack(fieldName);
674 
675                             if (fieldName == getFieldName!(T, i)) {
676                                 unpack(object.tupleof[i]);
677                             } else {
678                                 assert(false, "Invalid field name: '" ~ fieldName ~ "', expect '" ~ getFieldName!(T, i) ~ "'");
679                             }
680                         }
681                     }
682                 } else {
683                     foreach (i, member; object.tupleof) {
684                         static if (isPackedField!(T.tupleof[i]))
685                             unpack(object.tupleof[i]);
686                     }
687                 }
688             }
689         }
690 
691         return this;
692     }
693 
694 
695     void unpackObject(T)(ref T object) if (is(Unqual!T == class))
696     {
697         alias SerializingClasses!(T) Classes;
698 
699         size_t length = withFieldName_ ? beginMap() : beginArray();
700         if (length == 0)
701             return;
702 
703         if (length != SerializingMemberNumbers!(Classes))
704             rollback(calculateSize(length));
705 
706         if (withFieldName_) {
707             foreach (_; 0..length) {
708                 string fieldName;
709                 unpack(fieldName);
710 
711                 foreach (Class; Classes) {
712                     Class obj = cast(Class)object;
713 
714                     foreach (i, member; obj.tupleof) {
715                         static if (isPackedField!(Class.tupleof[i]))
716                         {
717                             if (fieldName == getFieldName!(Class, i)) {
718                                 unpack(obj.tupleof[i]);
719                                 goto endLoop;
720                             }
721                         }
722                     }
723                 }
724                 assert(false, "Invalid field name: '" ~ fieldName~"' ");
725 
726             endLoop:
727                 continue;
728             }
729         } else {
730             foreach (Class; Classes) {
731                 Class obj = cast(Class)object;
732 
733                 foreach (i, member; obj.tupleof) {
734                     static if (isPackedField!(Class.tupleof[i]))
735                         unpack(obj.tupleof[i]);
736                 }
737             }
738         }
739     }
740 
741 
742     /**
743      * Deserializes the container object and assigns to each argument.
744      *
745      * These methods check the length. Do rollback if
746      * the length of arguments is different from length of deserialized object.
747      *
748      * In unpackMap, the number of arguments must be even.
749      *
750      * Params:
751      *  objects = the references of object to assign.
752      *
753      * Returns:
754      *  self, i.e. for method chaining.
755      */
756     ref Unpacker unpackArray(Types...)(ref Types objects)
757     {
758         auto length = beginArray();
759         if (length != Types.length)
760             rollback(calculateSize(length));
761 
762         foreach (i, T; Types)
763             unpack(objects[i]);
764         // unpack(objects);  // slow :(
765 
766         return this;
767     }
768 
769 
770     /// ditto
771     ref Unpacker unpackMap(Types...)(ref Types objects)
772     {
773         static assert(Types.length % 2 == 0, "The number of arguments must be even");
774 
775         auto length = beginMap();
776         if (length != Types.length / 2)
777             rollback(calculateSize(length));
778 
779         foreach (i, T; Types)
780             unpack(objects[i]);
781 
782         return this;
783     }
784 
785 
786     /**
787      * Deserializes the type-information of container.
788      *
789      * These methods don't deserialize contents.
790      * You need to call unpack method to deserialize contents at your own risk.
791      * -----
792      * // serialized data is [1, "Hi!"];
793      * int num;
794      * unpacker.beginArray(2).unpack(num);  // num is 1
795      *
796      * // other operation
797      *
798      * string str;
799      * unpacker.unpack(str);  // str is "Hi!"
800      * -----
801      *
802      * Returns:
803      *  the container size.
804      */
805     @safe
806     size_t beginArray()
807     {
808         canRead(Offset, 0);
809         const  header = read();
810         size_t length;
811 
812         if (0x90 <= header && header <= 0x9f) {
813             length = header & 0x0f;
814         } else {
815             switch (header) {
816             case Format.ARRAY16:
817                 canRead(ushort.sizeof);
818                 length = load16To!size_t(read(ushort.sizeof));
819                 break;
820             case Format.ARRAY32:
821                 canRead(uint.sizeof);
822                 length = load32To!size_t(read(uint.sizeof));
823                 break;
824             case Format.NIL:
825                 break;
826             default:
827                 rollback();
828             }
829         }
830 
831         return length;
832     }
833 
834 
835     /// ditto
836     @safe
837     size_t beginMap()
838     {
839         canRead(Offset, 0);
840         const  header = read();
841         size_t length;
842 
843         if (0x80 <= header && header <= 0x8f) {
844             length = header & 0x0f;
845         } else {
846             switch (header) {
847             case Format.MAP16:
848                 canRead(ushort.sizeof);
849                 length = load16To!size_t(read(ushort.sizeof));
850                 break;
851             case Format.MAP32:
852                 canRead(uint.sizeof);
853                 length = load32To!size_t(read(uint.sizeof));
854                 break;
855             case Format.NIL:
856                 break;
857             default:
858                 rollback();
859             }
860         }
861 
862         return length;
863     }
864 
865 
866     /**
867      * Unpacks an EXT value into $(D type) and $(D data).
868      * $(D type) is checked and a $(D MessagePackException) is thrown if it does
869      *  not match. The length of $(D data) is checked and a $(D MessagePackException)
870      *  is thrown if the lengths do not match.  If $(D data) is null, a new slice
871      *  is returned.
872      */
873     ref Unpacker unpackExt(ref byte type, ref ubyte[] data)
874     {
875         import std.conv : text;
876 
877         canRead(Offset, 0);
878         const header = read();
879 
880         uint length;
881         uint rollbackLength = 0;
882         if (header >= Format.EXT && header <= Format.EXT + 4)
883         {
884             // Fixed
885             length = 2^^(header - Format.EXT);
886 
887         } else {
888             // Dynamic length
889             switch (header) with (Format)
890             {
891                 case EXT8:
892                     canRead(1);
893                     length = read();
894                     rollbackLength = 1;
895                     break;
896                 case EXT16:
897                     canRead(2);
898                     length = load16To!ushort(read(2));
899                     rollbackLength = 2;
900                     break;
901                 case EXT32:
902                     canRead(4);
903                     length = load32To!uint(read(4));
904                     rollbackLength = 4;
905                     break;
906                 default:
907                     rollback();
908             }
909 
910         }
911 
912         canRead(1 + length);
913 
914         // Read and check the type
915         byte type_ = read();
916         rollbackLength += 1;
917         if (type_ != type)
918         {
919             rollback(rollbackLength);
920             throw new MessagePackException(text("Cannot unpack EXT of type ",
921                                                 type_, " into type ", type));
922         }
923 
924         // Read and check data
925         if (data is null)
926             data = new ubyte[](length);
927         else if (data.length != length) {
928             rollback(rollbackLength);
929             throw new MessagePackException(text("Length mismatch while unpacking EXT: ",
930                             data.length, " was given, actual length is ", length));
931         }
932         data[] = read(length);
933         return this;
934     }
935 
936     /**
937      * Scans an entire buffer and converts each objects.
938      *
939      * This method is used for unpacking record-like objects.
940      *
941      * Example:
942      * -----
943      * // serialized data is "[1, 2][3, 4][5, 6][...".
944      * auto unpacker = Unpacker(serializedData);
945      * foreach (n, d; &unpacker.scan!(int, int))  // == "foreach (int n, int d; unpacker)"
946      *     writeln(n, d); // 1st loop "1, 2", 2nd loop "3, 4"...
947      * -----
948      */
949     int scan(Types...)(scope int delegate(ref Types) dg)
950     {
951         return opApply!(Types)(delegate int(ref Types objects) { return dg(objects); });
952     }
953 
954 
955     /// ditto
956     int opApply(Types...)(scope int delegate(ref Types) dg)
957     {
958         int result;
959 
960         while (used_ - offset_) {
961             auto length = beginArray();
962             if (length != Types.length)
963                 rollback(calculateSize(length));
964 
965             Types objects;
966             foreach (i, T; Types)
967                 unpack(objects[i]);
968 
969             result = dg(objects);
970             if (result)
971                 return result;
972         }
973 
974         return result;
975     }
976 
977 
978   private:
979     /*
980      * Deserializes nil object and assigns to $(D_PARAM value).
981      *
982      * Params:
983      *  value = the reference of value to assign.
984      *
985      * Returns:
986      *  self, i.e. for method chaining.
987      *
988      * Throws:
989      *  UnpackException when doesn't read from buffer or precision loss occurs and
990      *  MessagePackException when $(D_PARAM T) type doesn't match serialized type.
991      */
992     @safe
993     ref Unpacker unpackNil(T)(ref T value)
994     {
995         canRead(Offset, 0);
996         const header = read();
997 
998         if (header == Format.NIL)
999             value = null;
1000         else
1001             rollback();
1002 
1003         return this;
1004     }
1005 
1006 
1007     /*
1008      * Next object is nil?
1009      *
1010      * Returns:
1011      *  true if next object is nil.
1012      */
1013     @safe
1014     bool checkNil()
1015     {
1016         canRead(Offset, 0);
1017 
1018         return buffer_[offset_] == Format.NIL;
1019     }
1020 
1021 
1022     /*
1023      * Calculates the format size of container length.
1024      */
1025     size_t calculateSize(bool rawType = false)(in size_t length)
1026     {
1027         static if (rawType)
1028             return length < 32 ? 0 : length < 65536 ? ushort.sizeof : uint.sizeof;
1029         else
1030             return length < 16 ? 0 : length < 65536 ? ushort.sizeof : uint.sizeof;
1031     }
1032 
1033 
1034     /*
1035      * Reading test.
1036      *
1037      * Params:
1038      *  size   = the size to read.
1039      *  offset = the offset to subtract when doesn't read from buffer.
1040      *
1041      * Throws:
1042      *  UnpackException when doesn't read from buffer.
1043      */
1044     @safe
1045     void canRead(in size_t size, in size_t offset = Offset)
1046     {
1047         if (used_ - offset_ < size) {
1048             if (offset)
1049                 offset_ -= offset;
1050 
1051             throw new UnpackException("Insufficient buffer");
1052         }
1053     }
1054 
1055 
1056     /*
1057      * Reads value from buffer and advances offset.
1058      */
1059     @safe
1060     nothrow ubyte read()
1061     {
1062         return buffer_[offset_++];
1063     }
1064 
1065 
1066     /*
1067      * Reads value from buffer and advances offset.
1068      */
1069     @safe
1070     nothrow ubyte[] read(in size_t size)
1071     {
1072         auto result = buffer_[offset_..offset_ + size];
1073 
1074         offset_ += size;
1075 
1076         return result;
1077     }
1078 
1079 
1080     /*
1081      * Do rollback and throws exception.
1082      */
1083     @safe
1084     void rollback(in size_t size = 0)
1085     {
1086         offset_ -= size + Offset;
1087         onInvalidType();
1088     }
1089 }
1090 
1091 
1092 private:
1093 
1094 
1095 /*
1096  * A callback for type-mismatched error in deserialization process.
1097  */
1098 @safe
1099 pure void onInvalidType()
1100 {
1101     throw new MessagePackException("Attempt to unpack with non-compatible type");
1102 }
1103 
1104 
1105 unittest
1106 {
1107     import msgpack.packer;
1108 
1109     { // unique
1110         mixin DefinePacker;
1111 
1112         Tuple!(bool, bool) result;
1113         Tuple!(bool, bool) test = tuple(true, false);
1114 
1115         packer.pack(test);
1116 
1117         auto unpacker = Unpacker(packer.stream.data);
1118 
1119         unpacker.unpack(result);
1120         assert(test == result);
1121     }
1122     { // uint *
1123         mixin DefinePacker;
1124 
1125         Tuple!(ubyte, ushort, uint, ulong) result;
1126         Tuple!(ubyte, ushort, uint, ulong) test = tuple(cast(ubyte)ubyte.max, cast(ushort)ushort.max,
1127                                                         cast(uint)uint.max,   cast(ulong)ulong.max);
1128 
1129         packer.pack(test);
1130 
1131         auto unpacker = Unpacker(packer.stream.data);
1132 
1133         unpacker.unpack(result);
1134         assert(test == result);
1135     }
1136     { // int *
1137         mixin DefinePacker;
1138 
1139         Tuple!(byte, short, int, long) result;
1140         Tuple!(byte, short, int, long) test = tuple(cast(byte)byte.min, cast(short)short.min,
1141                                                     cast(int)int.min,   cast(long)long.min);
1142 
1143         packer.pack(test);
1144 
1145         auto unpacker = Unpacker(packer.stream.data);
1146 
1147         unpacker.unpack(result);
1148         assert(test == result);
1149     }
1150     { // floating point
1151         mixin DefinePacker;
1152 
1153         static if (real.sizeof == double.sizeof || !EnableReal)
1154         {
1155             Tuple!(float, double, double) result;
1156             Tuple!(float, double, double) test = tuple(cast(float)float.min_normal, cast(double)double.max, cast(real)double.min_normal);
1157         }
1158         else
1159         {
1160             Tuple!(float, double, real) result;
1161             Tuple!(float, double, real) test = tuple(cast(float)float.min_normal, cast(double)double.max, cast(real)real.min_normal);
1162         }
1163 
1164         packer.pack(test);
1165 
1166         auto unpacker = Unpacker(packer.stream.data);
1167 
1168         unpacker.unpack(result);
1169         assert(test == result);
1170     }
1171     { // pointer
1172         mixin DefinePacker;
1173 
1174         Tuple!(ulong, long, double) origin;
1175         Tuple!(ulong, long, double) values = tuple(ulong.max, long.min, double.min_normal);
1176         Tuple!(ulong*, long*, double*) result = tuple(&origin.field[0], &origin.field[1], &origin.field[2]);
1177         Tuple!(ulong*, long*, double*) test = tuple(&values.field[0], &values.field[1], &values.field[2]);
1178 
1179         packer.pack(test);
1180 
1181         auto unpacker = Unpacker(packer.stream.data);
1182 
1183         unpacker.unpack(result);
1184         foreach (i, v; test.field)
1185             assert(*v == *result.field[i]);
1186         assert(origin == values);
1187     }
1188     { // enum
1189         enum   : float { D = 0.5 }
1190         enum E : ulong { U = 100 }
1191 
1192         mixin DefinePacker;
1193 
1194         float f = D,   resultF;
1195         E     e = E.U, resultE;
1196 
1197         packer.pack(D, e);
1198 
1199         auto unpacker = Unpacker(packer.stream.data);
1200 
1201         unpacker.unpack(resultF, resultE);
1202         assert(f == resultF);
1203         assert(e == resultE);
1204     }
1205     { // container
1206         mixin DefinePacker;
1207 
1208         Tuple!(ulong[], double[uint], string, bool[2], char[2]) test
1209             = tuple([1UL, 2], [3U:4.0, 5:6.0, 7:8.0], "MessagePack is nice!", [true, false], "D!");
1210 
1211         packer.pack(test);
1212 
1213         auto unpacker = Unpacker(packer.stream.data);
1214         Tuple!(ulong[], double[uint], string, bool[2], char[2]) result;
1215 
1216         unpacker.unpack(result);
1217         assert(test == result);
1218     }
1219     { // ext
1220 
1221         // Try a variety of lengths, making sure to hit all the fixexts
1222         foreach (L; TypeTuple!(1, 2, 3, 4, 5, 8, 9, 16, 32, 512, 2^^16))
1223         {
1224             mixin DefinePacker;
1225 
1226             ubyte[] data = new ubyte[](L);
1227             ExtValue ext = ExtValue(7, data);
1228             packer.pack(ext);
1229 
1230             auto unpacker1 = Unpacker(packer.stream.data);
1231             ExtValue witness;
1232 
1233             unpacker1.unpack(witness);
1234             assert(ext == witness);
1235 
1236             // And try unpackExt
1237             auto unpacker2 = Unpacker(packer.stream.data);
1238             byte type = 1;
1239             ubyte[] deserializedData = new ubyte[](7);
1240 
1241             // This should be a type mismatch (1 != 7)
1242             assertThrown!MessagePackException(
1243                 unpacker2.unpackExt(type, deserializedData));
1244             type = 7;
1245 
1246             // A data size mismatch
1247             assertThrown!MessagePackException(
1248                 unpacker2.unpackExt(type, deserializedData));
1249             deserializedData = new ubyte[](L);
1250 
1251             // And this should succeed
1252             unpacker2.unpackExt(type, deserializedData);
1253             assert(deserializedData == data);
1254         }
1255     }
1256     { // user defined
1257         {
1258             static struct S
1259             {
1260                 uint num;
1261 
1262                 void toMsgpack(P)(ref P p) const { p.packArray(num); }
1263                 void fromMsgpack(ref Unpacker u)
1264                 {
1265                     assert(u.beginArray() == 1);
1266                     u.unpack(num);
1267                 }
1268             }
1269 
1270             mixin DefinePacker; S result, test = S(uint.max);
1271 
1272             packer.pack(test);
1273 
1274             auto unpacker = Unpacker(packer.stream.data);
1275             unpacker.unpack(result);
1276 
1277             assert(test.num == result.num);
1278         }
1279         {
1280             static class C
1281             {
1282                 uint num;
1283 
1284                 this(uint n) { num = n; }
1285 
1286                 void toMsgpack(P)(ref P p) const { p.packArray(num - 1); }
1287                 void fromMsgpack(ref Unpacker u)
1288                 {
1289                     assert(u.beginArray() == 1);
1290                     u.unpack(num);
1291                 }
1292             }
1293 
1294             mixin DefinePacker; C result, test = new C(ushort.max);
1295 
1296             packer.pack(test);
1297 
1298             auto unpacker = Unpacker(packer.stream.data);
1299             unpacker.unpack(result, ushort.max);
1300 
1301             assert(test.num == result.num + 1);
1302         }
1303     }
1304     { // simple struct and class
1305         {
1306             static struct Simple
1307             {
1308                 uint num;
1309                 @nonPacked string str;
1310             }
1311 
1312             static struct Simple2
1313             {
1314                 @nonPacked string str;
1315                 uint num;
1316             }
1317 
1318             foreach (Type; TypeTuple!(Simple, Simple2)) {
1319                 mixin DefinePacker;
1320                 Type result, test;
1321                 test.num = uint.max;
1322                 test.str = "ignored";
1323 
1324                 packer.pack(test);
1325                 auto unpacker = Unpacker(packer.stream.data);
1326                 unpacker.unpack(result);
1327 
1328                 assert(test.num == result.num);
1329                 assert(test.str != result.str);
1330             }
1331         }
1332 
1333         static class SimpleA
1334         {
1335             bool flag = true;
1336         }
1337 
1338         static class SimpleB : SimpleA
1339         {
1340             ubyte type = 100;
1341         }
1342 
1343         static class SimpleC : SimpleB
1344         {
1345             uint num = uint.max;
1346             @nonPacked string str;
1347         }
1348 
1349         static class SimpleC2 : SimpleB
1350         {
1351             @nonPacked string str;
1352             uint num = uint.max;
1353         }
1354 
1355         { // from derived class
1356             foreach (Type; TypeTuple!(SimpleC, SimpleC2)) {
1357                 mixin DefinePacker;
1358                 Type result, test = new Type();
1359                 test.flag = false;
1360                 test.type = 99;
1361                 test.num  = uint.max / 2;
1362                 test.str  = "ignored";
1363 
1364                 packer.pack(test);
1365                 auto unpacker = Unpacker(packer.stream.data);
1366                 unpacker.unpack(result);
1367 
1368                 assert(test.flag == result.flag);
1369                 assert(test.type == result.type);
1370                 assert(test.num  == result.num);
1371                 assert(test.str  != result.str);
1372             }
1373         }
1374         { // from base class
1375             mixin DefinePacker; SimpleC test = new SimpleC();
1376 
1377             packer.pack(test);
1378 
1379             SimpleB result = new SimpleC();
1380             auto unpacker  = Unpacker(packer.stream.data);
1381 
1382             try {
1383                 unpacker.unpack(result);
1384                 assert(false);
1385             } catch (Exception e) { }
1386         }
1387         { // https://github.com/msgpack/msgpack-d/issues/16
1388             mixin DefinePacker;
1389 
1390             static class Issue16
1391             {
1392                 int i;
1393                 this(int i) { this.i = i; }
1394             }
1395 
1396             Issue16 c1 = new Issue16(10);
1397 
1398             // change behaviour to accept null with new object without constructor
1399             Issue16 c2 = null;
1400             packer.pack(c1);
1401             auto unpacker1 = Unpacker(packer.stream.data);
1402             unpacker1.unpack(c2);
1403             //unpack(pack(c1), c2);
1404             assert(c2.i == c1.i);
1405 
1406             Issue16 c3 = new Issue16(20);
1407             packer.stream.clear();
1408             packer.pack(c1);
1409             auto unpacker2 = Unpacker(packer.stream.data);
1410             unpacker2.unpack(c3);            
1411             //unpack(pack(c1), c3);
1412             assert(c3.i == c1.i);
1413         }
1414     }
1415     { // variadic
1416         mixin DefinePacker;
1417 
1418         Tuple!(uint, long, double) test = tuple(uint.max, long.min, double.max);
1419 
1420         packer.pack(test);
1421 
1422         auto unpacker = Unpacker(packer.stream.data);
1423 
1424         uint u; long l; double d;
1425 
1426         unpacker.unpackArray(u, l, d);
1427         assert(test == tuple(u, l, d));
1428     }
1429     { // scan / opApply
1430         ubyte[] data;
1431         mixin DefinePacker;
1432 
1433         foreach (i; 0..2)
1434             packer.pack(tuple(1, 0.5, "Hi!"));
1435 
1436         foreach (n, d, s; &Unpacker(packer.stream.data).scan!(int, double, string)) {
1437             assert(n == 1);
1438             assert(d == 0.5);
1439             assert(s == "Hi!");
1440         }
1441     }
1442 }