Line data Source code
1 : // Copyright (c) 2009-2010 Satoshi Nakamoto
2 : // Copyright (c) 2009-2020 The Bitcoin Core developers
3 : // Distributed under the MIT software license, see the accompanying
4 : // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 :
6 : #ifndef BITCOIN_SERIALIZE_H
7 : #define BITCOIN_SERIALIZE_H
8 :
9 : #include <compat/endian.h>
10 :
11 : #include <algorithm>
12 : #include <cstdint>
13 : #include <cstring>
14 : #include <ios>
15 : #include <limits>
16 : #include <map>
17 : #include <memory>
18 : #include <set>
19 : #include <string>
20 : #include <string.h>
21 : #include <utility>
22 : #include <vector>
23 :
24 : #include <prevector.h>
25 : #include <span.h>
26 :
27 : static const unsigned int MAX_SIZE = 0x02000000;
28 :
29 : /** Maximum amount of memory (in bytes) to allocate at once when deserializing vectors. */
30 : static const unsigned int MAX_VECTOR_ALLOCATE = 5000000;
31 :
32 : /**
33 : * Dummy data type to identify deserializing constructors.
34 : *
35 : * By convention, a constructor of a type T with signature
36 : *
37 : * template <typename Stream> T::T(deserialize_type, Stream& s)
38 : *
39 : * is a deserializing constructor, which builds the type by
40 : * deserializing it from s. If T contains const fields, this
41 : * is likely the only way to do so.
42 : */
43 : struct deserialize_type {};
44 : constexpr deserialize_type deserialize {};
45 :
46 : //! Safely convert odd char pointer types to standard ones.
47 : inline char* CharCast(char* c) { return c; }
48 427092 : inline char* CharCast(unsigned char* c) { return (char*)c; }
49 : inline const char* CharCast(const char* c) { return c; }
50 518906 : inline const char* CharCast(const unsigned char* c) { return (const char*)c; }
51 :
52 : /*
53 : * Lowest-level serialization and conversion.
54 : * @note Sizes of these types are verified in the tests
55 : */
56 67725363 : template<typename Stream> inline void ser_writedata8(Stream &s, uint8_t obj)
57 : {
58 67725363 : s.write((char*)&obj, 1);
59 67725363 : }
60 2903796 : template<typename Stream> inline void ser_writedata16(Stream &s, uint16_t obj)
61 : {
62 2903796 : obj = htole16(obj);
63 2903796 : s.write((char*)&obj, 2);
64 2903796 : }
65 : template<typename Stream> inline void ser_writedata16be(Stream &s, uint16_t obj)
66 : {
67 : obj = htobe16(obj);
68 : s.write((char*)&obj, 2);
69 : }
70 68920056 : template<typename Stream> inline void ser_writedata32(Stream &s, uint32_t obj)
71 : {
72 68920056 : obj = htole32(obj);
73 68920056 : s.write((char*)&obj, 4);
74 68920056 : }
75 9531 : template<typename Stream> inline void ser_writedata32be(Stream &s, uint32_t obj)
76 : {
77 9531 : obj = htobe32(obj);
78 9531 : s.write((char*)&obj, 4);
79 9531 : }
80 41048240 : template<typename Stream> inline void ser_writedata64(Stream &s, uint64_t obj)
81 : {
82 41048240 : obj = htole64(obj);
83 41048240 : s.write((char*)&obj, 8);
84 41048240 : }
85 5569873 : template<typename Stream> inline uint8_t ser_readdata8(Stream &s)
86 : {
87 5569873 : uint8_t obj;
88 5569873 : s.read((char*)&obj, 1);
89 11139746 : return obj;
90 5569873 : }
91 639446 : template<typename Stream> inline uint16_t ser_readdata16(Stream &s)
92 : {
93 639446 : uint16_t obj;
94 639446 : s.read((char*)&obj, 2);
95 1278892 : return le16toh(obj);
96 639446 : }
97 : template<typename Stream> inline uint16_t ser_readdata16be(Stream &s)
98 : {
99 : uint16_t obj;
100 : s.read((char*)&obj, 2);
101 : return be16toh(obj);
102 : }
103 3708434 : template<typename Stream> inline uint32_t ser_readdata32(Stream &s)
104 : {
105 3708434 : uint32_t obj;
106 3708434 : s.read((char*)&obj, 4);
107 7416868 : return le32toh(obj);
108 3708434 : }
109 2462 : template<typename Stream> inline uint32_t ser_readdata32be(Stream &s)
110 : {
111 2462 : uint32_t obj;
112 2462 : s.read((char*)&obj, 4);
113 4924 : return be32toh(obj);
114 2462 : }
115 7254750 : template<typename Stream> inline uint64_t ser_readdata64(Stream &s)
116 : {
117 7254750 : uint64_t obj;
118 7254750 : s.read((char*)&obj, 8);
119 14509500 : return le64toh(obj);
120 7254750 : }
121 15394589 : inline uint64_t ser_double_to_uint64(double x)
122 : {
123 : uint64_t tmp;
124 : std::memcpy(&tmp, &x, sizeof(x));
125 : static_assert(sizeof(tmp) == sizeof(x), "double and uint64_t assumed to have the same size");
126 15394589 : return tmp;
127 : }
128 1008 : inline uint32_t ser_float_to_uint32(float x)
129 : {
130 : uint32_t tmp;
131 : std::memcpy(&tmp, &x, sizeof(x));
132 : static_assert(sizeof(tmp) == sizeof(x), "float and uint32_t assumed to have the same size");
133 1008 : return tmp;
134 : }
135 5978795 : inline double ser_uint64_to_double(uint64_t y)
136 : {
137 : double tmp;
138 5978795 : std::memcpy(&tmp, &y, sizeof(y));
139 : static_assert(sizeof(tmp) == sizeof(y), "double and uint64_t assumed to have the same size");
140 5978795 : return tmp;
141 : }
142 1006 : inline float ser_uint32_to_float(uint32_t y)
143 : {
144 : float tmp;
145 1006 : std::memcpy(&tmp, &y, sizeof(y));
146 : static_assert(sizeof(tmp) == sizeof(y), "float and uint32_t assumed to have the same size");
147 1006 : return tmp;
148 : }
149 :
150 :
151 : /////////////////////////////////////////////////////////////////
152 : //
153 : // Templates for serializing to anything that looks like a stream,
154 : // i.e. anything that supports .read(char*, size_t) and .write(char*, size_t)
155 : //
156 :
157 : class CSizeComputer;
158 :
159 : enum
160 : {
161 : // primary actions
162 : SER_NETWORK = (1 << 0),
163 : SER_DISK = (1 << 1),
164 : SER_GETHASH = (1 << 2),
165 : };
166 :
167 : //! Convert the reference base type to X, without changing constness or reference type.
168 1784598 : template<typename X> X& ReadWriteAsHelper(X& x) { return x; }
169 31434933 : template<typename X> const X& ReadWriteAsHelper(const X& x) { return x; }
170 :
171 : #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__))
172 : #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper<type>(obj)))
173 : #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const<Type>::type& obj) { code; })
174 : #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; })
175 :
176 : /**
177 : * Implement the Ser and Unser methods needed for implementing a formatter (see Using below).
178 : *
179 : * Both Ser and Unser are delegated to a single static method SerializationOps, which is polymorphic
180 : * in the serialized/deserialized type (allowing it to be const when serializing, and non-const when
181 : * deserializing).
182 : *
183 : * Example use:
184 : * struct FooFormatter {
185 : * FORMATTER_METHODS(Class, obj) { READWRITE(obj.val1, VARINT(obj.val2)); }
186 : * }
187 : * would define a class FooFormatter that defines a serialization of Class objects consisting
188 : * of serializing its val1 member using the default serialization, and its val2 member using
189 : * VARINT serialization. That FooFormatter can then be used in statements like
190 : * READWRITE(Using<FooFormatter>(obj.bla)).
191 : */
192 : #define FORMATTER_METHODS(cls, obj) \
193 : template<typename Stream> \
194 : static void Ser(Stream& s, const cls& obj) { SerializationOps(obj, s, CSerActionSerialize()); } \
195 : template<typename Stream> \
196 : static void Unser(Stream& s, cls& obj) { SerializationOps(obj, s, CSerActionUnserialize()); } \
197 : template<typename Stream, typename Type, typename Operation> \
198 : static inline void SerializationOps(Type& obj, Stream& s, Operation ser_action) \
199 :
200 : /**
201 : * Implement the Serialize and Unserialize methods by delegating to a single templated
202 : * static method that takes the to-be-(de)serialized object as a parameter. This approach
203 : * has the advantage that the constness of the object becomes a template parameter, and
204 : * thus allows a single implementation that sees the object as const for serializing
205 : * and non-const for deserializing, without casts.
206 : */
207 : #define SERIALIZE_METHODS(cls, obj) \
208 : template<typename Stream> \
209 : void Serialize(Stream& s) const \
210 : { \
211 : static_assert(std::is_same<const cls&, decltype(*this)>::value, "Serialize type mismatch"); \
212 : Ser(s, *this); \
213 : } \
214 : template<typename Stream> \
215 : void Unserialize(Stream& s) \
216 : { \
217 : static_assert(std::is_same<cls&, decltype(*this)>::value, "Unserialize type mismatch"); \
218 : Unser(s, *this); \
219 : } \
220 : FORMATTER_METHODS(cls, obj)
221 :
222 : #ifndef CHAR_EQUALS_INT8
223 10344844 : template<typename Stream> inline void Serialize(Stream& s, char a ) { ser_writedata8(s, a); } // TODO Get rid of bare char
224 : #endif
225 2 : template<typename Stream> inline void Serialize(Stream& s, int8_t a ) { ser_writedata8(s, a); }
226 1117681 : template<typename Stream> inline void Serialize(Stream& s, uint8_t a ) { ser_writedata8(s, a); }
227 2 : template<typename Stream> inline void Serialize(Stream& s, int16_t a ) { ser_writedata16(s, a); }
228 2 : template<typename Stream> inline void Serialize(Stream& s, uint16_t a) { ser_writedata16(s, a); }
229 16312489 : template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_writedata32(s, a); }
230 52602526 : template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); }
231 25232897 : template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); }
232 420715 : template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); }
233 1002 : template<typename Stream> inline void Serialize(Stream& s, float a ) { ser_writedata32(s, ser_float_to_uint32(a)); }
234 15394583 : template<typename Stream> inline void Serialize(Stream& s, double a ) { ser_writedata64(s, ser_double_to_uint64(a)); }
235 182539 : template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(a, N); }
236 267196 : template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(CharCast(a), N); }
237 251710 : template<typename Stream> inline void Serialize(Stream& s, const Span<const unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
238 191228 : template<typename Stream> inline void Serialize(Stream& s, const Span<unsigned char>& span) { s.write(CharCast(span.data()), span.size()); }
239 :
240 : #ifndef CHAR_EQUALS_INT8
241 76165 : template<typename Stream> inline void Unserialize(Stream& s, char& a ) { a = ser_readdata8(s); } // TODO Get rid of bare char
242 : #endif
243 1 : template<typename Stream> inline void Unserialize(Stream& s, int8_t& a ) { a = ser_readdata8(s); }
244 177776 : template<typename Stream> inline void Unserialize(Stream& s, uint8_t& a ) { a = ser_readdata8(s); }
245 : template<typename Stream> inline void Unserialize(Stream& s, int16_t& a ) { a = ser_readdata16(s); }
246 1 : template<typename Stream> inline void Unserialize(Stream& s, uint16_t& a) { a = ser_readdata16(s); }
247 1239297 : template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = ser_readdata32(s); }
248 2464844 : template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); }
249 1257149 : template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); }
250 18809 : template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); }
251 1000 : template<typename Stream> inline void Unserialize(Stream& s, float& a ) { a = ser_uint32_to_float(ser_readdata32(s)); }
252 5978789 : template<typename Stream> inline void Unserialize(Stream& s, double& a ) { a = ser_uint64_to_double(ser_readdata64(s)); }
253 173681 : template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(a, N); }
254 121282 : template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(CharCast(a), N); }
255 114582 : template<typename Stream> inline void Unserialize(Stream& s, Span<unsigned char>& span) { s.read(CharCast(span.data()), span.size()); }
256 :
257 370349 : template<typename Stream> inline void Serialize(Stream& s, bool a) { char f=a; ser_writedata8(s, f); }
258 70353 : template<typename Stream> inline void Unserialize(Stream& s, bool& a) { char f=ser_readdata8(s); a=f; }
259 :
260 :
261 :
262 :
263 :
264 :
265 : /**
266 : * Compact Size
267 : * size < 253 -- 1 byte
268 : * size <= USHRT_MAX -- 3 bytes (253 + 2 bytes)
269 : * size <= UINT_MAX -- 5 bytes (254 + 4 bytes)
270 : * size > UINT_MAX -- 9 bytes (255 + 8 bytes)
271 : */
272 12464810 : inline unsigned int GetSizeOfCompactSize(uint64_t nSize)
273 : {
274 12464810 : if (nSize < 253) return sizeof(unsigned char);
275 542950 : else if (nSize <= std::numeric_limits<uint16_t>::max()) return sizeof(unsigned char) + sizeof(uint16_t);
276 8826 : else if (nSize <= std::numeric_limits<unsigned int>::max()) return sizeof(unsigned char) + sizeof(unsigned int);
277 0 : else return sizeof(unsigned char) + sizeof(uint64_t);
278 12464810 : }
279 :
280 : inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize);
281 :
282 : template<typename Stream>
283 36751377 : void WriteCompactSize(Stream& os, uint64_t nSize)
284 : {
285 36751377 : if (nSize < 253)
286 : {
287 33843230 : ser_writedata8(os, nSize);
288 33843230 : }
289 2908174 : else if (nSize <= std::numeric_limits<uint16_t>::max())
290 : {
291 2903792 : ser_writedata8(os, 253);
292 2903792 : ser_writedata16(os, nSize);
293 2903792 : }
294 4382 : else if (nSize <= std::numeric_limits<unsigned int>::max())
295 : {
296 4382 : ser_writedata8(os, 254);
297 4382 : ser_writedata32(os, nSize);
298 4382 : }
299 : else
300 : {
301 0 : ser_writedata8(os, 255);
302 0 : ser_writedata64(os, nSize);
303 : }
304 36751629 : return;
305 : }
306 :
307 : template<typename Stream>
308 3506286 : uint64_t ReadCompactSize(Stream& is)
309 : {
310 3506286 : uint8_t chSize = ser_readdata8(is);
311 : uint64_t nSizeRet = 0;
312 3506286 : if (chSize < 253)
313 : {
314 2863538 : nSizeRet = chSize;
315 2863538 : }
316 642741 : else if (chSize == 253)
317 : {
318 639445 : nSizeRet = ser_readdata16(is);
319 639445 : if (nSizeRet < 253)
320 2 : throw std::ios_base::failure("non-canonical ReadCompactSize()");
321 : }
322 3296 : else if (chSize == 254)
323 : {
324 3293 : nSizeRet = ser_readdata32(is);
325 3293 : if (nSizeRet < 0x10000u)
326 2 : throw std::ios_base::failure("non-canonical ReadCompactSize()");
327 : }
328 : else
329 : {
330 3 : nSizeRet = ser_readdata64(is);
331 3 : if (nSizeRet < 0x100000000ULL)
332 3 : throw std::ios_base::failure("non-canonical ReadCompactSize()");
333 : }
334 3506273 : if (nSizeRet > (uint64_t)MAX_SIZE)
335 0 : throw std::ios_base::failure("ReadCompactSize(): size too large");
336 3506273 : return nSizeRet;
337 7 : }
338 :
339 : /**
340 : * Variable-length integers: bytes are a MSB base-128 encoding of the number.
341 : * The high bit in each byte signifies whether another digit follows. To make
342 : * sure the encoding is one-to-one, one is subtracted from all but the last digit.
343 : * Thus, the byte sequence a[] with length len, where all but the last byte
344 : * has bit 128 set, encodes the number:
345 : *
346 : * (a[len-1] & 0x7F) + sum(i=1..len-1, 128^i*((a[len-i-1] & 0x7F)+1))
347 : *
348 : * Properties:
349 : * * Very small (0-127: 1 byte, 128-16511: 2 bytes, 16512-2113663: 3 bytes)
350 : * * Every integer has exactly one encoding
351 : * * Encoding does not depend on size of original integer type
352 : * * No redundancy: every (infinite) byte sequence corresponds to a list
353 : * of encoded integers.
354 : *
355 : * 0: [0x00] 256: [0x81 0x00]
356 : * 1: [0x01] 16383: [0xFE 0x7F]
357 : * 127: [0x7F] 16384: [0xFF 0x00]
358 : * 128: [0x80 0x00] 16511: [0xFF 0x7F]
359 : * 255: [0x80 0x7F] 65535: [0x82 0xFE 0x7F]
360 : * 2^32: [0x8E 0xFE 0xFE 0xFF 0x00]
361 : */
362 :
363 : /**
364 : * Mode for encoding VarInts.
365 : *
366 : * Currently there is no support for signed encodings. The default mode will not
367 : * compile with signed values, and the legacy "nonnegative signed" mode will
368 : * accept signed values, but improperly encode and decode them if they are
369 : * negative. In the future, the DEFAULT mode could be extended to support
370 : * negative numbers in a backwards compatible way, and additional modes could be
371 : * added to support different varint formats (e.g. zigzag encoding).
372 : */
373 : enum class VarIntMode { DEFAULT, NONNEGATIVE_SIGNED };
374 :
375 : template <VarIntMode Mode, typename I>
376 : struct CheckVarIntMode {
377 16872870 : constexpr CheckVarIntMode()
378 8436435 : {
379 : static_assert(Mode != VarIntMode::DEFAULT || std::is_unsigned<I>::value, "Unsigned type required with mode DEFAULT.");
380 : static_assert(Mode != VarIntMode::NONNEGATIVE_SIGNED || std::is_signed<I>::value, "Signed type required with mode NONNEGATIVE_SIGNED.");
381 16872870 : }
382 : };
383 :
384 : template<VarIntMode Mode, typename I>
385 : inline unsigned int GetSizeOfVarInt(I n)
386 : {
387 : CheckVarIntMode<Mode, I>();
388 : int nRet = 0;
389 : while(true) {
390 : nRet++;
391 : if (n <= 0x7F)
392 : break;
393 : n = (n >> 7) - 1;
394 : }
395 : return nRet;
396 : }
397 :
398 : template<typename I>
399 : inline void WriteVarInt(CSizeComputer& os, I n);
400 :
401 : template<typename Stream, VarIntMode Mode, typename I>
402 7597664 : void WriteVarInt(Stream& os, I n)
403 : {
404 7597664 : CheckVarIntMode<Mode, I>();
405 7597664 : unsigned char tmp[(sizeof(n)*8+6)/7];
406 : int len=0;
407 19131681 : while(true) {
408 19131681 : tmp[len] = (n & 0x7F) | (len ? 0x80 : 0x00);
409 19131681 : if (n <= 0x7F)
410 : break;
411 11534017 : n = (n >> 7) - 1;
412 11534017 : len++;
413 : }
414 : do {
415 19131681 : ser_writedata8(os, tmp[len]);
416 19131681 : } while(len--);
417 7597664 : }
418 :
419 : template<typename Stream, VarIntMode Mode, typename I>
420 838771 : I ReadVarInt(Stream& is)
421 : {
422 838771 : CheckVarIntMode<Mode, I>();
423 : I n = 0;
424 838771 : while(true) {
425 1736831 : unsigned char chData = ser_readdata8(is);
426 1736831 : if (n > (std::numeric_limits<I>::max() >> 7)) {
427 0 : throw std::ios_base::failure("ReadVarInt(): size too large");
428 : }
429 1736831 : n = (n << 7) | (chData & 0x7F);
430 1736831 : if (chData & 0x80) {
431 898060 : if (n == std::numeric_limits<I>::max()) {
432 0 : throw std::ios_base::failure("ReadVarInt(): size too large");
433 : }
434 898060 : n++;
435 : } else {
436 838771 : return n;
437 : }
438 898060 : }
439 838771 : }
440 :
441 : /** Simple wrapper class to serialize objects using a formatter; used by Using(). */
442 : template<typename Formatter, typename T>
443 : class Wrapper
444 : {
445 : static_assert(std::is_lvalue_reference<T>::value, "Wrapper needs an lvalue reference type T");
446 : protected:
447 : T m_object;
448 : public:
449 41089044 : explicit Wrapper(T obj) : m_object(obj) {}
450 18204882 : template<typename Stream> void Serialize(Stream &s) const { Formatter().Ser(s, m_object); }
451 2339711 : template<typename Stream> void Unserialize(Stream &s) { Formatter().Unser(s, m_object); }
452 : };
453 :
454 : /** Cause serialization/deserialization of an object to be done using a specified formatter class.
455 : *
456 : * To use this, you need a class Formatter that has public functions Ser(stream, const object&) for
457 : * serialization, and Unser(stream, object&) for deserialization. Serialization routines (inside
458 : * READWRITE, or directly with << and >> operators), can then use Using<Formatter>(object).
459 : *
460 : * This works by constructing a Wrapper<Formatter, T>-wrapped version of object, where T is
461 : * const during serialization, and non-const during deserialization, which maintains const
462 : * correctness.
463 : */
464 : template<typename Formatter, typename T>
465 20544592 : static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&>(t); }
466 :
467 : #define VARINT_MODE(obj, mode) Using<VarIntFormatter<mode>>(obj)
468 : #define VARINT(obj) Using<VarIntFormatter<VarIntMode::DEFAULT>>(obj)
469 : #define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
470 : #define LIMITED_STRING(obj,n) Using<LimitedStringFormatter<n>>(obj)
471 :
472 : /** Serialization wrapper class for integers in VarInt format. */
473 : template<VarIntMode Mode>
474 : struct VarIntFormatter
475 : {
476 7597664 : template<typename Stream, typename I> void Ser(Stream &s, I v)
477 : {
478 7597664 : WriteVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s, v);
479 7597664 : }
480 :
481 838771 : template<typename Stream, typename I> void Unser(Stream& s, I& v)
482 : {
483 838771 : v = ReadVarInt<Stream,Mode,typename std::remove_cv<I>::type>(s);
484 838771 : }
485 : };
486 :
487 : /** Serialization wrapper class for custom integers and enums.
488 : *
489 : * It permits specifying the serialized size (1 to 8 bytes) and endianness.
490 : *
491 : * Use the big endian mode for values that are stored in memory in native
492 : * byte order, but serialized in big endian notation. This is only intended
493 : * to implement serializers that are compatible with existing formats, and
494 : * its use is not recommended for new data structures.
495 : */
496 : template<int Bytes, bool BigEndian = false>
497 : struct CustomUintFormatter
498 : {
499 : static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
500 : static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
501 :
502 96482 : template <typename Stream, typename I> void Ser(Stream& s, I v)
503 : {
504 96482 : if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
505 : if (BigEndian) {
506 31313 : uint64_t raw = htobe64(v);
507 31313 : s.write(((const char*)&raw) + 8 - Bytes, Bytes);
508 31313 : } else {
509 65169 : uint64_t raw = htole64(v);
510 65169 : s.write((const char*)&raw, Bytes);
511 65169 : }
512 96482 : }
513 :
514 45781 : template <typename Stream, typename I> void Unser(Stream& s, I& v)
515 : {
516 : using U = typename std::conditional<std::is_enum<I>::value, std::underlying_type<I>, std::common_type<I>>::type::type;
517 : static_assert(std::numeric_limits<U>::max() >= MAX && std::numeric_limits<U>::min() <= 0, "Assigned type too small");
518 45781 : uint64_t raw = 0;
519 : if (BigEndian) {
520 12433 : s.read(((char*)&raw) + 8 - Bytes, Bytes);
521 12433 : v = static_cast<I>(be64toh(raw));
522 : } else {
523 33348 : s.read((char*)&raw, Bytes);
524 33348 : v = static_cast<I>(le64toh(raw));
525 : }
526 45781 : }
527 : };
528 :
529 : template<int Bytes> using BigEndianFormatter = CustomUintFormatter<Bytes, true>;
530 :
531 : /** Formatter for integers in CompactSize format. */
532 : struct CompactSizeFormatter
533 : {
534 : template<typename Stream, typename I>
535 12000 : void Unser(Stream& s, I& v)
536 : {
537 12000 : uint64_t n = ReadCompactSize<Stream>(s);
538 12000 : if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
539 0 : throw std::ios_base::failure("CompactSize exceeds limit of type");
540 : }
541 12000 : v = n;
542 12000 : }
543 :
544 : template<typename Stream, typename I>
545 22411 : void Ser(Stream& s, I v)
546 : {
547 : static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
548 : static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
549 :
550 22411 : WriteCompactSize<Stream>(s, v);
551 22411 : }
552 : };
553 :
554 : template<size_t Limit>
555 : struct LimitedStringFormatter
556 : {
557 : template<typename Stream>
558 702 : void Unser(Stream& s, std::string& v)
559 : {
560 702 : size_t size = ReadCompactSize(s);
561 702 : if (size > Limit) {
562 0 : throw std::ios_base::failure("String length limit exceeded");
563 : }
564 701 : v.resize(size);
565 701 : if (size != 0) s.read((char*)v.data(), size);
566 701 : }
567 :
568 : template<typename Stream>
569 3 : void Ser(Stream& s, const std::string& v)
570 : {
571 3 : s << v;
572 3 : }
573 : };
574 :
575 : /** Formatter to serialize/deserialize vector elements using another formatter
576 : *
577 : * Example:
578 : * struct X {
579 : * std::vector<uint64_t> v;
580 : * SERIALIZE_METHODS(X, obj) { READWRITE(Using<VectorFormatter<VarInt>>(obj.v)); }
581 : * };
582 : * will define a struct that contains a vector of uint64_t, which is serialized
583 : * as a vector of VarInt-encoded integers.
584 : *
585 : * V is not required to be an std::vector type. It works for any class that
586 : * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
587 : */
588 : template<class Formatter>
589 : struct VectorFormatter
590 : {
591 : template<typename Stream, typename V>
592 9192508 : void Ser(Stream& s, const V& v)
593 : {
594 197572 : Formatter formatter;
595 9192508 : WriteCompactSize(s, v.size());
596 47123734 : for (const typename V::value_type& elem : v) {
597 37931228 : formatter.Ser(s, elem);
598 : }
599 9192507 : }
600 :
601 : template<typename Stream, typename V>
602 1107623 : void Unser(Stream& s, V& v)
603 : {
604 21578 : Formatter formatter;
605 1107623 : v.clear();
606 1107623 : size_t size = ReadCompactSize(s);
607 : size_t allocated = 0;
608 2003001 : while (allocated < size) {
609 : // For DoS prevention, do not blindly allocate as much as the stream claims to contain.
610 : // Instead, allocate in 5MiB batches, so that an attacker actually needs to provide
611 : // X MiB of data to make us allocate X+5 Mib.
612 : static_assert(sizeof(typename V::value_type) <= MAX_VECTOR_ALLOCATE, "Vector element size too large");
613 895395 : allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
614 895395 : v.reserve(allocated);
615 9790483 : while (v.size() < allocated) {
616 8895106 : v.emplace_back();
617 8895106 : formatter.Unser(s, v.back());
618 : }
619 : }
620 1107599 : };
621 : };
622 :
623 : /**
624 : * Forward declarations
625 : */
626 :
627 : /**
628 : * string
629 : */
630 : template<typename Stream, typename C> void Serialize(Stream& os, const std::basic_string<C>& str);
631 : template<typename Stream, typename C> void Unserialize(Stream& is, std::basic_string<C>& str);
632 :
633 : /**
634 : * prevector
635 : * prevectors of unsigned char are a special case and are intended to be serialized as a single opaque blob.
636 : */
637 : template<typename Stream, unsigned int N, typename T> void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&);
638 : template<typename Stream, unsigned int N, typename T, typename V> void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&);
639 : template<typename Stream, unsigned int N, typename T> inline void Serialize(Stream& os, const prevector<N, T>& v);
640 : template<typename Stream, unsigned int N, typename T> void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&);
641 : template<typename Stream, unsigned int N, typename T, typename V> void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&);
642 : template<typename Stream, unsigned int N, typename T> inline void Unserialize(Stream& is, prevector<N, T>& v);
643 :
644 : /**
645 : * vector
646 : * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob.
647 : */
648 : template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&);
649 : template<typename Stream, typename T, typename A> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&);
650 : template<typename Stream, typename T, typename A, typename V> void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&);
651 : template<typename Stream, typename T, typename A> inline void Serialize(Stream& os, const std::vector<T, A>& v);
652 : template<typename Stream, typename T, typename A> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&);
653 : template<typename Stream, typename T, typename A, typename V> void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&);
654 : template<typename Stream, typename T, typename A> inline void Unserialize(Stream& is, std::vector<T, A>& v);
655 :
656 : /**
657 : * pair
658 : */
659 : template<typename Stream, typename K, typename T> void Serialize(Stream& os, const std::pair<K, T>& item);
660 : template<typename Stream, typename K, typename T> void Unserialize(Stream& is, std::pair<K, T>& item);
661 :
662 : /**
663 : * map
664 : */
665 : template<typename Stream, typename K, typename T, typename Pred, typename A> void Serialize(Stream& os, const std::map<K, T, Pred, A>& m);
666 : template<typename Stream, typename K, typename T, typename Pred, typename A> void Unserialize(Stream& is, std::map<K, T, Pred, A>& m);
667 :
668 : /**
669 : * set
670 : */
671 : template<typename Stream, typename K, typename Pred, typename A> void Serialize(Stream& os, const std::set<K, Pred, A>& m);
672 : template<typename Stream, typename K, typename Pred, typename A> void Unserialize(Stream& is, std::set<K, Pred, A>& m);
673 :
674 : /**
675 : * shared_ptr
676 : */
677 : template<typename Stream, typename T> void Serialize(Stream& os, const std::shared_ptr<const T>& p);
678 : template<typename Stream, typename T> void Unserialize(Stream& os, std::shared_ptr<const T>& p);
679 :
680 : /**
681 : * unique_ptr
682 : */
683 : template<typename Stream, typename T> void Serialize(Stream& os, const std::unique_ptr<const T>& p);
684 : template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p);
685 :
686 :
687 :
688 : /**
689 : * If none of the specialized versions above matched, default to calling member function.
690 : */
691 : template<typename Stream, typename T>
692 164097129 : inline void Serialize(Stream& os, const T& a)
693 : {
694 164097129 : a.Serialize(os);
695 164097129 : }
696 :
697 : template<typename Stream, typename T>
698 8955961 : inline void Unserialize(Stream& is, T&& a)
699 : {
700 8955961 : a.Unserialize(is);
701 8955961 : }
702 :
703 : /** Default formatter. Serializes objects as themselves.
704 : *
705 : * The vector/prevector serialization code passes this to VectorFormatter
706 : * to enable reusing that logic. It shouldn't be needed elsewhere.
707 : */
708 : struct DefaultFormatter
709 : {
710 : template<typename Stream, typename T>
711 37620114 : static void Ser(Stream& s, const T& t) { Serialize(s, t); }
712 :
713 : template<typename Stream, typename T>
714 8864477 : static void Unser(Stream& s, T& t) { Unserialize(s, t); }
715 : };
716 :
717 :
718 :
719 :
720 :
721 : /**
722 : * string
723 : */
724 : template<typename Stream, typename C>
725 1386582 : void Serialize(Stream& os, const std::basic_string<C>& str)
726 : {
727 1386582 : WriteCompactSize(os, str.size());
728 1386582 : if (!str.empty())
729 1230570 : os.write((char*)str.data(), str.size() * sizeof(C));
730 1386581 : }
731 :
732 : template<typename Stream, typename C>
733 138622 : void Unserialize(Stream& is, std::basic_string<C>& str)
734 : {
735 138622 : unsigned int nSize = ReadCompactSize(is);
736 138622 : str.resize(nSize);
737 138622 : if (nSize != 0)
738 126204 : is.read((char*)str.data(), nSize * sizeof(C));
739 138622 : }
740 :
741 :
742 :
743 : /**
744 : * prevector
745 : */
746 : template<typename Stream, unsigned int N, typename T>
747 30749034 : void Serialize_impl(Stream& os, const prevector<N, T>& v, const unsigned char&)
748 : {
749 30749034 : WriteCompactSize(os, v.size());
750 30749034 : if (!v.empty())
751 27422962 : os.write((char*)v.data(), v.size() * sizeof(T));
752 30749237 : }
753 :
754 : template<typename Stream, unsigned int N, typename T, typename V>
755 269033 : void Serialize_impl(Stream& os, const prevector<N, T>& v, const V&)
756 : {
757 269033 : Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
758 269033 : }
759 :
760 : template<typename Stream, unsigned int N, typename T>
761 31018025 : inline void Serialize(Stream& os, const prevector<N, T>& v)
762 : {
763 31018025 : Serialize_impl(os, v, T());
764 31018025 : }
765 :
766 :
767 : template<typename Stream, unsigned int N, typename T>
768 1656487 : void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
769 : {
770 : // Limit size per read so bogus size value won't cause out of memory
771 1656487 : v.clear();
772 1656487 : unsigned int nSize = ReadCompactSize(is);
773 : unsigned int i = 0;
774 3192760 : while (i < nSize)
775 : {
776 1536282 : unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
777 1536282 : v.resize_uninitialized(i + blk);
778 1536282 : is.read((char*)&v[i], blk * sizeof(T));
779 : i += blk;
780 : }
781 1656479 : }
782 :
783 : template<typename Stream, unsigned int N, typename T, typename V>
784 11000 : void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
785 : {
786 11000 : Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
787 11000 : }
788 :
789 : template<typename Stream, unsigned int N, typename T>
790 1667488 : inline void Unserialize(Stream& is, prevector<N, T>& v)
791 : {
792 1667488 : Unserialize_impl(is, v, T());
793 1667488 : }
794 :
795 :
796 :
797 : /**
798 : * vector
799 : */
800 : template<typename Stream, typename T, typename A>
801 7061074 : void Serialize_impl(Stream& os, const std::vector<T, A>& v, const unsigned char&)
802 : {
803 7061074 : WriteCompactSize(os, v.size());
804 7061074 : if (!v.empty())
805 7056986 : os.write((char*)v.data(), v.size() * sizeof(T));
806 7061074 : }
807 :
808 : template<typename Stream, typename T, typename A>
809 22 : void Serialize_impl(Stream& os, const std::vector<T, A>& v, const bool&)
810 : {
811 : // A special case for std::vector<bool>, as dereferencing
812 : // std::vector<bool>::const_iterator does not result in a const bool&
813 : // due to std::vector's special casing for bool arguments.
814 22 : WriteCompactSize(os, v.size());
815 9961 : for (bool elem : v) {
816 9939 : ::Serialize(os, elem);
817 : }
818 22 : }
819 :
820 : template<typename Stream, typename T, typename A, typename V>
821 8715836 : void Serialize_impl(Stream& os, const std::vector<T, A>& v, const V&)
822 : {
823 8715836 : Serialize(os, Using<VectorFormatter<DefaultFormatter>>(v));
824 8715836 : }
825 :
826 : template<typename Stream, typename T, typename A>
827 15776930 : inline void Serialize(Stream& os, const std::vector<T, A>& v)
828 : {
829 15776930 : Serialize_impl(os, v, T());
830 15776932 : }
831 :
832 :
833 : template<typename Stream, typename T, typename A>
834 351446 : void Unserialize_impl(Stream& is, std::vector<T, A>& v, const unsigned char&)
835 : {
836 : // Limit size per read so bogus size value won't cause out of memory
837 351446 : v.clear();
838 351446 : unsigned int nSize = ReadCompactSize(is);
839 : unsigned int i = 0;
840 700836 : while (i < nSize)
841 : {
842 349390 : unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
843 349390 : v.resize(i + blk);
844 349390 : is.read((char*)&v[i], blk * sizeof(T));
845 : i += blk;
846 : }
847 351446 : }
848 :
849 : template<typename Stream, typename T, typename A, typename V>
850 1064992 : void Unserialize_impl(Stream& is, std::vector<T, A>& v, const V&)
851 : {
852 1064992 : Unserialize(is, Using<VectorFormatter<DefaultFormatter>>(v));
853 1064992 : }
854 :
855 : template<typename Stream, typename T, typename A>
856 1416437 : inline void Unserialize(Stream& is, std::vector<T, A>& v)
857 : {
858 1416437 : Unserialize_impl(is, v, T());
859 1416436 : }
860 :
861 :
862 :
863 : /**
864 : * pair
865 : */
866 : template<typename Stream, typename K, typename T>
867 793328 : void Serialize(Stream& os, const std::pair<K, T>& item)
868 : {
869 793328 : Serialize(os, item.first);
870 793328 : Serialize(os, item.second);
871 793328 : }
872 :
873 : template<typename Stream, typename K, typename T>
874 86586 : void Unserialize(Stream& is, std::pair<K, T>& item)
875 : {
876 86586 : Unserialize(is, item.first);
877 86586 : Unserialize(is, item.second);
878 86586 : }
879 :
880 :
881 :
882 : /**
883 : * map
884 : */
885 : template<typename Stream, typename K, typename T, typename Pred, typename A>
886 142206 : void Serialize(Stream& os, const std::map<K, T, Pred, A>& m)
887 : {
888 142206 : WriteCompactSize(os, m.size());
889 563961 : for (const auto& entry : m)
890 421755 : Serialize(os, entry);
891 142206 : }
892 :
893 : template<typename Stream, typename K, typename T, typename Pred, typename A>
894 10159 : void Unserialize(Stream& is, std::map<K, T, Pred, A>& m)
895 : {
896 10159 : m.clear();
897 10159 : unsigned int nSize = ReadCompactSize(is);
898 10159 : typename std::map<K, T, Pred, A>::iterator mi = m.begin();
899 39063 : for (unsigned int i = 0; i < nSize; i++)
900 : {
901 28904 : std::pair<K, T> item;
902 28904 : Unserialize(is, item);
903 28904 : mi = m.insert(mi, item);
904 28904 : }
905 10158 : }
906 :
907 :
908 :
909 : /**
910 : * set
911 : */
912 : template<typename Stream, typename K, typename Pred, typename A>
913 : void Serialize(Stream& os, const std::set<K, Pred, A>& m)
914 : {
915 : WriteCompactSize(os, m.size());
916 : for (typename std::set<K, Pred, A>::const_iterator it = m.begin(); it != m.end(); ++it)
917 : Serialize(os, (*it));
918 : }
919 :
920 : template<typename Stream, typename K, typename Pred, typename A>
921 : void Unserialize(Stream& is, std::set<K, Pred, A>& m)
922 : {
923 : m.clear();
924 : unsigned int nSize = ReadCompactSize(is);
925 : typename std::set<K, Pred, A>::iterator it = m.begin();
926 : for (unsigned int i = 0; i < nSize; i++)
927 : {
928 : K key;
929 : Unserialize(is, key);
930 : it = m.insert(it, key);
931 : }
932 : }
933 :
934 :
935 :
936 : /**
937 : * unique_ptr
938 : */
939 : template<typename Stream, typename T> void
940 : Serialize(Stream& os, const std::unique_ptr<const T>& p)
941 : {
942 : Serialize(os, *p);
943 : }
944 :
945 : template<typename Stream, typename T>
946 : void Unserialize(Stream& is, std::unique_ptr<const T>& p)
947 : {
948 : p.reset(new T(deserialize, is));
949 : }
950 :
951 :
952 :
953 : /**
954 : * shared_ptr
955 : */
956 : template<typename Stream, typename T> void
957 1066909 : Serialize(Stream& os, const std::shared_ptr<const T>& p)
958 : {
959 1066909 : Serialize(os, *p);
960 1066909 : }
961 :
962 : template<typename Stream, typename T>
963 240474 : void Unserialize(Stream& is, std::shared_ptr<const T>& p)
964 : {
965 240474 : p = std::make_shared<const T>(deserialize, is);
966 240474 : }
967 :
968 :
969 :
970 : /**
971 : * Support for SERIALIZE_METHODS and READWRITE macro.
972 : */
973 : struct CSerActionSerialize
974 : {
975 : constexpr bool ForRead() const { return false; }
976 : };
977 : struct CSerActionUnserialize
978 : {
979 : constexpr bool ForRead() const { return true; }
980 : };
981 :
982 :
983 :
984 :
985 :
986 :
987 :
988 :
989 : /* ::GetSerializeSize implementations
990 : *
991 : * Computing the serialized size of objects is done through a special stream
992 : * object of type CSizeComputer, which only records the number of bytes written
993 : * to it.
994 : *
995 : * If your Serialize or SerializationOp method has non-trivial overhead for
996 : * serialization, it may be worthwhile to implement a specialized version for
997 : * CSizeComputer, which uses the s.seek() method to record bytes that would
998 : * be written instead.
999 : */
1000 : class CSizeComputer
1001 : {
1002 : protected:
1003 : size_t nSize;
1004 :
1005 : const int nVersion;
1006 : public:
1007 3829412 : explicit CSizeComputer(int nVersionIn) : nSize(0), nVersion(nVersionIn) {}
1008 :
1009 28059184 : void write(const char *psz, size_t _nSize)
1010 : {
1011 28059184 : this->nSize += _nSize;
1012 28059184 : }
1013 :
1014 : /** Pretend _nSize bytes are written, without specifying them. */
1015 12463847 : void seek(size_t _nSize)
1016 : {
1017 12463847 : this->nSize += _nSize;
1018 12463847 : }
1019 :
1020 : template<typename T>
1021 8232345 : CSizeComputer& operator<<(const T& obj)
1022 : {
1023 8232345 : ::Serialize(*this, obj);
1024 8232345 : return (*this);
1025 : }
1026 :
1027 1914705 : size_t size() const {
1028 1914705 : return nSize;
1029 : }
1030 :
1031 1264719 : int GetVersion() const { return nVersion; }
1032 : };
1033 :
1034 : template<typename Stream>
1035 100213819 : void SerializeMany(Stream& s)
1036 : {
1037 100213819 : }
1038 :
1039 : template<typename Stream, typename Arg, typename... Args>
1040 187191566 : void SerializeMany(Stream& s, const Arg& arg, const Args&... args)
1041 : {
1042 187191566 : ::Serialize(s, arg);
1043 187191566 : ::SerializeMany(s, args...);
1044 187191566 : }
1045 :
1046 : template<typename Stream>
1047 5831157 : inline void UnserializeMany(Stream& s)
1048 : {
1049 5831157 : }
1050 :
1051 : template<typename Stream, typename Arg, typename... Args>
1052 10494062 : inline void UnserializeMany(Stream& s, Arg&& arg, Args&&... args)
1053 : {
1054 10494062 : ::Unserialize(s, arg);
1055 10494062 : ::UnserializeMany(s, args...);
1056 10494062 : }
1057 :
1058 : template<typename Stream, typename... Args>
1059 100029147 : inline void SerReadWriteMany(Stream& s, CSerActionSerialize ser_action, const Args&... args)
1060 : {
1061 100029147 : ::SerializeMany(s, args...);
1062 100029147 : }
1063 :
1064 : template<typename Stream, typename... Args>
1065 5830336 : inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&&... args)
1066 : {
1067 5830336 : ::UnserializeMany(s, args...);
1068 5830336 : }
1069 :
1070 : template<typename Stream, typename Type, typename Fn>
1071 39648 : inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&)
1072 : {
1073 39648 : }
1074 :
1075 : template<typename Stream, typename Type, typename Fn>
1076 13031 : inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn)
1077 : {
1078 13031 : fn(s, std::forward<Type>(obj));
1079 13031 : }
1080 :
1081 : template<typename Stream, typename Type, typename Fn>
1082 8149 : inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn)
1083 : {
1084 8149 : fn(s, std::forward<Type>(obj));
1085 8149 : }
1086 :
1087 : template<typename Stream, typename Type, typename Fn>
1088 412 : inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&)
1089 : {
1090 412 : }
1091 :
1092 : template<typename I>
1093 : inline void WriteVarInt(CSizeComputer &s, I n)
1094 : {
1095 : s.seek(GetSizeOfVarInt<I>(n));
1096 : }
1097 :
1098 12463847 : inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize)
1099 : {
1100 12463847 : s.seek(GetSizeOfCompactSize(nSize));
1101 12463847 : }
1102 :
1103 : template <typename T>
1104 1912438 : size_t GetSerializeSize(const T& t, int nVersion = 0)
1105 : {
1106 1912438 : return (CSizeComputer(nVersion) << t).size();
1107 : }
1108 :
1109 : template <typename... T>
1110 2268 : size_t GetSerializeSizeMany(int nVersion, const T&... t)
1111 : {
1112 2268 : CSizeComputer sc(nVersion);
1113 2268 : SerializeMany(sc, t...);
1114 4536 : return sc.size();
1115 2268 : }
1116 :
1117 : #endif // BITCOIN_SERIALIZE_H
|
