1 /*
2 * Copyright (c) 2009-2021, Google LLC
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 * * Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * * Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 * * Neither the name of Google LLC nor the
13 * names of its contributors may be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL Google LLC BE LIABLE FOR ANY DIRECT,
20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
23 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
25 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28 // Fast decoder: ~3x the speed of decode.c, but requires x86-64/ARM64.
29 // Also the table size grows by 2x.
30 //
31 // Could potentially be ported to other 64-bit archs that pass at least six
32 // arguments in registers and have 8 unused high bits in pointers.
33 //
34 // The overall design is to create specialized functions for every possible
35 // field type (eg. oneof boolean field with a 1 byte tag) and then dispatch
36 // to the specialized function as quickly as possible.
37
38 #include "upb/wire/decode_fast.h"
39
40 #include "upb/collections/array_internal.h"
41 #include "upb/wire/decode_internal.h"
42
43 // Must be last.
44 #include "upb/port/def.inc"
45
46 #if UPB_FASTTABLE
47
48 // The standard set of arguments passed to each parsing function.
49 // Thanks to x86-64 calling conventions, these will stay in registers.
50 #define UPB_PARSE_PARAMS \
51 upb_Decoder *d, const char *ptr, upb_Message *msg, intptr_t table, \
52 uint64_t hasbits, uint64_t data
53
54 #define UPB_PARSE_ARGS d, ptr, msg, table, hasbits, data
55
56 #define RETURN_GENERIC(m) \
57 /* Uncomment either of these for debugging purposes. */ \
58 /* fprintf(stderr, m); */ \
59 /*__builtin_trap(); */ \
60 return _upb_FastDecoder_DecodeGeneric(d, ptr, msg, table, hasbits, 0);
61
62 typedef enum {
63 CARD_s = 0, /* Singular (optional, non-repeated) */
64 CARD_o = 1, /* Oneof */
65 CARD_r = 2, /* Repeated */
66 CARD_p = 3 /* Packed Repeated */
67 } upb_card;
68
69 UPB_NOINLINE
fastdecode_isdonefallback(UPB_PARSE_PARAMS)70 static const char* fastdecode_isdonefallback(UPB_PARSE_PARAMS) {
71 int overrun = data;
72 ptr = _upb_EpsCopyInputStream_IsDoneFallbackInline(
73 &d->input, ptr, overrun, _upb_Decoder_BufferFlipCallback);
74 data = _upb_FastDecoder_LoadTag(ptr);
75 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
76 }
77
78 UPB_FORCEINLINE
fastdecode_dispatch(UPB_PARSE_PARAMS)79 static const char* fastdecode_dispatch(UPB_PARSE_PARAMS) {
80 int overrun;
81 switch (upb_EpsCopyInputStream_IsDoneStatus(&d->input, ptr, &overrun)) {
82 case kUpb_IsDoneStatus_Done:
83 *(uint32_t*)msg |= hasbits; // Sync hasbits.
84 const upb_MiniTable* l = decode_totablep(table);
85 return UPB_UNLIKELY(l->required_count)
86 ? _upb_Decoder_CheckRequired(d, ptr, msg, l)
87 : ptr;
88 case kUpb_IsDoneStatus_NotDone:
89 break;
90 case kUpb_IsDoneStatus_NeedFallback:
91 data = overrun;
92 UPB_MUSTTAIL return fastdecode_isdonefallback(UPB_PARSE_ARGS);
93 }
94
95 // Read two bytes of tag data (for a one-byte tag, the high byte is junk).
96 data = _upb_FastDecoder_LoadTag(ptr);
97 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS);
98 }
99
100 UPB_FORCEINLINE
fastdecode_checktag(uint16_t data,int tagbytes)101 static bool fastdecode_checktag(uint16_t data, int tagbytes) {
102 if (tagbytes == 1) {
103 return (data & 0xff) == 0;
104 } else {
105 return data == 0;
106 }
107 }
108
109 UPB_FORCEINLINE
fastdecode_longsize(const char * ptr,int * size)110 static const char* fastdecode_longsize(const char* ptr, int* size) {
111 int i;
112 UPB_ASSERT(*size & 0x80);
113 *size &= 0xff;
114 for (i = 0; i < 3; i++) {
115 ptr++;
116 size_t byte = (uint8_t)ptr[-1];
117 *size += (byte - 1) << (7 + 7 * i);
118 if (UPB_LIKELY((byte & 0x80) == 0)) return ptr;
119 }
120 ptr++;
121 size_t byte = (uint8_t)ptr[-1];
122 // len is limited by 2gb not 4gb, hence 8 and not 16 as normally expected
123 // for a 32 bit varint.
124 if (UPB_UNLIKELY(byte >= 8)) return NULL;
125 *size += (byte - 1) << 28;
126 return ptr;
127 }
128
129 UPB_FORCEINLINE
fastdecode_delimited(upb_Decoder * d,const char * ptr,upb_EpsCopyInputStream_ParseDelimitedFunc * func,void * ctx)130 static const char* fastdecode_delimited(
131 upb_Decoder* d, const char* ptr,
132 upb_EpsCopyInputStream_ParseDelimitedFunc* func, void* ctx) {
133 ptr++;
134
135 // Sign-extend so varint greater than one byte becomes negative, causing
136 // fast delimited parse to fail.
137 int len = (int8_t)ptr[-1];
138
139 if (!upb_EpsCopyInputStream_TryParseDelimitedFast(&d->input, &ptr, len, func,
140 ctx)) {
141 // Slow case: Sub-message is >=128 bytes and/or exceeds the current buffer.
142 // If it exceeds the buffer limit, limit/limit_ptr will change during
143 // sub-message parsing, so we need to preserve delta, not limit.
144 if (UPB_UNLIKELY(len & 0x80)) {
145 // Size varint >1 byte (length >= 128).
146 ptr = fastdecode_longsize(ptr, &len);
147 if (!ptr) {
148 // Corrupt wire format: size exceeded INT_MAX.
149 return NULL;
150 }
151 }
152 if (!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, len)) {
153 // Corrupt wire format: invalid limit.
154 return NULL;
155 }
156 int delta = upb_EpsCopyInputStream_PushLimit(&d->input, ptr, len);
157 ptr = func(&d->input, ptr, ctx);
158 upb_EpsCopyInputStream_PopLimit(&d->input, ptr, delta);
159 }
160 return ptr;
161 }
162
163 /* singular, oneof, repeated field handling ***********************************/
164
165 typedef struct {
166 upb_Array* arr;
167 void* end;
168 } fastdecode_arr;
169
170 typedef enum {
171 FD_NEXT_ATLIMIT,
172 FD_NEXT_SAMEFIELD,
173 FD_NEXT_OTHERFIELD
174 } fastdecode_next;
175
176 typedef struct {
177 void* dst;
178 fastdecode_next next;
179 uint32_t tag;
180 } fastdecode_nextret;
181
182 UPB_FORCEINLINE
fastdecode_resizearr(upb_Decoder * d,void * dst,fastdecode_arr * farr,int valbytes)183 static void* fastdecode_resizearr(upb_Decoder* d, void* dst,
184 fastdecode_arr* farr, int valbytes) {
185 if (UPB_UNLIKELY(dst == farr->end)) {
186 size_t old_size = farr->arr->capacity;
187 size_t old_bytes = old_size * valbytes;
188 size_t new_size = old_size * 2;
189 size_t new_bytes = new_size * valbytes;
190 char* old_ptr = _upb_array_ptr(farr->arr);
191 char* new_ptr = upb_Arena_Realloc(&d->arena, old_ptr, old_bytes, new_bytes);
192 uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
193 farr->arr->capacity = new_size;
194 farr->arr->data = _upb_array_tagptr(new_ptr, elem_size_lg2);
195 dst = (void*)(new_ptr + (old_size * valbytes));
196 farr->end = (void*)(new_ptr + (new_size * valbytes));
197 }
198 return dst;
199 }
200
201 UPB_FORCEINLINE
fastdecode_tagmatch(uint32_t tag,uint64_t data,int tagbytes)202 static bool fastdecode_tagmatch(uint32_t tag, uint64_t data, int tagbytes) {
203 if (tagbytes == 1) {
204 return (uint8_t)tag == (uint8_t)data;
205 } else {
206 return (uint16_t)tag == (uint16_t)data;
207 }
208 }
209
210 UPB_FORCEINLINE
fastdecode_commitarr(void * dst,fastdecode_arr * farr,int valbytes)211 static void fastdecode_commitarr(void* dst, fastdecode_arr* farr,
212 int valbytes) {
213 farr->arr->size =
214 (size_t)((char*)dst - (char*)_upb_array_ptr(farr->arr)) / valbytes;
215 }
216
217 UPB_FORCEINLINE
fastdecode_nextrepeated(upb_Decoder * d,void * dst,const char ** ptr,fastdecode_arr * farr,uint64_t data,int tagbytes,int valbytes)218 static fastdecode_nextret fastdecode_nextrepeated(upb_Decoder* d, void* dst,
219 const char** ptr,
220 fastdecode_arr* farr,
221 uint64_t data, int tagbytes,
222 int valbytes) {
223 fastdecode_nextret ret;
224 dst = (char*)dst + valbytes;
225
226 if (UPB_LIKELY(!_upb_Decoder_IsDone(d, ptr))) {
227 ret.tag = _upb_FastDecoder_LoadTag(*ptr);
228 if (fastdecode_tagmatch(ret.tag, data, tagbytes)) {
229 ret.next = FD_NEXT_SAMEFIELD;
230 } else {
231 fastdecode_commitarr(dst, farr, valbytes);
232 ret.next = FD_NEXT_OTHERFIELD;
233 }
234 } else {
235 fastdecode_commitarr(dst, farr, valbytes);
236 ret.next = FD_NEXT_ATLIMIT;
237 }
238
239 ret.dst = dst;
240 return ret;
241 }
242
243 UPB_FORCEINLINE
fastdecode_fieldmem(upb_Message * msg,uint64_t data)244 static void* fastdecode_fieldmem(upb_Message* msg, uint64_t data) {
245 size_t ofs = data >> 48;
246 return (char*)msg + ofs;
247 }
248
249 UPB_FORCEINLINE
fastdecode_getfield(upb_Decoder * d,const char * ptr,upb_Message * msg,uint64_t * data,uint64_t * hasbits,fastdecode_arr * farr,int valbytes,upb_card card)250 static void* fastdecode_getfield(upb_Decoder* d, const char* ptr,
251 upb_Message* msg, uint64_t* data,
252 uint64_t* hasbits, fastdecode_arr* farr,
253 int valbytes, upb_card card) {
254 switch (card) {
255 case CARD_s: {
256 uint8_t hasbit_index = *data >> 24;
257 // Set hasbit and return pointer to scalar field.
258 *hasbits |= 1ull << hasbit_index;
259 return fastdecode_fieldmem(msg, *data);
260 }
261 case CARD_o: {
262 uint16_t case_ofs = *data >> 32;
263 uint32_t* oneof_case = UPB_PTR_AT(msg, case_ofs, uint32_t);
264 uint8_t field_number = *data >> 24;
265 *oneof_case = field_number;
266 return fastdecode_fieldmem(msg, *data);
267 }
268 case CARD_r: {
269 // Get pointer to upb_Array and allocate/expand if necessary.
270 uint8_t elem_size_lg2 = __builtin_ctz(valbytes);
271 upb_Array** arr_p = fastdecode_fieldmem(msg, *data);
272 char* begin;
273 *(uint32_t*)msg |= *hasbits;
274 *hasbits = 0;
275 if (UPB_LIKELY(!*arr_p)) {
276 farr->arr = _upb_Array_New(&d->arena, 8, elem_size_lg2);
277 *arr_p = farr->arr;
278 } else {
279 farr->arr = *arr_p;
280 }
281 begin = _upb_array_ptr(farr->arr);
282 farr->end = begin + (farr->arr->capacity * valbytes);
283 *data = _upb_FastDecoder_LoadTag(ptr);
284 return begin + (farr->arr->size * valbytes);
285 }
286 default:
287 UPB_UNREACHABLE();
288 }
289 }
290
291 UPB_FORCEINLINE
fastdecode_flippacked(uint64_t * data,int tagbytes)292 static bool fastdecode_flippacked(uint64_t* data, int tagbytes) {
293 *data ^= (0x2 ^ 0x0); // Patch data to match packed wiretype.
294 return fastdecode_checktag(*data, tagbytes);
295 }
296
297 #define FASTDECODE_CHECKPACKED(tagbytes, card, func) \
298 if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
299 if (card == CARD_r && fastdecode_flippacked(&data, tagbytes)) { \
300 UPB_MUSTTAIL return func(UPB_PARSE_ARGS); \
301 } \
302 RETURN_GENERIC("packed check tag mismatch\n"); \
303 }
304
305 /* varint fields **************************************************************/
306
307 UPB_FORCEINLINE
fastdecode_munge(uint64_t val,int valbytes,bool zigzag)308 static uint64_t fastdecode_munge(uint64_t val, int valbytes, bool zigzag) {
309 if (valbytes == 1) {
310 return val != 0;
311 } else if (zigzag) {
312 if (valbytes == 4) {
313 uint32_t n = val;
314 return (n >> 1) ^ -(int32_t)(n & 1);
315 } else if (valbytes == 8) {
316 return (val >> 1) ^ -(int64_t)(val & 1);
317 }
318 UPB_UNREACHABLE();
319 }
320 return val;
321 }
322
323 UPB_FORCEINLINE
fastdecode_varint64(const char * ptr,uint64_t * val)324 static const char* fastdecode_varint64(const char* ptr, uint64_t* val) {
325 ptr++;
326 *val = (uint8_t)ptr[-1];
327 if (UPB_UNLIKELY(*val & 0x80)) {
328 int i;
329 for (i = 0; i < 8; i++) {
330 ptr++;
331 uint64_t byte = (uint8_t)ptr[-1];
332 *val += (byte - 1) << (7 + 7 * i);
333 if (UPB_LIKELY((byte & 0x80) == 0)) goto done;
334 }
335 ptr++;
336 uint64_t byte = (uint8_t)ptr[-1];
337 if (byte > 1) {
338 return NULL;
339 }
340 *val += (byte - 1) << 63;
341 }
342 done:
343 UPB_ASSUME(ptr != NULL);
344 return ptr;
345 }
346
347 #define FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
348 valbytes, card, zigzag, packed) \
349 uint64_t val; \
350 void* dst; \
351 fastdecode_arr farr; \
352 \
353 FASTDECODE_CHECKPACKED(tagbytes, card, packed); \
354 \
355 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \
356 card); \
357 if (card == CARD_r) { \
358 if (UPB_UNLIKELY(!dst)) { \
359 RETURN_GENERIC("need array resize\n"); \
360 } \
361 } \
362 \
363 again: \
364 if (card == CARD_r) { \
365 dst = fastdecode_resizearr(d, dst, &farr, valbytes); \
366 } \
367 \
368 ptr += tagbytes; \
369 ptr = fastdecode_varint64(ptr, &val); \
370 if (ptr == NULL) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
371 val = fastdecode_munge(val, valbytes, zigzag); \
372 memcpy(dst, &val, valbytes); \
373 \
374 if (card == CARD_r) { \
375 fastdecode_nextret ret = fastdecode_nextrepeated( \
376 d, dst, &ptr, &farr, data, tagbytes, valbytes); \
377 switch (ret.next) { \
378 case FD_NEXT_SAMEFIELD: \
379 dst = ret.dst; \
380 goto again; \
381 case FD_NEXT_OTHERFIELD: \
382 data = ret.tag; \
383 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
384 case FD_NEXT_ATLIMIT: \
385 return ptr; \
386 } \
387 } \
388 \
389 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
390
391 typedef struct {
392 uint8_t valbytes;
393 bool zigzag;
394 void* dst;
395 fastdecode_arr farr;
396 } fastdecode_varintdata;
397
398 UPB_FORCEINLINE
fastdecode_topackedvarint(upb_EpsCopyInputStream * e,const char * ptr,void * ctx)399 static const char* fastdecode_topackedvarint(upb_EpsCopyInputStream* e,
400 const char* ptr, void* ctx) {
401 upb_Decoder* d = (upb_Decoder*)e;
402 fastdecode_varintdata* data = ctx;
403 void* dst = data->dst;
404 uint64_t val;
405
406 while (!_upb_Decoder_IsDone(d, &ptr)) {
407 dst = fastdecode_resizearr(d, dst, &data->farr, data->valbytes);
408 ptr = fastdecode_varint64(ptr, &val);
409 if (ptr == NULL) return NULL;
410 val = fastdecode_munge(val, data->valbytes, data->zigzag);
411 memcpy(dst, &val, data->valbytes);
412 dst = (char*)dst + data->valbytes;
413 }
414
415 fastdecode_commitarr(dst, &data->farr, data->valbytes);
416 return ptr;
417 }
418
419 #define FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
420 valbytes, zigzag, unpacked) \
421 fastdecode_varintdata ctx = {valbytes, zigzag}; \
422 \
423 FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked); \
424 \
425 ctx.dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &ctx.farr, \
426 valbytes, CARD_r); \
427 if (UPB_UNLIKELY(!ctx.dst)) { \
428 RETURN_GENERIC("need array resize\n"); \
429 } \
430 \
431 ptr += tagbytes; \
432 ptr = fastdecode_delimited(d, ptr, &fastdecode_topackedvarint, &ctx); \
433 \
434 if (UPB_UNLIKELY(ptr == NULL)) { \
435 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
436 } \
437 \
438 UPB_MUSTTAIL return fastdecode_dispatch(d, ptr, msg, table, hasbits, 0);
439
440 #define FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
441 valbytes, card, zigzag, unpacked, packed) \
442 if (card == CARD_p) { \
443 FASTDECODE_PACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
444 valbytes, zigzag, unpacked); \
445 } else { \
446 FASTDECODE_UNPACKEDVARINT(d, ptr, msg, table, hasbits, data, tagbytes, \
447 valbytes, card, zigzag, packed); \
448 }
449
450 #define z_ZZ true
451 #define b_ZZ false
452 #define v_ZZ false
453
454 /* Generate all combinations:
455 * {s,o,r,p} x {b1,v4,z4,v8,z8} x {1bt,2bt} */
456
457 #define F(card, type, valbytes, tagbytes) \
458 UPB_NOINLINE \
459 const char* upb_p##card##type##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
460 FASTDECODE_VARINT(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \
461 CARD_##card, type##_ZZ, \
462 upb_pr##type##valbytes##_##tagbytes##bt, \
463 upb_pp##type##valbytes##_##tagbytes##bt); \
464 }
465
466 #define TYPES(card, tagbytes) \
467 F(card, b, 1, tagbytes) \
468 F(card, v, 4, tagbytes) \
469 F(card, v, 8, tagbytes) \
470 F(card, z, 4, tagbytes) \
471 F(card, z, 8, tagbytes)
472
473 #define TAGBYTES(card) \
474 TYPES(card, 1) \
475 TYPES(card, 2)
476
477 TAGBYTES(s)
478 TAGBYTES(o)
479 TAGBYTES(r)
480 TAGBYTES(p)
481
482 #undef z_ZZ
483 #undef b_ZZ
484 #undef v_ZZ
485 #undef o_ONEOF
486 #undef s_ONEOF
487 #undef r_ONEOF
488 #undef F
489 #undef TYPES
490 #undef TAGBYTES
491 #undef FASTDECODE_UNPACKEDVARINT
492 #undef FASTDECODE_PACKEDVARINT
493 #undef FASTDECODE_VARINT
494
495 /* fixed fields ***************************************************************/
496
497 #define FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
498 valbytes, card, packed) \
499 void* dst; \
500 fastdecode_arr farr; \
501 \
502 FASTDECODE_CHECKPACKED(tagbytes, card, packed) \
503 \
504 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, valbytes, \
505 card); \
506 if (card == CARD_r) { \
507 if (UPB_UNLIKELY(!dst)) { \
508 RETURN_GENERIC("couldn't allocate array in arena\n"); \
509 } \
510 } \
511 \
512 again: \
513 if (card == CARD_r) { \
514 dst = fastdecode_resizearr(d, dst, &farr, valbytes); \
515 } \
516 \
517 ptr += tagbytes; \
518 memcpy(dst, ptr, valbytes); \
519 ptr += valbytes; \
520 \
521 if (card == CARD_r) { \
522 fastdecode_nextret ret = fastdecode_nextrepeated( \
523 d, dst, &ptr, &farr, data, tagbytes, valbytes); \
524 switch (ret.next) { \
525 case FD_NEXT_SAMEFIELD: \
526 dst = ret.dst; \
527 goto again; \
528 case FD_NEXT_OTHERFIELD: \
529 data = ret.tag; \
530 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
531 case FD_NEXT_ATLIMIT: \
532 return ptr; \
533 } \
534 } \
535 \
536 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
537
538 #define FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
539 valbytes, unpacked) \
540 FASTDECODE_CHECKPACKED(tagbytes, CARD_r, unpacked) \
541 \
542 ptr += tagbytes; \
543 int size = (uint8_t)ptr[0]; \
544 ptr++; \
545 if (size & 0x80) { \
546 ptr = fastdecode_longsize(ptr, &size); \
547 } \
548 \
549 if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckDataSizeAvailable( \
550 &d->input, ptr, size) || \
551 (size % valbytes) != 0)) { \
552 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
553 } \
554 \
555 upb_Array** arr_p = fastdecode_fieldmem(msg, data); \
556 upb_Array* arr = *arr_p; \
557 uint8_t elem_size_lg2 = __builtin_ctz(valbytes); \
558 int elems = size / valbytes; \
559 \
560 if (UPB_LIKELY(!arr)) { \
561 *arr_p = arr = _upb_Array_New(&d->arena, elems, elem_size_lg2); \
562 if (!arr) { \
563 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
564 } \
565 } else { \
566 _upb_Array_ResizeUninitialized(arr, elems, &d->arena); \
567 } \
568 \
569 char* dst = _upb_array_ptr(arr); \
570 memcpy(dst, ptr, size); \
571 arr->size = elems; \
572 \
573 ptr += size; \
574 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
575
576 #define FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
577 valbytes, card, unpacked, packed) \
578 if (card == CARD_p) { \
579 FASTDECODE_PACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
580 valbytes, unpacked); \
581 } else { \
582 FASTDECODE_UNPACKEDFIXED(d, ptr, msg, table, hasbits, data, tagbytes, \
583 valbytes, card, packed); \
584 }
585
586 /* Generate all combinations:
587 * {s,o,r,p} x {f4,f8} x {1bt,2bt} */
588
589 #define F(card, valbytes, tagbytes) \
590 UPB_NOINLINE \
591 const char* upb_p##card##f##valbytes##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
592 FASTDECODE_FIXED(d, ptr, msg, table, hasbits, data, tagbytes, valbytes, \
593 CARD_##card, upb_ppf##valbytes##_##tagbytes##bt, \
594 upb_prf##valbytes##_##tagbytes##bt); \
595 }
596
597 #define TYPES(card, tagbytes) \
598 F(card, 4, tagbytes) \
599 F(card, 8, tagbytes)
600
601 #define TAGBYTES(card) \
602 TYPES(card, 1) \
603 TYPES(card, 2)
604
605 TAGBYTES(s)
606 TAGBYTES(o)
607 TAGBYTES(r)
608 TAGBYTES(p)
609
610 #undef F
611 #undef TYPES
612 #undef TAGBYTES
613 #undef FASTDECODE_UNPACKEDFIXED
614 #undef FASTDECODE_PACKEDFIXED
615
616 /* string fields **************************************************************/
617
618 typedef const char* fastdecode_copystr_func(struct upb_Decoder* d,
619 const char* ptr, upb_Message* msg,
620 const upb_MiniTable* table,
621 uint64_t hasbits,
622 upb_StringView* dst);
623
624 UPB_NOINLINE
fastdecode_verifyutf8(upb_Decoder * d,const char * ptr,upb_Message * msg,intptr_t table,uint64_t hasbits,uint64_t data)625 static const char* fastdecode_verifyutf8(upb_Decoder* d, const char* ptr,
626 upb_Message* msg, intptr_t table,
627 uint64_t hasbits, uint64_t data) {
628 upb_StringView* dst = (upb_StringView*)data;
629 if (!_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) {
630 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8);
631 }
632 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
633 }
634
635 #define FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, validate_utf8) \
636 int size = (uint8_t)ptr[0]; /* Could plumb through hasbits. */ \
637 ptr++; \
638 if (size & 0x80) { \
639 ptr = fastdecode_longsize(ptr, &size); \
640 } \
641 \
642 if (UPB_UNLIKELY(!upb_EpsCopyInputStream_CheckSize(&d->input, ptr, size))) { \
643 dst->size = 0; \
644 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
645 } \
646 \
647 const char* s_ptr = ptr; \
648 ptr = upb_EpsCopyInputStream_ReadString(&d->input, &s_ptr, size, &d->arena); \
649 if (!ptr) _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_OutOfMemory); \
650 dst->data = s_ptr; \
651 dst->size = size; \
652 \
653 if (validate_utf8) { \
654 data = (uint64_t)dst; \
655 UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
656 } else { \
657 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \
658 }
659
660 UPB_NOINLINE
fastdecode_longstring_utf8(struct upb_Decoder * d,const char * ptr,upb_Message * msg,intptr_t table,uint64_t hasbits,uint64_t data)661 static const char* fastdecode_longstring_utf8(struct upb_Decoder* d,
662 const char* ptr, upb_Message* msg,
663 intptr_t table, uint64_t hasbits,
664 uint64_t data) {
665 upb_StringView* dst = (upb_StringView*)data;
666 FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, true);
667 }
668
669 UPB_NOINLINE
fastdecode_longstring_noutf8(struct upb_Decoder * d,const char * ptr,upb_Message * msg,intptr_t table,uint64_t hasbits,uint64_t data)670 static const char* fastdecode_longstring_noutf8(
671 struct upb_Decoder* d, const char* ptr, upb_Message* msg, intptr_t table,
672 uint64_t hasbits, uint64_t data) {
673 upb_StringView* dst = (upb_StringView*)data;
674 FASTDECODE_LONGSTRING(d, ptr, msg, table, hasbits, dst, false);
675 }
676
677 UPB_FORCEINLINE
fastdecode_docopy(upb_Decoder * d,const char * ptr,uint32_t size,int copy,char * data,upb_StringView * dst)678 static void fastdecode_docopy(upb_Decoder* d, const char* ptr, uint32_t size,
679 int copy, char* data, upb_StringView* dst) {
680 d->arena.head.ptr += copy;
681 dst->data = data;
682 UPB_UNPOISON_MEMORY_REGION(data, copy);
683 memcpy(data, ptr, copy);
684 UPB_POISON_MEMORY_REGION(data + size, copy - size);
685 }
686
687 #define FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \
688 card, validate_utf8) \
689 upb_StringView* dst; \
690 fastdecode_arr farr; \
691 int64_t size; \
692 size_t arena_has; \
693 size_t common_has; \
694 char* buf; \
695 \
696 UPB_ASSERT(!upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0)); \
697 UPB_ASSERT(fastdecode_checktag(data, tagbytes)); \
698 \
699 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
700 sizeof(upb_StringView), card); \
701 \
702 again: \
703 if (card == CARD_r) { \
704 dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \
705 } \
706 \
707 size = (uint8_t)ptr[tagbytes]; \
708 ptr += tagbytes + 1; \
709 dst->size = size; \
710 \
711 buf = d->arena.head.ptr; \
712 arena_has = _upb_ArenaHas(&d->arena); \
713 common_has = UPB_MIN(arena_has, \
714 upb_EpsCopyInputStream_BytesAvailable(&d->input, ptr)); \
715 \
716 if (UPB_LIKELY(size <= 15 - tagbytes)) { \
717 if (arena_has < 16) goto longstr; \
718 d->arena.head.ptr += 16; \
719 memcpy(buf, ptr - tagbytes - 1, 16); \
720 dst->data = buf + tagbytes + 1; \
721 } else if (UPB_LIKELY(size <= 32)) { \
722 if (UPB_UNLIKELY(common_has < 32)) goto longstr; \
723 fastdecode_docopy(d, ptr, size, 32, buf, dst); \
724 } else if (UPB_LIKELY(size <= 64)) { \
725 if (UPB_UNLIKELY(common_has < 64)) goto longstr; \
726 fastdecode_docopy(d, ptr, size, 64, buf, dst); \
727 } else if (UPB_LIKELY(size < 128)) { \
728 if (UPB_UNLIKELY(common_has < 128)) goto longstr; \
729 fastdecode_docopy(d, ptr, size, 128, buf, dst); \
730 } else { \
731 goto longstr; \
732 } \
733 \
734 ptr += size; \
735 \
736 if (card == CARD_r) { \
737 if (validate_utf8 && \
738 !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \
739 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \
740 } \
741 fastdecode_nextret ret = fastdecode_nextrepeated( \
742 d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \
743 switch (ret.next) { \
744 case FD_NEXT_SAMEFIELD: \
745 dst = ret.dst; \
746 goto again; \
747 case FD_NEXT_OTHERFIELD: \
748 data = ret.tag; \
749 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
750 case FD_NEXT_ATLIMIT: \
751 return ptr; \
752 } \
753 } \
754 \
755 if (card != CARD_r && validate_utf8) { \
756 data = (uint64_t)dst; \
757 UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
758 } \
759 \
760 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS); \
761 \
762 longstr: \
763 if (card == CARD_r) { \
764 fastdecode_commitarr(dst + 1, &farr, sizeof(upb_StringView)); \
765 } \
766 ptr--; \
767 if (validate_utf8) { \
768 UPB_MUSTTAIL return fastdecode_longstring_utf8(d, ptr, msg, table, \
769 hasbits, (uint64_t)dst); \
770 } else { \
771 UPB_MUSTTAIL return fastdecode_longstring_noutf8(d, ptr, msg, table, \
772 hasbits, (uint64_t)dst); \
773 }
774
775 #define FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, card, \
776 copyfunc, validate_utf8) \
777 upb_StringView* dst; \
778 fastdecode_arr farr; \
779 int64_t size; \
780 \
781 if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
782 RETURN_GENERIC("string field tag mismatch\n"); \
783 } \
784 \
785 if (UPB_UNLIKELY( \
786 !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, 0))) { \
787 UPB_MUSTTAIL return copyfunc(UPB_PARSE_ARGS); \
788 } \
789 \
790 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
791 sizeof(upb_StringView), card); \
792 \
793 again: \
794 if (card == CARD_r) { \
795 dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_StringView)); \
796 } \
797 \
798 size = (int8_t)ptr[tagbytes]; \
799 ptr += tagbytes + 1; \
800 \
801 if (UPB_UNLIKELY( \
802 !upb_EpsCopyInputStream_AliasingAvailable(&d->input, ptr, size))) { \
803 ptr--; \
804 if (validate_utf8) { \
805 return fastdecode_longstring_utf8(d, ptr, msg, table, hasbits, \
806 (uint64_t)dst); \
807 } else { \
808 return fastdecode_longstring_noutf8(d, ptr, msg, table, hasbits, \
809 (uint64_t)dst); \
810 } \
811 } \
812 \
813 dst->data = ptr; \
814 dst->size = size; \
815 ptr = upb_EpsCopyInputStream_ReadStringAliased(&d->input, &dst->data, \
816 dst->size); \
817 \
818 if (card == CARD_r) { \
819 if (validate_utf8 && \
820 !_upb_Decoder_VerifyUtf8Inline(dst->data, dst->size)) { \
821 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_BadUtf8); \
822 } \
823 fastdecode_nextret ret = fastdecode_nextrepeated( \
824 d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_StringView)); \
825 switch (ret.next) { \
826 case FD_NEXT_SAMEFIELD: \
827 dst = ret.dst; \
828 goto again; \
829 case FD_NEXT_OTHERFIELD: \
830 data = ret.tag; \
831 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
832 case FD_NEXT_ATLIMIT: \
833 return ptr; \
834 } \
835 } \
836 \
837 if (card != CARD_r && validate_utf8) { \
838 data = (uint64_t)dst; \
839 UPB_MUSTTAIL return fastdecode_verifyutf8(UPB_PARSE_ARGS); \
840 } \
841 \
842 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
843
844 /* Generate all combinations:
845 * {p,c} x {s,o,r} x {s, b} x {1bt,2bt} */
846
847 #define s_VALIDATE true
848 #define b_VALIDATE false
849
850 #define F(card, tagbytes, type) \
851 UPB_NOINLINE \
852 const char* upb_c##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
853 FASTDECODE_COPYSTRING(d, ptr, msg, table, hasbits, data, tagbytes, \
854 CARD_##card, type##_VALIDATE); \
855 } \
856 const char* upb_p##card##type##_##tagbytes##bt(UPB_PARSE_PARAMS) { \
857 FASTDECODE_STRING(d, ptr, msg, table, hasbits, data, tagbytes, \
858 CARD_##card, upb_c##card##type##_##tagbytes##bt, \
859 type##_VALIDATE); \
860 }
861
862 #define UTF8(card, tagbytes) \
863 F(card, tagbytes, s) \
864 F(card, tagbytes, b)
865
866 #define TAGBYTES(card) \
867 UTF8(card, 1) \
868 UTF8(card, 2)
869
870 TAGBYTES(s)
TAGBYTES(o)871 TAGBYTES(o)
872 TAGBYTES(r)
873
874 #undef s_VALIDATE
875 #undef b_VALIDATE
876 #undef F
877 #undef TAGBYTES
878 #undef FASTDECODE_LONGSTRING
879 #undef FASTDECODE_COPYSTRING
880 #undef FASTDECODE_STRING
881
882 /* message fields *************************************************************/
883
884 UPB_INLINE
885 upb_Message* decode_newmsg_ceil(upb_Decoder* d, const upb_MiniTable* l,
886 int msg_ceil_bytes) {
887 size_t size = l->size + sizeof(upb_Message_Internal);
888 char* msg_data;
889 if (UPB_LIKELY(msg_ceil_bytes > 0 &&
890 _upb_ArenaHas(&d->arena) >= msg_ceil_bytes)) {
891 UPB_ASSERT(size <= (size_t)msg_ceil_bytes);
892 msg_data = d->arena.head.ptr;
893 d->arena.head.ptr += size;
894 UPB_UNPOISON_MEMORY_REGION(msg_data, msg_ceil_bytes);
895 memset(msg_data, 0, msg_ceil_bytes);
896 UPB_POISON_MEMORY_REGION(msg_data + size, msg_ceil_bytes - size);
897 } else {
898 msg_data = (char*)upb_Arena_Malloc(&d->arena, size);
899 memset(msg_data, 0, size);
900 }
901 return msg_data + sizeof(upb_Message_Internal);
902 }
903
904 typedef struct {
905 intptr_t table;
906 upb_Message* msg;
907 } fastdecode_submsgdata;
908
909 UPB_FORCEINLINE
fastdecode_tosubmsg(upb_EpsCopyInputStream * e,const char * ptr,void * ctx)910 static const char* fastdecode_tosubmsg(upb_EpsCopyInputStream* e,
911 const char* ptr, void* ctx) {
912 upb_Decoder* d = (upb_Decoder*)e;
913 fastdecode_submsgdata* submsg = ctx;
914 ptr = fastdecode_dispatch(d, ptr, submsg->msg, submsg->table, 0, 0);
915 UPB_ASSUME(ptr != NULL);
916 return ptr;
917 }
918
919 #define FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, \
920 msg_ceil_bytes, card) \
921 \
922 if (UPB_UNLIKELY(!fastdecode_checktag(data, tagbytes))) { \
923 RETURN_GENERIC("submessage field tag mismatch\n"); \
924 } \
925 \
926 if (--d->depth == 0) { \
927 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_MaxDepthExceeded); \
928 } \
929 \
930 upb_Message** dst; \
931 uint32_t submsg_idx = (data >> 16) & 0xff; \
932 const upb_MiniTable* tablep = decode_totablep(table); \
933 const upb_MiniTable* subtablep = tablep->subs[submsg_idx].submsg; \
934 fastdecode_submsgdata submsg = {decode_totable(subtablep)}; \
935 fastdecode_arr farr; \
936 \
937 if (subtablep->table_mask == (uint8_t)-1) { \
938 RETURN_GENERIC("submessage doesn't have fast tables."); \
939 } \
940 \
941 dst = fastdecode_getfield(d, ptr, msg, &data, &hasbits, &farr, \
942 sizeof(upb_Message*), card); \
943 \
944 if (card == CARD_s) { \
945 *(uint32_t*)msg |= hasbits; \
946 hasbits = 0; \
947 } \
948 \
949 again: \
950 if (card == CARD_r) { \
951 dst = fastdecode_resizearr(d, dst, &farr, sizeof(upb_Message*)); \
952 } \
953 \
954 submsg.msg = *dst; \
955 \
956 if (card == CARD_r || UPB_LIKELY(!submsg.msg)) { \
957 *dst = submsg.msg = decode_newmsg_ceil(d, subtablep, msg_ceil_bytes); \
958 } \
959 \
960 ptr += tagbytes; \
961 ptr = fastdecode_delimited(d, ptr, fastdecode_tosubmsg, &submsg); \
962 \
963 if (UPB_UNLIKELY(ptr == NULL || d->end_group != DECODE_NOGROUP)) { \
964 _upb_FastDecoder_ErrorJmp(d, kUpb_DecodeStatus_Malformed); \
965 } \
966 \
967 if (card == CARD_r) { \
968 fastdecode_nextret ret = fastdecode_nextrepeated( \
969 d, dst, &ptr, &farr, data, tagbytes, sizeof(upb_Message*)); \
970 switch (ret.next) { \
971 case FD_NEXT_SAMEFIELD: \
972 dst = ret.dst; \
973 goto again; \
974 case FD_NEXT_OTHERFIELD: \
975 d->depth++; \
976 data = ret.tag; \
977 UPB_MUSTTAIL return _upb_FastDecoder_TagDispatch(UPB_PARSE_ARGS); \
978 case FD_NEXT_ATLIMIT: \
979 d->depth++; \
980 return ptr; \
981 } \
982 } \
983 \
984 d->depth++; \
985 UPB_MUSTTAIL return fastdecode_dispatch(UPB_PARSE_ARGS);
986
987 #define F(card, tagbytes, size_ceil, ceil_arg) \
988 const char* upb_p##card##m_##tagbytes##bt_max##size_ceil##b( \
989 UPB_PARSE_PARAMS) { \
990 FASTDECODE_SUBMSG(d, ptr, msg, table, hasbits, data, tagbytes, ceil_arg, \
991 CARD_##card); \
992 }
993
994 #define SIZES(card, tagbytes) \
995 F(card, tagbytes, 64, 64) \
996 F(card, tagbytes, 128, 128) \
997 F(card, tagbytes, 192, 192) \
998 F(card, tagbytes, 256, 256) \
999 F(card, tagbytes, max, -1)
1000
1001 #define TAGBYTES(card) \
1002 SIZES(card, 1) \
1003 SIZES(card, 2)
1004
1005 TAGBYTES(s)
1006 TAGBYTES(o)
1007 TAGBYTES(r)
1008
1009 #undef TAGBYTES
1010 #undef SIZES
1011 #undef F
1012 #undef FASTDECODE_SUBMSG
1013
1014 #endif /* UPB_FASTTABLE */
1015