1 #include "Python.h"
2 #include "pycore_code.h"
3 #include "pycore_dict.h"
4 #include "pycore_function.h" // _PyFunction_GetVersionForCurrentState()
5 #include "pycore_global_strings.h" // _Py_ID()
6 #include "pycore_long.h"
7 #include "pycore_moduleobject.h"
8 #include "pycore_object.h"
9 #include "pycore_opcode.h" // _PyOpcode_Caches
10 #include "structmember.h" // struct PyMemberDef, T_OFFSET_EX
11
12 #include <stdlib.h> // rand()
13
14 /* For guidance on adding or extending families of instructions see
15 * ./adaptive.md
16 */
17
18 /* Map from opcode to adaptive opcode.
19 Values of zero are ignored. */
20 uint8_t _PyOpcode_Adaptive[256] = {
21 [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE,
22 [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
23 [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
24 [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
25 [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
26 [CALL] = CALL_ADAPTIVE,
27 [PRECALL] = PRECALL_ADAPTIVE,
28 [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
29 [BINARY_OP] = BINARY_OP_ADAPTIVE,
30 [COMPARE_OP] = COMPARE_OP_ADAPTIVE,
31 [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE,
32 };
33
34 Py_ssize_t _Py_QuickenedCount = 0;
35 #ifdef Py_STATS
36 PyStats _py_stats = { 0 };
37
38 #define ADD_STAT_TO_DICT(res, field) \
39 do { \
40 PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \
41 if (val == NULL) { \
42 Py_DECREF(res); \
43 return NULL; \
44 } \
45 if (PyDict_SetItemString(res, #field, val) == -1) { \
46 Py_DECREF(res); \
47 Py_DECREF(val); \
48 return NULL; \
49 } \
50 Py_DECREF(val); \
51 } while(0);
52
53 static PyObject*
stats_to_dict(SpecializationStats * stats)54 stats_to_dict(SpecializationStats *stats)
55 {
56 PyObject *res = PyDict_New();
57 if (res == NULL) {
58 return NULL;
59 }
60 ADD_STAT_TO_DICT(res, success);
61 ADD_STAT_TO_DICT(res, failure);
62 ADD_STAT_TO_DICT(res, hit);
63 ADD_STAT_TO_DICT(res, deferred);
64 ADD_STAT_TO_DICT(res, miss);
65 ADD_STAT_TO_DICT(res, deopt);
66 PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);
67 if (failure_kinds == NULL) {
68 Py_DECREF(res);
69 return NULL;
70 }
71 for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {
72 PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]);
73 if (stat == NULL) {
74 Py_DECREF(res);
75 Py_DECREF(failure_kinds);
76 return NULL;
77 }
78 PyTuple_SET_ITEM(failure_kinds, i, stat);
79 }
80 if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) {
81 Py_DECREF(res);
82 Py_DECREF(failure_kinds);
83 return NULL;
84 }
85 Py_DECREF(failure_kinds);
86 return res;
87 }
88 #undef ADD_STAT_TO_DICT
89
90 static int
add_stat_dict(PyObject * res,int opcode,const char * name)91 add_stat_dict(
92 PyObject *res,
93 int opcode,
94 const char *name) {
95
96 SpecializationStats *stats = &_py_stats.opcode_stats[opcode].specialization;
97 PyObject *d = stats_to_dict(stats);
98 if (d == NULL) {
99 return -1;
100 }
101 int err = PyDict_SetItemString(res, name, d);
102 Py_DECREF(d);
103 return err;
104 }
105
106 #ifdef Py_STATS
107 PyObject*
_Py_GetSpecializationStats(void)108 _Py_GetSpecializationStats(void) {
109 PyObject *stats = PyDict_New();
110 if (stats == NULL) {
111 return NULL;
112 }
113 int err = 0;
114 err += add_stat_dict(stats, LOAD_ATTR, "load_attr");
115 err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
116 err += add_stat_dict(stats, LOAD_METHOD, "load_method");
117 err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
118 err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
119 err += add_stat_dict(stats, STORE_ATTR, "store_attr");
120 err += add_stat_dict(stats, CALL, "call");
121 err += add_stat_dict(stats, BINARY_OP, "binary_op");
122 err += add_stat_dict(stats, COMPARE_OP, "compare_op");
123 err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");
124 err += add_stat_dict(stats, PRECALL, "precall");
125 if (err < 0) {
126 Py_DECREF(stats);
127 return NULL;
128 }
129 return stats;
130 }
131 #endif
132
133
134 #define PRINT_STAT(i, field) \
135 if (stats[i].field) { \
136 fprintf(out, " opcode[%d]." #field " : %" PRIu64 "\n", i, stats[i].field); \
137 }
138
139 static void
print_spec_stats(FILE * out,OpcodeStats * stats)140 print_spec_stats(FILE *out, OpcodeStats *stats)
141 {
142 /* Mark some opcodes as specializable for stats,
143 * even though we don't specialize them yet. */
144 fprintf(out, "opcode[%d].specializable : 1\n", FOR_ITER);
145 for (int i = 0; i < 256; i++) {
146 if (_PyOpcode_Adaptive[i]) {
147 fprintf(out, "opcode[%d].specializable : 1\n", i);
148 }
149 PRINT_STAT(i, specialization.success);
150 PRINT_STAT(i, specialization.failure);
151 PRINT_STAT(i, specialization.hit);
152 PRINT_STAT(i, specialization.deferred);
153 PRINT_STAT(i, specialization.miss);
154 PRINT_STAT(i, specialization.deopt);
155 PRINT_STAT(i, execution_count);
156 for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) {
157 uint64_t val = stats[i].specialization.failure_kinds[j];
158 if (val) {
159 fprintf(out, " opcode[%d].specialization.failure_kinds[%d] : %"
160 PRIu64 "\n", i, j, val);
161 }
162 }
163 for(int j = 0; j < 256; j++) {
164 if (stats[i].pair_count[j]) {
165 fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n",
166 i, j, stats[i].pair_count[j]);
167 }
168 }
169 }
170 }
171 #undef PRINT_STAT
172
173
174 static void
print_call_stats(FILE * out,CallStats * stats)175 print_call_stats(FILE *out, CallStats *stats)
176 {
177 fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls);
178 fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
179 fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);
180 fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);
181 }
182
183 static void
print_object_stats(FILE * out,ObjectStats * stats)184 print_object_stats(FILE *out, ObjectStats *stats)
185 {
186 fprintf(out, "Object allocations from freelist: %" PRIu64 "\n", stats->from_freelist);
187 fprintf(out, "Object frees to freelist: %" PRIu64 "\n", stats->to_freelist);
188 fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);
189 fprintf(out, "Object allocations to 512 bytes: %" PRIu64 "\n", stats->allocations512);
190 fprintf(out, "Object allocations to 4 kbytes: %" PRIu64 "\n", stats->allocations4k);
191 fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
192 fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
193 fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
194 fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
195 fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
196 fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
197 fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
198 }
199
200 static void
print_stats(FILE * out,PyStats * stats)201 print_stats(FILE *out, PyStats *stats) {
202 print_spec_stats(out, stats->opcode_stats);
203 print_call_stats(out, &stats->call_stats);
204 print_object_stats(out, &stats->object_stats);
205 }
206
207 void
_Py_PrintSpecializationStats(int to_file)208 _Py_PrintSpecializationStats(int to_file)
209 {
210 FILE *out = stderr;
211 if (to_file) {
212 /* Write to a file instead of stderr. */
213 # ifdef MS_WINDOWS
214 const char *dirname = "c:\\temp\\py_stats\\";
215 # else
216 const char *dirname = "/tmp/py_stats/";
217 # endif
218 /* Use random 160 bit number as file name,
219 * to avoid both accidental collisions and
220 * symlink attacks. */
221 unsigned char rand[20];
222 char hex_name[41];
223 _PyOS_URandomNonblock(rand, 20);
224 for (int i = 0; i < 20; i++) {
225 hex_name[2*i] = "0123456789abcdef"[rand[i]&15];
226 hex_name[2*i+1] = "0123456789abcdef"[(rand[i]>>4)&15];
227 }
228 hex_name[40] = '\0';
229 char buf[64];
230 assert(strlen(dirname) + 40 + strlen(".txt") < 64);
231 sprintf(buf, "%s%s.txt", dirname, hex_name);
232 FILE *fout = fopen(buf, "w");
233 if (fout) {
234 out = fout;
235 }
236 }
237 else {
238 fprintf(out, "Specialization stats:\n");
239 }
240 print_stats(out, &_py_stats);
241 if (out != stderr) {
242 fclose(out);
243 }
244 }
245
246 #ifdef Py_STATS
247
248 #define SPECIALIZATION_FAIL(opcode, kind) _py_stats.opcode_stats[opcode].specialization.failure_kinds[kind]++
249
250
251 #endif
252 #endif
253
254 #ifndef SPECIALIZATION_FAIL
255 #define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
256 #endif
257
258 // Insert adaptive instructions and superinstructions. This cannot fail.
259 void
_PyCode_Quicken(PyCodeObject * code)260 _PyCode_Quicken(PyCodeObject *code)
261 {
262 _Py_QuickenedCount++;
263 int previous_opcode = -1;
264 _Py_CODEUNIT *instructions = _PyCode_CODE(code);
265 for (int i = 0; i < Py_SIZE(code); i++) {
266 int opcode = _Py_OPCODE(instructions[i]);
267 uint8_t adaptive_opcode = _PyOpcode_Adaptive[opcode];
268 if (adaptive_opcode) {
269 _Py_SET_OPCODE(instructions[i], adaptive_opcode);
270 // Make sure the adaptive counter is zero:
271 assert(instructions[i + 1] == 0);
272 previous_opcode = -1;
273 i += _PyOpcode_Caches[opcode];
274 }
275 else {
276 assert(!_PyOpcode_Caches[opcode]);
277 switch (opcode) {
278 case EXTENDED_ARG:
279 _Py_SET_OPCODE(instructions[i], EXTENDED_ARG_QUICK);
280 break;
281 case JUMP_BACKWARD:
282 _Py_SET_OPCODE(instructions[i], JUMP_BACKWARD_QUICK);
283 break;
284 case RESUME:
285 _Py_SET_OPCODE(instructions[i], RESUME_QUICK);
286 break;
287 case LOAD_FAST:
288 switch(previous_opcode) {
289 case LOAD_FAST:
290 _Py_SET_OPCODE(instructions[i - 1],
291 LOAD_FAST__LOAD_FAST);
292 break;
293 case STORE_FAST:
294 _Py_SET_OPCODE(instructions[i - 1],
295 STORE_FAST__LOAD_FAST);
296 break;
297 case LOAD_CONST:
298 _Py_SET_OPCODE(instructions[i - 1],
299 LOAD_CONST__LOAD_FAST);
300 break;
301 }
302 break;
303 case STORE_FAST:
304 if (previous_opcode == STORE_FAST) {
305 _Py_SET_OPCODE(instructions[i - 1],
306 STORE_FAST__STORE_FAST);
307 }
308 break;
309 case LOAD_CONST:
310 if (previous_opcode == LOAD_FAST) {
311 _Py_SET_OPCODE(instructions[i - 1],
312 LOAD_FAST__LOAD_CONST);
313 }
314 break;
315 }
316 previous_opcode = opcode;
317 }
318 }
319 }
320
321 static inline int
miss_counter_start(void)322 miss_counter_start(void) {
323 /* Starting value for the counter.
324 * This value needs to be not too low, otherwise
325 * it would cause excessive de-optimization.
326 * Neither should it be too high, or that would delay
327 * de-optimization excessively when it is needed.
328 * A value around 50 seems to work, and we choose a
329 * prime number to avoid artifacts.
330 */
331 return 53;
332 }
333
334 /* Common */
335
336 #define SPEC_FAIL_OTHER 0
337 #define SPEC_FAIL_NO_DICT 1
338 #define SPEC_FAIL_OVERRIDDEN 2
339 #define SPEC_FAIL_OUT_OF_VERSIONS 3
340 #define SPEC_FAIL_OUT_OF_RANGE 4
341 #define SPEC_FAIL_EXPECTED_ERROR 5
342 #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6
343
344 #define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18
345
346 /* Attributes */
347
348 #define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8
349 #define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9
350 #define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10
351 #define SPEC_FAIL_ATTR_METHOD 11
352 #define SPEC_FAIL_ATTR_MUTABLE_CLASS 12
353 #define SPEC_FAIL_ATTR_PROPERTY 13
354 #define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14
355 #define SPEC_FAIL_ATTR_READ_ONLY 15
356 #define SPEC_FAIL_ATTR_AUDITED_SLOT 16
357 #define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17
358 #define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18
359 #define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19
360
361 /* Methods */
362
363 #define SPEC_FAIL_LOAD_METHOD_OVERRIDING_DESCRIPTOR 8
364 #define SPEC_FAIL_LOAD_METHOD_NON_OVERRIDING_DESCRIPTOR 9
365 #define SPEC_FAIL_LOAD_METHOD_NOT_DESCRIPTOR 10
366 #define SPEC_FAIL_LOAD_METHOD_METHOD 11
367 #define SPEC_FAIL_LOAD_METHOD_MUTABLE_CLASS 12
368 #define SPEC_FAIL_LOAD_METHOD_PROPERTY 13
369 #define SPEC_FAIL_LOAD_METHOD_NON_OBJECT_SLOT 14
370 #define SPEC_FAIL_LOAD_METHOD_IS_ATTR 15
371 #define SPEC_FAIL_LOAD_METHOD_DICT_SUBCLASS 16
372 #define SPEC_FAIL_LOAD_METHOD_BUILTIN_CLASS_METHOD 17
373 #define SPEC_FAIL_LOAD_METHOD_CLASS_METHOD_OBJ 18
374 #define SPEC_FAIL_LOAD_METHOD_OBJECT_SLOT 19
375 #define SPEC_FAIL_LOAD_METHOD_HAS_DICT 20
376 #define SPEC_FAIL_LOAD_METHOD_HAS_MANAGED_DICT 21
377 #define SPEC_FAIL_LOAD_METHOD_INSTANCE_ATTRIBUTE 22
378 #define SPEC_FAIL_LOAD_METHOD_METACLASS_ATTRIBUTE 23
379
380 /* Binary subscr and store subscr */
381
382 #define SPEC_FAIL_SUBSCR_ARRAY_INT 8
383 #define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9
384 #define SPEC_FAIL_SUBSCR_LIST_SLICE 10
385 #define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11
386 #define SPEC_FAIL_SUBSCR_STRING_INT 12
387 #define SPEC_FAIL_SUBSCR_STRING_SLICE 13
388 #define SPEC_FAIL_SUBSCR_BUFFER_INT 15
389 #define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16
390 #define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17
391
392 /* Store subscr */
393 #define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18
394 #define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19
395 #define SPEC_FAIL_SUBSCR_PY_SIMPLE 20
396 #define SPEC_FAIL_SUBSCR_PY_OTHER 21
397 #define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22
398 #define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23
399
400 /* Binary op */
401
402 #define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES 8
403 #define SPEC_FAIL_BINARY_OP_ADD_OTHER 9
404 #define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES 10
405 #define SPEC_FAIL_BINARY_OP_AND_INT 11
406 #define SPEC_FAIL_BINARY_OP_AND_OTHER 12
407 #define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE 13
408 #define SPEC_FAIL_BINARY_OP_LSHIFT 14
409 #define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY 15
410 #define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES 16
411 #define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER 17
412 #define SPEC_FAIL_BINARY_OP_OR 18
413 #define SPEC_FAIL_BINARY_OP_POWER 19
414 #define SPEC_FAIL_BINARY_OP_REMAINDER 20
415 #define SPEC_FAIL_BINARY_OP_RSHIFT 21
416 #define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES 22
417 #define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER 23
418 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24
419 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT 25
420 #define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER 26
421 #define SPEC_FAIL_BINARY_OP_XOR 27
422
423 /* Calls */
424 #define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9
425 #define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10
426 /* SPEC_FAIL_METHOD defined as 11 above */
427
428 #define SPEC_FAIL_CALL_INSTANCE_METHOD 11
429 #define SPEC_FAIL_CALL_CMETHOD 12
430 #define SPEC_FAIL_CALL_PYCFUNCTION 13
431 #define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14
432 #define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15
433 #define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16
434 #define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17
435 #define SPEC_FAIL_CALL_CLASS 18
436 #define SPEC_FAIL_CALL_PYTHON_CLASS 19
437 #define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20
438 #define SPEC_FAIL_CALL_BOUND_METHOD 21
439 #define SPEC_FAIL_CALL_STR 22
440 #define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23
441 #define SPEC_FAIL_CALL_CLASS_MUTABLE 24
442 #define SPEC_FAIL_CALL_KWNAMES 25
443 #define SPEC_FAIL_CALL_METHOD_WRAPPER 26
444 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27
445 #define SPEC_FAIL_CALL_PYFUNCTION 28
446 #define SPEC_FAIL_CALL_PEP_523 29
447
448 /* COMPARE_OP */
449 #define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
450 #define SPEC_FAIL_COMPARE_OP_STRING 13
451 #define SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP 14
452 #define SPEC_FAIL_COMPARE_OP_BIG_INT 15
453 #define SPEC_FAIL_COMPARE_OP_BYTES 16
454 #define SPEC_FAIL_COMPARE_OP_TUPLE 17
455 #define SPEC_FAIL_COMPARE_OP_LIST 18
456 #define SPEC_FAIL_COMPARE_OP_SET 19
457 #define SPEC_FAIL_COMPARE_OP_BOOL 20
458 #define SPEC_FAIL_COMPARE_OP_BASEOBJECT 21
459 #define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 22
460 #define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 23
461 #define SPEC_FAIL_COMPARE_OP_EXTENDED_ARG 24
462
463 /* FOR_ITER */
464 #define SPEC_FAIL_FOR_ITER_GENERATOR 10
465 #define SPEC_FAIL_FOR_ITER_COROUTINE 11
466 #define SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR 12
467 #define SPEC_FAIL_FOR_ITER_LIST 13
468 #define SPEC_FAIL_FOR_ITER_TUPLE 14
469 #define SPEC_FAIL_FOR_ITER_SET 15
470 #define SPEC_FAIL_FOR_ITER_STRING 16
471 #define SPEC_FAIL_FOR_ITER_BYTES 17
472 #define SPEC_FAIL_FOR_ITER_RANGE 18
473 #define SPEC_FAIL_FOR_ITER_ITERTOOLS 19
474 #define SPEC_FAIL_FOR_ITER_DICT_KEYS 20
475 #define SPEC_FAIL_FOR_ITER_DICT_ITEMS 21
476 #define SPEC_FAIL_FOR_ITER_DICT_VALUES 22
477 #define SPEC_FAIL_FOR_ITER_ENUMERATE 23
478
479 // UNPACK_SEQUENCE
480
481 #define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8
482 #define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9
483
484
485 static int
specialize_module_load_attr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name,int opcode,int opcode_module)486 specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
487 PyObject *name, int opcode, int opcode_module)
488 {
489 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
490 PyModuleObject *m = (PyModuleObject *)owner;
491 PyObject *value = NULL;
492 assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
493 PyDictObject *dict = (PyDictObject *)m->md_dict;
494 if (dict == NULL) {
495 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT);
496 return -1;
497 }
498 if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
499 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
500 return -1;
501 }
502 Py_ssize_t index = _PyDict_GetItemHint(dict, &_Py_ID(__getattr__), -1,
503 &value);
504 assert(index != DKIX_ERROR);
505 if (index != DKIX_EMPTY) {
506 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);
507 return -1;
508 }
509 index = _PyDict_GetItemHint(dict, name, -1, &value);
510 assert (index != DKIX_ERROR);
511 if (index != (uint16_t)index) {
512 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE);
513 return -1;
514 }
515 uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys);
516 if (keys_version == 0) {
517 SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
518 return -1;
519 }
520 write_u32(cache->version, keys_version);
521 cache->index = (uint16_t)index;
522 _Py_SET_OPCODE(*instr, opcode_module);
523 return 0;
524 }
525
526
527
528 /* Attribute specialization */
529
530 typedef enum {
531 OVERRIDING, /* Is an overriding descriptor, and will remain so. */
532 METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
533 PROPERTY, /* Is a property */
534 OBJECT_SLOT, /* Is an object slot descriptor */
535 OTHER_SLOT, /* Is a slot descriptor of another type */
536 NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/
537 BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */
538 PYTHON_CLASSMETHOD, /* Python classmethod(func) object */
539 NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */
540 MUTABLE, /* Instance of a mutable class; might, or might not, be a descriptor */
541 ABSENT, /* Attribute is not present on the class */
542 DUNDER_CLASS, /* __class__ attribute */
543 GETSET_OVERRIDDEN /* __getattribute__ or __setattr__ has been overridden */
544 } DescriptorClassification;
545
546
547 static DescriptorClassification
analyze_descriptor(PyTypeObject * type,PyObject * name,PyObject ** descr,int store)548 analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
549 {
550 if (store) {
551 if (type->tp_setattro != PyObject_GenericSetAttr) {
552 *descr = NULL;
553 return GETSET_OVERRIDDEN;
554 }
555 }
556 else {
557 if (type->tp_getattro != PyObject_GenericGetAttr) {
558 *descr = NULL;
559 return GETSET_OVERRIDDEN;
560 }
561 }
562 PyObject *descriptor = _PyType_Lookup(type, name);
563 *descr = descriptor;
564 if (descriptor == NULL) {
565 return ABSENT;
566 }
567 PyTypeObject *desc_cls = Py_TYPE(descriptor);
568 if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {
569 return MUTABLE;
570 }
571 if (desc_cls->tp_descr_set) {
572 if (desc_cls == &PyMemberDescr_Type) {
573 PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;
574 struct PyMemberDef *dmem = member->d_member;
575 if (dmem->type == T_OBJECT_EX) {
576 return OBJECT_SLOT;
577 }
578 return OTHER_SLOT;
579 }
580 if (desc_cls == &PyProperty_Type) {
581 return PROPERTY;
582 }
583 if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
584 if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
585 return DUNDER_CLASS;
586 }
587 }
588 return OVERRIDING;
589 }
590 if (desc_cls->tp_descr_get) {
591 if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {
592 return METHOD;
593 }
594 if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {
595 return BUILTIN_CLASSMETHOD;
596 }
597 if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {
598 return PYTHON_CLASSMETHOD;
599 }
600 return NON_OVERRIDING;
601 }
602 return NON_DESCRIPTOR;
603 }
604
605 static int
specialize_dict_access(PyObject * owner,_Py_CODEUNIT * instr,PyTypeObject * type,DescriptorClassification kind,PyObject * name,int base_op,int values_op,int hint_op)606 specialize_dict_access(
607 PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
608 DescriptorClassification kind, PyObject *name,
609 int base_op, int values_op, int hint_op)
610 {
611 assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
612 kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD);
613 // No descriptor, or non overriding.
614 if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
615 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
616 return 0;
617 }
618 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
619 PyObject **dictptr = _PyObject_ManagedDictPointer(owner);
620 PyDictObject *dict = (PyDictObject *)*dictptr;
621 if (dict == NULL) {
622 // Virtual dictionary
623 PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
624 assert(PyUnicode_CheckExact(name));
625 Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
626 assert (index != DKIX_ERROR);
627 if (index != (uint16_t)index) {
628 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
629 return 0;
630 }
631 write_u32(cache->version, type->tp_version_tag);
632 cache->index = (uint16_t)index;
633 _Py_SET_OPCODE(*instr, values_op);
634 }
635 else {
636 if (!PyDict_CheckExact(dict)) {
637 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
638 return 0;
639 }
640 // We found an instance with a __dict__.
641 PyObject *value = NULL;
642 Py_ssize_t hint =
643 _PyDict_GetItemHint(dict, name, -1, &value);
644 if (hint != (uint16_t)hint) {
645 SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
646 return 0;
647 }
648 cache->index = (uint16_t)hint;
649 write_u32(cache->version, type->tp_version_tag);
650 _Py_SET_OPCODE(*instr, hint_op);
651 }
652 return 1;
653 }
654
655 int
_Py_Specialize_LoadAttr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)656 _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
657 {
658 assert(_PyOpcode_Caches[LOAD_ATTR] == INLINE_CACHE_ENTRIES_LOAD_ATTR);
659 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
660 if (PyModule_CheckExact(owner)) {
661 int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
662 LOAD_ATTR_MODULE);
663 if (err) {
664 goto fail;
665 }
666 goto success;
667 }
668 PyTypeObject *type = Py_TYPE(owner);
669 if (type->tp_dict == NULL) {
670 if (PyType_Ready(type) < 0) {
671 return -1;
672 }
673 }
674 PyObject *descr;
675 DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
676 switch(kind) {
677 case OVERRIDING:
678 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
679 goto fail;
680 case METHOD:
681 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
682 goto fail;
683 case PROPERTY:
684 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY);
685 goto fail;
686 case OBJECT_SLOT:
687 {
688 PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
689 struct PyMemberDef *dmem = member->d_member;
690 Py_ssize_t offset = dmem->offset;
691 if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {
692 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_EXPECTED_ERROR);
693 goto fail;
694 }
695 if (dmem->flags & PY_AUDIT_READ) {
696 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);
697 goto fail;
698 }
699 if (offset != (uint16_t)offset) {
700 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
701 goto fail;
702 }
703 assert(dmem->type == T_OBJECT_EX);
704 assert(offset > 0);
705 cache->index = (uint16_t)offset;
706 write_u32(cache->version, type->tp_version_tag);
707 _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
708 goto success;
709 }
710 case DUNDER_CLASS:
711 {
712 Py_ssize_t offset = offsetof(PyObject, ob_type);
713 assert(offset == (uint16_t)offset);
714 cache->index = (uint16_t)offset;
715 write_u32(cache->version, type->tp_version_tag);
716 _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
717 goto success;
718 }
719 case OTHER_SLOT:
720 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
721 goto fail;
722 case MUTABLE:
723 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
724 goto fail;
725 case GETSET_OVERRIDDEN:
726 SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
727 goto fail;
728 case BUILTIN_CLASSMETHOD:
729 case PYTHON_CLASSMETHOD:
730 case NON_OVERRIDING:
731 case NON_DESCRIPTOR:
732 case ABSENT:
733 break;
734 }
735 int err = specialize_dict_access(
736 owner, instr, type, kind, name,
737 LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT
738 );
739 if (err < 0) {
740 return -1;
741 }
742 if (err) {
743 goto success;
744 }
745 fail:
746 STAT_INC(LOAD_ATTR, failure);
747 assert(!PyErr_Occurred());
748 cache->counter = adaptive_counter_backoff(cache->counter);
749 return 0;
750 success:
751 STAT_INC(LOAD_ATTR, success);
752 assert(!PyErr_Occurred());
753 cache->counter = miss_counter_start();
754 return 0;
755 }
756
757 int
_Py_Specialize_StoreAttr(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)758 _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
759 {
760 assert(_PyOpcode_Caches[STORE_ATTR] == INLINE_CACHE_ENTRIES_STORE_ATTR);
761 _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
762 PyTypeObject *type = Py_TYPE(owner);
763 if (PyModule_CheckExact(owner)) {
764 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
765 goto fail;
766 }
767 PyObject *descr;
768 DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
769 switch(kind) {
770 case OVERRIDING:
771 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
772 goto fail;
773 case METHOD:
774 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD);
775 goto fail;
776 case PROPERTY:
777 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY);
778 goto fail;
779 case OBJECT_SLOT:
780 {
781 PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
782 struct PyMemberDef *dmem = member->d_member;
783 Py_ssize_t offset = dmem->offset;
784 if (!PyObject_TypeCheck(owner, member->d_common.d_type)) {
785 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_EXPECTED_ERROR);
786 goto fail;
787 }
788 if (dmem->flags & READONLY) {
789 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY);
790 goto fail;
791 }
792 if (offset != (uint16_t)offset) {
793 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE);
794 goto fail;
795 }
796 assert(dmem->type == T_OBJECT_EX);
797 assert(offset > 0);
798 cache->index = (uint16_t)offset;
799 write_u32(cache->version, type->tp_version_tag);
800 _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT);
801 goto success;
802 }
803 case DUNDER_CLASS:
804 case OTHER_SLOT:
805 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
806 goto fail;
807 case MUTABLE:
808 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
809 goto fail;
810 case GETSET_OVERRIDDEN:
811 SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
812 goto fail;
813 case BUILTIN_CLASSMETHOD:
814 case PYTHON_CLASSMETHOD:
815 case NON_OVERRIDING:
816 case NON_DESCRIPTOR:
817 case ABSENT:
818 break;
819 }
820
821 int err = specialize_dict_access(
822 owner, instr, type, kind, name,
823 STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT
824 );
825 if (err < 0) {
826 return -1;
827 }
828 if (err) {
829 goto success;
830 }
831 fail:
832 STAT_INC(STORE_ATTR, failure);
833 assert(!PyErr_Occurred());
834 cache->counter = adaptive_counter_backoff(cache->counter);
835 return 0;
836 success:
837 STAT_INC(STORE_ATTR, success);
838 assert(!PyErr_Occurred());
839 cache->counter = miss_counter_start();
840 return 0;
841 }
842
843
844 #ifdef Py_STATS
845 static int
load_method_fail_kind(DescriptorClassification kind)846 load_method_fail_kind(DescriptorClassification kind)
847 {
848 switch (kind) {
849 case OVERRIDING:
850 return SPEC_FAIL_LOAD_METHOD_OVERRIDING_DESCRIPTOR;
851 case METHOD:
852 return SPEC_FAIL_LOAD_METHOD_METHOD;
853 case PROPERTY:
854 return SPEC_FAIL_LOAD_METHOD_PROPERTY;
855 case OBJECT_SLOT:
856 return SPEC_FAIL_LOAD_METHOD_OBJECT_SLOT;
857 case OTHER_SLOT:
858 return SPEC_FAIL_LOAD_METHOD_NON_OBJECT_SLOT;
859 case DUNDER_CLASS:
860 return SPEC_FAIL_OTHER;
861 case MUTABLE:
862 return SPEC_FAIL_LOAD_METHOD_MUTABLE_CLASS;
863 case GETSET_OVERRIDDEN:
864 return SPEC_FAIL_OVERRIDDEN;
865 case BUILTIN_CLASSMETHOD:
866 return SPEC_FAIL_LOAD_METHOD_BUILTIN_CLASS_METHOD;
867 case PYTHON_CLASSMETHOD:
868 return SPEC_FAIL_LOAD_METHOD_CLASS_METHOD_OBJ;
869 case NON_OVERRIDING:
870 return SPEC_FAIL_LOAD_METHOD_NON_OVERRIDING_DESCRIPTOR;
871 case NON_DESCRIPTOR:
872 return SPEC_FAIL_LOAD_METHOD_NOT_DESCRIPTOR;
873 case ABSENT:
874 return SPEC_FAIL_LOAD_METHOD_INSTANCE_ATTRIBUTE;
875 }
876 Py_UNREACHABLE();
877 }
878 #endif
879
880 static int
specialize_class_load_method(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)881 specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
882 PyObject *name)
883 {
884 _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
885 if (!PyType_CheckExact(owner) || _PyType_Lookup(Py_TYPE(owner), name)) {
886 SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_METACLASS_ATTRIBUTE);
887 return -1;
888 }
889 PyObject *descr = NULL;
890 DescriptorClassification kind = 0;
891 kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0);
892 switch (kind) {
893 case METHOD:
894 case NON_DESCRIPTOR:
895 write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
896 write_obj(cache->descr, descr);
897 _Py_SET_OPCODE(*instr, LOAD_METHOD_CLASS);
898 return 0;
899 #ifdef Py_STATS
900 case ABSENT:
901 SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_EXPECTED_ERROR);
902 return -1;
903 #endif
904 default:
905 SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
906 return -1;
907 }
908 }
909
910 typedef enum {
911 MANAGED_VALUES = 1,
912 MANAGED_DICT = 2,
913 OFFSET_DICT = 3,
914 NO_DICT = 4
915 } ObjectDictKind;
916
917 // Please collect stats carefully before and after modifying. A subtle change
918 // can cause a significant drop in cache hits. A possible test is
919 // python.exe -m test_typing test_re test_dis test_zlib.
920 int
_Py_Specialize_LoadMethod(PyObject * owner,_Py_CODEUNIT * instr,PyObject * name)921 _Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
922 {
923 assert(_PyOpcode_Caches[LOAD_METHOD] == INLINE_CACHE_ENTRIES_LOAD_METHOD);
924 _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
925 PyTypeObject *owner_cls = Py_TYPE(owner);
926
927 if (PyModule_CheckExact(owner)) {
928 assert(INLINE_CACHE_ENTRIES_LOAD_ATTR <=
929 INLINE_CACHE_ENTRIES_LOAD_METHOD);
930 int err = specialize_module_load_attr(owner, instr, name, LOAD_METHOD,
931 LOAD_METHOD_MODULE);
932 if (err) {
933 goto fail;
934 }
935 goto success;
936 }
937 if (owner_cls->tp_dict == NULL) {
938 if (PyType_Ready(owner_cls) < 0) {
939 return -1;
940 }
941 }
942 if (PyType_Check(owner)) {
943 int err = specialize_class_load_method(owner, instr, name);
944 if (err) {
945 goto fail;
946 }
947 goto success;
948 }
949
950 PyObject *descr = NULL;
951 DescriptorClassification kind = 0;
952 kind = analyze_descriptor(owner_cls, name, &descr, 0);
953 assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
954 if (kind != METHOD) {
955 SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
956 goto fail;
957 }
958 ObjectDictKind dictkind;
959 PyDictKeysObject *keys;
960 if (owner_cls->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
961 PyObject *dict = *_PyObject_ManagedDictPointer(owner);
962 keys = ((PyHeapTypeObject *)owner_cls)->ht_cached_keys;
963 if (dict == NULL) {
964 dictkind = MANAGED_VALUES;
965 }
966 else {
967 dictkind = MANAGED_DICT;
968 }
969 }
970 else {
971 Py_ssize_t dictoffset = owner_cls->tp_dictoffset;
972 if (dictoffset < 0 || dictoffset > INT16_MAX) {
973 SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_RANGE);
974 goto fail;
975 }
976 if (dictoffset == 0) {
977 dictkind = NO_DICT;
978 keys = NULL;
979 }
980 else {
981 PyObject *dict = *(PyObject **) ((char *)owner + dictoffset);
982 if (dict == NULL) {
983 SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_NO_DICT);
984 goto fail;
985 }
986 keys = ((PyDictObject *)dict)->ma_keys;
987 dictkind = OFFSET_DICT;
988 }
989 }
990 if (dictkind != NO_DICT) {
991 Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
992 if (index != DKIX_EMPTY) {
993 SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_IS_ATTR);
994 goto fail;
995 }
996 uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(keys);
997 if (keys_version == 0) {
998 SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_OUT_OF_VERSIONS);
999 goto fail;
1000 }
1001 write_u32(cache->keys_version, keys_version);
1002 }
1003 switch(dictkind) {
1004 case NO_DICT:
1005 _Py_SET_OPCODE(*instr, LOAD_METHOD_NO_DICT);
1006 break;
1007 case MANAGED_VALUES:
1008 _Py_SET_OPCODE(*instr, LOAD_METHOD_WITH_VALUES);
1009 break;
1010 case MANAGED_DICT:
1011 *(int16_t *)&cache->dict_offset = (int16_t)MANAGED_DICT_OFFSET;
1012 _Py_SET_OPCODE(*instr, LOAD_METHOD_WITH_DICT);
1013 break;
1014 case OFFSET_DICT:
1015 assert(owner_cls->tp_dictoffset > 0 && owner_cls->tp_dictoffset <= INT16_MAX);
1016 cache->dict_offset = (uint16_t)owner_cls->tp_dictoffset;
1017 _Py_SET_OPCODE(*instr, LOAD_METHOD_WITH_DICT);
1018 break;
1019 }
1020 /* `descr` is borrowed. This is safe for methods (even inherited ones from
1021 * super classes!) as long as tp_version_tag is validated for two main reasons:
1022 *
1023 * 1. The class will always hold a reference to the method so it will
1024 * usually not be GC-ed. Should it be deleted in Python, e.g.
1025 * `del obj.meth`, tp_version_tag will be invalidated, because of reason 2.
1026 *
1027 * 2. The pre-existing type method cache (MCACHE) uses the same principles
1028 * of caching a borrowed descriptor. The MCACHE infrastructure does all the
1029 * heavy lifting for us. E.g. it invalidates tp_version_tag on any MRO
1030 * modification, on any type object change along said MRO, etc. (see
1031 * PyType_Modified usages in typeobject.c). The MCACHE has been
1032 * working since Python 2.6 and it's battle-tested.
1033 */
1034 write_u32(cache->type_version, owner_cls->tp_version_tag);
1035 write_obj(cache->descr, descr);
1036 // Fall through.
1037 success:
1038 STAT_INC(LOAD_METHOD, success);
1039 assert(!PyErr_Occurred());
1040 cache->counter = miss_counter_start();
1041 return 0;
1042 fail:
1043 STAT_INC(LOAD_METHOD, failure);
1044 assert(!PyErr_Occurred());
1045 cache->counter = adaptive_counter_backoff(cache->counter);
1046 return 0;
1047 }
1048
1049 int
_Py_Specialize_LoadGlobal(PyObject * globals,PyObject * builtins,_Py_CODEUNIT * instr,PyObject * name)1050 _Py_Specialize_LoadGlobal(
1051 PyObject *globals, PyObject *builtins,
1052 _Py_CODEUNIT *instr, PyObject *name)
1053 {
1054 assert(_PyOpcode_Caches[LOAD_GLOBAL] == INLINE_CACHE_ENTRIES_LOAD_GLOBAL);
1055 /* Use inline cache */
1056 _PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)(instr + 1);
1057 assert(PyUnicode_CheckExact(name));
1058 if (!PyDict_CheckExact(globals)) {
1059 goto fail;
1060 }
1061 PyDictKeysObject * globals_keys = ((PyDictObject *)globals)->ma_keys;
1062 if (!DK_IS_UNICODE(globals_keys)) {
1063 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1064 goto fail;
1065 }
1066 Py_ssize_t index = _PyDictKeys_StringLookup(globals_keys, name);
1067 if (index == DKIX_ERROR) {
1068 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1069 goto fail;
1070 }
1071 if (index != DKIX_EMPTY) {
1072 if (index != (uint16_t)index) {
1073 goto fail;
1074 }
1075 uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(globals_keys);
1076 if (keys_version == 0) {
1077 goto fail;
1078 }
1079 cache->index = (uint16_t)index;
1080 write_u32(cache->module_keys_version, keys_version);
1081 _Py_SET_OPCODE(*instr, LOAD_GLOBAL_MODULE);
1082 goto success;
1083 }
1084 if (!PyDict_CheckExact(builtins)) {
1085 goto fail;
1086 }
1087 PyDictKeysObject * builtin_keys = ((PyDictObject *)builtins)->ma_keys;
1088 if (!DK_IS_UNICODE(builtin_keys)) {
1089 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1090 goto fail;
1091 }
1092 index = _PyDictKeys_StringLookup(builtin_keys, name);
1093 if (index == DKIX_ERROR) {
1094 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT);
1095 goto fail;
1096 }
1097 if (index != (uint16_t)index) {
1098 goto fail;
1099 }
1100 uint32_t globals_version = _PyDictKeys_GetVersionForCurrentState(globals_keys);
1101 if (globals_version == 0) {
1102 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1103 goto fail;
1104 }
1105 uint32_t builtins_version = _PyDictKeys_GetVersionForCurrentState(builtin_keys);
1106 if (builtins_version == 0) {
1107 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_VERSIONS);
1108 goto fail;
1109 }
1110 if (builtins_version > UINT16_MAX) {
1111 SPECIALIZATION_FAIL(LOAD_GLOBAL, SPEC_FAIL_OUT_OF_RANGE);
1112 goto fail;
1113 }
1114 cache->index = (uint16_t)index;
1115 write_u32(cache->module_keys_version, globals_version);
1116 cache->builtin_keys_version = (uint16_t)builtins_version;
1117 _Py_SET_OPCODE(*instr, LOAD_GLOBAL_BUILTIN);
1118 goto success;
1119 fail:
1120 STAT_INC(LOAD_GLOBAL, failure);
1121 assert(!PyErr_Occurred());
1122 cache->counter = adaptive_counter_backoff(cache->counter);
1123 return 0;
1124 success:
1125 STAT_INC(LOAD_GLOBAL, success);
1126 assert(!PyErr_Occurred());
1127 cache->counter = miss_counter_start();
1128 return 0;
1129 }
1130
1131 #ifdef Py_STATS
1132 static int
binary_subscr_fail_kind(PyTypeObject * container_type,PyObject * sub)1133 binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
1134 {
1135 if (container_type == &PyUnicode_Type) {
1136 if (PyLong_CheckExact(sub)) {
1137 return SPEC_FAIL_SUBSCR_STRING_INT;
1138 }
1139 if (PySlice_Check(sub)) {
1140 return SPEC_FAIL_SUBSCR_STRING_SLICE;
1141 }
1142 return SPEC_FAIL_OTHER;
1143 }
1144 else if (strcmp(container_type->tp_name, "array.array") == 0) {
1145 if (PyLong_CheckExact(sub)) {
1146 return SPEC_FAIL_SUBSCR_ARRAY_INT;
1147 }
1148 if (PySlice_Check(sub)) {
1149 return SPEC_FAIL_SUBSCR_ARRAY_SLICE;
1150 }
1151 return SPEC_FAIL_OTHER;
1152 }
1153 else if (container_type->tp_as_buffer) {
1154 if (PyLong_CheckExact(sub)) {
1155 return SPEC_FAIL_SUBSCR_BUFFER_INT;
1156 }
1157 if (PySlice_Check(sub)) {
1158 return SPEC_FAIL_SUBSCR_BUFFER_SLICE;
1159 }
1160 return SPEC_FAIL_OTHER;
1161 }
1162 else if (container_type->tp_as_sequence) {
1163 if (PyLong_CheckExact(sub) && container_type->tp_as_sequence->sq_item) {
1164 return SPEC_FAIL_SUBSCR_SEQUENCE_INT;
1165 }
1166 }
1167 return SPEC_FAIL_OTHER;
1168 }
1169 #endif
1170
1171
1172 #define SIMPLE_FUNCTION 0
1173
1174 static int
function_kind(PyCodeObject * code)1175 function_kind(PyCodeObject *code) {
1176 int flags = code->co_flags;
1177 if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) {
1178 return SPEC_FAIL_CALL_COMPLEX_PARAMETERS;
1179 }
1180 if ((flags & CO_OPTIMIZED) == 0) {
1181 return SPEC_FAIL_CALL_CO_NOT_OPTIMIZED;
1182 }
1183 return SIMPLE_FUNCTION;
1184 }
1185
1186 int
_Py_Specialize_BinarySubscr(PyObject * container,PyObject * sub,_Py_CODEUNIT * instr)1187 _Py_Specialize_BinarySubscr(
1188 PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
1189 {
1190 assert(_PyOpcode_Caches[BINARY_SUBSCR] ==
1191 INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
1192 _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1);
1193 PyTypeObject *container_type = Py_TYPE(container);
1194 if (container_type == &PyList_Type) {
1195 if (PyLong_CheckExact(sub)) {
1196 _Py_SET_OPCODE(*instr, BINARY_SUBSCR_LIST_INT);
1197 goto success;
1198 }
1199 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1200 PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_LIST_SLICE : SPEC_FAIL_OTHER);
1201 goto fail;
1202 }
1203 if (container_type == &PyTuple_Type) {
1204 if (PyLong_CheckExact(sub)) {
1205 _Py_SET_OPCODE(*instr, BINARY_SUBSCR_TUPLE_INT);
1206 goto success;
1207 }
1208 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1209 PySlice_Check(sub) ? SPEC_FAIL_SUBSCR_TUPLE_SLICE : SPEC_FAIL_OTHER);
1210 goto fail;
1211 }
1212 if (container_type == &PyDict_Type) {
1213 _Py_SET_OPCODE(*instr, BINARY_SUBSCR_DICT);
1214 goto success;
1215 }
1216 PyTypeObject *cls = Py_TYPE(container);
1217 PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__));
1218 if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
1219 if (!(container_type->tp_flags & Py_TPFLAGS_HEAPTYPE)) {
1220 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE);
1221 goto fail;
1222 }
1223 PyFunctionObject *func = (PyFunctionObject *)descriptor;
1224 PyCodeObject *fcode = (PyCodeObject *)func->func_code;
1225 int kind = function_kind(fcode);
1226 if (kind != SIMPLE_FUNCTION) {
1227 SPECIALIZATION_FAIL(BINARY_SUBSCR, kind);
1228 goto fail;
1229 }
1230 if (fcode->co_argcount != 2) {
1231 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1232 goto fail;
1233 }
1234 assert(cls->tp_version_tag != 0);
1235 write_u32(cache->type_version, cls->tp_version_tag);
1236 int version = _PyFunction_GetVersionForCurrentState(func);
1237 if (version == 0 || version != (uint16_t)version) {
1238 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_VERSIONS);
1239 goto fail;
1240 }
1241 if (_PyInterpreterState_GET()->eval_frame) {
1242 SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OTHER);
1243 goto fail;
1244 }
1245 cache->func_version = version;
1246 ((PyHeapTypeObject *)container_type)->_spec_cache.getitem = descriptor;
1247 _Py_SET_OPCODE(*instr, BINARY_SUBSCR_GETITEM);
1248 goto success;
1249 }
1250 SPECIALIZATION_FAIL(BINARY_SUBSCR,
1251 binary_subscr_fail_kind(container_type, sub));
1252 fail:
1253 STAT_INC(BINARY_SUBSCR, failure);
1254 assert(!PyErr_Occurred());
1255 cache->counter = adaptive_counter_backoff(cache->counter);
1256 return 0;
1257 success:
1258 STAT_INC(BINARY_SUBSCR, success);
1259 assert(!PyErr_Occurred());
1260 cache->counter = miss_counter_start();
1261 return 0;
1262 }
1263
1264 int
_Py_Specialize_StoreSubscr(PyObject * container,PyObject * sub,_Py_CODEUNIT * instr)1265 _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *instr)
1266 {
1267 _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)(instr + 1);
1268 PyTypeObject *container_type = Py_TYPE(container);
1269 if (container_type == &PyList_Type) {
1270 if (PyLong_CheckExact(sub)) {
1271 if ((Py_SIZE(sub) == 0 || Py_SIZE(sub) == 1)
1272 && ((PyLongObject *)sub)->ob_digit[0] < (size_t)PyList_GET_SIZE(container))
1273 {
1274 _Py_SET_OPCODE(*instr, STORE_SUBSCR_LIST_INT);
1275 goto success;
1276 }
1277 else {
1278 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1279 goto fail;
1280 }
1281 }
1282 else if (PySlice_Check(sub)) {
1283 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_LIST_SLICE);
1284 goto fail;
1285 }
1286 else {
1287 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1288 goto fail;
1289 }
1290 }
1291 if (container_type == &PyDict_Type) {
1292 _Py_SET_OPCODE(*instr, STORE_SUBSCR_DICT);
1293 goto success;
1294 }
1295 #ifdef Py_STATS
1296 PyMappingMethods *as_mapping = container_type->tp_as_mapping;
1297 if (as_mapping && (as_mapping->mp_ass_subscript
1298 == PyDict_Type.tp_as_mapping->mp_ass_subscript)) {
1299 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE);
1300 goto fail;
1301 }
1302 if (PyObject_CheckBuffer(container)) {
1303 if (PyLong_CheckExact(sub) && (((size_t)Py_SIZE(sub)) > 1)) {
1304 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OUT_OF_RANGE);
1305 }
1306 else if (strcmp(container_type->tp_name, "array.array") == 0) {
1307 if (PyLong_CheckExact(sub)) {
1308 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_INT);
1309 }
1310 else if (PySlice_Check(sub)) {
1311 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_ARRAY_SLICE);
1312 }
1313 else {
1314 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1315 }
1316 }
1317 else if (PyByteArray_CheckExact(container)) {
1318 if (PyLong_CheckExact(sub)) {
1319 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_INT);
1320 }
1321 else if (PySlice_Check(sub)) {
1322 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE);
1323 }
1324 else {
1325 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1326 }
1327 }
1328 else {
1329 if (PyLong_CheckExact(sub)) {
1330 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_INT);
1331 }
1332 else if (PySlice_Check(sub)) {
1333 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_BUFFER_SLICE);
1334 }
1335 else {
1336 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1337 }
1338 }
1339 goto fail;
1340 }
1341 PyObject *descriptor = _PyType_Lookup(container_type, &_Py_ID(__setitem__));
1342 if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) {
1343 PyFunctionObject *func = (PyFunctionObject *)descriptor;
1344 PyCodeObject *code = (PyCodeObject *)func->func_code;
1345 int kind = function_kind(code);
1346 if (kind == SIMPLE_FUNCTION) {
1347 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_SIMPLE);
1348 }
1349 else {
1350 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_SUBSCR_PY_OTHER);
1351 }
1352 goto fail;
1353 }
1354 #endif
1355 SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
1356 fail:
1357 STAT_INC(STORE_SUBSCR, failure);
1358 assert(!PyErr_Occurred());
1359 cache->counter = adaptive_counter_backoff(cache->counter);
1360 return 0;
1361 success:
1362 STAT_INC(STORE_SUBSCR, success);
1363 assert(!PyErr_Occurred());
1364 cache->counter = miss_counter_start();
1365 return 0;
1366 }
1367
1368 static int
specialize_class_call(PyObject * callable,_Py_CODEUNIT * instr,int nargs,PyObject * kwnames,int oparg)1369 specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1370 PyObject *kwnames, int oparg)
1371 {
1372 assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
1373 PyTypeObject *tp = _PyType_CAST(callable);
1374 if (tp->tp_new == PyBaseObject_Type.tp_new) {
1375 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_PYTHON_CLASS);
1376 return -1;
1377 }
1378 if (tp->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) {
1379 if (nargs == 1 && kwnames == NULL && oparg == 1) {
1380 if (tp == &PyUnicode_Type) {
1381 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_STR_1);
1382 return 0;
1383 }
1384 else if (tp == &PyType_Type) {
1385 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_TYPE_1);
1386 return 0;
1387 }
1388 else if (tp == &PyTuple_Type) {
1389 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_TUPLE_1);
1390 return 0;
1391 }
1392 }
1393 if (tp->tp_vectorcall != NULL) {
1394 _Py_SET_OPCODE(*instr, PRECALL_BUILTIN_CLASS);
1395 return 0;
1396 }
1397 SPECIALIZATION_FAIL(PRECALL, tp == &PyUnicode_Type ?
1398 SPEC_FAIL_CALL_STR : SPEC_FAIL_CALL_CLASS_NO_VECTORCALL);
1399 return -1;
1400 }
1401 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_CLASS_MUTABLE);
1402 return -1;
1403 }
1404
1405 #ifdef Py_STATS
1406 static int
builtin_call_fail_kind(int ml_flags)1407 builtin_call_fail_kind(int ml_flags)
1408 {
1409 switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1410 METH_KEYWORDS | METH_METHOD)) {
1411 case METH_VARARGS:
1412 return SPEC_FAIL_CALL_PYCFUNCTION;
1413 case METH_VARARGS | METH_KEYWORDS:
1414 return SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS;
1415 case METH_FASTCALL | METH_KEYWORDS:
1416 return SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS;
1417 case METH_NOARGS:
1418 return SPEC_FAIL_CALL_PYCFUNCTION_NOARGS;
1419 /* This case should never happen with PyCFunctionObject -- only
1420 PyMethodObject. See zlib.compressobj()'s methods for an example.
1421 */
1422 case METH_METHOD | METH_FASTCALL | METH_KEYWORDS:
1423 default:
1424 return SPEC_FAIL_CALL_BAD_CALL_FLAGS;
1425 }
1426 }
1427 #endif
1428
1429 static int
specialize_method_descriptor(PyMethodDescrObject * descr,_Py_CODEUNIT * instr,int nargs,PyObject * kwnames,int oparg)1430 specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
1431 int nargs, PyObject *kwnames, int oparg)
1432 {
1433 assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
1434 if (kwnames) {
1435 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
1436 return -1;
1437 }
1438
1439 switch (descr->d_method->ml_flags &
1440 (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1441 METH_KEYWORDS | METH_METHOD)) {
1442 case METH_NOARGS: {
1443 if (nargs != 1) {
1444 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1445 return -1;
1446 }
1447 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS);
1448 return 0;
1449 }
1450 case METH_O: {
1451 if (nargs != 2) {
1452 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1453 return -1;
1454 }
1455 PyInterpreterState *interp = _PyInterpreterState_GET();
1456 PyObject *list_append = interp->callable_cache.list_append;
1457 _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_PRECALL + 1
1458 + INLINE_CACHE_ENTRIES_CALL + 1];
1459 bool pop = (_Py_OPCODE(next) == POP_TOP);
1460 if ((PyObject *)descr == list_append && oparg == 1 && pop) {
1461 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_LIST_APPEND);
1462 return 0;
1463 }
1464 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_O);
1465 return 0;
1466 }
1467 case METH_FASTCALL: {
1468 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST);
1469 return 0;
1470 }
1471 case METH_FASTCALL|METH_KEYWORDS: {
1472 _Py_SET_OPCODE(*instr, PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS);
1473 return 0;
1474 }
1475 }
1476 SPECIALIZATION_FAIL(PRECALL, builtin_call_fail_kind(descr->d_method->ml_flags));
1477 return -1;
1478 }
1479
1480 static int
specialize_py_call(PyFunctionObject * func,_Py_CODEUNIT * instr,int nargs,PyObject * kwnames)1481 specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs,
1482 PyObject *kwnames)
1483 {
1484 _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1485 assert(_Py_OPCODE(*instr) == CALL_ADAPTIVE);
1486 PyCodeObject *code = (PyCodeObject *)func->func_code;
1487 int kind = function_kind(code);
1488 /* Don't specialize if PEP 523 is active */
1489 if (_PyInterpreterState_GET()->eval_frame) {
1490 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523);
1491 return -1;
1492 }
1493 if (kwnames) {
1494 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_KWNAMES);
1495 return -1;
1496 }
1497 if (kind != SIMPLE_FUNCTION) {
1498 SPECIALIZATION_FAIL(CALL, kind);
1499 return -1;
1500 }
1501 int argcount = code->co_argcount;
1502 int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults);
1503 assert(defcount <= argcount);
1504 int min_args = argcount-defcount;
1505 if (nargs > argcount || nargs < min_args) {
1506 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1507 return -1;
1508 }
1509 assert(nargs <= argcount && nargs >= min_args);
1510 assert(min_args >= 0 && defcount >= 0);
1511 assert(defcount == 0 || func->func_defaults != NULL);
1512 if (min_args > 0xffff) {
1513 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE);
1514 return -1;
1515 }
1516 int version = _PyFunction_GetVersionForCurrentState(func);
1517 if (version == 0) {
1518 SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS);
1519 return -1;
1520 }
1521 write_u32(cache->func_version, version);
1522 cache->min_args = min_args;
1523 if (argcount == nargs) {
1524 _Py_SET_OPCODE(*instr, CALL_PY_EXACT_ARGS);
1525 }
1526 else {
1527 _Py_SET_OPCODE(*instr, CALL_PY_WITH_DEFAULTS);
1528 }
1529 return 0;
1530 }
1531
1532 static int
specialize_c_call(PyObject * callable,_Py_CODEUNIT * instr,int nargs,PyObject * kwnames)1533 specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1534 PyObject *kwnames)
1535 {
1536 assert(_Py_OPCODE(*instr) == PRECALL_ADAPTIVE);
1537 if (PyCFunction_GET_FUNCTION(callable) == NULL) {
1538 return 1;
1539 }
1540 switch (PyCFunction_GET_FLAGS(callable) &
1541 (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O |
1542 METH_KEYWORDS | METH_METHOD)) {
1543 case METH_O: {
1544 if (kwnames) {
1545 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
1546 return -1;
1547 }
1548 if (nargs != 1) {
1549 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS);
1550 return 1;
1551 }
1552 /* len(o) */
1553 PyInterpreterState *interp = _PyInterpreterState_GET();
1554 if (callable == interp->callable_cache.len) {
1555 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_LEN);
1556 return 0;
1557 }
1558 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_BUILTIN_O);
1559 return 0;
1560 }
1561 case METH_FASTCALL: {
1562 if (kwnames) {
1563 SPECIALIZATION_FAIL(PRECALL, SPEC_FAIL_CALL_KWNAMES);
1564 return -1;
1565 }
1566 if (nargs == 2) {
1567 /* isinstance(o1, o2) */
1568 PyInterpreterState *interp = _PyInterpreterState_GET();
1569 if (callable == interp->callable_cache.isinstance) {
1570 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_ISINSTANCE);
1571 return 0;
1572 }
1573 }
1574 _Py_SET_OPCODE(*instr, PRECALL_NO_KW_BUILTIN_FAST);
1575 return 0;
1576 }
1577 case METH_FASTCALL | METH_KEYWORDS: {
1578 _Py_SET_OPCODE(*instr, PRECALL_BUILTIN_FAST_WITH_KEYWORDS);
1579 return 0;
1580 }
1581 default:
1582 SPECIALIZATION_FAIL(PRECALL,
1583 builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable)));
1584 return 1;
1585 }
1586 }
1587
1588 #ifdef Py_STATS
1589 static int
call_fail_kind(PyObject * callable)1590 call_fail_kind(PyObject *callable)
1591 {
1592 if (PyCFunction_CheckExact(callable)) {
1593 return SPEC_FAIL_CALL_PYCFUNCTION;
1594 }
1595 else if (PyFunction_Check(callable)) {
1596 return SPEC_FAIL_CALL_PYFUNCTION;
1597 }
1598 else if (PyInstanceMethod_Check(callable)) {
1599 return SPEC_FAIL_CALL_INSTANCE_METHOD;
1600 }
1601 else if (PyMethod_Check(callable)) {
1602 return SPEC_FAIL_CALL_BOUND_METHOD;
1603 }
1604 // builtin method
1605 else if (PyCMethod_Check(callable)) {
1606 return SPEC_FAIL_CALL_CMETHOD;
1607 }
1608 else if (PyType_Check(callable)) {
1609 if (((PyTypeObject *)callable)->tp_new == PyBaseObject_Type.tp_new) {
1610 return SPEC_FAIL_CALL_PYTHON_CLASS;
1611 }
1612 else {
1613 return SPEC_FAIL_CALL_CLASS;
1614 }
1615 }
1616 else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
1617 return SPEC_FAIL_CALL_METHOD_DESCRIPTOR;
1618 }
1619 else if (Py_TYPE(callable) == &PyWrapperDescr_Type) {
1620 return SPEC_FAIL_CALL_OPERATOR_WRAPPER;
1621 }
1622 else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) {
1623 return SPEC_FAIL_CALL_METHOD_WRAPPER;
1624 }
1625 return SPEC_FAIL_OTHER;
1626 }
1627 #endif
1628
1629
1630 int
_Py_Specialize_Precall(PyObject * callable,_Py_CODEUNIT * instr,int nargs,PyObject * kwnames,int oparg)1631 _Py_Specialize_Precall(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1632 PyObject *kwnames, int oparg)
1633 {
1634 assert(_PyOpcode_Caches[PRECALL] == INLINE_CACHE_ENTRIES_PRECALL);
1635 _PyPrecallCache *cache = (_PyPrecallCache *)(instr + 1);
1636 int fail;
1637 if (PyCFunction_CheckExact(callable)) {
1638 fail = specialize_c_call(callable, instr, nargs, kwnames);
1639 }
1640 else if (PyFunction_Check(callable)) {
1641 _Py_SET_OPCODE(*instr, PRECALL_PYFUNC);
1642 fail = 0;
1643 }
1644 else if (PyType_Check(callable)) {
1645 fail = specialize_class_call(callable, instr, nargs, kwnames, oparg);
1646 }
1647 else if (Py_IS_TYPE(callable, &PyMethodDescr_Type)) {
1648 fail = specialize_method_descriptor((PyMethodDescrObject *)callable,
1649 instr, nargs, kwnames, oparg);
1650 }
1651 else if (Py_TYPE(callable) == &PyMethod_Type) {
1652 _Py_SET_OPCODE(*instr, PRECALL_BOUND_METHOD);
1653 fail = 0;
1654 }
1655 else {
1656 SPECIALIZATION_FAIL(PRECALL, call_fail_kind(callable));
1657 fail = -1;
1658 }
1659 if (fail) {
1660 STAT_INC(PRECALL, failure);
1661 assert(!PyErr_Occurred());
1662 cache->counter = adaptive_counter_backoff(cache->counter);
1663 }
1664 else {
1665 STAT_INC(PRECALL, success);
1666 assert(!PyErr_Occurred());
1667 cache->counter = miss_counter_start();
1668 }
1669 return 0;
1670 }
1671
1672
1673 /* TODO:
1674 - Specialize calling classes.
1675 */
1676 int
_Py_Specialize_Call(PyObject * callable,_Py_CODEUNIT * instr,int nargs,PyObject * kwnames)1677 _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
1678 PyObject *kwnames)
1679 {
1680 assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL);
1681 _PyCallCache *cache = (_PyCallCache *)(instr + 1);
1682 int fail;
1683 if (PyFunction_Check(callable)) {
1684 fail = specialize_py_call((PyFunctionObject *)callable, instr, nargs,
1685 kwnames);
1686 }
1687 else {
1688 SPECIALIZATION_FAIL(CALL, call_fail_kind(callable));
1689 fail = -1;
1690 }
1691 if (fail) {
1692 STAT_INC(CALL, failure);
1693 assert(!PyErr_Occurred());
1694 cache->counter = adaptive_counter_backoff(cache->counter);
1695 }
1696 else {
1697 STAT_INC(CALL, success);
1698 assert(!PyErr_Occurred());
1699 cache->counter = miss_counter_start();
1700 }
1701 return 0;
1702 }
1703
1704 #ifdef Py_STATS
1705 static int
binary_op_fail_kind(int oparg,PyObject * lhs,PyObject * rhs)1706 binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
1707 {
1708 switch (oparg) {
1709 case NB_ADD:
1710 case NB_INPLACE_ADD:
1711 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1712 return SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES;
1713 }
1714 return SPEC_FAIL_BINARY_OP_ADD_OTHER;
1715 case NB_AND:
1716 case NB_INPLACE_AND:
1717 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1718 return SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES;
1719 }
1720 if (PyLong_CheckExact(lhs)) {
1721 return SPEC_FAIL_BINARY_OP_AND_INT;
1722 }
1723 return SPEC_FAIL_BINARY_OP_AND_OTHER;
1724 case NB_FLOOR_DIVIDE:
1725 case NB_INPLACE_FLOOR_DIVIDE:
1726 return SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE;
1727 case NB_LSHIFT:
1728 case NB_INPLACE_LSHIFT:
1729 return SPEC_FAIL_BINARY_OP_LSHIFT;
1730 case NB_MATRIX_MULTIPLY:
1731 case NB_INPLACE_MATRIX_MULTIPLY:
1732 return SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY;
1733 case NB_MULTIPLY:
1734 case NB_INPLACE_MULTIPLY:
1735 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1736 return SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES;
1737 }
1738 return SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER;
1739 case NB_OR:
1740 case NB_INPLACE_OR:
1741 return SPEC_FAIL_BINARY_OP_OR;
1742 case NB_POWER:
1743 case NB_INPLACE_POWER:
1744 return SPEC_FAIL_BINARY_OP_POWER;
1745 case NB_REMAINDER:
1746 case NB_INPLACE_REMAINDER:
1747 return SPEC_FAIL_BINARY_OP_REMAINDER;
1748 case NB_RSHIFT:
1749 case NB_INPLACE_RSHIFT:
1750 return SPEC_FAIL_BINARY_OP_RSHIFT;
1751 case NB_SUBTRACT:
1752 case NB_INPLACE_SUBTRACT:
1753 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1754 return SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES;
1755 }
1756 return SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER;
1757 case NB_TRUE_DIVIDE:
1758 case NB_INPLACE_TRUE_DIVIDE:
1759 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1760 return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES;
1761 }
1762 if (PyFloat_CheckExact(lhs)) {
1763 return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT;
1764 }
1765 return SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER;
1766 case NB_XOR:
1767 case NB_INPLACE_XOR:
1768 return SPEC_FAIL_BINARY_OP_XOR;
1769 }
1770 Py_UNREACHABLE();
1771 }
1772 #endif
1773
1774 void
_Py_Specialize_BinaryOp(PyObject * lhs,PyObject * rhs,_Py_CODEUNIT * instr,int oparg,PyObject ** locals)1775 _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
1776 int oparg, PyObject **locals)
1777 {
1778 assert(_PyOpcode_Caches[BINARY_OP] == INLINE_CACHE_ENTRIES_BINARY_OP);
1779 _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(instr + 1);
1780 switch (oparg) {
1781 case NB_ADD:
1782 case NB_INPLACE_ADD:
1783 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1784 break;
1785 }
1786 if (PyUnicode_CheckExact(lhs)) {
1787 _Py_CODEUNIT next = instr[INLINE_CACHE_ENTRIES_BINARY_OP + 1];
1788 bool to_store = (_Py_OPCODE(next) == STORE_FAST ||
1789 _Py_OPCODE(next) == STORE_FAST__LOAD_FAST);
1790 if (to_store && locals[_Py_OPARG(next)] == lhs) {
1791 _Py_SET_OPCODE(*instr, BINARY_OP_INPLACE_ADD_UNICODE);
1792 goto success;
1793 }
1794 _Py_SET_OPCODE(*instr, BINARY_OP_ADD_UNICODE);
1795 goto success;
1796 }
1797 if (PyLong_CheckExact(lhs)) {
1798 _Py_SET_OPCODE(*instr, BINARY_OP_ADD_INT);
1799 goto success;
1800 }
1801 if (PyFloat_CheckExact(lhs)) {
1802 _Py_SET_OPCODE(*instr, BINARY_OP_ADD_FLOAT);
1803 goto success;
1804 }
1805 break;
1806 case NB_MULTIPLY:
1807 case NB_INPLACE_MULTIPLY:
1808 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1809 break;
1810 }
1811 if (PyLong_CheckExact(lhs)) {
1812 _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_INT);
1813 goto success;
1814 }
1815 if (PyFloat_CheckExact(lhs)) {
1816 _Py_SET_OPCODE(*instr, BINARY_OP_MULTIPLY_FLOAT);
1817 goto success;
1818 }
1819 break;
1820 case NB_SUBTRACT:
1821 case NB_INPLACE_SUBTRACT:
1822 if (!Py_IS_TYPE(lhs, Py_TYPE(rhs))) {
1823 break;
1824 }
1825 if (PyLong_CheckExact(lhs)) {
1826 _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_INT);
1827 goto success;
1828 }
1829 if (PyFloat_CheckExact(lhs)) {
1830 _Py_SET_OPCODE(*instr, BINARY_OP_SUBTRACT_FLOAT);
1831 goto success;
1832 }
1833 break;
1834 #ifndef Py_STATS
1835 default:
1836 // These operators don't have any available specializations. Rather
1837 // than repeatedly attempting to specialize them, just convert them
1838 // back to BINARY_OP (unless we're collecting stats, where it's more
1839 // important to get accurate hit counts for the unadaptive version
1840 // and each of the different failure types):
1841 _Py_SET_OPCODE(*instr, BINARY_OP);
1842 return;
1843 #endif
1844 }
1845 SPECIALIZATION_FAIL(BINARY_OP, binary_op_fail_kind(oparg, lhs, rhs));
1846 STAT_INC(BINARY_OP, failure);
1847 cache->counter = adaptive_counter_backoff(cache->counter);
1848 return;
1849 success:
1850 STAT_INC(BINARY_OP, success);
1851 cache->counter = miss_counter_start();
1852 }
1853
1854
1855 #ifdef Py_STATS
1856 static int
compare_op_fail_kind(PyObject * lhs,PyObject * rhs)1857 compare_op_fail_kind(PyObject *lhs, PyObject *rhs)
1858 {
1859 if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
1860 if (PyFloat_CheckExact(lhs) && PyLong_CheckExact(rhs)) {
1861 return SPEC_FAIL_COMPARE_OP_FLOAT_LONG;
1862 }
1863 if (PyLong_CheckExact(lhs) && PyFloat_CheckExact(rhs)) {
1864 return SPEC_FAIL_COMPARE_OP_LONG_FLOAT;
1865 }
1866 return SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES;
1867 }
1868 if (PyBytes_CheckExact(lhs)) {
1869 return SPEC_FAIL_COMPARE_OP_BYTES;
1870 }
1871 if (PyTuple_CheckExact(lhs)) {
1872 return SPEC_FAIL_COMPARE_OP_TUPLE;
1873 }
1874 if (PyList_CheckExact(lhs)) {
1875 return SPEC_FAIL_COMPARE_OP_LIST;
1876 }
1877 if (PySet_CheckExact(lhs) || PyFrozenSet_CheckExact(lhs)) {
1878 return SPEC_FAIL_COMPARE_OP_SET;
1879 }
1880 if (PyBool_Check(lhs)) {
1881 return SPEC_FAIL_COMPARE_OP_BOOL;
1882 }
1883 if (Py_TYPE(lhs)->tp_richcompare == PyBaseObject_Type.tp_richcompare) {
1884 return SPEC_FAIL_COMPARE_OP_BASEOBJECT;
1885 }
1886 return SPEC_FAIL_OTHER;
1887 }
1888 #endif
1889
1890
1891 static int compare_masks[] = {
1892 // 1-bit: jump if less than
1893 // 2-bit: jump if equal
1894 // 4-bit: jump if greater
1895 [Py_LT] = 1 | 0 | 0,
1896 [Py_LE] = 1 | 2 | 0,
1897 [Py_EQ] = 0 | 2 | 0,
1898 [Py_NE] = 1 | 0 | 4,
1899 [Py_GT] = 0 | 0 | 4,
1900 [Py_GE] = 0 | 2 | 4,
1901 };
1902
1903 void
_Py_Specialize_CompareOp(PyObject * lhs,PyObject * rhs,_Py_CODEUNIT * instr,int oparg)1904 _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
1905 int oparg)
1906 {
1907 assert(_PyOpcode_Caches[COMPARE_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP);
1908 _PyCompareOpCache *cache = (_PyCompareOpCache *)(instr + 1);
1909 int next_opcode = _Py_OPCODE(instr[INLINE_CACHE_ENTRIES_COMPARE_OP + 1]);
1910 if (next_opcode != POP_JUMP_FORWARD_IF_FALSE &&
1911 next_opcode != POP_JUMP_BACKWARD_IF_FALSE &&
1912 next_opcode != POP_JUMP_FORWARD_IF_TRUE &&
1913 next_opcode != POP_JUMP_BACKWARD_IF_TRUE) {
1914 // Can't ever combine, so don't don't bother being adaptive (unless
1915 // we're collecting stats, where it's more important to get accurate hit
1916 // counts for the unadaptive version and each of the different failure
1917 // types):
1918 #ifndef Py_STATS
1919 _Py_SET_OPCODE(*instr, COMPARE_OP);
1920 return;
1921 #else
1922 if (next_opcode == EXTENDED_ARG) {
1923 SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_EXTENDED_ARG);
1924 goto failure;
1925 }
1926 SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP);
1927 goto failure;
1928 #endif
1929 }
1930 assert(oparg <= Py_GE);
1931 int when_to_jump_mask = compare_masks[oparg];
1932 if (next_opcode == POP_JUMP_FORWARD_IF_FALSE ||
1933 next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
1934 when_to_jump_mask = (1 | 2 | 4) & ~when_to_jump_mask;
1935 }
1936 if (next_opcode == POP_JUMP_BACKWARD_IF_TRUE ||
1937 next_opcode == POP_JUMP_BACKWARD_IF_FALSE) {
1938 when_to_jump_mask <<= 3;
1939 }
1940 if (Py_TYPE(lhs) != Py_TYPE(rhs)) {
1941 SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
1942 goto failure;
1943 }
1944 if (PyFloat_CheckExact(lhs)) {
1945 _Py_SET_OPCODE(*instr, COMPARE_OP_FLOAT_JUMP);
1946 cache->mask = when_to_jump_mask;
1947 goto success;
1948 }
1949 if (PyLong_CheckExact(lhs)) {
1950 if (Py_ABS(Py_SIZE(lhs)) <= 1 && Py_ABS(Py_SIZE(rhs)) <= 1) {
1951 _Py_SET_OPCODE(*instr, COMPARE_OP_INT_JUMP);
1952 cache->mask = when_to_jump_mask;
1953 goto success;
1954 }
1955 else {
1956 SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_BIG_INT);
1957 goto failure;
1958 }
1959 }
1960 if (PyUnicode_CheckExact(lhs)) {
1961 if (oparg != Py_EQ && oparg != Py_NE) {
1962 SPECIALIZATION_FAIL(COMPARE_OP, SPEC_FAIL_COMPARE_OP_STRING);
1963 goto failure;
1964 }
1965 else {
1966 _Py_SET_OPCODE(*instr, COMPARE_OP_STR_JUMP);
1967 cache->mask = when_to_jump_mask;
1968 goto success;
1969 }
1970 }
1971 SPECIALIZATION_FAIL(COMPARE_OP, compare_op_fail_kind(lhs, rhs));
1972 failure:
1973 STAT_INC(COMPARE_OP, failure);
1974 cache->counter = adaptive_counter_backoff(cache->counter);
1975 return;
1976 success:
1977 STAT_INC(COMPARE_OP, success);
1978 cache->counter = miss_counter_start();
1979 }
1980
1981 #ifdef Py_STATS
1982 static int
unpack_sequence_fail_kind(PyObject * seq)1983 unpack_sequence_fail_kind(PyObject *seq)
1984 {
1985 if (PySequence_Check(seq)) {
1986 return SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE;
1987 }
1988 if (PyIter_Check(seq)) {
1989 return SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR;
1990 }
1991 return SPEC_FAIL_OTHER;
1992 }
1993 #endif
1994
1995 void
_Py_Specialize_UnpackSequence(PyObject * seq,_Py_CODEUNIT * instr,int oparg)1996 _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
1997 {
1998 assert(_PyOpcode_Caches[UNPACK_SEQUENCE] ==
1999 INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE);
2000 _PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)(instr + 1);
2001 if (PyTuple_CheckExact(seq)) {
2002 if (PyTuple_GET_SIZE(seq) != oparg) {
2003 SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
2004 goto failure;
2005 }
2006 if (PyTuple_GET_SIZE(seq) == 2) {
2007 _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TWO_TUPLE);
2008 goto success;
2009 }
2010 _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_TUPLE);
2011 goto success;
2012 }
2013 if (PyList_CheckExact(seq)) {
2014 if (PyList_GET_SIZE(seq) != oparg) {
2015 SPECIALIZATION_FAIL(UNPACK_SEQUENCE, SPEC_FAIL_EXPECTED_ERROR);
2016 goto failure;
2017 }
2018 _Py_SET_OPCODE(*instr, UNPACK_SEQUENCE_LIST);
2019 goto success;
2020 }
2021 SPECIALIZATION_FAIL(UNPACK_SEQUENCE, unpack_sequence_fail_kind(seq));
2022 failure:
2023 STAT_INC(UNPACK_SEQUENCE, failure);
2024 cache->counter = adaptive_counter_backoff(cache->counter);
2025 return;
2026 success:
2027 STAT_INC(UNPACK_SEQUENCE, success);
2028 cache->counter = miss_counter_start();
2029 }
2030
2031 #ifdef Py_STATS
2032
2033 int
_PySpecialization_ClassifyIterator(PyObject * iter)2034 _PySpecialization_ClassifyIterator(PyObject *iter)
2035 {
2036 if (PyGen_CheckExact(iter)) {
2037 return SPEC_FAIL_FOR_ITER_GENERATOR;
2038 }
2039 if (PyCoro_CheckExact(iter)) {
2040 return SPEC_FAIL_FOR_ITER_COROUTINE;
2041 }
2042 if (PyAsyncGen_CheckExact(iter)) {
2043 return SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR;
2044 }
2045 PyTypeObject *t = Py_TYPE(iter);
2046 if (t == &PyListIter_Type) {
2047 return SPEC_FAIL_FOR_ITER_LIST;
2048 }
2049 if (t == &PyTupleIter_Type) {
2050 return SPEC_FAIL_FOR_ITER_TUPLE;
2051 }
2052 if (t == &PyDictIterKey_Type) {
2053 return SPEC_FAIL_FOR_ITER_DICT_KEYS;
2054 }
2055 if (t == &PyDictIterValue_Type) {
2056 return SPEC_FAIL_FOR_ITER_DICT_VALUES;
2057 }
2058 if (t == &PyDictIterItem_Type) {
2059 return SPEC_FAIL_FOR_ITER_DICT_ITEMS;
2060 }
2061 if (t == &PySetIter_Type) {
2062 return SPEC_FAIL_FOR_ITER_SET;
2063 }
2064 if (t == &PyUnicodeIter_Type) {
2065 return SPEC_FAIL_FOR_ITER_STRING;
2066 }
2067 if (t == &PyBytesIter_Type) {
2068 return SPEC_FAIL_FOR_ITER_BYTES;
2069 }
2070 if (t == &PyRangeIter_Type) {
2071 return SPEC_FAIL_FOR_ITER_RANGE;
2072 }
2073 if (t == &PyEnum_Type) {
2074 return SPEC_FAIL_FOR_ITER_ENUMERATE;
2075 }
2076
2077 if (strncmp(t->tp_name, "itertools", 8) == 0) {
2078 return SPEC_FAIL_FOR_ITER_ITERTOOLS;
2079 }
2080 return SPEC_FAIL_OTHER;
2081 }
2082
2083 #endif
2084