1 #include "Python.h"
2 #include "pycore_ast.h" // identifier, stmt_ty
3 #include "pycore_compile.h" // _Py_Mangle(), _PyFuture_FromAST()
4 #include "pycore_parser.h" // _PyParser_ASTFromString()
5 #include "pycore_pystate.h" // _PyThreadState_GET()
6 #include "pycore_symtable.h" // PySTEntryObject
7 #include "structmember.h" // PyMemberDef
8
9 /* error strings used for warnings */
10 #define GLOBAL_PARAM \
11 "name '%U' is parameter and global"
12
13 #define NONLOCAL_PARAM \
14 "name '%U' is parameter and nonlocal"
15
16 #define GLOBAL_AFTER_ASSIGN \
17 "name '%U' is assigned to before global declaration"
18
19 #define NONLOCAL_AFTER_ASSIGN \
20 "name '%U' is assigned to before nonlocal declaration"
21
22 #define GLOBAL_AFTER_USE \
23 "name '%U' is used prior to global declaration"
24
25 #define NONLOCAL_AFTER_USE \
26 "name '%U' is used prior to nonlocal declaration"
27
28 #define GLOBAL_ANNOT \
29 "annotated name '%U' can't be global"
30
31 #define NONLOCAL_ANNOT \
32 "annotated name '%U' can't be nonlocal"
33
34 #define IMPORT_STAR_WARNING "import * only allowed at module level"
35
36 #define NAMED_EXPR_COMP_IN_CLASS \
37 "assignment expression within a comprehension cannot be used in a class body"
38
39 #define NAMED_EXPR_COMP_CONFLICT \
40 "assignment expression cannot rebind comprehension iteration variable '%U'"
41
42 #define NAMED_EXPR_COMP_INNER_LOOP_CONFLICT \
43 "comprehension inner loop cannot rebind assignment expression target '%U'"
44
45 #define NAMED_EXPR_COMP_ITER_EXPR \
46 "assignment expression cannot be used in a comprehension iterable expression"
47
48 #define ANNOTATION_NOT_ALLOWED \
49 "'%s' can not be used within an annotation"
50
51
52 #define LOCATION(x) \
53 (x)->lineno, (x)->col_offset, (x)->end_lineno, (x)->end_col_offset
54
55 #define ST_LOCATION(x) \
56 (x)->ste_lineno, (x)->ste_col_offset, (x)->ste_end_lineno, (x)->ste_end_col_offset
57
58 static PySTEntryObject *
ste_new(struct symtable * st,identifier name,_Py_block_ty block,void * key,int lineno,int col_offset,int end_lineno,int end_col_offset)59 ste_new(struct symtable *st, identifier name, _Py_block_ty block,
60 void *key, int lineno, int col_offset,
61 int end_lineno, int end_col_offset)
62 {
63 PySTEntryObject *ste = NULL;
64 PyObject *k = NULL;
65
66 k = PyLong_FromVoidPtr(key);
67 if (k == NULL)
68 goto fail;
69 ste = PyObject_New(PySTEntryObject, &PySTEntry_Type);
70 if (ste == NULL) {
71 Py_DECREF(k);
72 goto fail;
73 }
74 ste->ste_table = st;
75 ste->ste_id = k; /* ste owns reference to k */
76
77 Py_INCREF(name);
78 ste->ste_name = name;
79
80 ste->ste_symbols = NULL;
81 ste->ste_varnames = NULL;
82 ste->ste_children = NULL;
83
84 ste->ste_directives = NULL;
85
86 ste->ste_type = block;
87 ste->ste_nested = 0;
88 ste->ste_free = 0;
89 ste->ste_varargs = 0;
90 ste->ste_varkeywords = 0;
91 ste->ste_opt_lineno = 0;
92 ste->ste_opt_col_offset = 0;
93 ste->ste_lineno = lineno;
94 ste->ste_col_offset = col_offset;
95 ste->ste_end_lineno = end_lineno;
96 ste->ste_end_col_offset = end_col_offset;
97
98 if (st->st_cur != NULL &&
99 (st->st_cur->ste_nested ||
100 st->st_cur->ste_type == FunctionBlock))
101 ste->ste_nested = 1;
102 ste->ste_child_free = 0;
103 ste->ste_generator = 0;
104 ste->ste_coroutine = 0;
105 ste->ste_comprehension = NoComprehension;
106 ste->ste_returns_value = 0;
107 ste->ste_needs_class_closure = 0;
108 ste->ste_comp_iter_target = 0;
109 ste->ste_comp_iter_expr = 0;
110
111 ste->ste_symbols = PyDict_New();
112 ste->ste_varnames = PyList_New(0);
113 ste->ste_children = PyList_New(0);
114 if (ste->ste_symbols == NULL
115 || ste->ste_varnames == NULL
116 || ste->ste_children == NULL)
117 goto fail;
118
119 if (PyDict_SetItem(st->st_blocks, ste->ste_id, (PyObject *)ste) < 0)
120 goto fail;
121
122 return ste;
123 fail:
124 Py_XDECREF(ste);
125 return NULL;
126 }
127
128 static PyObject *
ste_repr(PySTEntryObject * ste)129 ste_repr(PySTEntryObject *ste)
130 {
131 return PyUnicode_FromFormat("<symtable entry %U(%ld), line %d>",
132 ste->ste_name,
133 PyLong_AS_LONG(ste->ste_id), ste->ste_lineno);
134 }
135
136 static void
ste_dealloc(PySTEntryObject * ste)137 ste_dealloc(PySTEntryObject *ste)
138 {
139 ste->ste_table = NULL;
140 Py_XDECREF(ste->ste_id);
141 Py_XDECREF(ste->ste_name);
142 Py_XDECREF(ste->ste_symbols);
143 Py_XDECREF(ste->ste_varnames);
144 Py_XDECREF(ste->ste_children);
145 Py_XDECREF(ste->ste_directives);
146 PyObject_Free(ste);
147 }
148
149 #define OFF(x) offsetof(PySTEntryObject, x)
150
151 static PyMemberDef ste_memberlist[] = {
152 {"id", T_OBJECT, OFF(ste_id), READONLY},
153 {"name", T_OBJECT, OFF(ste_name), READONLY},
154 {"symbols", T_OBJECT, OFF(ste_symbols), READONLY},
155 {"varnames", T_OBJECT, OFF(ste_varnames), READONLY},
156 {"children", T_OBJECT, OFF(ste_children), READONLY},
157 {"nested", T_INT, OFF(ste_nested), READONLY},
158 {"type", T_INT, OFF(ste_type), READONLY},
159 {"lineno", T_INT, OFF(ste_lineno), READONLY},
160 {NULL}
161 };
162
163 PyTypeObject PySTEntry_Type = {
164 PyVarObject_HEAD_INIT(&PyType_Type, 0)
165 "symtable entry",
166 sizeof(PySTEntryObject),
167 0,
168 (destructor)ste_dealloc, /* tp_dealloc */
169 0, /* tp_vectorcall_offset */
170 0, /* tp_getattr */
171 0, /* tp_setattr */
172 0, /* tp_as_async */
173 (reprfunc)ste_repr, /* tp_repr */
174 0, /* tp_as_number */
175 0, /* tp_as_sequence */
176 0, /* tp_as_mapping */
177 0, /* tp_hash */
178 0, /* tp_call */
179 0, /* tp_str */
180 PyObject_GenericGetAttr, /* tp_getattro */
181 0, /* tp_setattro */
182 0, /* tp_as_buffer */
183 Py_TPFLAGS_DEFAULT, /* tp_flags */
184 0, /* tp_doc */
185 0, /* tp_traverse */
186 0, /* tp_clear */
187 0, /* tp_richcompare */
188 0, /* tp_weaklistoffset */
189 0, /* tp_iter */
190 0, /* tp_iternext */
191 0, /* tp_methods */
192 ste_memberlist, /* tp_members */
193 0, /* tp_getset */
194 0, /* tp_base */
195 0, /* tp_dict */
196 0, /* tp_descr_get */
197 0, /* tp_descr_set */
198 0, /* tp_dictoffset */
199 0, /* tp_init */
200 0, /* tp_alloc */
201 0, /* tp_new */
202 };
203
204 static int symtable_analyze(struct symtable *st);
205 static int symtable_enter_block(struct symtable *st, identifier name,
206 _Py_block_ty block, void *ast,
207 int lineno, int col_offset,
208 int end_lineno, int end_col_offset);
209 static int symtable_exit_block(struct symtable *st);
210 static int symtable_visit_stmt(struct symtable *st, stmt_ty s);
211 static int symtable_visit_expr(struct symtable *st, expr_ty s);
212 static int symtable_visit_genexp(struct symtable *st, expr_ty s);
213 static int symtable_visit_listcomp(struct symtable *st, expr_ty s);
214 static int symtable_visit_setcomp(struct symtable *st, expr_ty s);
215 static int symtable_visit_dictcomp(struct symtable *st, expr_ty s);
216 static int symtable_visit_arguments(struct symtable *st, arguments_ty);
217 static int symtable_visit_excepthandler(struct symtable *st, excepthandler_ty);
218 static int symtable_visit_alias(struct symtable *st, alias_ty);
219 static int symtable_visit_comprehension(struct symtable *st, comprehension_ty);
220 static int symtable_visit_keyword(struct symtable *st, keyword_ty);
221 static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args);
222 static int symtable_visit_annotation(struct symtable *st, expr_ty annotation);
223 static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args);
224 static int symtable_implicit_arg(struct symtable *st, int pos);
225 static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty);
226 static int symtable_visit_withitem(struct symtable *st, withitem_ty item);
227 static int symtable_visit_match_case(struct symtable *st, match_case_ty m);
228 static int symtable_visit_pattern(struct symtable *st, pattern_ty s);
229 static int symtable_raise_if_annotation_block(struct symtable *st, const char *, expr_ty);
230 static int symtable_raise_if_comprehension_block(struct symtable *st, expr_ty);
231
232
233 #define DUPLICATE_ARGUMENT \
234 "duplicate argument '%U' in function definition"
235
236 static struct symtable *
symtable_new(void)237 symtable_new(void)
238 {
239 struct symtable *st;
240
241 st = (struct symtable *)PyMem_Malloc(sizeof(struct symtable));
242 if (st == NULL) {
243 PyErr_NoMemory();
244 return NULL;
245 }
246
247 st->st_filename = NULL;
248 st->st_blocks = NULL;
249
250 if ((st->st_stack = PyList_New(0)) == NULL)
251 goto fail;
252 if ((st->st_blocks = PyDict_New()) == NULL)
253 goto fail;
254 st->st_cur = NULL;
255 st->st_private = NULL;
256 return st;
257 fail:
258 _PySymtable_Free(st);
259 return NULL;
260 }
261
262 /* When compiling the use of C stack is probably going to be a lot
263 lighter than when executing Python code but still can overflow
264 and causing a Python crash if not checked (e.g. eval("()"*300000)).
265 Using the current recursion limit for the compiler seems too
266 restrictive (it caused at least one test to fail) so a factor is
267 used to allow deeper recursion when compiling an expression.
268
269 Using a scaling factor means this should automatically adjust when
270 the recursion limit is adjusted for small or large C stack allocations.
271 */
272 #define COMPILER_STACK_FRAME_SCALE 3
273
274 struct symtable *
_PySymtable_Build(mod_ty mod,PyObject * filename,PyFutureFeatures * future)275 _PySymtable_Build(mod_ty mod, PyObject *filename, PyFutureFeatures *future)
276 {
277 struct symtable *st = symtable_new();
278 asdl_stmt_seq *seq;
279 int i;
280 PyThreadState *tstate;
281 int recursion_limit = Py_GetRecursionLimit();
282 int starting_recursion_depth;
283
284 if (st == NULL)
285 return NULL;
286 if (filename == NULL) {
287 _PySymtable_Free(st);
288 return NULL;
289 }
290 Py_INCREF(filename);
291 st->st_filename = filename;
292 st->st_future = future;
293
294 /* Setup recursion depth check counters */
295 tstate = _PyThreadState_GET();
296 if (!tstate) {
297 _PySymtable_Free(st);
298 return NULL;
299 }
300 /* Be careful here to prevent overflow. */
301 int recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
302 starting_recursion_depth = (recursion_depth < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
303 recursion_depth * COMPILER_STACK_FRAME_SCALE : recursion_depth;
304 st->recursion_depth = starting_recursion_depth;
305 st->recursion_limit = (recursion_limit < INT_MAX / COMPILER_STACK_FRAME_SCALE) ?
306 recursion_limit * COMPILER_STACK_FRAME_SCALE : recursion_limit;
307
308 /* Make the initial symbol information gathering pass */
309 if (!symtable_enter_block(st, &_Py_ID(top), ModuleBlock, (void *)mod, 0, 0, 0, 0)) {
310 _PySymtable_Free(st);
311 return NULL;
312 }
313
314 st->st_top = st->st_cur;
315 switch (mod->kind) {
316 case Module_kind:
317 seq = mod->v.Module.body;
318 for (i = 0; i < asdl_seq_LEN(seq); i++)
319 if (!symtable_visit_stmt(st,
320 (stmt_ty)asdl_seq_GET(seq, i)))
321 goto error;
322 break;
323 case Expression_kind:
324 if (!symtable_visit_expr(st, mod->v.Expression.body))
325 goto error;
326 break;
327 case Interactive_kind:
328 seq = mod->v.Interactive.body;
329 for (i = 0; i < asdl_seq_LEN(seq); i++)
330 if (!symtable_visit_stmt(st,
331 (stmt_ty)asdl_seq_GET(seq, i)))
332 goto error;
333 break;
334 case FunctionType_kind:
335 PyErr_SetString(PyExc_RuntimeError,
336 "this compiler does not handle FunctionTypes");
337 goto error;
338 }
339 if (!symtable_exit_block(st)) {
340 _PySymtable_Free(st);
341 return NULL;
342 }
343 /* Check that the recursion depth counting balanced correctly */
344 if (st->recursion_depth != starting_recursion_depth) {
345 PyErr_Format(PyExc_SystemError,
346 "symtable analysis recursion depth mismatch (before=%d, after=%d)",
347 starting_recursion_depth, st->recursion_depth);
348 _PySymtable_Free(st);
349 return NULL;
350 }
351 /* Make the second symbol analysis pass */
352 if (symtable_analyze(st))
353 return st;
354 _PySymtable_Free(st);
355 return NULL;
356 error:
357 (void) symtable_exit_block(st);
358 _PySymtable_Free(st);
359 return NULL;
360 }
361
362
363 void
_PySymtable_Free(struct symtable * st)364 _PySymtable_Free(struct symtable *st)
365 {
366 Py_XDECREF(st->st_filename);
367 Py_XDECREF(st->st_blocks);
368 Py_XDECREF(st->st_stack);
369 PyMem_Free((void *)st);
370 }
371
372 PySTEntryObject *
PySymtable_Lookup(struct symtable * st,void * key)373 PySymtable_Lookup(struct symtable *st, void *key)
374 {
375 PyObject *k, *v;
376
377 k = PyLong_FromVoidPtr(key);
378 if (k == NULL)
379 return NULL;
380 v = PyDict_GetItemWithError(st->st_blocks, k);
381 if (v) {
382 assert(PySTEntry_Check(v));
383 Py_INCREF(v);
384 }
385 else if (!PyErr_Occurred()) {
386 PyErr_SetString(PyExc_KeyError,
387 "unknown symbol table entry");
388 }
389
390 Py_DECREF(k);
391 return (PySTEntryObject *)v;
392 }
393
394 long
_PyST_GetSymbol(PySTEntryObject * ste,PyObject * name)395 _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name)
396 {
397 PyObject *v = PyDict_GetItemWithError(ste->ste_symbols, name);
398 if (!v)
399 return 0;
400 assert(PyLong_Check(v));
401 return PyLong_AS_LONG(v);
402 }
403
404 int
_PyST_GetScope(PySTEntryObject * ste,PyObject * name)405 _PyST_GetScope(PySTEntryObject *ste, PyObject *name)
406 {
407 long symbol = _PyST_GetSymbol(ste, name);
408 return (symbol >> SCOPE_OFFSET) & SCOPE_MASK;
409 }
410
411 static int
error_at_directive(PySTEntryObject * ste,PyObject * name)412 error_at_directive(PySTEntryObject *ste, PyObject *name)
413 {
414 Py_ssize_t i;
415 PyObject *data;
416 assert(ste->ste_directives);
417 for (i = 0; i < PyList_GET_SIZE(ste->ste_directives); i++) {
418 data = PyList_GET_ITEM(ste->ste_directives, i);
419 assert(PyTuple_CheckExact(data));
420 assert(PyUnicode_CheckExact(PyTuple_GET_ITEM(data, 0)));
421 if (PyUnicode_Compare(PyTuple_GET_ITEM(data, 0), name) == 0) {
422 PyErr_RangedSyntaxLocationObject(ste->ste_table->st_filename,
423 PyLong_AsLong(PyTuple_GET_ITEM(data, 1)),
424 PyLong_AsLong(PyTuple_GET_ITEM(data, 2)) + 1,
425 PyLong_AsLong(PyTuple_GET_ITEM(data, 3)),
426 PyLong_AsLong(PyTuple_GET_ITEM(data, 4)) + 1);
427
428 return 0;
429 }
430 }
431 PyErr_SetString(PyExc_RuntimeError,
432 "BUG: internal directive bookkeeping broken");
433 return 0;
434 }
435
436
437 /* Analyze raw symbol information to determine scope of each name.
438
439 The next several functions are helpers for symtable_analyze(),
440 which determines whether a name is local, global, or free. In addition,
441 it determines which local variables are cell variables; they provide
442 bindings that are used for free variables in enclosed blocks.
443
444 There are also two kinds of global variables, implicit and explicit. An
445 explicit global is declared with the global statement. An implicit
446 global is a free variable for which the compiler has found no binding
447 in an enclosing function scope. The implicit global is either a global
448 or a builtin. Python's module and class blocks use the xxx_NAME opcodes
449 to handle these names to implement slightly odd semantics. In such a
450 block, the name is treated as global until it is assigned to; then it
451 is treated as a local.
452
453 The symbol table requires two passes to determine the scope of each name.
454 The first pass collects raw facts from the AST via the symtable_visit_*
455 functions: the name is a parameter here, the name is used but not defined
456 here, etc. The second pass analyzes these facts during a pass over the
457 PySTEntryObjects created during pass 1.
458
459 When a function is entered during the second pass, the parent passes
460 the set of all name bindings visible to its children. These bindings
461 are used to determine if non-local variables are free or implicit globals.
462 Names which are explicitly declared nonlocal must exist in this set of
463 visible names - if they do not, a syntax error is raised. After doing
464 the local analysis, it analyzes each of its child blocks using an
465 updated set of name bindings.
466
467 The children update the free variable set. If a local variable is added to
468 the free variable set by the child, the variable is marked as a cell. The
469 function object being defined must provide runtime storage for the variable
470 that may outlive the function's frame. Cell variables are removed from the
471 free set before the analyze function returns to its parent.
472
473 During analysis, the names are:
474 symbols: dict mapping from symbol names to flag values (including offset scope values)
475 scopes: dict mapping from symbol names to scope values (no offset)
476 local: set of all symbol names local to the current scope
477 bound: set of all symbol names local to a containing function scope
478 free: set of all symbol names referenced but not bound in child scopes
479 global: set of all symbol names explicitly declared as global
480 */
481
482 #define SET_SCOPE(DICT, NAME, I) { \
483 PyObject *o = PyLong_FromLong(I); \
484 if (!o) \
485 return 0; \
486 if (PyDict_SetItem((DICT), (NAME), o) < 0) { \
487 Py_DECREF(o); \
488 return 0; \
489 } \
490 Py_DECREF(o); \
491 }
492
493 /* Decide on scope of name, given flags.
494
495 The namespace dictionaries may be modified to record information
496 about the new name. For example, a new global will add an entry to
497 global. A name that was global can be changed to local.
498 */
499
500 static int
analyze_name(PySTEntryObject * ste,PyObject * scopes,PyObject * name,long flags,PyObject * bound,PyObject * local,PyObject * free,PyObject * global)501 analyze_name(PySTEntryObject *ste, PyObject *scopes, PyObject *name, long flags,
502 PyObject *bound, PyObject *local, PyObject *free,
503 PyObject *global)
504 {
505 if (flags & DEF_GLOBAL) {
506 if (flags & DEF_NONLOCAL) {
507 PyErr_Format(PyExc_SyntaxError,
508 "name '%U' is nonlocal and global",
509 name);
510 return error_at_directive(ste, name);
511 }
512 SET_SCOPE(scopes, name, GLOBAL_EXPLICIT);
513 if (PySet_Add(global, name) < 0)
514 return 0;
515 if (bound && (PySet_Discard(bound, name) < 0))
516 return 0;
517 return 1;
518 }
519 if (flags & DEF_NONLOCAL) {
520 if (!bound) {
521 PyErr_Format(PyExc_SyntaxError,
522 "nonlocal declaration not allowed at module level");
523 return error_at_directive(ste, name);
524 }
525 if (!PySet_Contains(bound, name)) {
526 PyErr_Format(PyExc_SyntaxError,
527 "no binding for nonlocal '%U' found",
528 name);
529
530 return error_at_directive(ste, name);
531 }
532 SET_SCOPE(scopes, name, FREE);
533 ste->ste_free = 1;
534 return PySet_Add(free, name) >= 0;
535 }
536 if (flags & DEF_BOUND) {
537 SET_SCOPE(scopes, name, LOCAL);
538 if (PySet_Add(local, name) < 0)
539 return 0;
540 if (PySet_Discard(global, name) < 0)
541 return 0;
542 return 1;
543 }
544 /* If an enclosing block has a binding for this name, it
545 is a free variable rather than a global variable.
546 Note that having a non-NULL bound implies that the block
547 is nested.
548 */
549 if (bound && PySet_Contains(bound, name)) {
550 SET_SCOPE(scopes, name, FREE);
551 ste->ste_free = 1;
552 return PySet_Add(free, name) >= 0;
553 }
554 /* If a parent has a global statement, then call it global
555 explicit? It could also be global implicit.
556 */
557 if (global && PySet_Contains(global, name)) {
558 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
559 return 1;
560 }
561 if (ste->ste_nested)
562 ste->ste_free = 1;
563 SET_SCOPE(scopes, name, GLOBAL_IMPLICIT);
564 return 1;
565 }
566
567 #undef SET_SCOPE
568
569 /* If a name is defined in free and also in locals, then this block
570 provides the binding for the free variable. The name should be
571 marked CELL in this block and removed from the free list.
572
573 Note that the current block's free variables are included in free.
574 That's safe because no name can be free and local in the same scope.
575 */
576
577 static int
analyze_cells(PyObject * scopes,PyObject * free)578 analyze_cells(PyObject *scopes, PyObject *free)
579 {
580 PyObject *name, *v, *v_cell;
581 int success = 0;
582 Py_ssize_t pos = 0;
583
584 v_cell = PyLong_FromLong(CELL);
585 if (!v_cell)
586 return 0;
587 while (PyDict_Next(scopes, &pos, &name, &v)) {
588 long scope;
589 assert(PyLong_Check(v));
590 scope = PyLong_AS_LONG(v);
591 if (scope != LOCAL)
592 continue;
593 if (!PySet_Contains(free, name))
594 continue;
595 /* Replace LOCAL with CELL for this name, and remove
596 from free. It is safe to replace the value of name
597 in the dict, because it will not cause a resize.
598 */
599 if (PyDict_SetItem(scopes, name, v_cell) < 0)
600 goto error;
601 if (PySet_Discard(free, name) < 0)
602 goto error;
603 }
604 success = 1;
605 error:
606 Py_DECREF(v_cell);
607 return success;
608 }
609
610 static int
drop_class_free(PySTEntryObject * ste,PyObject * free)611 drop_class_free(PySTEntryObject *ste, PyObject *free)
612 {
613 int res;
614 res = PySet_Discard(free, &_Py_ID(__class__));
615 if (res < 0)
616 return 0;
617 if (res)
618 ste->ste_needs_class_closure = 1;
619 return 1;
620 }
621
622 /* Enter the final scope information into the ste_symbols dict.
623 *
624 * All arguments are dicts. Modifies symbols, others are read-only.
625 */
626 static int
update_symbols(PyObject * symbols,PyObject * scopes,PyObject * bound,PyObject * free,int classflag)627 update_symbols(PyObject *symbols, PyObject *scopes,
628 PyObject *bound, PyObject *free, int classflag)
629 {
630 PyObject *name = NULL, *itr = NULL;
631 PyObject *v = NULL, *v_scope = NULL, *v_new = NULL, *v_free = NULL;
632 Py_ssize_t pos = 0;
633
634 /* Update scope information for all symbols in this scope */
635 while (PyDict_Next(symbols, &pos, &name, &v)) {
636 long scope, flags;
637 assert(PyLong_Check(v));
638 flags = PyLong_AS_LONG(v);
639 v_scope = PyDict_GetItemWithError(scopes, name);
640 assert(v_scope && PyLong_Check(v_scope));
641 scope = PyLong_AS_LONG(v_scope);
642 flags |= (scope << SCOPE_OFFSET);
643 v_new = PyLong_FromLong(flags);
644 if (!v_new)
645 return 0;
646 if (PyDict_SetItem(symbols, name, v_new) < 0) {
647 Py_DECREF(v_new);
648 return 0;
649 }
650 Py_DECREF(v_new);
651 }
652
653 /* Record not yet resolved free variables from children (if any) */
654 v_free = PyLong_FromLong(FREE << SCOPE_OFFSET);
655 if (!v_free)
656 return 0;
657
658 itr = PyObject_GetIter(free);
659 if (itr == NULL) {
660 Py_DECREF(v_free);
661 return 0;
662 }
663
664 while ((name = PyIter_Next(itr))) {
665 v = PyDict_GetItemWithError(symbols, name);
666
667 /* Handle symbol that already exists in this scope */
668 if (v) {
669 /* Handle a free variable in a method of
670 the class that has the same name as a local
671 or global in the class scope.
672 */
673 if (classflag &&
674 PyLong_AS_LONG(v) & (DEF_BOUND | DEF_GLOBAL)) {
675 long flags = PyLong_AS_LONG(v) | DEF_FREE_CLASS;
676 v_new = PyLong_FromLong(flags);
677 if (!v_new) {
678 goto error;
679 }
680 if (PyDict_SetItem(symbols, name, v_new) < 0) {
681 Py_DECREF(v_new);
682 goto error;
683 }
684 Py_DECREF(v_new);
685 }
686 /* It's a cell, or already free in this scope */
687 Py_DECREF(name);
688 continue;
689 }
690 else if (PyErr_Occurred()) {
691 goto error;
692 }
693 /* Handle global symbol */
694 if (bound && !PySet_Contains(bound, name)) {
695 Py_DECREF(name);
696 continue; /* it's a global */
697 }
698 /* Propagate new free symbol up the lexical stack */
699 if (PyDict_SetItem(symbols, name, v_free) < 0) {
700 goto error;
701 }
702 Py_DECREF(name);
703 }
704 Py_DECREF(itr);
705 Py_DECREF(v_free);
706 return 1;
707 error:
708 Py_XDECREF(v_free);
709 Py_XDECREF(itr);
710 Py_XDECREF(name);
711 return 0;
712 }
713
714 /* Make final symbol table decisions for block of ste.
715
716 Arguments:
717 ste -- current symtable entry (input/output)
718 bound -- set of variables bound in enclosing scopes (input). bound
719 is NULL for module blocks.
720 free -- set of free variables in enclosed scopes (output)
721 globals -- set of declared global variables in enclosing scopes (input)
722
723 The implementation uses two mutually recursive functions,
724 analyze_block() and analyze_child_block(). analyze_block() is
725 responsible for analyzing the individual names defined in a block.
726 analyze_child_block() prepares temporary namespace dictionaries
727 used to evaluated nested blocks.
728
729 The two functions exist because a child block should see the name
730 bindings of its enclosing blocks, but those bindings should not
731 propagate back to a parent block.
732 */
733
734 static int
735 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
736 PyObject *global, PyObject* child_free);
737
738 static int
analyze_block(PySTEntryObject * ste,PyObject * bound,PyObject * free,PyObject * global)739 analyze_block(PySTEntryObject *ste, PyObject *bound, PyObject *free,
740 PyObject *global)
741 {
742 PyObject *name, *v, *local = NULL, *scopes = NULL, *newbound = NULL;
743 PyObject *newglobal = NULL, *newfree = NULL, *allfree = NULL;
744 PyObject *temp;
745 int i, success = 0;
746 Py_ssize_t pos = 0;
747
748 local = PySet_New(NULL); /* collect new names bound in block */
749 if (!local)
750 goto error;
751 scopes = PyDict_New(); /* collect scopes defined for each name */
752 if (!scopes)
753 goto error;
754
755 /* Allocate new global and bound variable dictionaries. These
756 dictionaries hold the names visible in nested blocks. For
757 ClassBlocks, the bound and global names are initialized
758 before analyzing names, because class bindings aren't
759 visible in methods. For other blocks, they are initialized
760 after names are analyzed.
761 */
762
763 /* TODO(jhylton): Package these dicts in a struct so that we
764 can write reasonable helper functions?
765 */
766 newglobal = PySet_New(NULL);
767 if (!newglobal)
768 goto error;
769 newfree = PySet_New(NULL);
770 if (!newfree)
771 goto error;
772 newbound = PySet_New(NULL);
773 if (!newbound)
774 goto error;
775
776 /* Class namespace has no effect on names visible in
777 nested functions, so populate the global and bound
778 sets to be passed to child blocks before analyzing
779 this one.
780 */
781 if (ste->ste_type == ClassBlock) {
782 /* Pass down known globals */
783 temp = PyNumber_InPlaceOr(newglobal, global);
784 if (!temp)
785 goto error;
786 Py_DECREF(temp);
787 /* Pass down previously bound symbols */
788 if (bound) {
789 temp = PyNumber_InPlaceOr(newbound, bound);
790 if (!temp)
791 goto error;
792 Py_DECREF(temp);
793 }
794 }
795
796 while (PyDict_Next(ste->ste_symbols, &pos, &name, &v)) {
797 long flags = PyLong_AS_LONG(v);
798 if (!analyze_name(ste, scopes, name, flags,
799 bound, local, free, global))
800 goto error;
801 }
802
803 /* Populate global and bound sets to be passed to children. */
804 if (ste->ste_type != ClassBlock) {
805 /* Add function locals to bound set */
806 if (ste->ste_type == FunctionBlock) {
807 temp = PyNumber_InPlaceOr(newbound, local);
808 if (!temp)
809 goto error;
810 Py_DECREF(temp);
811 }
812 /* Pass down previously bound symbols */
813 if (bound) {
814 temp = PyNumber_InPlaceOr(newbound, bound);
815 if (!temp)
816 goto error;
817 Py_DECREF(temp);
818 }
819 /* Pass down known globals */
820 temp = PyNumber_InPlaceOr(newglobal, global);
821 if (!temp)
822 goto error;
823 Py_DECREF(temp);
824 }
825 else {
826 /* Special-case __class__ */
827 if (PySet_Add(newbound, &_Py_ID(__class__)) < 0)
828 goto error;
829 }
830
831 /* Recursively call analyze_child_block() on each child block.
832
833 newbound, newglobal now contain the names visible in
834 nested blocks. The free variables in the children will
835 be collected in allfree.
836 */
837 allfree = PySet_New(NULL);
838 if (!allfree)
839 goto error;
840 for (i = 0; i < PyList_GET_SIZE(ste->ste_children); ++i) {
841 PyObject *c = PyList_GET_ITEM(ste->ste_children, i);
842 PySTEntryObject* entry;
843 assert(c && PySTEntry_Check(c));
844 entry = (PySTEntryObject*)c;
845 if (!analyze_child_block(entry, newbound, newfree, newglobal,
846 allfree))
847 goto error;
848 /* Check if any children have free variables */
849 if (entry->ste_free || entry->ste_child_free)
850 ste->ste_child_free = 1;
851 }
852
853 temp = PyNumber_InPlaceOr(newfree, allfree);
854 if (!temp)
855 goto error;
856 Py_DECREF(temp);
857
858 /* Check if any local variables must be converted to cell variables */
859 if (ste->ste_type == FunctionBlock && !analyze_cells(scopes, newfree))
860 goto error;
861 else if (ste->ste_type == ClassBlock && !drop_class_free(ste, newfree))
862 goto error;
863 /* Records the results of the analysis in the symbol table entry */
864 if (!update_symbols(ste->ste_symbols, scopes, bound, newfree,
865 ste->ste_type == ClassBlock))
866 goto error;
867
868 temp = PyNumber_InPlaceOr(free, newfree);
869 if (!temp)
870 goto error;
871 Py_DECREF(temp);
872 success = 1;
873 error:
874 Py_XDECREF(scopes);
875 Py_XDECREF(local);
876 Py_XDECREF(newbound);
877 Py_XDECREF(newglobal);
878 Py_XDECREF(newfree);
879 Py_XDECREF(allfree);
880 if (!success)
881 assert(PyErr_Occurred());
882 return success;
883 }
884
885 static int
analyze_child_block(PySTEntryObject * entry,PyObject * bound,PyObject * free,PyObject * global,PyObject * child_free)886 analyze_child_block(PySTEntryObject *entry, PyObject *bound, PyObject *free,
887 PyObject *global, PyObject* child_free)
888 {
889 PyObject *temp_bound = NULL, *temp_global = NULL, *temp_free = NULL;
890 PyObject *temp;
891
892 /* Copy the bound and global dictionaries.
893
894 These dictionaries are used by all blocks enclosed by the
895 current block. The analyze_block() call modifies these
896 dictionaries.
897
898 */
899 temp_bound = PySet_New(bound);
900 if (!temp_bound)
901 goto error;
902 temp_free = PySet_New(free);
903 if (!temp_free)
904 goto error;
905 temp_global = PySet_New(global);
906 if (!temp_global)
907 goto error;
908
909 if (!analyze_block(entry, temp_bound, temp_free, temp_global))
910 goto error;
911 temp = PyNumber_InPlaceOr(child_free, temp_free);
912 if (!temp)
913 goto error;
914 Py_DECREF(temp);
915 Py_DECREF(temp_bound);
916 Py_DECREF(temp_free);
917 Py_DECREF(temp_global);
918 return 1;
919 error:
920 Py_XDECREF(temp_bound);
921 Py_XDECREF(temp_free);
922 Py_XDECREF(temp_global);
923 return 0;
924 }
925
926 static int
symtable_analyze(struct symtable * st)927 symtable_analyze(struct symtable *st)
928 {
929 PyObject *free, *global;
930 int r;
931
932 free = PySet_New(NULL);
933 if (!free)
934 return 0;
935 global = PySet_New(NULL);
936 if (!global) {
937 Py_DECREF(free);
938 return 0;
939 }
940 r = analyze_block(st->st_top, NULL, free, global);
941 Py_DECREF(free);
942 Py_DECREF(global);
943 return r;
944 }
945
946 /* symtable_enter_block() gets a reference via ste_new.
947 This reference is released when the block is exited, via the DECREF
948 in symtable_exit_block().
949 */
950
951 static int
symtable_exit_block(struct symtable * st)952 symtable_exit_block(struct symtable *st)
953 {
954 Py_ssize_t size;
955
956 st->st_cur = NULL;
957 size = PyList_GET_SIZE(st->st_stack);
958 if (size) {
959 if (PyList_SetSlice(st->st_stack, size - 1, size, NULL) < 0)
960 return 0;
961 if (--size)
962 st->st_cur = (PySTEntryObject *)PyList_GET_ITEM(st->st_stack, size - 1);
963 }
964 return 1;
965 }
966
967 static int
symtable_enter_block(struct symtable * st,identifier name,_Py_block_ty block,void * ast,int lineno,int col_offset,int end_lineno,int end_col_offset)968 symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block,
969 void *ast, int lineno, int col_offset,
970 int end_lineno, int end_col_offset)
971 {
972 PySTEntryObject *prev = NULL, *ste;
973
974 ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset);
975 if (ste == NULL)
976 return 0;
977 if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) {
978 Py_DECREF(ste);
979 return 0;
980 }
981 prev = st->st_cur;
982 /* bpo-37757: For now, disallow *all* assignment expressions in the
983 * outermost iterator expression of a comprehension, even those inside
984 * a nested comprehension or a lambda expression.
985 */
986 if (prev) {
987 ste->ste_comp_iter_expr = prev->ste_comp_iter_expr;
988 }
989 /* The entry is owned by the stack. Borrow it for st_cur. */
990 Py_DECREF(ste);
991 st->st_cur = ste;
992
993 /* Annotation blocks shouldn't have any affect on the symbol table since in
994 * the compilation stage, they will all be transformed to strings. They are
995 * only created if future 'annotations' feature is activated. */
996 if (block == AnnotationBlock) {
997 return 1;
998 }
999
1000 if (block == ModuleBlock)
1001 st->st_global = st->st_cur->ste_symbols;
1002
1003 if (prev) {
1004 if (PyList_Append(prev->ste_children, (PyObject *)ste) < 0) {
1005 return 0;
1006 }
1007 }
1008 return 1;
1009 }
1010
1011 static long
symtable_lookup(struct symtable * st,PyObject * name)1012 symtable_lookup(struct symtable *st, PyObject *name)
1013 {
1014 PyObject *mangled = _Py_Mangle(st->st_private, name);
1015 if (!mangled)
1016 return 0;
1017 long ret = _PyST_GetSymbol(st->st_cur, mangled);
1018 Py_DECREF(mangled);
1019 return ret;
1020 }
1021
1022 static int
symtable_add_def_helper(struct symtable * st,PyObject * name,int flag,struct _symtable_entry * ste,int lineno,int col_offset,int end_lineno,int end_col_offset)1023 symtable_add_def_helper(struct symtable *st, PyObject *name, int flag, struct _symtable_entry *ste,
1024 int lineno, int col_offset, int end_lineno, int end_col_offset)
1025 {
1026 PyObject *o;
1027 PyObject *dict;
1028 long val;
1029 PyObject *mangled = _Py_Mangle(st->st_private, name);
1030
1031
1032 if (!mangled)
1033 return 0;
1034 dict = ste->ste_symbols;
1035 if ((o = PyDict_GetItemWithError(dict, mangled))) {
1036 val = PyLong_AS_LONG(o);
1037 if ((flag & DEF_PARAM) && (val & DEF_PARAM)) {
1038 /* Is it better to use 'mangled' or 'name' here? */
1039 PyErr_Format(PyExc_SyntaxError, DUPLICATE_ARGUMENT, name);
1040 PyErr_RangedSyntaxLocationObject(st->st_filename,
1041 lineno, col_offset + 1,
1042 end_lineno, end_col_offset + 1);
1043 goto error;
1044 }
1045 val |= flag;
1046 }
1047 else if (PyErr_Occurred()) {
1048 goto error;
1049 }
1050 else {
1051 val = flag;
1052 }
1053 if (ste->ste_comp_iter_target) {
1054 /* This name is an iteration variable in a comprehension,
1055 * so check for a binding conflict with any named expressions.
1056 * Otherwise, mark it as an iteration variable so subsequent
1057 * named expressions can check for conflicts.
1058 */
1059 if (val & (DEF_GLOBAL | DEF_NONLOCAL)) {
1060 PyErr_Format(PyExc_SyntaxError,
1061 NAMED_EXPR_COMP_INNER_LOOP_CONFLICT, name);
1062 PyErr_RangedSyntaxLocationObject(st->st_filename,
1063 lineno, col_offset + 1,
1064 end_lineno, end_col_offset + 1);
1065 goto error;
1066 }
1067 val |= DEF_COMP_ITER;
1068 }
1069 o = PyLong_FromLong(val);
1070 if (o == NULL)
1071 goto error;
1072 if (PyDict_SetItem(dict, mangled, o) < 0) {
1073 Py_DECREF(o);
1074 goto error;
1075 }
1076 Py_DECREF(o);
1077
1078 if (flag & DEF_PARAM) {
1079 if (PyList_Append(ste->ste_varnames, mangled) < 0)
1080 goto error;
1081 } else if (flag & DEF_GLOBAL) {
1082 /* XXX need to update DEF_GLOBAL for other flags too;
1083 perhaps only DEF_FREE_GLOBAL */
1084 val = flag;
1085 if ((o = PyDict_GetItemWithError(st->st_global, mangled))) {
1086 val |= PyLong_AS_LONG(o);
1087 }
1088 else if (PyErr_Occurred()) {
1089 goto error;
1090 }
1091 o = PyLong_FromLong(val);
1092 if (o == NULL)
1093 goto error;
1094 if (PyDict_SetItem(st->st_global, mangled, o) < 0) {
1095 Py_DECREF(o);
1096 goto error;
1097 }
1098 Py_DECREF(o);
1099 }
1100 Py_DECREF(mangled);
1101 return 1;
1102
1103 error:
1104 Py_DECREF(mangled);
1105 return 0;
1106 }
1107
1108 static int
symtable_add_def(struct symtable * st,PyObject * name,int flag,int lineno,int col_offset,int end_lineno,int end_col_offset)1109 symtable_add_def(struct symtable *st, PyObject *name, int flag,
1110 int lineno, int col_offset, int end_lineno, int end_col_offset)
1111 {
1112 return symtable_add_def_helper(st, name, flag, st->st_cur,
1113 lineno, col_offset, end_lineno, end_col_offset);
1114 }
1115
1116 /* VISIT, VISIT_SEQ and VIST_SEQ_TAIL take an ASDL type as their second argument.
1117 They use the ASDL name to synthesize the name of the C type and the visit
1118 function.
1119
1120 VISIT_SEQ_TAIL permits the start of an ASDL sequence to be skipped, which is
1121 useful if the first node in the sequence requires special treatment.
1122
1123 VISIT_QUIT macro returns the specified value exiting from the function but
1124 first adjusts current recursion counter depth.
1125 */
1126
1127 #define VISIT_QUIT(ST, X) \
1128 return --(ST)->recursion_depth,(X)
1129
1130 #define VISIT(ST, TYPE, V) \
1131 if (!symtable_visit_ ## TYPE((ST), (V))) \
1132 VISIT_QUIT((ST), 0);
1133
1134 #define VISIT_SEQ(ST, TYPE, SEQ) { \
1135 int i; \
1136 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1137 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1138 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1139 if (!symtable_visit_ ## TYPE((ST), elt)) \
1140 VISIT_QUIT((ST), 0); \
1141 } \
1142 }
1143
1144 #define VISIT_SEQ_TAIL(ST, TYPE, SEQ, START) { \
1145 int i; \
1146 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1147 for (i = (START); i < asdl_seq_LEN(seq); i++) { \
1148 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1149 if (!symtable_visit_ ## TYPE((ST), elt)) \
1150 VISIT_QUIT((ST), 0); \
1151 } \
1152 }
1153
1154 #define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) { \
1155 int i = 0; \
1156 asdl_ ## TYPE ## _seq *seq = (SEQ); /* avoid variable capture */ \
1157 for (i = 0; i < asdl_seq_LEN(seq); i++) { \
1158 TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
1159 if (!elt) continue; /* can be NULL */ \
1160 if (!symtable_visit_ ## TYPE((ST), elt)) \
1161 VISIT_QUIT((ST), 0); \
1162 } \
1163 }
1164
1165 static int
symtable_record_directive(struct symtable * st,identifier name,int lineno,int col_offset,int end_lineno,int end_col_offset)1166 symtable_record_directive(struct symtable *st, identifier name, int lineno,
1167 int col_offset, int end_lineno, int end_col_offset)
1168 {
1169 PyObject *data, *mangled;
1170 int res;
1171 if (!st->st_cur->ste_directives) {
1172 st->st_cur->ste_directives = PyList_New(0);
1173 if (!st->st_cur->ste_directives)
1174 return 0;
1175 }
1176 mangled = _Py_Mangle(st->st_private, name);
1177 if (!mangled)
1178 return 0;
1179 data = Py_BuildValue("(Niiii)", mangled, lineno, col_offset, end_lineno, end_col_offset);
1180 if (!data)
1181 return 0;
1182 res = PyList_Append(st->st_cur->ste_directives, data);
1183 Py_DECREF(data);
1184 return res == 0;
1185 }
1186
1187
1188 static int
symtable_visit_stmt(struct symtable * st,stmt_ty s)1189 symtable_visit_stmt(struct symtable *st, stmt_ty s)
1190 {
1191 if (++st->recursion_depth > st->recursion_limit) {
1192 PyErr_SetString(PyExc_RecursionError,
1193 "maximum recursion depth exceeded during compilation");
1194 VISIT_QUIT(st, 0);
1195 }
1196 switch (s->kind) {
1197 case FunctionDef_kind:
1198 if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s)))
1199 VISIT_QUIT(st, 0);
1200 if (s->v.FunctionDef.args->defaults)
1201 VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
1202 if (s->v.FunctionDef.args->kw_defaults)
1203 VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
1204 if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args,
1205 s->v.FunctionDef.returns))
1206 VISIT_QUIT(st, 0);
1207 if (s->v.FunctionDef.decorator_list)
1208 VISIT_SEQ(st, expr, s->v.FunctionDef.decorator_list);
1209 if (!symtable_enter_block(st, s->v.FunctionDef.name,
1210 FunctionBlock, (void *)s,
1211 LOCATION(s)))
1212 VISIT_QUIT(st, 0);
1213 VISIT(st, arguments, s->v.FunctionDef.args);
1214 VISIT_SEQ(st, stmt, s->v.FunctionDef.body);
1215 if (!symtable_exit_block(st))
1216 VISIT_QUIT(st, 0);
1217 break;
1218 case ClassDef_kind: {
1219 PyObject *tmp;
1220 if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s)))
1221 VISIT_QUIT(st, 0);
1222 VISIT_SEQ(st, expr, s->v.ClassDef.bases);
1223 VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
1224 if (s->v.ClassDef.decorator_list)
1225 VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
1226 if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
1227 (void *)s, s->lineno, s->col_offset,
1228 s->end_lineno, s->end_col_offset))
1229 VISIT_QUIT(st, 0);
1230 tmp = st->st_private;
1231 st->st_private = s->v.ClassDef.name;
1232 VISIT_SEQ(st, stmt, s->v.ClassDef.body);
1233 st->st_private = tmp;
1234 if (!symtable_exit_block(st))
1235 VISIT_QUIT(st, 0);
1236 break;
1237 }
1238 case Return_kind:
1239 if (s->v.Return.value) {
1240 VISIT(st, expr, s->v.Return.value);
1241 st->st_cur->ste_returns_value = 1;
1242 }
1243 break;
1244 case Delete_kind:
1245 VISIT_SEQ(st, expr, s->v.Delete.targets);
1246 break;
1247 case Assign_kind:
1248 VISIT_SEQ(st, expr, s->v.Assign.targets);
1249 VISIT(st, expr, s->v.Assign.value);
1250 break;
1251 case AnnAssign_kind:
1252 if (s->v.AnnAssign.target->kind == Name_kind) {
1253 expr_ty e_name = s->v.AnnAssign.target;
1254 long cur = symtable_lookup(st, e_name->v.Name.id);
1255 if (cur < 0) {
1256 VISIT_QUIT(st, 0);
1257 }
1258 if ((cur & (DEF_GLOBAL | DEF_NONLOCAL))
1259 && (st->st_cur->ste_symbols != st->st_global)
1260 && s->v.AnnAssign.simple) {
1261 PyErr_Format(PyExc_SyntaxError,
1262 cur & DEF_GLOBAL ? GLOBAL_ANNOT : NONLOCAL_ANNOT,
1263 e_name->v.Name.id);
1264 PyErr_RangedSyntaxLocationObject(st->st_filename,
1265 s->lineno,
1266 s->col_offset + 1,
1267 s->end_lineno,
1268 s->end_col_offset + 1);
1269 VISIT_QUIT(st, 0);
1270 }
1271 if (s->v.AnnAssign.simple &&
1272 !symtable_add_def(st, e_name->v.Name.id,
1273 DEF_ANNOT | DEF_LOCAL, LOCATION(e_name))) {
1274 VISIT_QUIT(st, 0);
1275 }
1276 else {
1277 if (s->v.AnnAssign.value
1278 && !symtable_add_def(st, e_name->v.Name.id, DEF_LOCAL, LOCATION(e_name))) {
1279 VISIT_QUIT(st, 0);
1280 }
1281 }
1282 }
1283 else {
1284 VISIT(st, expr, s->v.AnnAssign.target);
1285 }
1286 if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) {
1287 VISIT_QUIT(st, 0);
1288 }
1289
1290 if (s->v.AnnAssign.value) {
1291 VISIT(st, expr, s->v.AnnAssign.value);
1292 }
1293 break;
1294 case AugAssign_kind:
1295 VISIT(st, expr, s->v.AugAssign.target);
1296 VISIT(st, expr, s->v.AugAssign.value);
1297 break;
1298 case For_kind:
1299 VISIT(st, expr, s->v.For.target);
1300 VISIT(st, expr, s->v.For.iter);
1301 VISIT_SEQ(st, stmt, s->v.For.body);
1302 if (s->v.For.orelse)
1303 VISIT_SEQ(st, stmt, s->v.For.orelse);
1304 break;
1305 case While_kind:
1306 VISIT(st, expr, s->v.While.test);
1307 VISIT_SEQ(st, stmt, s->v.While.body);
1308 if (s->v.While.orelse)
1309 VISIT_SEQ(st, stmt, s->v.While.orelse);
1310 break;
1311 case If_kind:
1312 /* XXX if 0: and lookup_yield() hacks */
1313 VISIT(st, expr, s->v.If.test);
1314 VISIT_SEQ(st, stmt, s->v.If.body);
1315 if (s->v.If.orelse)
1316 VISIT_SEQ(st, stmt, s->v.If.orelse);
1317 break;
1318 case Match_kind:
1319 VISIT(st, expr, s->v.Match.subject);
1320 VISIT_SEQ(st, match_case, s->v.Match.cases);
1321 break;
1322 case Raise_kind:
1323 if (s->v.Raise.exc) {
1324 VISIT(st, expr, s->v.Raise.exc);
1325 if (s->v.Raise.cause) {
1326 VISIT(st, expr, s->v.Raise.cause);
1327 }
1328 }
1329 break;
1330 case Try_kind:
1331 VISIT_SEQ(st, stmt, s->v.Try.body);
1332 VISIT_SEQ(st, stmt, s->v.Try.orelse);
1333 VISIT_SEQ(st, excepthandler, s->v.Try.handlers);
1334 VISIT_SEQ(st, stmt, s->v.Try.finalbody);
1335 break;
1336 case TryStar_kind:
1337 VISIT_SEQ(st, stmt, s->v.TryStar.body);
1338 VISIT_SEQ(st, stmt, s->v.TryStar.orelse);
1339 VISIT_SEQ(st, excepthandler, s->v.TryStar.handlers);
1340 VISIT_SEQ(st, stmt, s->v.TryStar.finalbody);
1341 break;
1342 case Assert_kind:
1343 VISIT(st, expr, s->v.Assert.test);
1344 if (s->v.Assert.msg)
1345 VISIT(st, expr, s->v.Assert.msg);
1346 break;
1347 case Import_kind:
1348 VISIT_SEQ(st, alias, s->v.Import.names);
1349 break;
1350 case ImportFrom_kind:
1351 VISIT_SEQ(st, alias, s->v.ImportFrom.names);
1352 break;
1353 case Global_kind: {
1354 int i;
1355 asdl_identifier_seq *seq = s->v.Global.names;
1356 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1357 identifier name = (identifier)asdl_seq_GET(seq, i);
1358 long cur = symtable_lookup(st, name);
1359 if (cur < 0)
1360 VISIT_QUIT(st, 0);
1361 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1362 const char* msg;
1363 if (cur & DEF_PARAM) {
1364 msg = GLOBAL_PARAM;
1365 } else if (cur & USE) {
1366 msg = GLOBAL_AFTER_USE;
1367 } else if (cur & DEF_ANNOT) {
1368 msg = GLOBAL_ANNOT;
1369 } else { /* DEF_LOCAL */
1370 msg = GLOBAL_AFTER_ASSIGN;
1371 }
1372 PyErr_Format(PyExc_SyntaxError,
1373 msg, name);
1374 PyErr_RangedSyntaxLocationObject(st->st_filename,
1375 s->lineno,
1376 s->col_offset + 1,
1377 s->end_lineno,
1378 s->end_col_offset + 1);
1379 VISIT_QUIT(st, 0);
1380 }
1381 if (!symtable_add_def(st, name, DEF_GLOBAL, LOCATION(s)))
1382 VISIT_QUIT(st, 0);
1383 if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1384 s->end_lineno, s->end_col_offset))
1385 VISIT_QUIT(st, 0);
1386 }
1387 break;
1388 }
1389 case Nonlocal_kind: {
1390 int i;
1391 asdl_identifier_seq *seq = s->v.Nonlocal.names;
1392 for (i = 0; i < asdl_seq_LEN(seq); i++) {
1393 identifier name = (identifier)asdl_seq_GET(seq, i);
1394 long cur = symtable_lookup(st, name);
1395 if (cur < 0)
1396 VISIT_QUIT(st, 0);
1397 if (cur & (DEF_PARAM | DEF_LOCAL | USE | DEF_ANNOT)) {
1398 const char* msg;
1399 if (cur & DEF_PARAM) {
1400 msg = NONLOCAL_PARAM;
1401 } else if (cur & USE) {
1402 msg = NONLOCAL_AFTER_USE;
1403 } else if (cur & DEF_ANNOT) {
1404 msg = NONLOCAL_ANNOT;
1405 } else { /* DEF_LOCAL */
1406 msg = NONLOCAL_AFTER_ASSIGN;
1407 }
1408 PyErr_Format(PyExc_SyntaxError, msg, name);
1409 PyErr_RangedSyntaxLocationObject(st->st_filename,
1410 s->lineno,
1411 s->col_offset + 1,
1412 s->end_lineno,
1413 s->end_col_offset + 1);
1414 VISIT_QUIT(st, 0);
1415 }
1416 if (!symtable_add_def(st, name, DEF_NONLOCAL, LOCATION(s)))
1417 VISIT_QUIT(st, 0);
1418 if (!symtable_record_directive(st, name, s->lineno, s->col_offset,
1419 s->end_lineno, s->end_col_offset))
1420 VISIT_QUIT(st, 0);
1421 }
1422 break;
1423 }
1424 case Expr_kind:
1425 VISIT(st, expr, s->v.Expr.value);
1426 break;
1427 case Pass_kind:
1428 case Break_kind:
1429 case Continue_kind:
1430 /* nothing to do here */
1431 break;
1432 case With_kind:
1433 VISIT_SEQ(st, withitem, s->v.With.items);
1434 VISIT_SEQ(st, stmt, s->v.With.body);
1435 break;
1436 case AsyncFunctionDef_kind:
1437 if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s)))
1438 VISIT_QUIT(st, 0);
1439 if (s->v.AsyncFunctionDef.args->defaults)
1440 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.args->defaults);
1441 if (s->v.AsyncFunctionDef.args->kw_defaults)
1442 VISIT_SEQ_WITH_NULL(st, expr,
1443 s->v.AsyncFunctionDef.args->kw_defaults);
1444 if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args,
1445 s->v.AsyncFunctionDef.returns))
1446 VISIT_QUIT(st, 0);
1447 if (s->v.AsyncFunctionDef.decorator_list)
1448 VISIT_SEQ(st, expr, s->v.AsyncFunctionDef.decorator_list);
1449 if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name,
1450 FunctionBlock, (void *)s,
1451 s->lineno, s->col_offset,
1452 s->end_lineno, s->end_col_offset))
1453 VISIT_QUIT(st, 0);
1454 st->st_cur->ste_coroutine = 1;
1455 VISIT(st, arguments, s->v.AsyncFunctionDef.args);
1456 VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body);
1457 if (!symtable_exit_block(st))
1458 VISIT_QUIT(st, 0);
1459 break;
1460 case AsyncWith_kind:
1461 VISIT_SEQ(st, withitem, s->v.AsyncWith.items);
1462 VISIT_SEQ(st, stmt, s->v.AsyncWith.body);
1463 break;
1464 case AsyncFor_kind:
1465 VISIT(st, expr, s->v.AsyncFor.target);
1466 VISIT(st, expr, s->v.AsyncFor.iter);
1467 VISIT_SEQ(st, stmt, s->v.AsyncFor.body);
1468 if (s->v.AsyncFor.orelse)
1469 VISIT_SEQ(st, stmt, s->v.AsyncFor.orelse);
1470 break;
1471 }
1472 VISIT_QUIT(st, 1);
1473 }
1474
1475 static int
symtable_extend_namedexpr_scope(struct symtable * st,expr_ty e)1476 symtable_extend_namedexpr_scope(struct symtable *st, expr_ty e)
1477 {
1478 assert(st->st_stack);
1479 assert(e->kind == Name_kind);
1480
1481 PyObject *target_name = e->v.Name.id;
1482 Py_ssize_t i, size;
1483 struct _symtable_entry *ste;
1484 size = PyList_GET_SIZE(st->st_stack);
1485 assert(size);
1486
1487 /* Iterate over the stack in reverse and add to the nearest adequate scope */
1488 for (i = size - 1; i >= 0; i--) {
1489 ste = (struct _symtable_entry *) PyList_GET_ITEM(st->st_stack, i);
1490
1491 /* If we find a comprehension scope, check for a target
1492 * binding conflict with iteration variables, otherwise skip it
1493 */
1494 if (ste->ste_comprehension) {
1495 long target_in_scope = _PyST_GetSymbol(ste, target_name);
1496 if (target_in_scope & DEF_COMP_ITER) {
1497 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_CONFLICT, target_name);
1498 PyErr_RangedSyntaxLocationObject(st->st_filename,
1499 e->lineno,
1500 e->col_offset + 1,
1501 e->end_lineno,
1502 e->end_col_offset + 1);
1503 VISIT_QUIT(st, 0);
1504 }
1505 continue;
1506 }
1507
1508 /* If we find a FunctionBlock entry, add as GLOBAL/LOCAL or NONLOCAL/LOCAL */
1509 if (ste->ste_type == FunctionBlock) {
1510 long target_in_scope = _PyST_GetSymbol(ste, target_name);
1511 if (target_in_scope & DEF_GLOBAL) {
1512 if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
1513 VISIT_QUIT(st, 0);
1514 } else {
1515 if (!symtable_add_def(st, target_name, DEF_NONLOCAL, LOCATION(e)))
1516 VISIT_QUIT(st, 0);
1517 }
1518 if (!symtable_record_directive(st, target_name, LOCATION(e)))
1519 VISIT_QUIT(st, 0);
1520
1521 return symtable_add_def_helper(st, target_name, DEF_LOCAL, ste, LOCATION(e));
1522 }
1523 /* If we find a ModuleBlock entry, add as GLOBAL */
1524 if (ste->ste_type == ModuleBlock) {
1525 if (!symtable_add_def(st, target_name, DEF_GLOBAL, LOCATION(e)))
1526 VISIT_QUIT(st, 0);
1527 if (!symtable_record_directive(st, target_name, LOCATION(e)))
1528 VISIT_QUIT(st, 0);
1529
1530 return symtable_add_def_helper(st, target_name, DEF_GLOBAL, ste, LOCATION(e));
1531 }
1532 /* Disallow usage in ClassBlock */
1533 if (ste->ste_type == ClassBlock) {
1534 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_IN_CLASS);
1535 PyErr_RangedSyntaxLocationObject(st->st_filename,
1536 e->lineno,
1537 e->col_offset + 1,
1538 e->end_lineno,
1539 e->end_col_offset + 1);
1540 VISIT_QUIT(st, 0);
1541 }
1542 }
1543
1544 /* We should always find either a FunctionBlock, ModuleBlock or ClassBlock
1545 and should never fall to this case
1546 */
1547 assert(0);
1548 return 0;
1549 }
1550
1551 static int
symtable_handle_namedexpr(struct symtable * st,expr_ty e)1552 symtable_handle_namedexpr(struct symtable *st, expr_ty e)
1553 {
1554 if (st->st_cur->ste_comp_iter_expr > 0) {
1555 /* Assignment isn't allowed in a comprehension iterable expression */
1556 PyErr_Format(PyExc_SyntaxError, NAMED_EXPR_COMP_ITER_EXPR);
1557 PyErr_RangedSyntaxLocationObject(st->st_filename,
1558 e->lineno,
1559 e->col_offset + 1,
1560 e->end_lineno,
1561 e->end_col_offset + 1);
1562 return 0;
1563 }
1564 if (st->st_cur->ste_comprehension) {
1565 /* Inside a comprehension body, so find the right target scope */
1566 if (!symtable_extend_namedexpr_scope(st, e->v.NamedExpr.target))
1567 return 0;
1568 }
1569 VISIT(st, expr, e->v.NamedExpr.value);
1570 VISIT(st, expr, e->v.NamedExpr.target);
1571 return 1;
1572 }
1573
1574 static int
symtable_visit_expr(struct symtable * st,expr_ty e)1575 symtable_visit_expr(struct symtable *st, expr_ty e)
1576 {
1577 if (++st->recursion_depth > st->recursion_limit) {
1578 PyErr_SetString(PyExc_RecursionError,
1579 "maximum recursion depth exceeded during compilation");
1580 VISIT_QUIT(st, 0);
1581 }
1582 switch (e->kind) {
1583 case NamedExpr_kind:
1584 if (!symtable_raise_if_annotation_block(st, "named expression", e)) {
1585 VISIT_QUIT(st, 0);
1586 }
1587 if(!symtable_handle_namedexpr(st, e))
1588 VISIT_QUIT(st, 0);
1589 break;
1590 case BoolOp_kind:
1591 VISIT_SEQ(st, expr, e->v.BoolOp.values);
1592 break;
1593 case BinOp_kind:
1594 VISIT(st, expr, e->v.BinOp.left);
1595 VISIT(st, expr, e->v.BinOp.right);
1596 break;
1597 case UnaryOp_kind:
1598 VISIT(st, expr, e->v.UnaryOp.operand);
1599 break;
1600 case Lambda_kind: {
1601 if (e->v.Lambda.args->defaults)
1602 VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
1603 if (e->v.Lambda.args->kw_defaults)
1604 VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults);
1605 if (!symtable_enter_block(st, &_Py_ID(lambda),
1606 FunctionBlock, (void *)e,
1607 e->lineno, e->col_offset,
1608 e->end_lineno, e->end_col_offset))
1609 VISIT_QUIT(st, 0);
1610 VISIT(st, arguments, e->v.Lambda.args);
1611 VISIT(st, expr, e->v.Lambda.body);
1612 if (!symtable_exit_block(st))
1613 VISIT_QUIT(st, 0);
1614 break;
1615 }
1616 case IfExp_kind:
1617 VISIT(st, expr, e->v.IfExp.test);
1618 VISIT(st, expr, e->v.IfExp.body);
1619 VISIT(st, expr, e->v.IfExp.orelse);
1620 break;
1621 case Dict_kind:
1622 VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys);
1623 VISIT_SEQ(st, expr, e->v.Dict.values);
1624 break;
1625 case Set_kind:
1626 VISIT_SEQ(st, expr, e->v.Set.elts);
1627 break;
1628 case GeneratorExp_kind:
1629 if (!symtable_visit_genexp(st, e))
1630 VISIT_QUIT(st, 0);
1631 break;
1632 case ListComp_kind:
1633 if (!symtable_visit_listcomp(st, e))
1634 VISIT_QUIT(st, 0);
1635 break;
1636 case SetComp_kind:
1637 if (!symtable_visit_setcomp(st, e))
1638 VISIT_QUIT(st, 0);
1639 break;
1640 case DictComp_kind:
1641 if (!symtable_visit_dictcomp(st, e))
1642 VISIT_QUIT(st, 0);
1643 break;
1644 case Yield_kind:
1645 if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1646 VISIT_QUIT(st, 0);
1647 }
1648 if (e->v.Yield.value)
1649 VISIT(st, expr, e->v.Yield.value);
1650 st->st_cur->ste_generator = 1;
1651 if (st->st_cur->ste_comprehension) {
1652 return symtable_raise_if_comprehension_block(st, e);
1653 }
1654 break;
1655 case YieldFrom_kind:
1656 if (!symtable_raise_if_annotation_block(st, "yield expression", e)) {
1657 VISIT_QUIT(st, 0);
1658 }
1659 VISIT(st, expr, e->v.YieldFrom.value);
1660 st->st_cur->ste_generator = 1;
1661 if (st->st_cur->ste_comprehension) {
1662 return symtable_raise_if_comprehension_block(st, e);
1663 }
1664 break;
1665 case Await_kind:
1666 if (!symtable_raise_if_annotation_block(st, "await expression", e)) {
1667 VISIT_QUIT(st, 0);
1668 }
1669 VISIT(st, expr, e->v.Await.value);
1670 st->st_cur->ste_coroutine = 1;
1671 break;
1672 case Compare_kind:
1673 VISIT(st, expr, e->v.Compare.left);
1674 VISIT_SEQ(st, expr, e->v.Compare.comparators);
1675 break;
1676 case Call_kind:
1677 VISIT(st, expr, e->v.Call.func);
1678 VISIT_SEQ(st, expr, e->v.Call.args);
1679 VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
1680 break;
1681 case FormattedValue_kind:
1682 VISIT(st, expr, e->v.FormattedValue.value);
1683 if (e->v.FormattedValue.format_spec)
1684 VISIT(st, expr, e->v.FormattedValue.format_spec);
1685 break;
1686 case JoinedStr_kind:
1687 VISIT_SEQ(st, expr, e->v.JoinedStr.values);
1688 break;
1689 case Constant_kind:
1690 /* Nothing to do here. */
1691 break;
1692 /* The following exprs can be assignment targets. */
1693 case Attribute_kind:
1694 VISIT(st, expr, e->v.Attribute.value);
1695 break;
1696 case Subscript_kind:
1697 VISIT(st, expr, e->v.Subscript.value);
1698 VISIT(st, expr, e->v.Subscript.slice);
1699 break;
1700 case Starred_kind:
1701 VISIT(st, expr, e->v.Starred.value);
1702 break;
1703 case Slice_kind:
1704 if (e->v.Slice.lower)
1705 VISIT(st, expr, e->v.Slice.lower)
1706 if (e->v.Slice.upper)
1707 VISIT(st, expr, e->v.Slice.upper)
1708 if (e->v.Slice.step)
1709 VISIT(st, expr, e->v.Slice.step)
1710 break;
1711 case Name_kind:
1712 if (!symtable_add_def(st, e->v.Name.id,
1713 e->v.Name.ctx == Load ? USE : DEF_LOCAL, LOCATION(e)))
1714 VISIT_QUIT(st, 0);
1715 /* Special-case super: it counts as a use of __class__ */
1716 if (e->v.Name.ctx == Load &&
1717 st->st_cur->ste_type == FunctionBlock &&
1718 _PyUnicode_EqualToASCIIString(e->v.Name.id, "super")) {
1719 if (!symtable_add_def(st, &_Py_ID(__class__), USE, LOCATION(e)))
1720 VISIT_QUIT(st, 0);
1721 }
1722 break;
1723 /* child nodes of List and Tuple will have expr_context set */
1724 case List_kind:
1725 VISIT_SEQ(st, expr, e->v.List.elts);
1726 break;
1727 case Tuple_kind:
1728 VISIT_SEQ(st, expr, e->v.Tuple.elts);
1729 break;
1730 }
1731 VISIT_QUIT(st, 1);
1732 }
1733
1734 static int
symtable_visit_pattern(struct symtable * st,pattern_ty p)1735 symtable_visit_pattern(struct symtable *st, pattern_ty p)
1736 {
1737 if (++st->recursion_depth > st->recursion_limit) {
1738 PyErr_SetString(PyExc_RecursionError,
1739 "maximum recursion depth exceeded during compilation");
1740 VISIT_QUIT(st, 0);
1741 }
1742 switch (p->kind) {
1743 case MatchValue_kind:
1744 VISIT(st, expr, p->v.MatchValue.value);
1745 break;
1746 case MatchSingleton_kind:
1747 /* Nothing to do here. */
1748 break;
1749 case MatchSequence_kind:
1750 VISIT_SEQ(st, pattern, p->v.MatchSequence.patterns);
1751 break;
1752 case MatchStar_kind:
1753 if (p->v.MatchStar.name) {
1754 symtable_add_def(st, p->v.MatchStar.name, DEF_LOCAL, LOCATION(p));
1755 }
1756 break;
1757 case MatchMapping_kind:
1758 VISIT_SEQ(st, expr, p->v.MatchMapping.keys);
1759 VISIT_SEQ(st, pattern, p->v.MatchMapping.patterns);
1760 if (p->v.MatchMapping.rest) {
1761 symtable_add_def(st, p->v.MatchMapping.rest, DEF_LOCAL, LOCATION(p));
1762 }
1763 break;
1764 case MatchClass_kind:
1765 VISIT(st, expr, p->v.MatchClass.cls);
1766 VISIT_SEQ(st, pattern, p->v.MatchClass.patterns);
1767 VISIT_SEQ(st, pattern, p->v.MatchClass.kwd_patterns);
1768 break;
1769 case MatchAs_kind:
1770 if (p->v.MatchAs.pattern) {
1771 VISIT(st, pattern, p->v.MatchAs.pattern);
1772 }
1773 if (p->v.MatchAs.name) {
1774 symtable_add_def(st, p->v.MatchAs.name, DEF_LOCAL, LOCATION(p));
1775 }
1776 break;
1777 case MatchOr_kind:
1778 VISIT_SEQ(st, pattern, p->v.MatchOr.patterns);
1779 break;
1780 }
1781 VISIT_QUIT(st, 1);
1782 }
1783
1784 static int
symtable_implicit_arg(struct symtable * st,int pos)1785 symtable_implicit_arg(struct symtable *st, int pos)
1786 {
1787 PyObject *id = PyUnicode_FromFormat(".%d", pos);
1788 if (id == NULL)
1789 return 0;
1790 if (!symtable_add_def(st, id, DEF_PARAM, ST_LOCATION(st->st_cur))) {
1791 Py_DECREF(id);
1792 return 0;
1793 }
1794 Py_DECREF(id);
1795 return 1;
1796 }
1797
1798 static int
symtable_visit_params(struct symtable * st,asdl_arg_seq * args)1799 symtable_visit_params(struct symtable *st, asdl_arg_seq *args)
1800 {
1801 int i;
1802
1803 if (!args)
1804 return -1;
1805
1806 for (i = 0; i < asdl_seq_LEN(args); i++) {
1807 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1808 if (!symtable_add_def(st, arg->arg, DEF_PARAM, LOCATION(arg)))
1809 return 0;
1810 }
1811
1812 return 1;
1813 }
1814
1815 static int
symtable_visit_annotation(struct symtable * st,expr_ty annotation)1816 symtable_visit_annotation(struct symtable *st, expr_ty annotation)
1817 {
1818 int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1819 if (future_annotations &&
1820 !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
1821 (void *)annotation, annotation->lineno,
1822 annotation->col_offset, annotation->end_lineno,
1823 annotation->end_col_offset)) {
1824 VISIT_QUIT(st, 0);
1825 }
1826 VISIT(st, expr, annotation);
1827 if (future_annotations && !symtable_exit_block(st)) {
1828 VISIT_QUIT(st, 0);
1829 }
1830 return 1;
1831 }
1832
1833 static int
symtable_visit_argannotations(struct symtable * st,asdl_arg_seq * args)1834 symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args)
1835 {
1836 int i;
1837
1838 if (!args)
1839 return -1;
1840
1841 for (i = 0; i < asdl_seq_LEN(args); i++) {
1842 arg_ty arg = (arg_ty)asdl_seq_GET(args, i);
1843 if (arg->annotation)
1844 VISIT(st, expr, arg->annotation);
1845 }
1846
1847 return 1;
1848 }
1849
1850 static int
symtable_visit_annotations(struct symtable * st,stmt_ty o,arguments_ty a,expr_ty returns)1851 symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns)
1852 {
1853 int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS;
1854 if (future_annotations &&
1855 !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock,
1856 (void *)o, o->lineno, o->col_offset, o->end_lineno,
1857 o->end_col_offset)) {
1858 VISIT_QUIT(st, 0);
1859 }
1860 if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs))
1861 return 0;
1862 if (a->args && !symtable_visit_argannotations(st, a->args))
1863 return 0;
1864 if (a->vararg && a->vararg->annotation)
1865 VISIT(st, expr, a->vararg->annotation);
1866 if (a->kwarg && a->kwarg->annotation)
1867 VISIT(st, expr, a->kwarg->annotation);
1868 if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs))
1869 return 0;
1870 if (future_annotations && !symtable_exit_block(st)) {
1871 VISIT_QUIT(st, 0);
1872 }
1873 if (returns && !symtable_visit_annotation(st, returns)) {
1874 VISIT_QUIT(st, 0);
1875 }
1876 return 1;
1877 }
1878
1879 static int
symtable_visit_arguments(struct symtable * st,arguments_ty a)1880 symtable_visit_arguments(struct symtable *st, arguments_ty a)
1881 {
1882 /* skip default arguments inside function block
1883 XXX should ast be different?
1884 */
1885 if (a->posonlyargs && !symtable_visit_params(st, a->posonlyargs))
1886 return 0;
1887 if (a->args && !symtable_visit_params(st, a->args))
1888 return 0;
1889 if (a->kwonlyargs && !symtable_visit_params(st, a->kwonlyargs))
1890 return 0;
1891 if (a->vararg) {
1892 if (!symtable_add_def(st, a->vararg->arg, DEF_PARAM, LOCATION(a->vararg)))
1893 return 0;
1894 st->st_cur->ste_varargs = 1;
1895 }
1896 if (a->kwarg) {
1897 if (!symtable_add_def(st, a->kwarg->arg, DEF_PARAM, LOCATION(a->kwarg)))
1898 return 0;
1899 st->st_cur->ste_varkeywords = 1;
1900 }
1901 return 1;
1902 }
1903
1904
1905 static int
symtable_visit_excepthandler(struct symtable * st,excepthandler_ty eh)1906 symtable_visit_excepthandler(struct symtable *st, excepthandler_ty eh)
1907 {
1908 if (eh->v.ExceptHandler.type)
1909 VISIT(st, expr, eh->v.ExceptHandler.type);
1910 if (eh->v.ExceptHandler.name)
1911 if (!symtable_add_def(st, eh->v.ExceptHandler.name, DEF_LOCAL, LOCATION(eh)))
1912 return 0;
1913 VISIT_SEQ(st, stmt, eh->v.ExceptHandler.body);
1914 return 1;
1915 }
1916
1917 static int
symtable_visit_withitem(struct symtable * st,withitem_ty item)1918 symtable_visit_withitem(struct symtable *st, withitem_ty item)
1919 {
1920 VISIT(st, expr, item->context_expr);
1921 if (item->optional_vars) {
1922 VISIT(st, expr, item->optional_vars);
1923 }
1924 return 1;
1925 }
1926
1927 static int
symtable_visit_match_case(struct symtable * st,match_case_ty m)1928 symtable_visit_match_case(struct symtable *st, match_case_ty m)
1929 {
1930 VISIT(st, pattern, m->pattern);
1931 if (m->guard) {
1932 VISIT(st, expr, m->guard);
1933 }
1934 VISIT_SEQ(st, stmt, m->body);
1935 return 1;
1936 }
1937
1938 static int
symtable_visit_alias(struct symtable * st,alias_ty a)1939 symtable_visit_alias(struct symtable *st, alias_ty a)
1940 {
1941 /* Compute store_name, the name actually bound by the import
1942 operation. It is different than a->name when a->name is a
1943 dotted package name (e.g. spam.eggs)
1944 */
1945 PyObject *store_name;
1946 PyObject *name = (a->asname == NULL) ? a->name : a->asname;
1947 Py_ssize_t dot = PyUnicode_FindChar(name, '.', 0,
1948 PyUnicode_GET_LENGTH(name), 1);
1949 if (dot != -1) {
1950 store_name = PyUnicode_Substring(name, 0, dot);
1951 if (!store_name)
1952 return 0;
1953 }
1954 else {
1955 store_name = name;
1956 Py_INCREF(store_name);
1957 }
1958 if (!_PyUnicode_EqualToASCIIString(name, "*")) {
1959 int r = symtable_add_def(st, store_name, DEF_IMPORT, LOCATION(a));
1960 Py_DECREF(store_name);
1961 return r;
1962 }
1963 else {
1964 if (st->st_cur->ste_type != ModuleBlock) {
1965 int lineno = a->lineno;
1966 int col_offset = a->col_offset;
1967 int end_lineno = a->end_lineno;
1968 int end_col_offset = a->end_col_offset;
1969 PyErr_SetString(PyExc_SyntaxError, IMPORT_STAR_WARNING);
1970 PyErr_RangedSyntaxLocationObject(st->st_filename,
1971 lineno, col_offset + 1,
1972 end_lineno, end_col_offset + 1);
1973 Py_DECREF(store_name);
1974 return 0;
1975 }
1976 Py_DECREF(store_name);
1977 return 1;
1978 }
1979 }
1980
1981
1982 static int
symtable_visit_comprehension(struct symtable * st,comprehension_ty lc)1983 symtable_visit_comprehension(struct symtable *st, comprehension_ty lc)
1984 {
1985 st->st_cur->ste_comp_iter_target = 1;
1986 VISIT(st, expr, lc->target);
1987 st->st_cur->ste_comp_iter_target = 0;
1988 st->st_cur->ste_comp_iter_expr++;
1989 VISIT(st, expr, lc->iter);
1990 st->st_cur->ste_comp_iter_expr--;
1991 VISIT_SEQ(st, expr, lc->ifs);
1992 if (lc->is_async) {
1993 st->st_cur->ste_coroutine = 1;
1994 }
1995 return 1;
1996 }
1997
1998
1999 static int
symtable_visit_keyword(struct symtable * st,keyword_ty k)2000 symtable_visit_keyword(struct symtable *st, keyword_ty k)
2001 {
2002 VISIT(st, expr, k->value);
2003 return 1;
2004 }
2005
2006
2007 static int
symtable_handle_comprehension(struct symtable * st,expr_ty e,identifier scope_name,asdl_comprehension_seq * generators,expr_ty elt,expr_ty value)2008 symtable_handle_comprehension(struct symtable *st, expr_ty e,
2009 identifier scope_name, asdl_comprehension_seq *generators,
2010 expr_ty elt, expr_ty value)
2011 {
2012 int is_generator = (e->kind == GeneratorExp_kind);
2013 comprehension_ty outermost = ((comprehension_ty)
2014 asdl_seq_GET(generators, 0));
2015 /* Outermost iterator is evaluated in current scope */
2016 st->st_cur->ste_comp_iter_expr++;
2017 VISIT(st, expr, outermost->iter);
2018 st->st_cur->ste_comp_iter_expr--;
2019 /* Create comprehension scope for the rest */
2020 if (!scope_name ||
2021 !symtable_enter_block(st, scope_name, FunctionBlock, (void *)e,
2022 e->lineno, e->col_offset,
2023 e->end_lineno, e->end_col_offset)) {
2024 return 0;
2025 }
2026 switch(e->kind) {
2027 case ListComp_kind:
2028 st->st_cur->ste_comprehension = ListComprehension;
2029 break;
2030 case SetComp_kind:
2031 st->st_cur->ste_comprehension = SetComprehension;
2032 break;
2033 case DictComp_kind:
2034 st->st_cur->ste_comprehension = DictComprehension;
2035 break;
2036 default:
2037 st->st_cur->ste_comprehension = GeneratorExpression;
2038 break;
2039 }
2040 if (outermost->is_async) {
2041 st->st_cur->ste_coroutine = 1;
2042 }
2043
2044 /* Outermost iter is received as an argument */
2045 if (!symtable_implicit_arg(st, 0)) {
2046 symtable_exit_block(st);
2047 return 0;
2048 }
2049 /* Visit iteration variable target, and mark them as such */
2050 st->st_cur->ste_comp_iter_target = 1;
2051 VISIT(st, expr, outermost->target);
2052 st->st_cur->ste_comp_iter_target = 0;
2053 /* Visit the rest of the comprehension body */
2054 VISIT_SEQ(st, expr, outermost->ifs);
2055 VISIT_SEQ_TAIL(st, comprehension, generators, 1);
2056 if (value)
2057 VISIT(st, expr, value);
2058 VISIT(st, expr, elt);
2059 st->st_cur->ste_generator = is_generator;
2060 int is_async = st->st_cur->ste_coroutine && !is_generator;
2061 if (!symtable_exit_block(st)) {
2062 return 0;
2063 }
2064 if (is_async) {
2065 st->st_cur->ste_coroutine = 1;
2066 }
2067 return 1;
2068 }
2069
2070 static int
symtable_visit_genexp(struct symtable * st,expr_ty e)2071 symtable_visit_genexp(struct symtable *st, expr_ty e)
2072 {
2073 return symtable_handle_comprehension(st, e, &_Py_ID(genexpr),
2074 e->v.GeneratorExp.generators,
2075 e->v.GeneratorExp.elt, NULL);
2076 }
2077
2078 static int
symtable_visit_listcomp(struct symtable * st,expr_ty e)2079 symtable_visit_listcomp(struct symtable *st, expr_ty e)
2080 {
2081 return symtable_handle_comprehension(st, e, &_Py_ID(listcomp),
2082 e->v.ListComp.generators,
2083 e->v.ListComp.elt, NULL);
2084 }
2085
2086 static int
symtable_visit_setcomp(struct symtable * st,expr_ty e)2087 symtable_visit_setcomp(struct symtable *st, expr_ty e)
2088 {
2089 return symtable_handle_comprehension(st, e, &_Py_ID(setcomp),
2090 e->v.SetComp.generators,
2091 e->v.SetComp.elt, NULL);
2092 }
2093
2094 static int
symtable_visit_dictcomp(struct symtable * st,expr_ty e)2095 symtable_visit_dictcomp(struct symtable *st, expr_ty e)
2096 {
2097 return symtable_handle_comprehension(st, e, &_Py_ID(dictcomp),
2098 e->v.DictComp.generators,
2099 e->v.DictComp.key,
2100 e->v.DictComp.value);
2101 }
2102
2103 static int
symtable_raise_if_annotation_block(struct symtable * st,const char * name,expr_ty e)2104 symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e)
2105 {
2106 if (st->st_cur->ste_type != AnnotationBlock) {
2107 return 1;
2108 }
2109
2110 PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name);
2111 PyErr_RangedSyntaxLocationObject(st->st_filename,
2112 e->lineno,
2113 e->col_offset + 1,
2114 e->end_lineno,
2115 e->end_col_offset + 1);
2116 return 0;
2117 }
2118
2119 static int
symtable_raise_if_comprehension_block(struct symtable * st,expr_ty e)2120 symtable_raise_if_comprehension_block(struct symtable *st, expr_ty e) {
2121 _Py_comprehension_ty type = st->st_cur->ste_comprehension;
2122 PyErr_SetString(PyExc_SyntaxError,
2123 (type == ListComprehension) ? "'yield' inside list comprehension" :
2124 (type == SetComprehension) ? "'yield' inside set comprehension" :
2125 (type == DictComprehension) ? "'yield' inside dict comprehension" :
2126 "'yield' inside generator expression");
2127 PyErr_RangedSyntaxLocationObject(st->st_filename,
2128 e->lineno, e->col_offset + 1,
2129 e->end_lineno, e->end_col_offset + 1);
2130 VISIT_QUIT(st, 0);
2131 }
2132
2133 struct symtable *
_Py_SymtableStringObjectFlags(const char * str,PyObject * filename,int start,PyCompilerFlags * flags)2134 _Py_SymtableStringObjectFlags(const char *str, PyObject *filename,
2135 int start, PyCompilerFlags *flags)
2136 {
2137 struct symtable *st;
2138 mod_ty mod;
2139 PyArena *arena;
2140
2141 arena = _PyArena_New();
2142 if (arena == NULL)
2143 return NULL;
2144
2145 mod = _PyParser_ASTFromString(str, filename, start, flags, arena);
2146 if (mod == NULL) {
2147 _PyArena_Free(arena);
2148 return NULL;
2149 }
2150 PyFutureFeatures *future = _PyFuture_FromAST(mod, filename);
2151 if (future == NULL) {
2152 _PyArena_Free(arena);
2153 return NULL;
2154 }
2155 future->ff_features |= flags->cf_flags;
2156 st = _PySymtable_Build(mod, filename, future);
2157 PyObject_Free((void *)future);
2158 _PyArena_Free(arena);
2159 return st;
2160 }
2161