diff -r 1e00b161f5f5 Include/Python-ast.h --- a/Include/Python-ast.h Wed Mar 09 12:53:30 2011 +0100 +++ b/Include/Python-ast.h Wed May 11 11:21:24 2016 +0300 @@ -44,6 +44,7 @@ struct _mod { union { struct { asdl_seq *body; + string docstring; } Module; struct { @@ -77,6 +78,7 @@ struct _stmt { asdl_seq *body; asdl_seq *decorator_list; expr_ty returns; + string docstring; } FunctionDef; struct { @@ -87,6 +89,7 @@ struct _stmt { expr_ty kwargs; asdl_seq *body; asdl_seq *decorator_list; + string docstring; } ClassDef; struct { @@ -184,10 +187,9 @@ struct _stmt { enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4, IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8, SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11, - Yield_kind=12, Compare_kind=13, Call_kind=14, Num_kind=15, - Str_kind=16, Bytes_kind=17, Ellipsis_kind=18, - Attribute_kind=19, Subscript_kind=20, Starred_kind=21, - Name_kind=22, List_kind=23, Tuple_kind=24}; + Yield_kind=12, Compare_kind=13, Call_kind=14, Lit_kind=15, + Attribute_kind=16, Subscript_kind=17, Starred_kind=18, + Name_kind=19, List_kind=20, Tuple_kind=21}; struct _expr { enum _expr_kind kind; union { @@ -267,16 +269,8 @@ struct _expr { } Call; struct { - object n; - } Num; - - struct { - string s; - } Str; - - struct { - string s; - } Bytes; + object v; + } Lit; struct { expr_ty value; @@ -384,23 +378,23 @@ struct _alias { }; -#define Module(a0, a1) _Py_Module(a0, a1) -mod_ty _Py_Module(asdl_seq * body, PyArena *arena); +#define Module(a0, a1, a2) _Py_Module(a0, a1, a2) +mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena); #define Interactive(a0, a1) _Py_Interactive(a0, a1) mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena); #define Expression(a0, a1) _Py_Expression(a0, a1) mod_ty _Py_Expression(expr_ty body, PyArena *arena); #define Suite(a0, a1) _Py_Suite(a0, a1) mod_ty _Py_Suite(asdl_seq * body, PyArena *arena); -#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) +#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body, - asdl_seq * decorator_list, expr_ty returns, int lineno, - int col_offset, PyArena *arena); -#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) + asdl_seq * decorator_list, expr_ty returns, string + docstring, int lineno, int col_offset, PyArena *arena); +#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, expr_ty starargs, expr_ty kwargs, asdl_seq * body, - asdl_seq * decorator_list, int lineno, int col_offset, - PyArena *arena); + asdl_seq * decorator_list, string docstring, int lineno, + int col_offset, PyArena *arena); #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3) stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena); #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3) @@ -497,14 +491,8 @@ expr_ty _Py_Compare(expr_ty left, asdl_i expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs, expr_ty kwargs, int lineno, int col_offset, PyArena *arena); -#define Num(a0, a1, a2, a3) _Py_Num(a0, a1, a2, a3) -expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena); -#define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3) -expr_ty _Py_Str(string s, int lineno, int col_offset, PyArena *arena); -#define Bytes(a0, a1, a2, a3) _Py_Bytes(a0, a1, a2, a3) -expr_ty _Py_Bytes(string s, int lineno, int col_offset, PyArena *arena); -#define Ellipsis(a0, a1, a2) _Py_Ellipsis(a0, a1, a2) -expr_ty _Py_Ellipsis(int lineno, int col_offset, PyArena *arena); +#define Lit(a0, a1, a2, a3) _Py_Lit(a0, a1, a2, a3) +expr_ty _Py_Lit(object v, int lineno, int col_offset, PyArena *arena); #define Attribute(a0, a1, a2, a3, a4, a5) _Py_Attribute(a0, a1, a2, a3, a4, a5) expr_ty _Py_Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int lineno, int col_offset, PyArena *arena); diff -r 1e00b161f5f5 Include/compile.h --- a/Include/compile.h Wed Mar 09 12:53:30 2011 +0100 +++ b/Include/compile.h Wed May 11 11:21:24 2016 +0300 @@ -38,6 +38,9 @@ PyAPI_FUNC(PyCodeObject *) PyAST_Compile PyArena *arena); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *); +#ifndef Py_LIMITED_API +int _PyAST_Optimize(struct _mod *, PyArena *arena); +#endif #ifdef __cplusplus } diff -r 1e00b161f5f5 Lib/ast.py --- a/Lib/ast.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Lib/ast.py Wed May 11 11:21:24 2016 +0300 @@ -49,10 +49,8 @@ def literal_eval(node_or_string): if isinstance(node_or_string, Expression): node_or_string = node_or_string.body def _convert(node): - if isinstance(node, (Str, Bytes)): - return node.s - elif isinstance(node, Num): - return node.n + if isinstance(node, Lit): + return node.v elif isinstance(node, Tuple): return tuple(map(_convert, node.elts)) elif isinstance(node, List): @@ -67,7 +65,7 @@ def literal_eval(node_or_string): return _safe_names[node.id] elif isinstance(node, UnaryOp) and \ isinstance(node.op, (UAdd, USub)) and \ - isinstance(node.operand, (Num, UnaryOp, BinOp)): + isinstance(node.operand, (Lit, UnaryOp, BinOp)): operand = _convert(node.operand) if isinstance(node.op, UAdd): return + operand @@ -75,8 +73,8 @@ def literal_eval(node_or_string): return - operand elif isinstance(node, BinOp) and \ isinstance(node.op, (Add, Sub)) and \ - isinstance(node.right, (Num, UnaryOp, BinOp)) and \ - isinstance(node.left, (Num, UnaryOp, BinOp)): + isinstance(node.right, (Lit, UnaryOp, BinOp)) and \ + isinstance(node.left, (Lit, UnaryOp, BinOp)): left = _convert(node.left) right = _convert(node.right) if isinstance(node.op, Add): @@ -199,12 +197,10 @@ def get_docstring(node, clean=True): """ if not isinstance(node, (FunctionDef, ClassDef, Module)): raise TypeError("%r can't have docstrings" % node.__class__.__name__) - if node.body and isinstance(node.body[0], Expr) and \ - isinstance(node.body[0].value, Str): - if clean: - import inspect - return inspect.cleandoc(node.body[0].value.s) - return node.body[0].value.s + if clean: + import inspect + return inspect.cleandoc(node.docstring) + return node.docstring def walk(node): @@ -277,7 +273,7 @@ class NodeTransformer(NodeVisitor): def visit_Name(self, node): return copy_location(Subscript( value=Name(id='data', ctx=Load()), - slice=Index(value=Str(s=node.id)), + slice=Index(value=Lit(v=node.id)), ctx=node.ctx ), node) diff -r 1e00b161f5f5 Lib/test/disutil.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/disutil.py Wed May 11 11:21:24 2016 +0300 @@ -0,0 +1,19 @@ +import dis +import sys +from io import StringIO + +def disassemble(func): + f = StringIO() + tmp = sys.stdout + sys.stdout = f + try: + dis.dis(func) + result = f.getvalue() + finally: + sys.stdout = tmp + f.close() + return result + +def dis_single(line): + return disassemble(compile(line, '', 'single')) + diff -r 1e00b161f5f5 Lib/test/test_ast.py --- a/Lib/test/test_ast.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Lib/test/test_ast.py Wed May 11 11:21:24 2016 +0300 @@ -3,7 +3,7 @@ from test import support import ast def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)): + if t is None or isinstance(t, (str, bytes, int, complex)): return t elif isinstance(t, list): return [to_tuple(e) for e in t] @@ -96,10 +96,10 @@ eval_tests = [ "1 < 2 < 3", # Call "f(1,2,c=3,*d,**e)", - # Num + # Literals "10", - # Str "'string'", + "b'string'", # Attribute "a.b", # Subscript @@ -199,7 +199,7 @@ class AST_Tests(unittest.TestCase): def test_invalid_sum(self): pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)]) + m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], "doc") with self.assertRaises(TypeError) as cm: compile(m, "", "exec") self.assertIn("but got <_ast.expr", str(cm.exception)) @@ -216,60 +216,61 @@ class ASTHelpers_Test(unittest.TestCase) node = ast.parse('spam(eggs, "and cheese")') self.assertEqual(ast.dump(node), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], " - "keywords=[], starargs=None, kwargs=None))])" + "args=[Name(id='eggs', ctx=Load()), Lit(v='and cheese')], " + "keywords=[], starargs=None, kwargs=None))], docstring=None)" ) self.assertEqual(ast.dump(node, annotate_fields=False), "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Str('and cheese')], [], None, None))])" + "Lit('and cheese')], [], None, None))], None)" ) self.assertEqual(ast.dump(node, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), " - "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, " + "lineno=1, col_offset=5), Lit(v='and cheese', lineno=1, " "col_offset=11)], keywords=[], starargs=None, kwargs=None, " - "lineno=1, col_offset=0), lineno=1, col_offset=0)])" + "lineno=1, col_offset=0), lineno=1, col_offset=0)], " + "docstring=None)" ) def test_copy_location(self): src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Num(2), src.body.right) + src.body.right = ast.copy_location(ast.Lit(2), src.body.right) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Num(n=1, lineno=1, col_offset=0), ' - 'op=Add(), right=Num(n=2, lineno=1, col_offset=4), lineno=1, ' + 'Expression(body=BinOp(left=Lit(v=1, lineno=1, col_offset=0), ' + 'op=Add(), right=Lit(v=2, lineno=1, col_offset=4), lineno=1, ' 'col_offset=0))' ) def test_fix_missing_locations(self): src = ast.parse('write("spam")') src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Str('eggs')], [], None, None))) + [ast.Lit('eggs')], [], None, None))) self.assertEqual(src, ast.fix_missing_locations(src)) self.assertEqual(ast.dump(src, include_attributes=True), "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " - "lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, " + "lineno=1, col_offset=0), args=[Lit(v='spam', lineno=1, " "col_offset=6)], keywords=[], starargs=None, kwargs=None, " "lineno=1, col_offset=0), lineno=1, col_offset=0), " "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, " - "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], " + "col_offset=0), args=[Lit(v='eggs', lineno=1, col_offset=0)], " "keywords=[], starargs=None, kwargs=None, lineno=1, " - "col_offset=0), lineno=1, col_offset=0)])" + "col_offset=0), lineno=1, col_offset=0)], docstring=None)" ) def test_increment_lineno(self): src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src, n=3), src) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Num(n=1, lineno=4, col_offset=0), ' - 'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, ' + 'Expression(body=BinOp(left=Lit(v=1, lineno=4, col_offset=0), ' + 'op=Add(), right=Lit(v=1, lineno=4, col_offset=4), lineno=4, ' 'col_offset=0))' ) # issue10869: do not increment lineno of root twice src = ast.parse('1 + 1', mode='eval') self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Num(n=1, lineno=4, col_offset=0), ' - 'op=Add(), right=Num(n=1, lineno=4, col_offset=4), lineno=4, ' + 'Expression(body=BinOp(left=Lit(v=1, lineno=4, col_offset=0), ' + 'op=Add(), right=Lit(v=1, lineno=4, col_offset=4), lineno=4, ' 'col_offset=0))' ) @@ -285,10 +286,10 @@ class ASTHelpers_Test(unittest.TestCase) self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) iterator = ast.iter_child_nodes(node.body) self.assertEqual(next(iterator).id, 'spam') - self.assertEqual(next(iterator).n, 23) - self.assertEqual(next(iterator).n, 42) + self.assertEqual(next(iterator).v, 23) + self.assertEqual(next(iterator).v, 42) self.assertEqual(ast.dump(next(iterator)), - "keyword(arg='eggs', value=Str(s='leek'))" + "keyword(arg='eggs', value=Lit(v='leek'))" ) def test_get_docstring(self): @@ -332,50 +333,51 @@ def main(): #### EVERYTHING BELOW IS GENERATED ##### exec_results = [ -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None)]), -('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]), -('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]), -('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]), -('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]), -('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Num', (1, 5), 1))]), -('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), -('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), -('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), -('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), -('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]), -('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]), -('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), -('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), -('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), -('Module', [('Global', (1, 0), ['v'])]), -('Module', [('Expr', (1, 0), ('Num', (1, 0), 1))]), -('Module', [('Pass', (1, 0))]), -('Module', [('Break', (1, 0))]), -('Module', [('Continue', (1, 0))]), -('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])]), -('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]), -('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))]), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Pass', (1, 9))], [], None, None)], None), +('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [], None)], None), +('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, None, [], None, None, [], []), [('Return', (1, 8), ('Lit', (1, 15), 1))], [], None, None)], None), +('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])], None), +('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Lit', (1, 4), 1))], None), +('Module', [('AugAssign', (1, 0), ('Name', (1, 0), 'v', ('Store',)), ('Add',), ('Lit', (1, 5), 1))], None), +('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])], None), +('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])], None), +('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])], None), +('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Lit', (1, 16), 'string')], [], None, None), None)], None), +('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])], None), +('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])], None), +('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)], None), +('Module', [('Import', (1, 0), [('alias', 'sys', None)])], None), +('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)], None), +('Module', [('Global', (1, 0), ['v'])], None), +('Module', [('Expr', (1, 0), ('Lit', (1, 0), 1))], None), +('Module', [('Pass', (1, 0))], None), +('Module', [('Break', (1, 0))], None), +('Module', [('Continue', (1, 0))], None), +('Module', [('For', (1, 0), ('Tuple', (1, 4), [('Name', (1, 4), 'a', ('Store',)), ('Name', (1, 6), 'b', ('Store',))], ('Store',)), ('Name', (1, 11), 'c', ('Load',)), [('Pass', (1, 14))], [])], None), +('Module', [('Expr', (1, 0), ('ListComp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))], None), +('Module', [('Expr', (1, 0), ('GeneratorExp', (1, 1), ('Tuple', (1, 2), [('Name', (1, 2), 'a', ('Load',)), ('Name', (1, 4), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'a', ('Store',)), ('Name', (1, 13), 'b', ('Store',))], ('Store',)), ('Name', (1, 18), 'c', ('Load',)), [])]))], None), ] single_results = [ -('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Num', (1, 0), 1), ('Add',), ('Num', (1, 2), 2)))]), +('Interactive', [('Expr', (1, 0), ('BinOp', (1, 0), ('Lit', (1, 0), 1), ('Add',), ('Lit', (1, 2), 2)))]), ] eval_results = [ ('Expression', ('BoolOp', (1, 0), ('And',), [('Name', (1, 0), 'a', ('Load',)), ('Name', (1, 6), 'b', ('Load',))])), ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))), ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Name', (1, 7), 'None', ('Load',)))), -('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])), +('Expression', ('Lambda', (1, 0), ('arguments', [], None, None, [], None, None, [], []), ('Lit', (1, 7), None))), +('Expression', ('Dict', (1, 0), [('Lit', (1, 2), 1)], [('Lit', (1, 4), 2)])), ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])), -('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])), -('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))), -('Expression', ('Num', (1, 0), 10)), -('Expression', ('Str', (1, 0), 'string')), +('Expression', ('Compare', (1, 0), ('Lit', (1, 0), 1), [('Lt',), ('Lt',)], [('Lit', (1, 4), 2), ('Lit', (1, 8), 3)])), +('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Lit', (1, 2), 1), ('Lit', (1, 4), 2)], [('keyword', 'c', ('Lit', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))), +('Expression', ('Lit', (1, 0), 10)), +('Expression', ('Lit', (1, 0), 'string')), +('Expression', ('Lit', (1, 0), b'string')), ('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))), ('Expression', ('Subscript', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Slice', ('Name', (1, 2), 'b', ('Load',)), ('Name', (1, 4), 'c', ('Load',)), None), ('Load',))), ('Expression', ('Name', (1, 0), 'v', ('Load',))), -('Expression', ('List', (1, 0), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))), -('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))), -('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)), +('Expression', ('List', (1, 0), [('Lit', (1, 1), 1), ('Lit', (1, 3), 2), ('Lit', (1, 5), 3)], ('Load',))), +('Expression', ('Tuple', (1, 0), [('Lit', (1, 0), 1), ('Lit', (1, 2), 2), ('Lit', (1, 4), 3)], ('Load',))), +('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Lit', (1, 12), 1), ('Lit', (1, 14), 2), None), ('Load',))], [], None, None)), ] main() diff -r 1e00b161f5f5 Lib/test/test_compile.py --- a/Lib/test/test_compile.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Lib/test/test_compile.py Wed May 11 11:21:24 2016 +0300 @@ -2,6 +2,7 @@ import unittest import sys import _ast from test import support +from test.disutil import disassemble class TestSpecifics(unittest.TestCase): @@ -433,6 +434,16 @@ if 1: ast.body = [_ast.BoolOp()] self.assertRaises(TypeError, compile, ast, '', 'exec') + def test_if_no_else(self): + def f1(x): + if x: + x = 1 + return x + + asm = disassemble(f1) + for elem in ('JUMP_FORWARD', 'JUMP_ABSOLUTE'): + self.assertNotIn(elem, asm) + def test_main(): support.run_unittest(TestSpecifics) diff -r 1e00b161f5f5 Lib/test/test_dis.py --- a/Lib/test/test_dis.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Lib/test/test_dis.py Wed May 11 11:21:24 2016 +0300 @@ -4,7 +4,7 @@ from test.support import run_unittest, c import unittest import sys import dis -import io +from test.disutil import disassemble def _f(a): @@ -139,12 +139,7 @@ dis_compound_stmt_str = """\ class DisTests(unittest.TestCase): def do_disassembly_test(self, func, expected): - s = io.StringIO() - save_stdout = sys.stdout - sys.stdout = s - dis.dis(func) - sys.stdout = save_stdout - got = s.getvalue() + got = disassemble(func) # Trim trailing blanks (if any). lines = got.split('\n') lines = [line.rstrip() for line in lines] diff -r 1e00b161f5f5 Lib/test/test_peepholer.py --- a/Lib/test/test_peepholer.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Lib/test/test_peepholer.py Wed May 11 11:21:24 2016 +0300 @@ -1,28 +1,14 @@ -import dis +from test.disutil import disassemble, dis_single import re import sys -from io import StringIO import unittest -def disassemble(func): - f = StringIO() - tmp = sys.stdout - sys.stdout = f - dis.dis(func) - sys.stdout = tmp - result = f.getvalue() - f.close() - return result - -def dis_single(line): - return disassemble(compile(line, '', 'single')) - class TestTranforms(unittest.TestCase): def test_unot(self): # UNARY_NOT POP_JUMP_IF_FALSE --> POP_JUMP_IF_TRUE' def unot(x): - if not x == 2: + if not x: del x asm = disassemble(unot) for elem in ('UNARY_NOT', 'POP_JUMP_IF_FALSE'): @@ -52,12 +38,10 @@ class TestTranforms(unittest.TestCase): def h(x): False return x - for func, name in ((f, 'None'), (g, 'True'), (h, 'False')): + for func in (f, g, h): asm = disassemble(func) for elem in ('LOAD_GLOBAL',): self.assertNotIn(elem, asm) - for elem in ('LOAD_CONST', '('+name+')'): - self.assertIn(elem, asm) def f(): 'Adding a docstring made this test fail in Py2.5.0' return None @@ -81,11 +65,26 @@ class TestTranforms(unittest.TestCase): ('a, = a,', 'LOAD_CONST',), ('a, b = a, b', 'ROT_TWO',), ('a, b, c = a, b, c', 'ROT_THREE',), + ('[a] = [a]', 'LOAD_CONST',), + ('[a, b] = [a, b]', 'ROT_TWO',), + ('[a, b, c] = [a, b, c]', 'ROT_THREE',), ): asm = dis_single(line) self.assertIn(elem, asm) self.assertNotIn('BUILD_TUPLE', asm) - self.assertNotIn('UNPACK_TUPLE', asm) + self.assertNotIn('BUILD_LIST', asm) + self.assertNotIn('UNPACK_SEQUENCE', asm) + + # Optimizing sets this way changes semantics -- sets + # remove duplicate items. So don't do it. + for line in ( + 'a, = {a}', + 'a, b = {a, b}', + 'a, b, c = {a, b, c}', + ): + asm = dis_single(line) + self.assertIn('BUILD_SET', asm) + self.assertIn('UNPACK_SEQUENCE', asm) def test_folding_of_tuples_of_constants(self): for line, elem in ( diff -r 1e00b161f5f5 Lib/test/test_sys_settrace.py --- a/Lib/test/test_sys_settrace.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Lib/test/test_sys_settrace.py Wed May 11 11:21:24 2016 +0300 @@ -337,9 +337,9 @@ class TraceTestCase(unittest.TestCase): tracer.events, generator_example.events) def test_14_onliner_if(self): - def onliners(): - if True: False - else: True + def onliners(x=1): + if True: x + else: x return 0 self.run_and_compare( onliners, diff -r 1e00b161f5f5 Makefile.pre.in --- a/Makefile.pre.in Wed Mar 09 12:53:30 2011 +0100 +++ b/Makefile.pre.in Wed May 11 11:21:24 2016 +0300 @@ -274,6 +274,9 @@ ASDLGEN_FILES= $(srcdir)/Parser/asdl.py # XXX Note that a build now requires Python exist before the build starts ASDLGEN= $(srcdir)/Parser/asdl_c.py +ASDLTX= $(srcdir)/Parser/asdl_ct.py +ASDLTX_FILES= $(srcdir)/Python/ast_opt.ct + ########################################################################## # Python @@ -291,6 +294,7 @@ PYTHON_OBJS= \ Python/Python-ast.o \ Python/asdl.o \ Python/ast.o \ + Python/ast_opt.o \ Python/bltinmodule.o \ Python/ceval.o \ Python/compile.o \ @@ -608,6 +612,9 @@ Parser/pgenmain.o: $(srcdir)/Include/par $(AST_C): $(AST_ASDL) $(ASDLGEN_FILES) $(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL) +$(ASDLTX_FILES:.ct=.c): %.c: %.ct $(ASDLTX) $(AST_ASDL) + $(ASDLTX) $(AST_ASDL) $< $@ + Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H) Python/getplatform.o: $(srcdir)/Python/getplatform.c diff -r 1e00b161f5f5 PC/os2emx/python33.def --- a/PC/os2emx/python33.def Wed Mar 09 12:53:30 2011 +0100 +++ b/PC/os2emx/python33.def Wed May 11 11:21:24 2016 +0300 @@ -719,14 +719,12 @@ EXPORTS "Compare" "Call" "Repr" - "Num" - "Str" + "Lit" "Attribute" "Subscript" "Name" "List" "Tuple" - "Ellipsis" "Slice" "ExtSlice" "Index" diff -r 1e00b161f5f5 Parser/Python.asdl --- a/Parser/Python.asdl Wed Mar 09 12:53:30 2011 +0100 +++ b/Parser/Python.asdl Wed May 11 11:21:24 2016 +0300 @@ -2,7 +2,7 @@ module Python version "$Revision$" { - mod = Module(stmt* body) + mod = Module(stmt* body, string? docstring) | Interactive(stmt* body) | Expression(expr body) @@ -10,14 +10,16 @@ module Python version "$Revision$" | Suite(stmt* body) stmt = FunctionDef(identifier name, arguments args, - stmt* body, expr* decorator_list, expr? returns) + stmt* body, expr* decorator_list, expr? returns, + string? docstring) | ClassDef(identifier name, expr* bases, keyword* keywords, expr? starargs, expr? kwargs, stmt* body, - expr* decorator_list) + expr* decorator_list, + string? docstring) | Return(expr? value) | Delete(expr* targets) @@ -66,11 +68,7 @@ module Python version "$Revision$" | Compare(expr left, cmpop* ops, expr* comparators) | Call(expr func, expr* args, keyword* keywords, expr? starargs, expr? kwargs) - | Num(object n) -- a number as a PyObject. - | Str(string s) -- need to specify raw, unicode, etc? - | Bytes(string s) - | Ellipsis - -- other literals? bools? + | Lit(object v) -- a literal as a PyObject. -- the following expression can appear in assignment context | Attribute(expr value, identifier attr, expr_context ctx) diff -r 1e00b161f5f5 Parser/asdl_c.py --- a/Parser/asdl_c.py Wed Mar 09 12:53:30 2011 +0100 +++ b/Parser/asdl_c.py Wed May 11 11:21:24 2016 +0300 @@ -291,6 +291,10 @@ class FunctionVisitor(PrototypeVisitor): emit("%s(%s)" % (name, argstr)) emit("{") emit("%s p;" % ctype, 1) + # XXX: special hack for Lit. Lit value can be None and it + # should be stored as Py_None, not as NULL. + if name.value == 'Lit': + emit('if (!v) v = Py_None;', 1) for argtype, argname, opt in args: if not opt and argtype != "int": emit("if (!%s) {" % argname, 1) @@ -501,17 +505,21 @@ class Obj2ModVisitor(PickleVisitor): self.emit("goto failed;", depth+2) self.emit("}", depth+1) self.emit("len = PyList_GET_SIZE(tmp);", depth+1) + self.emit("if (len == 0)", depth+1) + self.emit("%s = NULL;" % field.name, depth+2) + self.emit("else {", depth+1) if self.isSimpleType(field): - self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1) + self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+2) else: - self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1) - self.emit("if (%s == NULL) goto failed;" % field.name, depth+1) - self.emit("for (i = 0; i < len; i++) {", depth+1) - self.emit("%s value;" % ctype, depth+2) + self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+2) + self.emit("if (%s == NULL) goto failed;" % field.name, depth+2) + self.emit("for (i = 0; i < len; i++) {", depth+2) + self.emit("%s value;" % ctype, depth+3) self.emit("res = obj2ast_%s(PyList_GET_ITEM(tmp, i), &value, arena);" % - field.type, depth+2, reflow=False) - self.emit("if (res != 0) goto failed;", depth+2) - self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+2) + field.type, depth+3, reflow=False) + self.emit("if (res != 0) goto failed;", depth+3) + self.emit("asdl_seq_SET(%s, i, value);" % field.name, depth+3) + self.emit("}", depth+2) self.emit("}", depth+1) else: self.emit("res = obj2ast_%s(tmp, &%s, arena);" % diff -r 1e00b161f5f5 Parser/asdl_ct.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Parser/asdl_ct.py Wed May 11 11:21:24 2016 +0300 @@ -0,0 +1,258 @@ +#!/usr/bin/env python +"Generate code for AST visitors." + +import asdl +import re +import sys + +def is_reachable(ty, types, rules): + "Are any 'rules' reachable from 'ty' given 'types'." + + visited = set() + def reachable(ty): + if ty in visited: + return False + visited.add(ty) + + if isinstance(ty, asdl.Id): + if ty.value in rules: + return True + if ty.value not in types: + return False + return reachable(types[ty.value]) + elif isinstance(ty, asdl.Sum): + return any(map(reachable, ty.types)) + elif isinstance(ty, asdl.Constructor): + if ty.name.value in rules: + return True + return any(reachable(field.type) for field in ty.fields) + elif isinstance(ty, asdl.Product): + return any(reachable(field.type) for field in ty.fields) + else: + raise TypeError(type(ty)) + + return False + + return reachable(ty) + + +MACRO_DEF = """ +#define CALL(FUNC, TYPE, ARG) \\ + if (!FUNC((ARG){0})) \\ + return 0; + +#define CALL_OPT(FUNC, TYPE, ARG) \\ + if ((ARG) != NULL && !FUNC((ARG){0})) \\ + return 0; + +#define CALL_SEQ(FUNC, TYPE, ARG) {{ \\ + int i; \\ + asdl_seq *seq = (ARG); /* avoid variable capture */ \\ + for (i = 0; i < asdl_seq_LEN(seq); i++) {{ \\ + TYPE elt = (TYPE)asdl_seq_GET(seq, i); \\ + /* XXX: kw_defaults has NULL elements, because it's \\ + sized to the number of kw args */ \\ + if (elt != NULL && !FUNC(elt{0})) \\ + return 0; \\ + }} \\ +}} +""" + +MACRO_UNDEF = """ +#undef CALL +#undef CALL_OPT +#undef CALL_SEQ +""" + +BANNER = '/* File automatically generated by Parser/asdl_ct.py. */\n' + +class Visitor: + "One visitor definition." + + def __init__(self, name, types): + self.name = name + self.types = types + self.rules = {} + self._funcs = {} + + def add_rule(self, name, func, kind): + "Add @kind(name, func) rule." + + if self._funcs: + raise RuntimeError('Visitor already generated.') + if name in self.rules: + raise NameError('{0} already registered to {1}' + .format(name, self.rules[name])) + self.rules[name] = (func, kind) + + def generate(self, out, start, ctx): + "Generate visitor function." + + arg = ', ctx_' if ctx else '' + out(MACRO_DEF.format(arg), depth=0) + + self._ctx = ctx + self._reach = {} + self._stack = [] + self._need_func(start) + while self._stack: + self._process(out, self._stack.pop()) + + out(MACRO_UNDEF, depth=0) + + def write_protos(self, out): + "Write prototypes for generated functions." + + if not self._funcs: + raise RuntimeError('Visitor not generated.') + + for proto in self._funcs.values(): + out(proto + ';', depth=0) + + def used(self): + return bool(self._funcs) + + def _process(self, out, name): + def worker(): + node = self.types[name] + if isinstance(node, (asdl.Constructor, asdl.Product)): + self._process_case(out, node, nodety, depth=1) + elif isinstance(node, asdl.Sum): + out('switch (node_->kind) {', depth=1) + for ty in node.types: + if self._can_reach(ty): + out('case ' + ty.name.value + '_kind:', depth=1) + self._process_case(out, ty, nodety, depth=2) + out('break;', depth=2) + out('default:', depth=1) + out('break;', depth=2) + out('}', depth=1) + else: + raise TypeError(type(node)) + + out(self._funcs[name], depth=0) + out('{', depth=0) + nodety = name + '_ty' + self._with_kind(out, name, nodety, 'node_', worker, depth=1) + out('return 1;\n}\n', depth=1) + + def _process_case(self, out, node, nodety, depth): + assert isinstance(node, (asdl.Constructor, asdl.Product)) + def worker(): + for field in node.fields: + ty = field.type + assert isinstance(ty, asdl.Id) + if not self._can_reach(ty): + continue + func_name = self._need_func(ty.value) + nodety = ty.value + "_ty" + kind = 'OPT' if field.opt else 'SEQ' if field.seq else '' + self._call(out, func_name, kind, nodety, prefix + field.name.value, + depth=depth) + + if isinstance(node, asdl.Constructor): + name = node.name.value + prefix = 'node_->v.' + name + '.' + else: + name = '' + prefix = 'node_->' + self._with_kind(out, name, nodety, 'node_', worker, depth=depth) + + def _with_kind(self, out, name, nodety, arg, func, depth): + rule, kind = self.rules.get(name, (None, None)) + if kind in ('pre', 'just'): + self._call(out, rule, '', nodety, arg, depth=depth) + if kind != 'just': + func() + if kind == 'post': + self._call(out, rule, '', nodety, arg, depth=depth) + + def _can_reach(self, ty): + if ty in self._reach: + return self._reach[ty] + self._reach[ty] = can = is_reachable(ty, self.types, self.rules) + return can + + def _call(self, out, func, kind, type, arg, depth): + if kind: + kind = '_' + kind + out('CALL{0}({1}, {2}, {3});'.format(kind, func, type, arg), + depth=depth) + + def _need_func(self, name): + func_name = self.name + '_' + name + if name not in self._funcs: + proto = "static int {0}({1}_ty node_".format(func_name, name) + if self._ctx: + proto += ', ' + self._ctx + ' ctx_' + proto += ')' + self._funcs[name] = proto + self._stack.append(name) + return func_name + + +class Processor: + def __init__(self, asdl_name): + self.mod = asdl.parse(asdl_name) + self.visitors = {} + + def process(self, infile, outfile): + with open(infile) as f: + s = re.sub('@(\w+)\(((?:[^,()]*,?)*)\)\n', + self._action, f.read()) + with open(outfile, 'w') as f: + f.write(BANNER) + f.write(s) + self._warn() + + def _action(self, match): + cmd = match.group(1) + args = list(map(str.strip, match.group(2).split(','))) + if cmd in ('pre', 'post', 'just'): + if len(args) != 3: + raise TypeError(cmd + ' expects 3 arguments') + + v = self.visitors.setdefault(args[0], + Visitor(args[0], self.mod.types)) + v.add_rule(args[1], args[2], cmd) + return '' + if cmd == 'visitor': + if len(args) < 3: + raise TypeError(cmd + ' expects at least 3 arguments') + if args[0] not in self.visitors: + raise KeyError('visitor ' + args[0] + ' is not defined') + + return self._gen(self.visitors[args[0]], args[2:], args[1]) + + raise NameError('unknown command ' + cmd) + + def _gen(self, visitor, starts, ctx): + def output(s, depth): + out.append(' ' * depth + s) + + out = [] + for start in starts: + visitor.generate(output, start, ctx) + code = '\n'.join(out) + + out = [] + visitor.write_protos(output) + protos = '\n'.join(out) + return protos + code + + def _warn(self): + for v in self.visitors.values(): + if not v.used(): + sys.stderr.write('warning: unused visitor ' + + v.name + '\n') + + +if __name__ == "__main__": + args = sys.argv + if len(args) != 4: + sys.stdout.write("usage: {0} \n" + .format(args[0])) + sys.exit(1) + p = Processor(args[1]) + p.process(args[2], args[3]) + diff -r 1e00b161f5f5 Python/Python-ast.c --- a/Python/Python-ast.c Wed Mar 09 12:53:30 2011 +0100 +++ b/Python/Python-ast.c Wed May 11 11:21:24 2016 +0300 @@ -2,7 +2,7 @@ /* - __version__ 82163. + __version__ . This module must be committed separately after each AST grammar change; The __version__ number is set to the revision number of the commit @@ -18,6 +18,7 @@ static PyObject* ast2obj_mod(void*); static PyTypeObject *Module_type; static char *Module_fields[]={ "body", + "docstring", }; static PyTypeObject *Interactive_type; static char *Interactive_fields[]={ @@ -44,6 +45,7 @@ static char *FunctionDef_fields[]={ "body", "decorator_list", "returns", + "docstring", }; static PyTypeObject *ClassDef_type; static char *ClassDef_fields[]={ @@ -54,6 +56,7 @@ static char *ClassDef_fields[]={ "kwargs", "body", "decorator_list", + "docstring", }; static PyTypeObject *Return_type; static char *Return_fields[]={ @@ -226,19 +229,10 @@ static char *Call_fields[]={ "starargs", "kwargs", }; -static PyTypeObject *Num_type; -static char *Num_fields[]={ - "n", +static PyTypeObject *Lit_type; +static char *Lit_fields[]={ + "v", }; -static PyTypeObject *Str_type; -static char *Str_fields[]={ - "s", -}; -static PyTypeObject *Bytes_type; -static char *Bytes_fields[]={ - "s", -}; -static PyTypeObject *Ellipsis_type; static PyTypeObject *Attribute_type; static char *Attribute_fields[]={ "value", @@ -648,7 +642,7 @@ static int init_types(void) mod_type = make_type("mod", &AST_type, NULL, 0); if (!mod_type) return 0; if (!add_attributes(mod_type, NULL, 0)) return 0; - Module_type = make_type("Module", mod_type, Module_fields, 1); + Module_type = make_type("Module", mod_type, Module_fields, 2); if (!Module_type) return 0; Interactive_type = make_type("Interactive", mod_type, Interactive_fields, 1); @@ -662,9 +656,9 @@ static int init_types(void) if (!stmt_type) return 0; if (!add_attributes(stmt_type, stmt_attributes, 2)) return 0; FunctionDef_type = make_type("FunctionDef", stmt_type, - FunctionDef_fields, 5); + FunctionDef_fields, 6); if (!FunctionDef_type) return 0; - ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 7); + ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 8); if (!ClassDef_type) return 0; Return_type = make_type("Return", stmt_type, Return_fields, 1); if (!Return_type) return 0; @@ -740,14 +734,8 @@ static int init_types(void) if (!Compare_type) return 0; Call_type = make_type("Call", expr_type, Call_fields, 5); if (!Call_type) return 0; - Num_type = make_type("Num", expr_type, Num_fields, 1); - if (!Num_type) return 0; - Str_type = make_type("Str", expr_type, Str_fields, 1); - if (!Str_type) return 0; - Bytes_type = make_type("Bytes", expr_type, Bytes_fields, 1); - if (!Bytes_type) return 0; - Ellipsis_type = make_type("Ellipsis", expr_type, NULL, 0); - if (!Ellipsis_type) return 0; + Lit_type = make_type("Lit", expr_type, Lit_fields, 1); + if (!Lit_type) return 0; Attribute_type = make_type("Attribute", expr_type, Attribute_fields, 3); if (!Attribute_type) return 0; Subscript_type = make_type("Subscript", expr_type, Subscript_fields, 3); @@ -962,7 +950,7 @@ static int obj2ast_keyword(PyObject* obj static int obj2ast_alias(PyObject* obj, alias_ty* out, PyArena* arena); mod_ty -Module(asdl_seq * body, PyArena *arena) +Module(asdl_seq * body, string docstring, PyArena *arena) { mod_ty p; p = (mod_ty)PyArena_Malloc(arena, sizeof(*p)); @@ -970,6 +958,7 @@ Module(asdl_seq * body, PyArena *arena) return NULL; p->kind = Module_kind; p->v.Module.body = body; + p->v.Module.docstring = docstring; return p; } @@ -1016,8 +1005,8 @@ Suite(asdl_seq * body, PyArena *arena) stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq * - decorator_list, expr_ty returns, int lineno, int col_offset, - PyArena *arena) + decorator_list, expr_ty returns, string docstring, int lineno, int + col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1039,6 +1028,7 @@ FunctionDef(identifier name, arguments_t p->v.FunctionDef.body = body; p->v.FunctionDef.decorator_list = decorator_list; p->v.FunctionDef.returns = returns; + p->v.FunctionDef.docstring = docstring; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -1047,7 +1037,7 @@ FunctionDef(identifier name, arguments_t stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, expr_ty starargs, expr_ty kwargs, asdl_seq * body, asdl_seq * decorator_list, - int lineno, int col_offset, PyArena *arena) + string docstring, int lineno, int col_offset, PyArena *arena) { stmt_ty p; if (!name) { @@ -1066,6 +1056,7 @@ ClassDef(identifier name, asdl_seq * bas p->v.ClassDef.kwargs = kwargs; p->v.ClassDef.body = body; p->v.ClassDef.decorator_list = decorator_list; + p->v.ClassDef.docstring = docstring; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -1749,70 +1740,20 @@ Call(expr_ty func, asdl_seq * args, asdl } expr_ty -Num(object n, int lineno, int col_offset, PyArena *arena) +Lit(object v, int lineno, int col_offset, PyArena *arena) { expr_ty p; - if (!n) { + if (!v) v = Py_None; + if (!v) { PyErr_SetString(PyExc_ValueError, - "field n is required for Num"); + "field v is required for Lit"); return NULL; } p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); if (!p) return NULL; - p->kind = Num_kind; - p->v.Num.n = n; - p->lineno = lineno; - p->col_offset = col_offset; - return p; -} - -expr_ty -Str(string s, int lineno, int col_offset, PyArena *arena) -{ - expr_ty p; - if (!s) { - PyErr_SetString(PyExc_ValueError, - "field s is required for Str"); - return NULL; - } - p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = Str_kind; - p->v.Str.s = s; - p->lineno = lineno; - p->col_offset = col_offset; - return p; -} - -expr_ty -Bytes(string s, int lineno, int col_offset, PyArena *arena) -{ - expr_ty p; - if (!s) { - PyErr_SetString(PyExc_ValueError, - "field s is required for Bytes"); - return NULL; - } - p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = Bytes_kind; - p->v.Bytes.s = s; - p->lineno = lineno; - p->col_offset = col_offset; - return p; -} - -expr_ty -Ellipsis(int lineno, int col_offset, PyArena *arena) -{ - expr_ty p; - p = (expr_ty)PyArena_Malloc(arena, sizeof(*p)); - if (!p) - return NULL; - p->kind = Ellipsis_kind; + p->kind = Lit_kind; + p->v.Lit.v = v; p->lineno = lineno; p->col_offset = col_offset; return p; @@ -2155,6 +2096,11 @@ ast2obj_mod(void* _o) if (PyObject_SetAttrString(result, "body", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.Module.docstring); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "docstring", value) == -1) + goto failed; + Py_DECREF(value); break; case Interactive_kind: result = PyType_GenericNew(Interactive_type, NULL, NULL); @@ -2232,6 +2178,11 @@ ast2obj_stmt(void* _o) if (PyObject_SetAttrString(result, "returns", value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.FunctionDef.docstring); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "docstring", value) == -1) + goto failed; + Py_DECREF(value); break; case ClassDef_kind: result = PyType_GenericNew(ClassDef_type, NULL, NULL); @@ -2273,6 +2224,11 @@ ast2obj_stmt(void* _o) -1) goto failed; Py_DECREF(value); + value = ast2obj_string(o->v.ClassDef.docstring); + if (!value) goto failed; + if (PyObject_SetAttrString(result, "docstring", value) == -1) + goto failed; + Py_DECREF(value); break; case Return_kind: result = PyType_GenericNew(Return_type, NULL, NULL); @@ -2796,37 +2752,15 @@ ast2obj_expr(void* _o) goto failed; Py_DECREF(value); break; - case Num_kind: - result = PyType_GenericNew(Num_type, NULL, NULL); + case Lit_kind: + result = PyType_GenericNew(Lit_type, NULL, NULL); if (!result) goto failed; - value = ast2obj_object(o->v.Num.n); + value = ast2obj_object(o->v.Lit.v); if (!value) goto failed; - if (PyObject_SetAttrString(result, "n", value) == -1) + if (PyObject_SetAttrString(result, "v", value) == -1) goto failed; Py_DECREF(value); break; - case Str_kind: - result = PyType_GenericNew(Str_type, NULL, NULL); - if (!result) goto failed; - value = ast2obj_string(o->v.Str.s); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "s", value) == -1) - goto failed; - Py_DECREF(value); - break; - case Bytes_kind: - result = PyType_GenericNew(Bytes_type, NULL, NULL); - if (!result) goto failed; - value = ast2obj_string(o->v.Bytes.s); - if (!value) goto failed; - if (PyObject_SetAttrString(result, "s", value) == -1) - goto failed; - Py_DECREF(value); - break; - case Ellipsis_kind: - result = PyType_GenericNew(Ellipsis_type, NULL, NULL); - if (!result) goto failed; - break; case Attribute_kind: result = PyType_GenericNew(Attribute_type, NULL, NULL); if (!result) goto failed; @@ -3388,6 +3322,7 @@ obj2ast_mod(PyObject* obj, mod_ty* out, } if (isinstance) { asdl_seq* body; + string docstring; if (PyObject_HasAttrString(obj, "body")) { int res; @@ -3400,13 +3335,17 @@ obj2ast_mod(PyObject* obj, mod_ty* out, goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3414,7 +3353,18 @@ obj2ast_mod(PyObject* obj, mod_ty* out, PyErr_SetString(PyExc_TypeError, "required field \"body\" missing from Module"); return 1; } - *out = Module(body, arena); + if (PyObject_HasAttrString(obj, "docstring")) { + int res; + tmp = PyObject_GetAttrString(obj, "docstring"); + if (tmp == NULL) goto failed; + res = obj2ast_string(tmp, &docstring, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + docstring = NULL; + } + *out = Module(body, docstring, arena); if (*out == NULL) goto failed; return 0; } @@ -3436,13 +3386,17 @@ obj2ast_mod(PyObject* obj, mod_ty* out, goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3495,13 +3449,17 @@ obj2ast_mod(PyObject* obj, mod_ty* out, goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3567,6 +3525,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out asdl_seq* body; asdl_seq* decorator_list; expr_ty returns; + string docstring; if (PyObject_HasAttrString(obj, "name")) { int res; @@ -3603,13 +3562,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3628,13 +3591,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - decorator_list = asdl_seq_new(len, arena); - if (decorator_list == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(decorator_list, i, value); + if (len == 0) + decorator_list = NULL; + else { + decorator_list = asdl_seq_new(len, arena); + if (decorator_list == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(decorator_list, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3653,8 +3620,19 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out } else { returns = NULL; } + if (PyObject_HasAttrString(obj, "docstring")) { + int res; + tmp = PyObject_GetAttrString(obj, "docstring"); + if (tmp == NULL) goto failed; + res = obj2ast_string(tmp, &docstring, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + docstring = NULL; + } *out = FunctionDef(name, args, body, decorator_list, returns, - lineno, col_offset, arena); + docstring, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -3670,6 +3648,7 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out expr_ty kwargs; asdl_seq* body; asdl_seq* decorator_list; + string docstring; if (PyObject_HasAttrString(obj, "name")) { int res; @@ -3694,13 +3673,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - bases = asdl_seq_new(len, arena); - if (bases == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(bases, i, value); + if (len == 0) + bases = NULL; + else { + bases = asdl_seq_new(len, arena); + if (bases == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(bases, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3719,13 +3702,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - keywords = asdl_seq_new(len, arena); - if (keywords == NULL) goto failed; - for (i = 0; i < len; i++) { - keyword_ty value; - res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(keywords, i, value); + if (len == 0) + keywords = NULL; + else { + keywords = asdl_seq_new(len, arena); + if (keywords == NULL) goto failed; + for (i = 0; i < len; i++) { + keyword_ty value; + res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(keywords, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3766,13 +3753,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3791,13 +3782,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - decorator_list = asdl_seq_new(len, arena); - if (decorator_list == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(decorator_list, i, value); + if (len == 0) + decorator_list = NULL; + else { + decorator_list = asdl_seq_new(len, arena); + if (decorator_list == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(decorator_list, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3805,8 +3800,20 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef"); return 1; } + if (PyObject_HasAttrString(obj, "docstring")) { + int res; + tmp = PyObject_GetAttrString(obj, "docstring"); + if (tmp == NULL) goto failed; + res = obj2ast_string(tmp, &docstring, arena); + if (res != 0) goto failed; + Py_XDECREF(tmp); + tmp = NULL; + } else { + docstring = NULL; + } *out = ClassDef(name, bases, keywords, starargs, kwargs, body, - decorator_list, lineno, col_offset, arena); + decorator_list, docstring, lineno, col_offset, + arena); if (*out == NULL) goto failed; return 0; } @@ -3850,13 +3857,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - targets = asdl_seq_new(len, arena); - if (targets == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(targets, i, value); + if (len == 0) + targets = NULL; + else { + targets = asdl_seq_new(len, arena); + if (targets == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(targets, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -3887,13 +3898,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - targets = asdl_seq_new(len, arena); - if (targets == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(targets, i, value); + if (len == 0) + targets = NULL; + else { + targets = asdl_seq_new(len, arena); + if (targets == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(targets, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4011,13 +4026,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4036,13 +4055,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - orelse = asdl_seq_new(len, arena); - if (orelse == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(orelse, i, value); + if (len == 0) + orelse = NULL; + else { + orelse = asdl_seq_new(len, arena); + if (orelse == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(orelse, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4087,13 +4110,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4112,13 +4139,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - orelse = asdl_seq_new(len, arena); - if (orelse == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(orelse, i, value); + if (len == 0) + orelse = NULL; + else { + orelse = asdl_seq_new(len, arena); + if (orelse == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(orelse, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4162,13 +4193,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4187,13 +4222,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - orelse = asdl_seq_new(len, arena); - if (orelse == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(orelse, i, value); + if (len == 0) + orelse = NULL; + else { + orelse = asdl_seq_new(len, arena); + if (orelse == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(orelse, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4248,13 +4287,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4321,13 +4364,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4346,13 +4393,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - handlers = asdl_seq_new(len, arena); - if (handlers == NULL) goto failed; - for (i = 0; i < len; i++) { - excepthandler_ty value; - res = obj2ast_excepthandler(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(handlers, i, value); + if (len == 0) + handlers = NULL; + else { + handlers = asdl_seq_new(len, arena); + if (handlers == NULL) goto failed; + for (i = 0; i < len; i++) { + excepthandler_ty value; + res = obj2ast_excepthandler(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(handlers, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4371,13 +4422,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - orelse = asdl_seq_new(len, arena); - if (orelse == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(orelse, i, value); + if (len == 0) + orelse = NULL; + else { + orelse = asdl_seq_new(len, arena); + if (orelse == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(orelse, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4409,13 +4464,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4434,13 +4493,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - finalbody = asdl_seq_new(len, arena); - if (finalbody == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(finalbody, i, value); + if (len == 0) + finalbody = NULL; + else { + finalbody = asdl_seq_new(len, arena); + if (finalbody == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(finalbody, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4505,13 +4568,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - names = asdl_seq_new(len, arena); - if (names == NULL) goto failed; - for (i = 0; i < len; i++) { - alias_ty value; - res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(names, i, value); + if (len == 0) + names = NULL; + else { + names = asdl_seq_new(len, arena); + if (names == NULL) goto failed; + for (i = 0; i < len; i++) { + alias_ty value; + res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(names, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4554,13 +4621,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - names = asdl_seq_new(len, arena); - if (names == NULL) goto failed; - for (i = 0; i < len; i++) { - alias_ty value; - res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(names, i, value); + if (len == 0) + names = NULL; + else { + names = asdl_seq_new(len, arena); + if (names == NULL) goto failed; + for (i = 0; i < len; i++) { + alias_ty value; + res = obj2ast_alias(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(names, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4602,13 +4673,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - names = asdl_seq_new(len, arena); - if (names == NULL) goto failed; - for (i = 0; i < len; i++) { - identifier value; - res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(names, i, value); + if (len == 0) + names = NULL; + else { + names = asdl_seq_new(len, arena); + if (names == NULL) goto failed; + for (i = 0; i < len; i++) { + identifier value; + res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(names, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4638,13 +4713,17 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - names = asdl_seq_new(len, arena); - if (names == NULL) goto failed; - for (i = 0; i < len; i++) { - identifier value; - res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(names, i, value); + if (len == 0) + names = NULL; + else { + names = asdl_seq_new(len, arena); + if (names == NULL) goto failed; + for (i = 0; i < len; i++) { + identifier value; + res = obj2ast_identifier(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(names, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4784,13 +4863,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - values = asdl_seq_new(len, arena); - if (values == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(values, i, value); + if (len == 0) + values = NULL; + else { + values = asdl_seq_new(len, arena); + if (values == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(values, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -4991,13 +5074,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - keys = asdl_seq_new(len, arena); - if (keys == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(keys, i, value); + if (len == 0) + keys = NULL; + else { + keys = asdl_seq_new(len, arena); + if (keys == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(keys, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5016,13 +5103,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - values = asdl_seq_new(len, arena); - if (values == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(values, i, value); + if (len == 0) + values = NULL; + else { + values = asdl_seq_new(len, arena); + if (values == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(values, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5052,13 +5143,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - elts = asdl_seq_new(len, arena); - if (elts == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(elts, i, value); + if (len == 0) + elts = NULL; + else { + elts = asdl_seq_new(len, arena); + if (elts == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(elts, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5101,13 +5196,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - generators = asdl_seq_new(len, arena); - if (generators == NULL) goto failed; - for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(generators, i, value); + if (len == 0) + generators = NULL; + else { + generators = asdl_seq_new(len, arena); + if (generators == NULL) goto failed; + for (i = 0; i < len; i++) { + comprehension_ty value; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(generators, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5150,13 +5249,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - generators = asdl_seq_new(len, arena); - if (generators == NULL) goto failed; - for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(generators, i, value); + if (len == 0) + generators = NULL; + else { + generators = asdl_seq_new(len, arena); + if (generators == NULL) goto failed; + for (i = 0; i < len; i++) { + comprehension_ty value; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(generators, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5212,13 +5315,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - generators = asdl_seq_new(len, arena); - if (generators == NULL) goto failed; - for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(generators, i, value); + if (len == 0) + generators = NULL; + else { + generators = asdl_seq_new(len, arena); + if (generators == NULL) goto failed; + for (i = 0; i < len; i++) { + comprehension_ty value; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(generators, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5262,13 +5369,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - generators = asdl_seq_new(len, arena); - if (generators == NULL) goto failed; - for (i = 0; i < len; i++) { - comprehension_ty value; - res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(generators, i, value); + if (len == 0) + generators = NULL; + else { + generators = asdl_seq_new(len, arena); + if (generators == NULL) goto failed; + for (i = 0; i < len; i++) { + comprehension_ty value; + res = obj2ast_comprehension(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(generators, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5334,13 +5445,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - ops = asdl_int_seq_new(len, arena); - if (ops == NULL) goto failed; - for (i = 0; i < len; i++) { - cmpop_ty value; - res = obj2ast_cmpop(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(ops, i, value); + if (len == 0) + ops = NULL; + else { + ops = asdl_int_seq_new(len, arena); + if (ops == NULL) goto failed; + for (i = 0; i < len; i++) { + cmpop_ty value; + res = obj2ast_cmpop(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(ops, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5359,13 +5474,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - comparators = asdl_seq_new(len, arena); - if (comparators == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(comparators, i, value); + if (len == 0) + comparators = NULL; + else { + comparators = asdl_seq_new(len, arena); + if (comparators == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(comparators, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5412,13 +5531,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - args = asdl_seq_new(len, arena); - if (args == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(args, i, value); + if (len == 0) + args = NULL; + else { + args = asdl_seq_new(len, arena); + if (args == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(args, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5437,13 +5560,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - keywords = asdl_seq_new(len, arena); - if (keywords == NULL) goto failed; - for (i = 0; i < len; i++) { - keyword_ty value; - res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(keywords, i, value); + if (len == 0) + keywords = NULL; + else { + keywords = asdl_seq_new(len, arena); + if (keywords == NULL) goto failed; + for (i = 0; i < len; i++) { + keyword_ty value; + res = obj2ast_keyword(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(keywords, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5478,82 +5605,26 @@ obj2ast_expr(PyObject* obj, expr_ty* out if (*out == NULL) goto failed; return 0; } - isinstance = PyObject_IsInstance(obj, (PyObject*)Num_type); + isinstance = PyObject_IsInstance(obj, (PyObject*)Lit_type); if (isinstance == -1) { return 1; } if (isinstance) { - object n; - - if (PyObject_HasAttrString(obj, "n")) { + object v; + + if (PyObject_HasAttrString(obj, "v")) { int res; - tmp = PyObject_GetAttrString(obj, "n"); + tmp = PyObject_GetAttrString(obj, "v"); if (tmp == NULL) goto failed; - res = obj2ast_object(tmp, &n, arena); + res = obj2ast_object(tmp, &v, arena); if (res != 0) goto failed; Py_XDECREF(tmp); tmp = NULL; } else { - PyErr_SetString(PyExc_TypeError, "required field \"n\" missing from Num"); + PyErr_SetString(PyExc_TypeError, "required field \"v\" missing from Lit"); return 1; } - *out = Num(n, lineno, col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } - isinstance = PyObject_IsInstance(obj, (PyObject*)Str_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - string s; - - if (PyObject_HasAttrString(obj, "s")) { - int res; - tmp = PyObject_GetAttrString(obj, "s"); - if (tmp == NULL) goto failed; - res = obj2ast_string(tmp, &s, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Str"); - return 1; - } - *out = Str(s, lineno, col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } - isinstance = PyObject_IsInstance(obj, (PyObject*)Bytes_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - string s; - - if (PyObject_HasAttrString(obj, "s")) { - int res; - tmp = PyObject_GetAttrString(obj, "s"); - if (tmp == NULL) goto failed; - res = obj2ast_string(tmp, &s, arena); - if (res != 0) goto failed; - Py_XDECREF(tmp); - tmp = NULL; - } else { - PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from Bytes"); - return 1; - } - *out = Bytes(s, lineno, col_offset, arena); - if (*out == NULL) goto failed; - return 0; - } - isinstance = PyObject_IsInstance(obj, (PyObject*)Ellipsis_type); - if (isinstance == -1) { - return 1; - } - if (isinstance) { - - *out = Ellipsis(lineno, col_offset, arena); + *out = Lit(v, lineno, col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -5746,13 +5817,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - elts = asdl_seq_new(len, arena); - if (elts == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(elts, i, value); + if (len == 0) + elts = NULL; + else { + elts = asdl_seq_new(len, arena); + if (elts == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(elts, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5795,13 +5870,17 @@ obj2ast_expr(PyObject* obj, expr_ty* out goto failed; } len = PyList_GET_SIZE(tmp); - elts = asdl_seq_new(len, arena); - if (elts == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(elts, i, value); + if (len == 0) + elts = NULL; + else { + elts = asdl_seq_new(len, arena); + if (elts == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(elts, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -5965,13 +6044,17 @@ obj2ast_slice(PyObject* obj, slice_ty* o goto failed; } len = PyList_GET_SIZE(tmp); - dims = asdl_seq_new(len, arena); - if (dims == NULL) goto failed; - for (i = 0; i < len; i++) { - slice_ty value; - res = obj2ast_slice(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(dims, i, value); + if (len == 0) + dims = NULL; + else { + dims = asdl_seq_new(len, arena); + if (dims == NULL) goto failed; + for (i = 0; i < len; i++) { + slice_ty value; + res = obj2ast_slice(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(dims, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6320,13 +6403,17 @@ obj2ast_comprehension(PyObject* obj, com goto failed; } len = PyList_GET_SIZE(tmp); - ifs = asdl_seq_new(len, arena); - if (ifs == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(ifs, i, value); + if (len == 0) + ifs = NULL; + else { + ifs = asdl_seq_new(len, arena); + if (ifs == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(ifs, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6420,13 +6507,17 @@ obj2ast_excepthandler(PyObject* obj, exc goto failed; } len = PyList_GET_SIZE(tmp); - body = asdl_seq_new(len, arena); - if (body == NULL) goto failed; - for (i = 0; i < len; i++) { - stmt_ty value; - res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(body, i, value); + if (len == 0) + body = NULL; + else { + body = asdl_seq_new(len, arena); + if (body == NULL) goto failed; + for (i = 0; i < len; i++) { + stmt_ty value; + res = obj2ast_stmt(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(body, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6470,13 +6561,17 @@ obj2ast_arguments(PyObject* obj, argumen goto failed; } len = PyList_GET_SIZE(tmp); - args = asdl_seq_new(len, arena); - if (args == NULL) goto failed; - for (i = 0; i < len; i++) { - arg_ty value; - res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(args, i, value); + if (len == 0) + args = NULL; + else { + args = asdl_seq_new(len, arena); + if (args == NULL) goto failed; + for (i = 0; i < len; i++) { + arg_ty value; + res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(args, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6517,13 +6612,17 @@ obj2ast_arguments(PyObject* obj, argumen goto failed; } len = PyList_GET_SIZE(tmp); - kwonlyargs = asdl_seq_new(len, arena); - if (kwonlyargs == NULL) goto failed; - for (i = 0; i < len; i++) { - arg_ty value; - res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(kwonlyargs, i, value); + if (len == 0) + kwonlyargs = NULL; + else { + kwonlyargs = asdl_seq_new(len, arena); + if (kwonlyargs == NULL) goto failed; + for (i = 0; i < len; i++) { + arg_ty value; + res = obj2ast_arg(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(kwonlyargs, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6564,13 +6663,17 @@ obj2ast_arguments(PyObject* obj, argumen goto failed; } len = PyList_GET_SIZE(tmp); - defaults = asdl_seq_new(len, arena); - if (defaults == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(defaults, i, value); + if (len == 0) + defaults = NULL; + else { + defaults = asdl_seq_new(len, arena); + if (defaults == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(defaults, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6589,13 +6692,17 @@ obj2ast_arguments(PyObject* obj, argumen goto failed; } len = PyList_GET_SIZE(tmp); - kw_defaults = asdl_seq_new(len, arena); - if (kw_defaults == NULL) goto failed; - for (i = 0; i < len; i++) { - expr_ty value; - res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); - if (res != 0) goto failed; - asdl_seq_SET(kw_defaults, i, value); + if (len == 0) + kw_defaults = NULL; + else { + kw_defaults = asdl_seq_new(len, arena); + if (kw_defaults == NULL) goto failed; + for (i = 0; i < len; i++) { + expr_ty value; + res = obj2ast_expr(PyList_GET_ITEM(tmp, i), &value, arena); + if (res != 0) goto failed; + asdl_seq_SET(kw_defaults, i, value); + } } Py_XDECREF(tmp); tmp = NULL; @@ -6739,7 +6846,7 @@ PyInit__ast(void) NULL; if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0) return NULL; - if (PyModule_AddStringConstant(m, "__version__", "82163") < 0) + if (PyModule_AddStringConstant(m, "__version__", "") < 0) return NULL; if (PyDict_SetItemString(d, "mod", (PyObject*)mod_type) < 0) return NULL; @@ -6826,14 +6933,8 @@ PyInit__ast(void) return NULL; if (PyDict_SetItemString(d, "Call", (PyObject*)Call_type) < 0) return NULL; - if (PyDict_SetItemString(d, "Num", (PyObject*)Num_type) < 0) return + if (PyDict_SetItemString(d, "Lit", (PyObject*)Lit_type) < 0) return NULL; - if (PyDict_SetItemString(d, "Str", (PyObject*)Str_type) < 0) return - NULL; - if (PyDict_SetItemString(d, "Bytes", (PyObject*)Bytes_type) < 0) return - NULL; - if (PyDict_SetItemString(d, "Ellipsis", (PyObject*)Ellipsis_type) < 0) - return NULL; if (PyDict_SetItemString(d, "Attribute", (PyObject*)Attribute_type) < 0) return NULL; if (PyDict_SetItemString(d, "Subscript", (PyObject*)Subscript_type) < diff -r 1e00b161f5f5 Python/ast.c --- a/Python/ast.c Wed Mar 09 12:53:30 2011 +0100 +++ b/Python/ast.c Wed May 11 11:21:24 2016 +0300 @@ -24,7 +24,9 @@ struct compiling { static asdl_seq *seq_for_testlist(struct compiling *, const node *); static expr_ty ast_for_expr(struct compiling *, const node *); static stmt_ty ast_for_stmt(struct compiling *, const node *); -static asdl_seq *ast_for_suite(struct compiling *, const node *); +static asdl_seq *ast_for_body(struct compiling *c, const node *n, + string *docstring); +static string docstring_from_stmts(asdl_seq *stmts); static asdl_seq *ast_for_exprlist(struct compiling *, const node *, expr_context_ty); static expr_ty ast_for_testlist(struct compiling *, const node *); @@ -275,7 +277,7 @@ PyAST_FromNode(const node *n, PyCompiler } } } - return Module(stmts, arena); + return Module(stmts, docstring_from_stmts(stmts), arena); case eval_input: { expr_ty testlist_ast; @@ -374,6 +376,12 @@ static const char* FORBIDDEN[] = { NULL, }; +static PyObject* FORBIDDEN_VALUES[] = { + Py_None, + Py_True, + Py_False +}; + static int forbidden_name(identifier name, const node *n, int full_checks) { @@ -394,6 +402,24 @@ forbidden_name(identifier name, const no return 0; } +static int +make_name_literal(expr_ty e) +{ + int i; + identifier name; + + assert(e->kind == Name_kind); + name = e->v.Name.id; + for (i = 0; FORBIDDEN[i]; i++) { + if (PyUnicode_CompareWithASCIIString(name, FORBIDDEN[i]) == 0) { + e->kind = Lit_kind; + e->v.Lit.v = FORBIDDEN_VALUES[i]; + return 1; + } + } + return 0; +} + /* Set the context ctx for expr_ty e, recursively traversing e. Only sets context for expr kinds that "can appear in assignment context" @@ -481,13 +507,14 @@ set_context(struct compiling *c, expr_ty break; case Dict_kind: case Set_kind: - case Num_kind: - case Str_kind: + case Lit_kind: + /* Tweak error message */ + if (e->v.Lit.v == Py_None || + e->v.Lit.v == Py_True || + e->v.Lit.v == Py_False) + return ast_error(n, "assignment to keyword"); expr_name = "literal"; break; - case Ellipsis_kind: - expr_name = "Ellipsis"; - break; case Compare_kind: expr_name = "comparison"; break; @@ -1026,6 +1053,7 @@ ast_for_funcdef(struct compiling *c, con arguments_ty args; asdl_seq *body; expr_ty returns = NULL; + string docstring; int name_i = 1; REQ(n, funcdef); @@ -1044,12 +1072,12 @@ ast_for_funcdef(struct compiling *c, con return NULL; name_i += 2; } - body = ast_for_suite(c, CHILD(n, name_i + 3)); + body = ast_for_body(c, CHILD(n, name_i + 3), &docstring); if (!body) return NULL; - return FunctionDef(name, args, body, decorator_seq, returns, LINENO(n), - n->n_col_offset, c->c_arena); + return FunctionDef(name, args, body, decorator_seq, returns, + docstring, LINENO(n), n->n_col_offset, c->c_arena); } static stmt_ty @@ -1357,10 +1385,13 @@ ast_for_atom(struct compiling *c, const case NAME: { /* All names start in Load context, but may later be changed. */ + expr_ty e; PyObject *name = NEW_IDENTIFIER(ch); if (!name) return NULL; - return Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); + e = Name(name, Load, LINENO(n), n->n_col_offset, c->c_arena); + make_name_literal(e); + return e; } case STRING: { PyObject *str = parsestrplus(c, n, &bytesmode); @@ -1386,10 +1417,7 @@ ast_for_atom(struct compiling *c, const return NULL; } PyArena_AddPyObject(c->c_arena, str); - if (bytesmode) - return Bytes(str, LINENO(n), n->n_col_offset, c->c_arena); - else - return Str(str, LINENO(n), n->n_col_offset, c->c_arena); + return Lit(str, LINENO(n), n->n_col_offset, c->c_arena); } case NUMBER: { PyObject *pynum = parsenumber(c, STR(ch)); @@ -1397,10 +1425,10 @@ ast_for_atom(struct compiling *c, const return NULL; PyArena_AddPyObject(c->c_arena, pynum); - return Num(pynum, LINENO(n), n->n_col_offset, c->c_arena); + return Lit(pynum, LINENO(n), n->n_col_offset, c->c_arena); } case ELLIPSIS: /* Ellipsis */ - return Ellipsis(LINENO(n), n->n_col_offset, c->c_arena); + return Lit(Py_Ellipsis, LINENO(n), n->n_col_offset, c->c_arena); case LPAR: /* some parenthesized expressions */ ch = CHILD(n, 1); @@ -2633,6 +2661,40 @@ ast_for_suite(struct compiling *c, const return seq; } +static string +docstring_from_stmts(asdl_seq *stmts) +{ + /* XXX not very efficient, but simple */ + if (stmts && stmts->size) { + stmt_ty s; + + s = (stmt_ty)asdl_seq_GET(stmts, 0); + if (s->kind == Expr_kind && + s->v.Expr.value->kind == Lit_kind && + PyUnicode_CheckExact(s->v.Expr.value->v.Lit.v)) { + /* If first statement is a literal string, it's the + doc string. */ + string doc = s->v.Expr.value->v.Lit.v; + memmove(&asdl_seq_GET(stmts, 0), &asdl_seq_GET(stmts, 1), + (stmts->size - 1) * sizeof(void*)); + stmts->size--; + return doc; + } + } + + return NULL; +} + +static asdl_seq * +ast_for_body(struct compiling *c, const node *n, string *docstring) +{ + asdl_seq *stmts; + + stmts = ast_for_suite(c, n); + *docstring = docstring_from_stmts(stmts); + return stmts; +} + static stmt_ty ast_for_if_stmt(struct compiling *c, const node *n) { @@ -3031,12 +3093,13 @@ ast_for_classdef(struct compiling *c, co /* classdef: 'class' NAME ['(' arglist ')'] ':' suite */ PyObject *classname; asdl_seq *s; + string docstring; expr_ty call; REQ(n, classdef); if (NCH(n) == 4) { /* class NAME ':' suite */ - s = ast_for_suite(c, CHILD(n, 3)); + s = ast_for_body(c, CHILD(n, 3), &docstring); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); @@ -3045,11 +3108,11 @@ ast_for_classdef(struct compiling *c, co if (forbidden_name(classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + docstring, LINENO(n), n->n_col_offset, c->c_arena); } if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */ - s = ast_for_suite(c, CHILD(n,5)); + s = ast_for_body(c, CHILD(n,5), &docstring); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); @@ -3058,7 +3121,7 @@ ast_for_classdef(struct compiling *c, co if (forbidden_name(classname, CHILD(n, 3), 0)) return NULL; return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq, - LINENO(n), n->n_col_offset, c->c_arena); + docstring, LINENO(n), n->n_col_offset, c->c_arena); } /* class NAME '(' arglist ')' ':' suite */ @@ -3074,7 +3137,7 @@ ast_for_classdef(struct compiling *c, co if (!call) return NULL; } - s = ast_for_suite(c, CHILD(n, 6)); + s = ast_for_body(c, CHILD(n, 6), &docstring); if (!s) return NULL; classname = NEW_IDENTIFIER(CHILD(n, 1)); @@ -3085,7 +3148,8 @@ ast_for_classdef(struct compiling *c, co return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, call->v.Call.starargs, call->v.Call.kwargs, s, - decorator_seq, LINENO(n), n->n_col_offset, c->c_arena); + decorator_seq, docstring, LINENO(n), n->n_col_offset, + c->c_arena); } static stmt_ty diff -r 1e00b161f5f5 Python/ast_opt.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Python/ast_opt.c Wed May 11 11:21:24 2016 +0300 @@ -0,0 +1,507 @@ +/* File automatically generated by Parser/asdl_ct.py. */ +#include "Python.h" +#include "Python-ast.h" + + +static int make_lit(expr_ty node, PyObject *val, PyArena *arena) +{ + if (val == NULL) { + if(!PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + PyErr_Clear(); + return 1; + } + if (PyArena_AddPyObject(arena, val) < 0) { + Py_DECREF(val); + return 1; + } + node->kind = Lit_kind; + node->v.Lit.v = val; + return 1; +} + +#define MOVE_NODE(TO, FROM) (memcpy((TO), (FROM), sizeof(struct _expr))) + +static PyObject *unary_not(PyObject *v) +{ + int r = PyObject_IsTrue(v); + if (r < 0) + return NULL; + return r ? Py_False : Py_True; +} + +static int fold_unaryop(expr_ty node, PyArena *arena) +{ + typedef PyObject *(*unary_op)(PyObject*); + static const unary_op ops[] = { + PyNumber_Invert, + unary_not, + PyNumber_Positive, + PyNumber_Negative + }; + /* Eq and NotEq are often implemented in terms of one another, so + folding not (self == other) into self != other breaks implementation + of !=. Detecting such cases doesn't seem worthwhile. + Python uses for 'is subset'/'is superset' operations on sets. + They don't satisfy not folding laws. */ + static const int negated_op[] = { + 0, 0, 0, 0, 0, 0, IsNot, Is, NotIn, In + }; + + expr_ty arg; + PyObject *newval; + + arg = node->v.UnaryOp.operand; + if (arg->kind != Lit_kind) { + /* Fold not into comparison */ + if (node->v.UnaryOp.op == Not && + arg->kind == Compare_kind && + asdl_seq_LEN(arg->v.Compare.ops) == 1) { + int op = asdl_seq_GET(arg->v.Compare.ops, 0); + op = negated_op[op-1]; + if (op) { + asdl_seq_SET(arg->v.Compare.ops, 0, op); + MOVE_NODE(node, arg); + return 1; + } + } + /* TODO: assume that all unary operations cancel out + themselves, i.e. op op x == x? */ + return 1; + } + + newval = ops[node->v.UnaryOp.op - 1](arg->v.Lit.v); + return make_lit(node, newval, arena); +} + +static PyObject *binary_power(PyObject *base, PyObject *power) +{ + return PyNumber_Power(base, power, Py_None); +} + +static int fold_binop(expr_ty node, PyArena *arena) +{ + typedef PyObject *(*bin_op)(PyObject*, PyObject*); + static const bin_op ops[] = { + PyNumber_Add, + PyNumber_Subtract, + PyNumber_Multiply, + PyNumber_TrueDivide, + PyNumber_Remainder, + binary_power, + PyNumber_Lshift, + PyNumber_Rshift, + PyNumber_Or, + PyNumber_Xor, + PyNumber_And, + PyNumber_FloorDivide + }; + + expr_ty lhs, rhs; + Py_ssize_t size; + PyObject *newval; + + lhs = node->v.BinOp.left; + rhs = node->v.BinOp.right; + if (lhs->kind != Lit_kind || rhs->kind != Lit_kind) + return 1; + + newval = ops[node->v.BinOp.op - 1](lhs->v.Lit.v, rhs->v.Lit.v); + /* Avoid creating large constants. */ + size = PyObject_Size(newval); + if (size == -1) { + if (PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + return 1; + PyErr_Clear(); + } + else if (size > 20) { + Py_DECREF(newval); + return 1; + } + return make_lit(node, newval, arena); +} + +static PyObject *make_const_tuple(asdl_seq *elts, int make_set) +{ + PyObject *newval; + int i; + + for (i = 0; i < asdl_seq_LEN(elts); i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + if (e->kind != Lit_kind) + return NULL; + } + + newval = PyTuple_New(asdl_seq_LEN(elts)); + if (newval == NULL) + return NULL; + + for (i = 0; i < asdl_seq_LEN(elts); i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + Py_INCREF(e->v.Lit.v); + PyTuple_SET_ITEM(newval, i, e->v.Lit.v); + } + + /* Need to create frozen_set instead. */ + if (make_set) { + PyObject *old = newval; + newval = PyFrozenSet_New(old); + Py_DECREF(old); + } + return newval; +} + +static int fold_tuple(expr_ty node, PyArena *arena) +{ + PyObject *newval; + + if (node->v.Tuple.ctx != Load) + return 1; + + newval = make_const_tuple(node->v.Tuple.elts, 0); + return make_lit(node, newval, arena); +} + +static int fold_subscr(expr_ty node, PyArena *arena) +{ + PyObject *newval; + expr_ty arg, idx; + slice_ty slice; + + arg = node->v.Subscript.value; + slice = node->v.Subscript.slice; + if (node->v.Subscript.ctx != Load || + arg->kind != Lit_kind || + /* TODO: handle other types of slices */ + slice->kind != Index_kind || + slice->v.Index.value->kind != Lit_kind) + return 1; + + idx = slice->v.Index.value; + newval = PyObject_GetItem(arg->v.Lit.v, idx->v.Lit.v); + return make_lit(node, newval, arena); +} + +static int fold_compare(expr_ty node, PyArena *arena) +{ + asdl_int_seq *ops; + asdl_seq *args; + PyObject *newval; + int i; + + ops = node->v.Compare.ops; + args = node->v.Compare.comparators; + /* TODO: optimize cases with literal arguments. */ + for (i = 0; i < asdl_seq_LEN(ops); i++) { + int op; + expr_ty arg; + asdl_seq *elts; + + op = asdl_seq_GET(ops, i); + arg = (expr_ty)asdl_seq_GET(args, i); + /* Change literal list or set in 'in' or 'not in' into + tuple or frozenset respectively. */ + /* TODO: do the same when list or set is used as iterable + in for loop and comprehensions? */ + if (op != In && op != NotIn) + continue; + if (arg->kind == List_kind) + elts = arg->v.List.elts; + else if (arg->kind == Set_kind) + elts = arg->v.Set.elts; + else continue; + + newval = make_const_tuple(elts, arg->kind == Set_kind); + make_lit(arg, newval, arena); + } + return 1; +} + +static int astfold_slice(slice_ty node_, PyArena* ctx_); +static int astfold_keyword(keyword_ty node_, PyArena* ctx_); +static int astfold_expr(expr_ty node_, PyArena* ctx_); +static int astfold_stmt(stmt_ty node_, PyArena* ctx_); +static int astfold_excepthandler(excepthandler_ty node_, PyArena* ctx_); +static int astfold_comprehension(comprehension_ty node_, PyArena* ctx_); +static int astfold_arg(arg_ty node_, PyArena* ctx_); +static int astfold_mod(mod_ty node_, PyArena* ctx_); +static int astfold_arguments(arguments_ty node_, PyArena* ctx_); +#define CALL(FUNC, TYPE, ARG) \ + if (!FUNC((ARG), ctx_)) \ + return 0; + +#define CALL_OPT(FUNC, TYPE, ARG) \ + if ((ARG) != NULL && !FUNC((ARG), ctx_)) \ + return 0; + +#define CALL_SEQ(FUNC, TYPE, ARG) { \ + int i; \ + asdl_seq *seq = (ARG); /* avoid variable capture */ \ + for (i = 0; i < asdl_seq_LEN(seq); i++) { \ + TYPE elt = (TYPE)asdl_seq_GET(seq, i); \ + /* XXX: kw_defaults has NULL elements, because it's \ + sized to the number of kw args */ \ + if (elt != NULL && !FUNC(elt, ctx_)) \ + return 0; \ + } \ +} + +static int astfold_mod(mod_ty node_, PyArena* ctx_) +{ + switch (node_->kind) { + case Module_kind: + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.Module.body); + break; + case Interactive_kind: + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.Interactive.body); + break; + case Expression_kind: + CALL(astfold_expr, expr_ty, node_->v.Expression.body); + break; + case Suite_kind: + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.Suite.body); + break; + default: + break; + } + return 1; +} + +static int astfold_expr(expr_ty node_, PyArena* ctx_) +{ + switch (node_->kind) { + case BoolOp_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.BoolOp.values); + break; + case BinOp_kind: + CALL(astfold_expr, expr_ty, node_->v.BinOp.left); + CALL(astfold_expr, expr_ty, node_->v.BinOp.right); + CALL(fold_binop, expr_ty, node_); + break; + case UnaryOp_kind: + CALL(astfold_expr, expr_ty, node_->v.UnaryOp.operand); + CALL(fold_unaryop, expr_ty, node_); + break; + case Lambda_kind: + CALL(astfold_arguments, arguments_ty, node_->v.Lambda.args); + CALL(astfold_expr, expr_ty, node_->v.Lambda.body); + break; + case IfExp_kind: + CALL(astfold_expr, expr_ty, node_->v.IfExp.test); + CALL(astfold_expr, expr_ty, node_->v.IfExp.body); + CALL(astfold_expr, expr_ty, node_->v.IfExp.orelse); + break; + case Dict_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.Dict.keys); + CALL_SEQ(astfold_expr, expr_ty, node_->v.Dict.values); + break; + case Set_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.Set.elts); + break; + case ListComp_kind: + CALL(astfold_expr, expr_ty, node_->v.ListComp.elt); + CALL_SEQ(astfold_comprehension, comprehension_ty, node_->v.ListComp.generators); + break; + case SetComp_kind: + CALL(astfold_expr, expr_ty, node_->v.SetComp.elt); + CALL_SEQ(astfold_comprehension, comprehension_ty, node_->v.SetComp.generators); + break; + case DictComp_kind: + CALL(astfold_expr, expr_ty, node_->v.DictComp.key); + CALL(astfold_expr, expr_ty, node_->v.DictComp.value); + CALL_SEQ(astfold_comprehension, comprehension_ty, node_->v.DictComp.generators); + break; + case GeneratorExp_kind: + CALL(astfold_expr, expr_ty, node_->v.GeneratorExp.elt); + CALL_SEQ(astfold_comprehension, comprehension_ty, node_->v.GeneratorExp.generators); + break; + case Yield_kind: + CALL_OPT(astfold_expr, expr_ty, node_->v.Yield.value); + break; + case Compare_kind: + CALL(astfold_expr, expr_ty, node_->v.Compare.left); + CALL_SEQ(astfold_expr, expr_ty, node_->v.Compare.comparators); + CALL(fold_compare, expr_ty, node_); + break; + case Call_kind: + CALL(astfold_expr, expr_ty, node_->v.Call.func); + CALL_SEQ(astfold_expr, expr_ty, node_->v.Call.args); + CALL_SEQ(astfold_keyword, keyword_ty, node_->v.Call.keywords); + CALL_OPT(astfold_expr, expr_ty, node_->v.Call.starargs); + CALL_OPT(astfold_expr, expr_ty, node_->v.Call.kwargs); + break; + case Attribute_kind: + CALL(astfold_expr, expr_ty, node_->v.Attribute.value); + break; + case Subscript_kind: + CALL(astfold_expr, expr_ty, node_->v.Subscript.value); + CALL(astfold_slice, slice_ty, node_->v.Subscript.slice); + CALL(fold_subscr, expr_ty, node_); + break; + case Starred_kind: + CALL(astfold_expr, expr_ty, node_->v.Starred.value); + break; + case List_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.List.elts); + break; + case Tuple_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.Tuple.elts); + CALL(fold_tuple, expr_ty, node_); + break; + default: + break; + } + return 1; +} + +static int astfold_slice(slice_ty node_, PyArena* ctx_) +{ + switch (node_->kind) { + case Slice_kind: + CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.lower); + CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.upper); + CALL_OPT(astfold_expr, expr_ty, node_->v.Slice.step); + break; + case ExtSlice_kind: + CALL_SEQ(astfold_slice, slice_ty, node_->v.ExtSlice.dims); + break; + case Index_kind: + CALL(astfold_expr, expr_ty, node_->v.Index.value); + break; + default: + break; + } + return 1; +} + +static int astfold_keyword(keyword_ty node_, PyArena* ctx_) +{ + CALL(astfold_expr, expr_ty, node_->value); + return 1; +} + +static int astfold_comprehension(comprehension_ty node_, PyArena* ctx_) +{ + CALL(astfold_expr, expr_ty, node_->target); + CALL(astfold_expr, expr_ty, node_->iter); + CALL_SEQ(astfold_expr, expr_ty, node_->ifs); + return 1; +} + +static int astfold_arguments(arguments_ty node_, PyArena* ctx_) +{ + CALL_SEQ(astfold_arg, arg_ty, node_->args); + CALL_OPT(astfold_expr, expr_ty, node_->varargannotation); + CALL_SEQ(astfold_arg, arg_ty, node_->kwonlyargs); + CALL_OPT(astfold_expr, expr_ty, node_->kwargannotation); + CALL_SEQ(astfold_expr, expr_ty, node_->defaults); + CALL_SEQ(astfold_expr, expr_ty, node_->kw_defaults); + return 1; +} + +static int astfold_arg(arg_ty node_, PyArena* ctx_) +{ + CALL_OPT(astfold_expr, expr_ty, node_->annotation); + return 1; +} + +static int astfold_stmt(stmt_ty node_, PyArena* ctx_) +{ + switch (node_->kind) { + case FunctionDef_kind: + CALL(astfold_arguments, arguments_ty, node_->v.FunctionDef.args); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.FunctionDef.body); + CALL_SEQ(astfold_expr, expr_ty, node_->v.FunctionDef.decorator_list); + CALL_OPT(astfold_expr, expr_ty, node_->v.FunctionDef.returns); + break; + case ClassDef_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.bases); + CALL_SEQ(astfold_keyword, keyword_ty, node_->v.ClassDef.keywords); + CALL_OPT(astfold_expr, expr_ty, node_->v.ClassDef.starargs); + CALL_OPT(astfold_expr, expr_ty, node_->v.ClassDef.kwargs); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.ClassDef.body); + CALL_SEQ(astfold_expr, expr_ty, node_->v.ClassDef.decorator_list); + break; + case Return_kind: + CALL_OPT(astfold_expr, expr_ty, node_->v.Return.value); + break; + case Delete_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.Delete.targets); + break; + case Assign_kind: + CALL_SEQ(astfold_expr, expr_ty, node_->v.Assign.targets); + CALL(astfold_expr, expr_ty, node_->v.Assign.value); + break; + case AugAssign_kind: + CALL(astfold_expr, expr_ty, node_->v.AugAssign.target); + CALL(astfold_expr, expr_ty, node_->v.AugAssign.value); + break; + case For_kind: + CALL(astfold_expr, expr_ty, node_->v.For.target); + CALL(astfold_expr, expr_ty, node_->v.For.iter); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.For.body); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.For.orelse); + break; + case While_kind: + CALL(astfold_expr, expr_ty, node_->v.While.test); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.While.body); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.While.orelse); + break; + case If_kind: + CALL(astfold_expr, expr_ty, node_->v.If.test); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.If.body); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.If.orelse); + break; + case With_kind: + CALL(astfold_expr, expr_ty, node_->v.With.context_expr); + CALL_OPT(astfold_expr, expr_ty, node_->v.With.optional_vars); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.With.body); + break; + case Raise_kind: + CALL_OPT(astfold_expr, expr_ty, node_->v.Raise.exc); + CALL_OPT(astfold_expr, expr_ty, node_->v.Raise.cause); + break; + case TryExcept_kind: + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.TryExcept.body); + CALL_SEQ(astfold_excepthandler, excepthandler_ty, node_->v.TryExcept.handlers); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.TryExcept.orelse); + break; + case TryFinally_kind: + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.TryFinally.body); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.TryFinally.finalbody); + break; + case Assert_kind: + CALL(astfold_expr, expr_ty, node_->v.Assert.test); + CALL_OPT(astfold_expr, expr_ty, node_->v.Assert.msg); + break; + case Expr_kind: + CALL(astfold_expr, expr_ty, node_->v.Expr.value); + break; + default: + break; + } + return 1; +} + +static int astfold_excepthandler(excepthandler_ty node_, PyArena* ctx_) +{ + switch (node_->kind) { + case ExceptHandler_kind: + CALL_OPT(astfold_expr, expr_ty, node_->v.ExceptHandler.type); + CALL_SEQ(astfold_stmt, stmt_ty, node_->v.ExceptHandler.body); + break; + default: + break; + } + return 1; +} + + +#undef CALL +#undef CALL_OPT +#undef CALL_SEQ + +int _PyAST_Optimize(mod_ty mod, PyArena *arena) +{ + return astfold_mod(mod, arena); +} + diff -r 1e00b161f5f5 Python/ast_opt.ct --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Python/ast_opt.ct Wed May 11 11:21:24 2016 +0300 @@ -0,0 +1,229 @@ +#include "Python.h" +#include "Python-ast.h" + + +static int make_lit(expr_ty node, PyObject *val, PyArena *arena) +{ + if (val == NULL) { + if(!PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + PyErr_Clear(); + return 1; + } + if (PyArena_AddPyObject(arena, val) < 0) { + Py_DECREF(val); + return 1; + } + node->kind = Lit_kind; + node->v.Lit.v = val; + return 1; +} + +#define MOVE_NODE(TO, FROM) (memcpy((TO), (FROM), sizeof(struct _expr))) + +static PyObject *unary_not(PyObject *v) +{ + int r = PyObject_IsTrue(v); + if (r < 0) + return NULL; + return r ? Py_False : Py_True; +} + +@post(astfold, UnaryOp, fold_unaryop) +static int fold_unaryop(expr_ty node, PyArena *arena) +{ + typedef PyObject *(*unary_op)(PyObject*); + static const unary_op ops[] = { + PyNumber_Invert, + unary_not, + PyNumber_Positive, + PyNumber_Negative + }; + /* Eq and NotEq are often implemented in terms of one another, so + folding not (self == other) into self != other breaks implementation + of !=. Detecting such cases doesn't seem worthwhile. + Python uses for 'is subset'/'is superset' operations on sets. + They don't satisfy not folding laws. */ + static const int negated_op[] = { + 0, 0, 0, 0, 0, 0, IsNot, Is, NotIn, In + }; + + expr_ty arg; + PyObject *newval; + + arg = node->v.UnaryOp.operand; + if (arg->kind != Lit_kind) { + /* Fold not into comparison */ + if (node->v.UnaryOp.op == Not && + arg->kind == Compare_kind && + asdl_seq_LEN(arg->v.Compare.ops) == 1) { + int op = asdl_seq_GET(arg->v.Compare.ops, 0); + op = negated_op[op-1]; + if (op) { + asdl_seq_SET(arg->v.Compare.ops, 0, op); + MOVE_NODE(node, arg); + return 1; + } + } + /* TODO: assume that all unary operations cancel out + themselves, i.e. op op x == x? */ + return 1; + } + + newval = ops[node->v.UnaryOp.op - 1](arg->v.Lit.v); + return make_lit(node, newval, arena); +} + +static PyObject *binary_power(PyObject *base, PyObject *power) +{ + return PyNumber_Power(base, power, Py_None); +} + +@post(astfold, BinOp, fold_binop) +static int fold_binop(expr_ty node, PyArena *arena) +{ + typedef PyObject *(*bin_op)(PyObject*, PyObject*); + static const bin_op ops[] = { + PyNumber_Add, + PyNumber_Subtract, + PyNumber_Multiply, + PyNumber_TrueDivide, + PyNumber_Remainder, + binary_power, + PyNumber_Lshift, + PyNumber_Rshift, + PyNumber_Or, + PyNumber_Xor, + PyNumber_And, + PyNumber_FloorDivide + }; + + expr_ty lhs, rhs; + Py_ssize_t size; + PyObject *newval; + + lhs = node->v.BinOp.left; + rhs = node->v.BinOp.right; + if (lhs->kind != Lit_kind || rhs->kind != Lit_kind) + return 1; + + newval = ops[node->v.BinOp.op - 1](lhs->v.Lit.v, rhs->v.Lit.v); + /* Avoid creating large constants. */ + size = PyObject_Size(newval); + if (size == -1) { + if (PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) + return 1; + PyErr_Clear(); + } + else if (size > 20) { + Py_DECREF(newval); + return 1; + } + return make_lit(node, newval, arena); +} + +static PyObject *make_const_tuple(asdl_seq *elts, int make_set) +{ + PyObject *newval; + int i; + + for (i = 0; i < asdl_seq_LEN(elts); i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + if (e->kind != Lit_kind) + return NULL; + } + + newval = PyTuple_New(asdl_seq_LEN(elts)); + if (newval == NULL) + return NULL; + + for (i = 0; i < asdl_seq_LEN(elts); i++) { + expr_ty e = (expr_ty)asdl_seq_GET(elts, i); + Py_INCREF(e->v.Lit.v); + PyTuple_SET_ITEM(newval, i, e->v.Lit.v); + } + + /* Need to create frozen_set instead. */ + if (make_set) { + PyObject *old = newval; + newval = PyFrozenSet_New(old); + Py_DECREF(old); + } + return newval; +} + +@post(astfold, Tuple, fold_tuple) +static int fold_tuple(expr_ty node, PyArena *arena) +{ + PyObject *newval; + + if (node->v.Tuple.ctx != Load) + return 1; + + newval = make_const_tuple(node->v.Tuple.elts, 0); + return make_lit(node, newval, arena); +} + +@post(astfold, Subscript, fold_subscr) +static int fold_subscr(expr_ty node, PyArena *arena) +{ + PyObject *newval; + expr_ty arg, idx; + slice_ty slice; + + arg = node->v.Subscript.value; + slice = node->v.Subscript.slice; + if (node->v.Subscript.ctx != Load || + arg->kind != Lit_kind || + /* TODO: handle other types of slices */ + slice->kind != Index_kind || + slice->v.Index.value->kind != Lit_kind) + return 1; + + idx = slice->v.Index.value; + newval = PyObject_GetItem(arg->v.Lit.v, idx->v.Lit.v); + return make_lit(node, newval, arena); +} + +@post(astfold, Compare, fold_compare) +static int fold_compare(expr_ty node, PyArena *arena) +{ + asdl_int_seq *ops; + asdl_seq *args; + PyObject *newval; + int i; + + ops = node->v.Compare.ops; + args = node->v.Compare.comparators; + /* TODO: optimize cases with literal arguments. */ + for (i = 0; i < asdl_seq_LEN(ops); i++) { + int op; + expr_ty arg; + asdl_seq *elts; + + op = asdl_seq_GET(ops, i); + arg = (expr_ty)asdl_seq_GET(args, i); + /* Change literal list or set in 'in' or 'not in' into + tuple or frozenset respectively. */ + /* TODO: do the same when list or set is used as iterable + in for loop and comprehensions? */ + if (op != In && op != NotIn) + continue; + if (arg->kind == List_kind) + elts = arg->v.List.elts; + else if (arg->kind == Set_kind) + elts = arg->v.Set.elts; + else continue; + + newval = make_const_tuple(elts, arg->kind == Set_kind); + make_lit(arg, newval, arena); + } + return 1; +} + +@visitor(astfold, PyArena*, mod) + +int _PyAST_Optimize(mod_ty mod, PyArena *arena) +{ + return astfold_mod(mod, arena); +} + diff -r 1e00b161f5f5 Python/compile.c --- a/Python/compile.c Wed Mar 09 12:53:30 2011 +0100 +++ b/Python/compile.c Wed May 11 11:21:24 2016 +0300 @@ -287,6 +287,9 @@ PyAST_CompileEx(mod_ty mod, const char * c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize; c.c_nestlevel = 0; + if (!_PyAST_Optimize(mod, arena)) + goto finally; + c.c_st = PySymtable_Build(mod, filename, c.c_future); if (c.c_st == NULL) { if (!PyErr_Occurred()) @@ -996,15 +999,19 @@ compiler_addop_name(struct compiler *c, static int compiler_addop_i(struct compiler *c, int opcode, int oparg) { + basicblock *b; struct instr *i; int off; off = compiler_next_instr(c, c->u->u_curblock); if (off < 0) return 0; - i = &c->u->u_curblock->b_instr[off]; + b = c->u->u_curblock; + i = &b->b_instr[off]; i->i_opcode = opcode; i->i_oparg = oparg; i->i_hasarg = 1; + if (opcode == RAISE_VARARGS) + b->b_return = 1; compiler_set_lineno(c, off); return 1; } @@ -1132,33 +1139,23 @@ compiler_addop_j(struct compiler *c, int } \ } +/* Compile a sequence of statements, setting a docstring. */ + static int -compiler_isdocstring(stmt_ty s) +compiler_body(struct compiler *c, asdl_seq *stmts, string docstring) { - if (s->kind != Expr_kind) - return 0; - return s->v.Expr.value->kind == Str_kind; -} - -/* Compile a sequence of statements, checking for a docstring. */ - -static int -compiler_body(struct compiler *c, asdl_seq *stmts) -{ - int i = 0; + int i; stmt_ty st; + /* if not -OO mode, set docstring */ + if (c->c_optimize < 2 && docstring) { + ADDOP_O(c, LOAD_CONST, docstring, consts); + ADDOP_NAME(c, STORE_NAME, __doc__, names); + } if (!asdl_seq_LEN(stmts)) return 1; st = (stmt_ty)asdl_seq_GET(stmts, 0); - if (compiler_isdocstring(st) && c->c_optimize < 2) { - /* don't generate docstrings if -OO */ - i = 1; - VISIT(c, expr, st->v.Expr.value); - if (!compiler_nameop(c, __doc__, Store)) - return 0; - } - for (; i < asdl_seq_LEN(stmts); i++) + for (i = 0; i < asdl_seq_LEN(stmts); i++) VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); return 1; } @@ -1179,7 +1176,7 @@ compiler_mod(struct compiler *c, mod_ty return NULL; switch (mod->kind) { case Module_kind: - if (!compiler_body(c, mod->v.Module.body)) { + if (!compiler_body(c, mod->v.Module.body, mod->v.Module.docstring)) { compiler_exit_scope(c); return 0; } @@ -1433,12 +1430,12 @@ static int compiler_function(struct compiler *c, stmt_ty s) { PyCodeObject *co; - PyObject *first_const = Py_None; + PyObject *docstring = Py_None; arguments_ty args = s->v.FunctionDef.args; expr_ty returns = s->v.FunctionDef.returns; asdl_seq* decos = s->v.FunctionDef.decorator_list; stmt_ty st; - int i, n, docstring, kw_default_count = 0, arglength; + int i, n, kw_default_count = 0, arglength; int num_annotations; assert(s->kind == FunctionDef_kind); @@ -1464,10 +1461,10 @@ compiler_function(struct compiler *c, st return 0; st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0); - docstring = compiler_isdocstring(st); - if (docstring && c->c_optimize < 2) - first_const = st->v.Expr.value->v.Str.s; - if (compiler_add_o(c, c->u->u_consts, first_const) < 0) { + /* if not -OO mode, add docstring */ + if (c->c_optimize < 2 && s->v.FunctionDef.docstring) + docstring = s->v.FunctionDef.docstring; + if (compiler_add_o(c, c->u->u_consts, docstring) < 0) { compiler_exit_scope(c); return 0; } @@ -1475,8 +1472,7 @@ compiler_function(struct compiler *c, st c->u->u_argcount = asdl_seq_LEN(args->args); c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); n = asdl_seq_LEN(s->v.FunctionDef.body); - /* if there was a docstring, we need to skip the first statement */ - for (i = docstring; i < n; i++) { + for (i = 0; i < n; i++) { st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, i); VISIT_IN_SCOPE(c, stmt, st); } @@ -1554,7 +1550,7 @@ compiler_class(struct compiler *c, stmt_ } Py_DECREF(str); /* compile the body proper */ - if (!compiler_body(c, s->v.ClassDef.body)) { + if (!compiler_body(c, s->v.ClassDef.body, s->v.ClassDef.docstring)) { compiler_exit_scope(c); return 0; } @@ -1614,20 +1610,104 @@ compiler_class(struct compiler *c, stmt_ return 1; } +/* Last jump needs special handling, since we're going to fall through + into 'next' block. If that's where we were going to jump, reverse + condition and jump to alternative instead. */ +static int +compiler_cond_jump(struct compiler *c, int if_true, + basicblock *target, basicblock *alt_target, + basicblock *next) +{ + int opcode; + + if (target == next) { + if_true = !if_true; + target = alt_target; + } + opcode = if_true ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE; + ADDOP_JABS(c, opcode, target); + return 1; +} + +static int +compiler_condition(struct compiler *c, expr_ty cond, + basicblock *if_true, basicblock *if_false, + basicblock *next) +{ + int i, op, len; + asdl_seq *vals; + basicblock *alt; + + /* Optimize logical operations inside condition. */ + switch (cond->kind) { + case UnaryOp_kind: + if (cond->v.UnaryOp.op == Not) + return compiler_condition(c, cond->v.UnaryOp.operand, + if_false, if_true, next); + break; + + case BoolOp_kind: + alt = NULL; + op = cond->v.BoolOp.op == And; + vals = cond->v.BoolOp.values; + len = asdl_seq_LEN(vals); + for (i = 0; i < len; i++) { + expr_ty e; + basicblock *t, *f; + + if (alt) + compiler_use_next_block(c, alt); + /* Make new 'next' block, unless we're at the last expr. */ + if (i < len-1) { + alt = compiler_new_block(c); + if (alt == NULL) + return 0; + } + else alt = next; + /* AND: if true try next expr, if false go to parent's false + OR: if false try next expr, if true go to parent's true. */ + if (op) { + t = (alt == next) ? if_true : alt; + f = if_false; + } + else { + t = if_true; + f = (alt == next) ? if_false : alt; + } + e = (expr_ty)asdl_seq_GET(vals, i); + if (!compiler_condition(c, e, t, f, alt)) + return 0; + } + return 1; + + default: + break; + } + + /* Fall back to simple handling. */ + VISIT(c, expr, cond); + return compiler_cond_jump(c, 0, if_false, if_true, next); +} + + static int compiler_ifexp(struct compiler *c, expr_ty e) { - basicblock *end, *next; + basicblock *end, *then, *next; assert(e->kind == IfExp_kind); end = compiler_new_block(c); if (end == NULL) return 0; + then = compiler_new_block(c); + if (then == NULL) + return 0; next = compiler_new_block(c); if (next == NULL) return 0; - VISIT(c, expr, e->v.IfExp.test); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, next); + if (!compiler_condition(c, e->v.IfExp.test, then, next, then)) + return 0; + compiler_use_next_block(c, then); VISIT(c, expr, e->v.IfExp.body); ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, next); @@ -1692,7 +1772,7 @@ compiler_lambda(struct compiler *c, expr static int compiler_if(struct compiler *c, stmt_ty s) { - basicblock *end, *next; + basicblock *then, *end, *next; int constant; assert(s->kind == If_kind); end = compiler_new_block(c); @@ -1709,6 +1789,9 @@ compiler_if(struct compiler *c, stmt_ty } else if (constant == 1) { VISIT_SEQ(c, stmt, s->v.If.body); } else { + then = compiler_new_block(c); + if (then == NULL) + return 0; if (s->v.If.orelse) { next = compiler_new_block(c); if (next == NULL) @@ -1716,11 +1799,12 @@ compiler_if(struct compiler *c, stmt_ty } else next = end; - VISIT(c, expr, s->v.If.test); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, next); + if (!compiler_condition(c, s->v.If.test, then, next, then)) + return 0; + compiler_use_next_block(c, then); VISIT_SEQ(c, stmt, s->v.If.body); - ADDOP_JREL(c, JUMP_FORWARD, end); if (s->v.If.orelse) { + ADDOP_JREL(c, JUMP_FORWARD, end); compiler_use_next_block(c, next); VISIT_SEQ(c, stmt, s->v.If.orelse); } @@ -1760,7 +1844,7 @@ compiler_for(struct compiler *c, stmt_ty static int compiler_while(struct compiler *c, stmt_ty s) { - basicblock *loop, *orelse, *end, *anchor = NULL; + basicblock *loop, *orelse, *end, *body = NULL, *anchor = NULL; int constant = expr_constant(c, s->v.While.test); if (constant == 0) { @@ -1771,8 +1855,9 @@ compiler_while(struct compiler *c, stmt_ loop = compiler_new_block(c); end = compiler_new_block(c); if (constant == -1) { + body = compiler_new_block(c); anchor = compiler_new_block(c); - if (anchor == NULL) + if (body == NULL || anchor == NULL) return 0; } if (loop == NULL || end == NULL) @@ -1790,8 +1875,9 @@ compiler_while(struct compiler *c, stmt_ if (!compiler_push_fblock(c, LOOP, loop)) return 0; if (constant == -1) { - VISIT(c, expr, s->v.While.test); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, anchor); + if (!compiler_condition(c, s->v.While.test, body, anchor, body)) + return 0; + compiler_use_next_block(c, body); } VISIT_SEQ(c, stmt, s->v.While.body); ADDOP_JABS(c, JUMP_ABSOLUTE, loop); @@ -2319,8 +2405,7 @@ compiler_visit_stmt(struct compiler *c, VISIT(c, expr, s->v.Expr.value); ADDOP(c, PRINT_EXPR); } - else if (s->v.Expr.value->kind != Str_kind && - s->v.Expr.value->kind != Num_kind) { + else if (s->v.Expr.value->kind != Lit_kind) { VISIT(c, expr, s->v.Expr.value); ADDOP(c, POP_TOP); } @@ -2604,6 +2689,41 @@ compiler_boolop(struct compiler *c, expr return 1; } +#define LAST_INSTR(C) \ + ((C)->u->u_curblock->b_iused > 0 ? \ + &(C)->u->u_curblock->b_instr[(C)->u->u_curblock->b_iused-1] : \ + NULL) + +static int +compiler_skip_unpack(struct compiler *c, int n) +{ + /* Skip over BUILD_SEQN 1 UNPACK_SEQN 1. + Replace BUILD_SEQN 2 UNPACK_SEQN 2 with ROT2. + Replace BUILD_SEQN 3 UNPACK_SEQN 3 with ROT3 ROT2. */ + struct instr *in; + + if (n < 1 || n > 3 || + (in = LAST_INSTR(c)) == NULL) + return 0; + + /* NOTE: do NOT do this for SET, set deletes duplicate + values, while replacement code doesn't. */ + if ((in->i_opcode != BUILD_TUPLE && + in->i_opcode != BUILD_LIST) || + in->i_oparg != n) + return 0; + + if (n == 1) + c->u->u_curblock->b_iused -= 1; + else { + in->i_opcode = n == 2 ? ROT_TWO : ROT_THREE; + in->i_hasarg = 0; + if (n == 3) + ADDOP(c, ROT_TWO); + } + return 1; +} + static int compiler_list(struct compiler *c, expr_ty e) { @@ -2627,7 +2747,8 @@ compiler_list(struct compiler *c, expr_t } } if (!seen_star) { - ADDOP_I(c, UNPACK_SEQUENCE, n); + if (!compiler_skip_unpack(c, n)) + ADDOP_I(c, UNPACK_SEQUENCE, n); } } VISIT_SEQ(c, expr, e->v.List.elts); @@ -2660,7 +2781,8 @@ compiler_tuple(struct compiler *c, expr_ } } if (!seen_star) { - ADDOP_I(c, UNPACK_SEQUENCE, n); + if (!compiler_skip_unpack(c, n)) + ADDOP_I(c, UNPACK_SEQUENCE, n); } } VISIT_SEQ(c, expr, e->v.Tuple.elts); @@ -2825,10 +2947,16 @@ compiler_comprehension_generator(struct /* XXX this needs to be cleaned up...a lot! */ n = asdl_seq_LEN(gen->ifs); for (i = 0; i < n; i++) { - expr_ty e = (expr_ty)asdl_seq_GET(gen->ifs, i); - VISIT(c, expr, e); - ADDOP_JABS(c, POP_JUMP_IF_FALSE, if_cleanup); - NEXT_BLOCK(c); + expr_ty e; + basicblock *next; + + next = compiler_new_block(c); + if (next == NULL) + return 0; + e = (expr_ty)asdl_seq_GET(gen->ifs, i); + if (!compiler_condition(c, e, next, if_cleanup, next)) + return 0; + compiler_use_next_block(c, next); } if (++gen_index < asdl_seq_LEN(generators)) @@ -3015,27 +3143,7 @@ compiler_visit_keyword(struct compiler * static int expr_constant(struct compiler *c, expr_ty e) { - char *id; - switch (e->kind) { - case Ellipsis_kind: - return 1; - case Num_kind: - return PyObject_IsTrue(e->v.Num.n); - case Str_kind: - return PyObject_IsTrue(e->v.Str.s); - case Name_kind: - /* optimize away names that can't be reassigned */ - id = PyBytes_AS_STRING( - _PyUnicode_AsDefaultEncodedString(e->v.Name.id)); - if (strcmp(id, "True") == 0) return 1; - if (strcmp(id, "False") == 0) return 0; - if (strcmp(id, "None") == 0) return 0; - if (strcmp(id, "__debug__") == 0) - return ! c->c_optimize; - /* fall through */ - default: - return -1; - } + return e->kind == Lit_kind ? PyObject_IsTrue(e->v.Lit.v) : -1; } /* @@ -3183,17 +3291,8 @@ compiler_visit_expr(struct compiler *c, return compiler_compare(c, e); case Call_kind: return compiler_call(c, e); - case Num_kind: - ADDOP_O(c, LOAD_CONST, e->v.Num.n, consts); - break; - case Str_kind: - ADDOP_O(c, LOAD_CONST, e->v.Str.s, consts); - break; - case Bytes_kind: - ADDOP_O(c, LOAD_CONST, e->v.Bytes.s, consts); - break; - case Ellipsis_kind: - ADDOP_O(c, LOAD_CONST, Py_Ellipsis, consts); + case Lit_kind: + ADDOP_O(c, LOAD_CONST, e->v.Lit.v, consts); break; /* The following exprs can be assignment targets. */ case Attribute_kind: @@ -3661,6 +3760,62 @@ blocksize(basicblock *b) return size; } +static void +optimize_jumps(struct compiler *c) +{ + basicblock *b = c->u->u_blocks; + while (b) { + int i; + for (i = 0; i < b->b_iused; i++) { + struct instr *instr = &b->b_instr[i]; + /* Turn unconditional jumps to RETURN into RETURNs. */ + if ((instr->i_opcode == JUMP_FORWARD || + instr->i_opcode == JUMP_ABSOLUTE) && + instr->i_target->b_iused > 0 && + instr->i_target->b_instr[0].i_opcode == RETURN_VALUE) { + instr->i_opcode = RETURN_VALUE; + instr->i_target = 0; + instr->i_hasarg = instr->i_jabs = instr->i_jrel = 0; + b->b_return = 1; + } + /* Retarget jumps to unconditional jumps. */ + if (instr->i_opcode == JUMP_FORWARD || + instr->i_opcode == JUMP_ABSOLUTE || + instr->i_opcode == POP_JUMP_IF_FALSE || + instr->i_opcode == POP_JUMP_IF_TRUE || + instr->i_opcode == JUMP_IF_FALSE_OR_POP || + instr->i_opcode == JUMP_IF_TRUE_OR_POP) { + for (;;) { + struct instr *target; + if (instr->i_target->b_iused == 0) + break; + target = &instr->i_target->b_instr[0]; + if (instr == target || + (target->i_opcode != JUMP_FORWARD && + target->i_opcode != JUMP_ABSOLUTE)) + break; + instr->i_target = target->i_target; + /* Change relative jump to absolute. */ + if (instr->i_opcode == JUMP_FORWARD) { + instr->i_opcode = JUMP_ABSOLUTE; + instr->i_jabs = 1; + instr->i_jrel = 0; + } + } + } + /* Code after unconditional jumps is dead. */ + if (instr->i_opcode == JUMP_FORWARD || + instr->i_opcode == JUMP_ABSOLUTE || + instr->i_opcode == RETURN_VALUE || + instr->i_opcode == RAISE_VARARGS) { + b->b_iused = i + 1; + break; + } + } + b = b->b_list; + } +} + /* Appends a pair to the end of the line number table, a_lnotab, representing the instruction's bytecode offset and line number. See Objects/lnotab_notes.txt for the description of the line number table. */ @@ -3931,13 +4086,12 @@ makecode(struct compiler *c, struct asse PyObject *name = NULL; PyObject *freevars = NULL; PyObject *cellvars = NULL; - PyObject *bytecode = NULL; int nlocals, flags; tmp = dict_keys_inorder(c->u->u_consts, 0); if (!tmp) goto error; - consts = PySequence_List(tmp); /* optimize_code requires a list */ + consts = PySequence_Tuple(tmp); Py_DECREF(tmp); names = dict_keys_inorder(c->u->u_names, 0); @@ -3960,19 +4114,9 @@ makecode(struct compiler *c, struct asse if (flags < 0) goto error; - bytecode = PyCode_Optimize(a->a_bytecode, consts, names, a->a_lnotab); - if (!bytecode) - goto error; - - tmp = PyList_AsTuple(consts); /* PyCode_New requires a tuple */ - if (!tmp) - goto error; - Py_DECREF(consts); - consts = tmp; - co = PyCode_New(c->u->u_argcount, c->u->u_kwonlyargcount, nlocals, stackdepth(c), flags, - bytecode, consts, names, varnames, + a->a_bytecode, consts, names, varnames, freevars, cellvars, filename, c->u->u_name, c->u->u_firstlineno, @@ -3985,7 +4129,6 @@ makecode(struct compiler *c, struct asse Py_XDECREF(name); Py_XDECREF(freevars); Py_XDECREF(cellvars); - Py_XDECREF(bytecode); return co; } @@ -4032,6 +4175,8 @@ assemble(struct compiler *c, int addNone int i, j, nblocks; PyCodeObject *co = NULL; + optimize_jumps(c); + /* Make sure every block that falls off the end returns None. XXX NEXT_BLOCK() isn't quite right, because if the last block ends with a jump or return b_next shouldn't set. diff -r 1e00b161f5f5 Python/future.c --- a/Python/future.c Wed Mar 09 12:53:30 2011 +0100 +++ b/Python/future.c Wed May 11 11:21:24 2016 +0300 @@ -58,7 +58,7 @@ future_check_features(PyFutureFeatures * static int future_parse(PyFutureFeatures *ff, mod_ty mod, const char *filename) { - int i, found_docstring = 0, done = 0, prev_line = 0; + int i, done = 0, prev_line = 0; static PyObject *future; if (!future) { @@ -107,12 +107,8 @@ future_parse(PyFutureFeatures *ff, mod_t else done = 1; } - else if (s->kind == Expr_kind && !found_docstring) { - expr_ty e = s->v.Expr.value; - if (e->kind != Str_kind) - done = 1; - else - found_docstring = 1; + else if (s->kind == Expr_kind) { + done = 1; } else done = 1; diff -r 1e00b161f5f5 Python/symtable.c --- a/Python/symtable.c Wed Mar 09 12:53:30 2011 +0100 +++ b/Python/symtable.c Wed May 11 11:21:24 2016 +0300 @@ -1397,10 +1397,7 @@ symtable_visit_expr(struct symtable *st, if (e->v.Call.kwargs) VISIT(st, expr, e->v.Call.kwargs); break; - case Num_kind: - case Str_kind: - case Bytes_kind: - case Ellipsis_kind: + case Lit_kind: /* Nothing to do here. */ break; /* The following exprs can be assignment targets. */