Author: Carl Friedrich Bolz-Tereick <[email protected]>
Branch: py3.6
Changeset: r97401:ce93612a2d0c
Date: 2019-09-07 17:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ce93612a2d0c/
Log: fix up the position of AST nodes of the expressions with an f-string
diff --git a/pypy/interpreter/astcompiler/ast.py
b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2740,9 +2740,9 @@
w_metavar = get_field(space, w_node, 'metavar', False)
w_lineno = get_field(space, w_node, 'lineno', False)
w_col_offset = get_field(space, w_node, 'col_offset', False)
- _metavar = space.int_w(w_metavar)
- _lineno = space.int_w(w_lineno)
- _col_offset = space.int_w(w_col_offset)
+ _metavar = obj_to_int(space, w_metavar, False)
+ _lineno = obj_to_int(space, w_lineno, False)
+ _col_offset = obj_to_int(space, w_col_offset, False)
return RevDBMetaVar(_metavar, _lineno, _col_offset)
State.ast_type('RevDBMetaVar', 'expr', ['metavar'])
diff --git a/pypy/interpreter/astcompiler/fstring.py
b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -25,7 +25,7 @@
def f_constant_string(astbuilder, joined_pieces, w_u, atom_node):
add_constant_string(astbuilder, joined_pieces, w_u, atom_node)
-def f_string_compile(astbuilder, source, atom_node):
+def f_string_compile(astbuilder, source, atom_node, fstr):
# Note: a f-string is kept as a single literal up to here.
# At this point only, we recursively call the AST compiler
# on all the '{expr}' parts. The 'expr' part is not parsed
@@ -44,16 +44,43 @@
astbuilder.error("internal error: parser not available for parsing "
"the expressions inside the f-string", atom_node)
assert isinstance(source, str) # utf-8 encoded
- source = '(%s)' % source
+
+ paren_source = '(%s)' % source # to deal with whitespace at the start of
source
+
+ lineno = 0
+ column_offset = 0
+ if fstr.stnode:
+ stnode = fstr.stnode
+ lineno = stnode.get_lineno() - 1 # one-based
+ # CPython has an equivalent hack :-(
+ column_offset = stnode.value.find(source) + stnode.get_column()
info = pyparse.CompileInfo("<fstring>", "eval",
consts.PyCF_SOURCE_IS_UTF8 |
consts.PyCF_IGNORE_COOKIE,
optimize=astbuilder.compile_info.optimize)
parser = astbuilder.recursive_parser
- parse_tree = parser.parse_source(source, info)
- return ast_from_node(astbuilder.space, parse_tree, info,
- recursive_parser=parser)
+ parse_tree = parser.parse_source(paren_source, info)
+
+ return fixup_fstring_positions(
+ ast_from_node(astbuilder.space, parse_tree, info,
+ recursive_parser=parser),
+ lineno, column_offset)
+
+def fixup_fstring_positions(ast, line_offset, column_offset):
+ visitor = FixPosVisitor(line_offset, column_offset)
+ return ast.mutate_over(visitor)
+
+class FixPosVisitor(ast.ASTVisitor):
+ def __init__(self, line_offset, column_offset):
+ self.line_offset = line_offset
+ self.column_offset = column_offset
+
+ def default_visitor(self, node):
+ if isinstance(node, ast.stmt) or isinstance(node, ast.expr):
+ node.lineno += self.line_offset
+ node.col_offset += self.column_offset
+ return node
def unexpected_end_of_string(astbuilder, atom_node):
@@ -177,7 +204,7 @@
# Compile the expression as soon as possible, so we show errors
# related to the expression before errors related to the
# conversion or format_spec.
- expr = f_string_compile(astbuilder, s[expr_start:i], atom_node)
+ expr = f_string_compile(astbuilder, s[expr_start:i], atom_node, fstr)
assert isinstance(expr, ast.Expression)
# Check for a conversion char, if present.
@@ -345,7 +372,7 @@
child = atom_node.get_child(i)
try:
w_next = parsestring.parsestr(
- space, encoding, child.get_value())
+ space, encoding, child.get_value(), child)
if not isinstance(w_next, parsestring.W_FString):
add_constant_string(astbuilder, joined_pieces, w_next,
atom_node)
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py
b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -22,7 +22,7 @@
flags = consts.CO_FUTURE_WITH_STATEMENT
info = pyparse.CompileInfo("<test>", p_mode, flags)
tree = self.parser.parse_source(source, info)
- ast_node = ast_from_node(self.space, tree, info)
+ ast_node = ast_from_node(self.space, tree, info, self.parser)
return ast_node
def get_first_expr(self, source, p_mode=None, flags=None):
@@ -1476,3 +1476,8 @@
" bytes in position 0-1: truncated \\xXX escape")
assert exc.lineno == 2
assert exc.offset == 6
+
+ def test_fstring_lineno(self):
+ mod = self.get_ast('x=1\nf"{ x + 1}"')
+ assert mod.body[1].value.values[0].value.lineno == 2
+ assert mod.body[1].value.values[0].value.col_offset == 8
diff --git a/pypy/interpreter/pyparser/parsestring.py
b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -7,14 +7,15 @@
class W_FString(W_Root):
- def __init__(self, unparsed, raw_mode):
+ def __init__(self, unparsed, raw_mode, stnode):
assert isinstance(unparsed, str) # utf-8 encoded string
self.unparsed = unparsed # but the quotes are removed
self.raw_mode = raw_mode
self.current_index = 0 # for astcompiler.fstring
+ self.stnode = stnode
-def parsestr(space, encoding, s):
+def parsestr(space, encoding, s, stnode=None):
"""Parses a string or unicode literal, and return usually
a wrapped value. If we get an f-string, then instead return
an unparsed but unquoted W_FString instance.
@@ -88,7 +89,7 @@
if unicode_literal and not rawmode: # XXX Py_UnicodeFlag is ignored for now
assert 0 <= ps <= q
if saw_f:
- return W_FString(s[ps:q], rawmode)
+ return W_FString(s[ps:q], rawmode, stnode)
if encoding is None:
substr = s[ps:q]
else:
@@ -112,7 +113,7 @@
if not unicode_literal:
return space.newbytes(substr)
elif saw_f:
- return W_FString(substr, rawmode)
+ return W_FString(substr, rawmode, stnode)
else:
v = unicodehelper.str_decode_utf8(substr, 'strict', True, None)
return space.newtext(*v)
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
