1"""Parse tokens from the lexer into nodes for the compiler.""" 2import typing 3import typing as t 4 5from . import nodes 6from .exceptions import TemplateAssertionError 7from .exceptions import TemplateSyntaxError 8from .lexer import describe_token 9from .lexer import describe_token_expr 10 11if t.TYPE_CHECKING: 12 import typing_extensions as te 13 from .environment import Environment 14 15_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include) 16_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock) 17 18_statement_keywords = frozenset( 19 [ 20 "for", 21 "if", 22 "block", 23 "extends", 24 "print", 25 "macro", 26 "include", 27 "from", 28 "import", 29 "set", 30 "with", 31 "autoescape", 32 ] 33) 34_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) 35 36_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = { 37 "add": nodes.Add, 38 "sub": nodes.Sub, 39 "mul": nodes.Mul, 40 "div": nodes.Div, 41 "floordiv": nodes.FloorDiv, 42 "mod": nodes.Mod, 43} 44 45 46class Parser: 47 """This is the central parsing class Jinja uses. It's passed to 48 extensions and can be used to parse expressions or statements. 49 """ 50 51 def __init__( 52 self, 53 environment: "Environment", 54 source: str, 55 name: t.Optional[str] = None, 56 filename: t.Optional[str] = None, 57 state: t.Optional[str] = None, 58 ) -> None: 59 self.environment = environment 60 self.stream = environment._tokenize(source, name, filename, state) 61 self.name = name 62 self.filename = filename 63 self.closed = False 64 self.extensions: t.Dict[ 65 str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]] 66 ] = {} 67 for extension in environment.iter_extensions(): 68 for tag in extension.tags: 69 self.extensions[tag] = extension.parse 70 self._last_identifier = 0 71 self._tag_stack: t.List[str] = [] 72 self._end_token_stack: t.List[t.Tuple[str, ...]] = [] 73 74 def fail( 75 self, 76 msg: str, 77 lineno: t.Optional[int] = None, 78 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError, 79 ) -> "te.NoReturn": 80 """Convenience method that raises `exc` with the message, passed 81 line number or last line number as well as the current name and 82 filename. 83 """ 84 if lineno is None: 85 lineno = self.stream.current.lineno 86 raise exc(msg, lineno, self.name, self.filename) 87 88 def _fail_ut_eof( 89 self, 90 name: t.Optional[str], 91 end_token_stack: t.List[t.Tuple[str, ...]], 92 lineno: t.Optional[int], 93 ) -> "te.NoReturn": 94 expected: t.Set[str] = set() 95 for exprs in end_token_stack: 96 expected.update(map(describe_token_expr, exprs)) 97 if end_token_stack: 98 currently_looking: t.Optional[str] = " or ".join( 99 map(repr, map(describe_token_expr, end_token_stack[-1])) 100 ) 101 else: 102 currently_looking = None 103 104 if name is None: 105 message = ["Unexpected end of template."] 106 else: 107 message = [f"Encountered unknown tag {name!r}."] 108 109 if currently_looking: 110 if name is not None and name in expected: 111 message.append( 112 "You probably made a nesting mistake. Jinja is expecting this tag," 113 f" but currently looking for {currently_looking}." 114 ) 115 else: 116 message.append( 117 f"Jinja was looking for the following tags: {currently_looking}." 118 ) 119 120 if self._tag_stack: 121 message.append( 122 "The innermost block that needs to be closed is" 123 f" {self._tag_stack[-1]!r}." 124 ) 125 126 self.fail(" ".join(message), lineno) 127 128 def fail_unknown_tag( 129 self, name: str, lineno: t.Optional[int] = None 130 ) -> "te.NoReturn": 131 """Called if the parser encounters an unknown tag. Tries to fail 132 with a human readable error message that could help to identify 133 the problem. 134 """ 135 self._fail_ut_eof(name, self._end_token_stack, lineno) 136 137 def fail_eof( 138 self, 139 end_tokens: t.Optional[t.Tuple[str, ...]] = None, 140 lineno: t.Optional[int] = None, 141 ) -> "te.NoReturn": 142 """Like fail_unknown_tag but for end of template situations.""" 143 stack = list(self._end_token_stack) 144 if end_tokens is not None: 145 stack.append(end_tokens) 146 self._fail_ut_eof(None, stack, lineno) 147 148 def is_tuple_end( 149 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None 150 ) -> bool: 151 """Are we at the end of a tuple?""" 152 if self.stream.current.type in ("variable_end", "block_end", "rparen"): 153 return True 154 elif extra_end_rules is not None: 155 return self.stream.current.test_any(extra_end_rules) # type: ignore 156 return False 157 158 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName: 159 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" 160 self._last_identifier += 1 161 rv = object.__new__(nodes.InternalName) 162 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno) 163 return rv 164 165 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]: 166 """Parse a single statement.""" 167 token = self.stream.current 168 if token.type != "name": 169 self.fail("tag name expected", token.lineno) 170 self._tag_stack.append(token.value) 171 pop_tag = True 172 try: 173 if token.value in _statement_keywords: 174 f = getattr(self, f"parse_{self.stream.current.value}") 175 return f() # type: ignore 176 if token.value == "call": 177 return self.parse_call_block() 178 if token.value == "filter": 179 return self.parse_filter_block() 180 ext = self.extensions.get(token.value) 181 if ext is not None: 182 return ext(self) 183 184 # did not work out, remove the token we pushed by accident 185 # from the stack so that the unknown tag fail function can 186 # produce a proper error message. 187 self._tag_stack.pop() 188 pop_tag = False 189 self.fail_unknown_tag(token.value, token.lineno) 190 finally: 191 if pop_tag: 192 self._tag_stack.pop() 193 194 def parse_statements( 195 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False 196 ) -> t.List[nodes.Node]: 197 """Parse multiple statements into a list until one of the end tokens 198 is reached. This is used to parse the body of statements as it also 199 parses template data if appropriate. The parser checks first if the 200 current token is a colon and skips it if there is one. Then it checks 201 for the block end and parses until if one of the `end_tokens` is 202 reached. Per default the active token in the stream at the end of 203 the call is the matched end token. If this is not wanted `drop_needle` 204 can be set to `True` and the end token is removed. 205 """ 206 # the first token may be a colon for python compatibility 207 self.stream.skip_if("colon") 208 209 # in the future it would be possible to add whole code sections 210 # by adding some sort of end of statement token and parsing those here. 211 self.stream.expect("block_end") 212 result = self.subparse(end_tokens) 213 214 # we reached the end of the template too early, the subparser 215 # does not check for this, so we do that now 216 if self.stream.current.type == "eof": 217 self.fail_eof(end_tokens) 218 219 if drop_needle: 220 next(self.stream) 221 return result 222 223 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]: 224 """Parse an assign statement.""" 225 lineno = next(self.stream).lineno 226 target = self.parse_assign_target(with_namespace=True) 227 if self.stream.skip_if("assign"): 228 expr = self.parse_tuple() 229 return nodes.Assign(target, expr, lineno=lineno) 230 filter_node = self.parse_filter(None) 231 body = self.parse_statements(("name:endset",), drop_needle=True) 232 return nodes.AssignBlock(target, filter_node, body, lineno=lineno) 233 234 def parse_for(self) -> nodes.For: 235 """Parse a for loop.""" 236 lineno = self.stream.expect("name:for").lineno 237 target = self.parse_assign_target(extra_end_rules=("name:in",)) 238 self.stream.expect("name:in") 239 iter = self.parse_tuple( 240 with_condexpr=False, extra_end_rules=("name:recursive",) 241 ) 242 test = None 243 if self.stream.skip_if("name:if"): 244 test = self.parse_expression() 245 recursive = self.stream.skip_if("name:recursive") 246 body = self.parse_statements(("name:endfor", "name:else")) 247 if next(self.stream).value == "endfor": 248 else_ = [] 249 else: 250 else_ = self.parse_statements(("name:endfor",), drop_needle=True) 251 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) 252 253 def parse_if(self) -> nodes.If: 254 """Parse an if construct.""" 255 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) 256 while True: 257 node.test = self.parse_tuple(with_condexpr=False) 258 node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) 259 node.elif_ = [] 260 node.else_ = [] 261 token = next(self.stream) 262 if token.test("name:elif"): 263 node = nodes.If(lineno=self.stream.current.lineno) 264 result.elif_.append(node) 265 continue 266 elif token.test("name:else"): 267 result.else_ = self.parse_statements(("name:endif",), drop_needle=True) 268 break 269 return result 270 271 def parse_with(self) -> nodes.With: 272 node = nodes.With(lineno=next(self.stream).lineno) 273 targets: t.List[nodes.Expr] = [] 274 values: t.List[nodes.Expr] = [] 275 while self.stream.current.type != "block_end": 276 if targets: 277 self.stream.expect("comma") 278 target = self.parse_assign_target() 279 target.set_ctx("param") 280 targets.append(target) 281 self.stream.expect("assign") 282 values.append(self.parse_expression()) 283 node.targets = targets 284 node.values = values 285 node.body = self.parse_statements(("name:endwith",), drop_needle=True) 286 return node 287 288 def parse_autoescape(self) -> nodes.Scope: 289 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) 290 node.options = [nodes.Keyword("autoescape", self.parse_expression())] 291 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) 292 return nodes.Scope([node]) 293 294 def parse_block(self) -> nodes.Block: 295 node = nodes.Block(lineno=next(self.stream).lineno) 296 node.name = self.stream.expect("name").value 297 node.scoped = self.stream.skip_if("name:scoped") 298 node.required = self.stream.skip_if("name:required") 299 300 # common problem people encounter when switching from django 301 # to jinja. we do not support hyphens in block names, so let's 302 # raise a nicer error message in that case. 303 if self.stream.current.type == "sub": 304 self.fail( 305 "Block names in Jinja have to be valid Python identifiers and may not" 306 " contain hyphens, use an underscore instead." 307 ) 308 309 node.body = self.parse_statements(("name:endblock",), drop_needle=True) 310 311 # enforce that required blocks only contain whitespace or comments 312 # by asserting that the body, if not empty, is just TemplateData nodes 313 # with whitespace data 314 if node.required: 315 for body_node in node.body: 316 if not isinstance(body_node, nodes.Output) or any( 317 not isinstance(output_node, nodes.TemplateData) 318 or not output_node.data.isspace() 319 for output_node in body_node.nodes 320 ): 321 self.fail("Required blocks can only contain comments or whitespace") 322 323 self.stream.skip_if("name:" + node.name) 324 return node 325 326 def parse_extends(self) -> nodes.Extends: 327 node = nodes.Extends(lineno=next(self.stream).lineno) 328 node.template = self.parse_expression() 329 return node 330 331 def parse_import_context( 332 self, node: _ImportInclude, default: bool 333 ) -> _ImportInclude: 334 if self.stream.current.test_any( 335 "name:with", "name:without" 336 ) and self.stream.look().test("name:context"): 337 node.with_context = next(self.stream).value == "with" 338 self.stream.skip() 339 else: 340 node.with_context = default 341 return node 342 343 def parse_include(self) -> nodes.Include: 344 node = nodes.Include(lineno=next(self.stream).lineno) 345 node.template = self.parse_expression() 346 if self.stream.current.test("name:ignore") and self.stream.look().test( 347 "name:missing" 348 ): 349 node.ignore_missing = True 350 self.stream.skip(2) 351 else: 352 node.ignore_missing = False 353 return self.parse_import_context(node, True) 354 355 def parse_import(self) -> nodes.Import: 356 node = nodes.Import(lineno=next(self.stream).lineno) 357 node.template = self.parse_expression() 358 self.stream.expect("name:as") 359 node.target = self.parse_assign_target(name_only=True).name 360 return self.parse_import_context(node, False) 361 362 def parse_from(self) -> nodes.FromImport: 363 node = nodes.FromImport(lineno=next(self.stream).lineno) 364 node.template = self.parse_expression() 365 self.stream.expect("name:import") 366 node.names = [] 367 368 def parse_context() -> bool: 369 if self.stream.current.value in { 370 "with", 371 "without", 372 } and self.stream.look().test("name:context"): 373 node.with_context = next(self.stream).value == "with" 374 self.stream.skip() 375 return True 376 return False 377 378 while True: 379 if node.names: 380 self.stream.expect("comma") 381 if self.stream.current.type == "name": 382 if parse_context(): 383 break 384 target = self.parse_assign_target(name_only=True) 385 if target.name.startswith("_"): 386 self.fail( 387 "names starting with an underline can not be imported", 388 target.lineno, 389 exc=TemplateAssertionError, 390 ) 391 if self.stream.skip_if("name:as"): 392 alias = self.parse_assign_target(name_only=True) 393 node.names.append((target.name, alias.name)) 394 else: 395 node.names.append(target.name) 396 if parse_context() or self.stream.current.type != "comma": 397 break 398 else: 399 self.stream.expect("name") 400 if not hasattr(node, "with_context"): 401 node.with_context = False 402 return node 403 404 def parse_signature(self, node: _MacroCall) -> None: 405 args = node.args = [] 406 defaults = node.defaults = [] 407 self.stream.expect("lparen") 408 while self.stream.current.type != "rparen": 409 if args: 410 self.stream.expect("comma") 411 arg = self.parse_assign_target(name_only=True) 412 arg.set_ctx("param") 413 if self.stream.skip_if("assign"): 414 defaults.append(self.parse_expression()) 415 elif defaults: 416 self.fail("non-default argument follows default argument") 417 args.append(arg) 418 self.stream.expect("rparen") 419 420 def parse_call_block(self) -> nodes.CallBlock: 421 node = nodes.CallBlock(lineno=next(self.stream).lineno) 422 if self.stream.current.type == "lparen": 423 self.parse_signature(node) 424 else: 425 node.args = [] 426 node.defaults = [] 427 428 call_node = self.parse_expression() 429 if not isinstance(call_node, nodes.Call): 430 self.fail("expected call", node.lineno) 431 node.call = call_node 432 node.body = self.parse_statements(("name:endcall",), drop_needle=True) 433 return node 434 435 def parse_filter_block(self) -> nodes.FilterBlock: 436 node = nodes.FilterBlock(lineno=next(self.stream).lineno) 437 node.filter = self.parse_filter(None, start_inline=True) # type: ignore 438 node.body = self.parse_statements(("name:endfilter",), drop_needle=True) 439 return node 440 441 def parse_macro(self) -> nodes.Macro: 442 node = nodes.Macro(lineno=next(self.stream).lineno) 443 node.name = self.parse_assign_target(name_only=True).name 444 self.parse_signature(node) 445 node.body = self.parse_statements(("name:endmacro",), drop_needle=True) 446 return node 447 448 def parse_print(self) -> nodes.Output: 449 node = nodes.Output(lineno=next(self.stream).lineno) 450 node.nodes = [] 451 while self.stream.current.type != "block_end": 452 if node.nodes: 453 self.stream.expect("comma") 454 node.nodes.append(self.parse_expression()) 455 return node 456 457 @typing.overload 458 def parse_assign_target( 459 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ... 460 ) -> nodes.Name: 461 ... 462 463 @typing.overload 464 def parse_assign_target( 465 self, 466 with_tuple: bool = True, 467 name_only: bool = False, 468 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 469 with_namespace: bool = False, 470 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: 471 ... 472 473 def parse_assign_target( 474 self, 475 with_tuple: bool = True, 476 name_only: bool = False, 477 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 478 with_namespace: bool = False, 479 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]: 480 """Parse an assignment target. As Jinja allows assignments to 481 tuples, this function can parse all allowed assignment targets. Per 482 default assignments to tuples are parsed, that can be disable however 483 by setting `with_tuple` to `False`. If only assignments to names are 484 wanted `name_only` can be set to `True`. The `extra_end_rules` 485 parameter is forwarded to the tuple parsing function. If 486 `with_namespace` is enabled, a namespace assignment may be parsed. 487 """ 488 target: nodes.Expr 489 490 if with_namespace and self.stream.look().type == "dot": 491 token = self.stream.expect("name") 492 next(self.stream) # dot 493 attr = self.stream.expect("name") 494 target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) 495 elif name_only: 496 token = self.stream.expect("name") 497 target = nodes.Name(token.value, "store", lineno=token.lineno) 498 else: 499 if with_tuple: 500 target = self.parse_tuple( 501 simplified=True, extra_end_rules=extra_end_rules 502 ) 503 else: 504 target = self.parse_primary() 505 506 target.set_ctx("store") 507 508 if not target.can_assign(): 509 self.fail( 510 f"can't assign to {type(target).__name__.lower()!r}", target.lineno 511 ) 512 513 return target # type: ignore 514 515 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr: 516 """Parse an expression. Per default all expressions are parsed, if 517 the optional `with_condexpr` parameter is set to `False` conditional 518 expressions are not parsed. 519 """ 520 if with_condexpr: 521 return self.parse_condexpr() 522 return self.parse_or() 523 524 def parse_condexpr(self) -> nodes.Expr: 525 lineno = self.stream.current.lineno 526 expr1 = self.parse_or() 527 expr3: t.Optional[nodes.Expr] 528 529 while self.stream.skip_if("name:if"): 530 expr2 = self.parse_or() 531 if self.stream.skip_if("name:else"): 532 expr3 = self.parse_condexpr() 533 else: 534 expr3 = None 535 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno) 536 lineno = self.stream.current.lineno 537 return expr1 538 539 def parse_or(self) -> nodes.Expr: 540 lineno = self.stream.current.lineno 541 left = self.parse_and() 542 while self.stream.skip_if("name:or"): 543 right = self.parse_and() 544 left = nodes.Or(left, right, lineno=lineno) 545 lineno = self.stream.current.lineno 546 return left 547 548 def parse_and(self) -> nodes.Expr: 549 lineno = self.stream.current.lineno 550 left = self.parse_not() 551 while self.stream.skip_if("name:and"): 552 right = self.parse_not() 553 left = nodes.And(left, right, lineno=lineno) 554 lineno = self.stream.current.lineno 555 return left 556 557 def parse_not(self) -> nodes.Expr: 558 if self.stream.current.test("name:not"): 559 lineno = next(self.stream).lineno 560 return nodes.Not(self.parse_not(), lineno=lineno) 561 return self.parse_compare() 562 563 def parse_compare(self) -> nodes.Expr: 564 lineno = self.stream.current.lineno 565 expr = self.parse_math1() 566 ops = [] 567 while True: 568 token_type = self.stream.current.type 569 if token_type in _compare_operators: 570 next(self.stream) 571 ops.append(nodes.Operand(token_type, self.parse_math1())) 572 elif self.stream.skip_if("name:in"): 573 ops.append(nodes.Operand("in", self.parse_math1())) 574 elif self.stream.current.test("name:not") and self.stream.look().test( 575 "name:in" 576 ): 577 self.stream.skip(2) 578 ops.append(nodes.Operand("notin", self.parse_math1())) 579 else: 580 break 581 lineno = self.stream.current.lineno 582 if not ops: 583 return expr 584 return nodes.Compare(expr, ops, lineno=lineno) 585 586 def parse_math1(self) -> nodes.Expr: 587 lineno = self.stream.current.lineno 588 left = self.parse_concat() 589 while self.stream.current.type in ("add", "sub"): 590 cls = _math_nodes[self.stream.current.type] 591 next(self.stream) 592 right = self.parse_concat() 593 left = cls(left, right, lineno=lineno) 594 lineno = self.stream.current.lineno 595 return left 596 597 def parse_concat(self) -> nodes.Expr: 598 lineno = self.stream.current.lineno 599 args = [self.parse_math2()] 600 while self.stream.current.type == "tilde": 601 next(self.stream) 602 args.append(self.parse_math2()) 603 if len(args) == 1: 604 return args[0] 605 return nodes.Concat(args, lineno=lineno) 606 607 def parse_math2(self) -> nodes.Expr: 608 lineno = self.stream.current.lineno 609 left = self.parse_pow() 610 while self.stream.current.type in ("mul", "div", "floordiv", "mod"): 611 cls = _math_nodes[self.stream.current.type] 612 next(self.stream) 613 right = self.parse_pow() 614 left = cls(left, right, lineno=lineno) 615 lineno = self.stream.current.lineno 616 return left 617 618 def parse_pow(self) -> nodes.Expr: 619 lineno = self.stream.current.lineno 620 left = self.parse_unary() 621 while self.stream.current.type == "pow": 622 next(self.stream) 623 right = self.parse_unary() 624 left = nodes.Pow(left, right, lineno=lineno) 625 lineno = self.stream.current.lineno 626 return left 627 628 def parse_unary(self, with_filter: bool = True) -> nodes.Expr: 629 token_type = self.stream.current.type 630 lineno = self.stream.current.lineno 631 node: nodes.Expr 632 633 if token_type == "sub": 634 next(self.stream) 635 node = nodes.Neg(self.parse_unary(False), lineno=lineno) 636 elif token_type == "add": 637 next(self.stream) 638 node = nodes.Pos(self.parse_unary(False), lineno=lineno) 639 else: 640 node = self.parse_primary() 641 node = self.parse_postfix(node) 642 if with_filter: 643 node = self.parse_filter_expr(node) 644 return node 645 646 def parse_primary(self) -> nodes.Expr: 647 token = self.stream.current 648 node: nodes.Expr 649 if token.type == "name": 650 if token.value in ("true", "false", "True", "False"): 651 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) 652 elif token.value in ("none", "None"): 653 node = nodes.Const(None, lineno=token.lineno) 654 else: 655 node = nodes.Name(token.value, "load", lineno=token.lineno) 656 next(self.stream) 657 elif token.type == "string": 658 next(self.stream) 659 buf = [token.value] 660 lineno = token.lineno 661 while self.stream.current.type == "string": 662 buf.append(self.stream.current.value) 663 next(self.stream) 664 node = nodes.Const("".join(buf), lineno=lineno) 665 elif token.type in ("integer", "float"): 666 next(self.stream) 667 node = nodes.Const(token.value, lineno=token.lineno) 668 elif token.type == "lparen": 669 next(self.stream) 670 node = self.parse_tuple(explicit_parentheses=True) 671 self.stream.expect("rparen") 672 elif token.type == "lbracket": 673 node = self.parse_list() 674 elif token.type == "lbrace": 675 node = self.parse_dict() 676 else: 677 self.fail(f"unexpected {describe_token(token)!r}", token.lineno) 678 return node 679 680 def parse_tuple( 681 self, 682 simplified: bool = False, 683 with_condexpr: bool = True, 684 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None, 685 explicit_parentheses: bool = False, 686 ) -> t.Union[nodes.Tuple, nodes.Expr]: 687 """Works like `parse_expression` but if multiple expressions are 688 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. 689 This method could also return a regular expression instead of a tuple 690 if no commas where found. 691 692 The default parsing mode is a full tuple. If `simplified` is `True` 693 only names and literals are parsed. The `no_condexpr` parameter is 694 forwarded to :meth:`parse_expression`. 695 696 Because tuples do not require delimiters and may end in a bogus comma 697 an extra hint is needed that marks the end of a tuple. For example 698 for loops support tuples between `for` and `in`. In that case the 699 `extra_end_rules` is set to ``['name:in']``. 700 701 `explicit_parentheses` is true if the parsing was triggered by an 702 expression in parentheses. This is used to figure out if an empty 703 tuple is a valid expression or not. 704 """ 705 lineno = self.stream.current.lineno 706 if simplified: 707 parse = self.parse_primary 708 elif with_condexpr: 709 parse = self.parse_expression 710 else: 711 712 def parse() -> nodes.Expr: 713 return self.parse_expression(with_condexpr=False) 714 715 args: t.List[nodes.Expr] = [] 716 is_tuple = False 717 718 while True: 719 if args: 720 self.stream.expect("comma") 721 if self.is_tuple_end(extra_end_rules): 722 break 723 args.append(parse()) 724 if self.stream.current.type == "comma": 725 is_tuple = True 726 else: 727 break 728 lineno = self.stream.current.lineno 729 730 if not is_tuple: 731 if args: 732 return args[0] 733 734 # if we don't have explicit parentheses, an empty tuple is 735 # not a valid expression. This would mean nothing (literally 736 # nothing) in the spot of an expression would be an empty 737 # tuple. 738 if not explicit_parentheses: 739 self.fail( 740 "Expected an expression," 741 f" got {describe_token(self.stream.current)!r}" 742 ) 743 744 return nodes.Tuple(args, "load", lineno=lineno) 745 746 def parse_list(self) -> nodes.List: 747 token = self.stream.expect("lbracket") 748 items: t.List[nodes.Expr] = [] 749 while self.stream.current.type != "rbracket": 750 if items: 751 self.stream.expect("comma") 752 if self.stream.current.type == "rbracket": 753 break 754 items.append(self.parse_expression()) 755 self.stream.expect("rbracket") 756 return nodes.List(items, lineno=token.lineno) 757 758 def parse_dict(self) -> nodes.Dict: 759 token = self.stream.expect("lbrace") 760 items: t.List[nodes.Pair] = [] 761 while self.stream.current.type != "rbrace": 762 if items: 763 self.stream.expect("comma") 764 if self.stream.current.type == "rbrace": 765 break 766 key = self.parse_expression() 767 self.stream.expect("colon") 768 value = self.parse_expression() 769 items.append(nodes.Pair(key, value, lineno=key.lineno)) 770 self.stream.expect("rbrace") 771 return nodes.Dict(items, lineno=token.lineno) 772 773 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr: 774 while True: 775 token_type = self.stream.current.type 776 if token_type == "dot" or token_type == "lbracket": 777 node = self.parse_subscript(node) 778 # calls are valid both after postfix expressions (getattr 779 # and getitem) as well as filters and tests 780 elif token_type == "lparen": 781 node = self.parse_call(node) 782 else: 783 break 784 return node 785 786 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr: 787 while True: 788 token_type = self.stream.current.type 789 if token_type == "pipe": 790 node = self.parse_filter(node) # type: ignore 791 elif token_type == "name" and self.stream.current.value == "is": 792 node = self.parse_test(node) 793 # calls are valid both after postfix expressions (getattr 794 # and getitem) as well as filters and tests 795 elif token_type == "lparen": 796 node = self.parse_call(node) 797 else: 798 break 799 return node 800 801 def parse_subscript( 802 self, node: nodes.Expr 803 ) -> t.Union[nodes.Getattr, nodes.Getitem]: 804 token = next(self.stream) 805 arg: nodes.Expr 806 807 if token.type == "dot": 808 attr_token = self.stream.current 809 next(self.stream) 810 if attr_token.type == "name": 811 return nodes.Getattr( 812 node, attr_token.value, "load", lineno=token.lineno 813 ) 814 elif attr_token.type != "integer": 815 self.fail("expected name or number", attr_token.lineno) 816 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) 817 return nodes.Getitem(node, arg, "load", lineno=token.lineno) 818 if token.type == "lbracket": 819 args: t.List[nodes.Expr] = [] 820 while self.stream.current.type != "rbracket": 821 if args: 822 self.stream.expect("comma") 823 args.append(self.parse_subscribed()) 824 self.stream.expect("rbracket") 825 if len(args) == 1: 826 arg = args[0] 827 else: 828 arg = nodes.Tuple(args, "load", lineno=token.lineno) 829 return nodes.Getitem(node, arg, "load", lineno=token.lineno) 830 self.fail("expected subscript expression", token.lineno) 831 832 def parse_subscribed(self) -> nodes.Expr: 833 lineno = self.stream.current.lineno 834 args: t.List[t.Optional[nodes.Expr]] 835 836 if self.stream.current.type == "colon": 837 next(self.stream) 838 args = [None] 839 else: 840 node = self.parse_expression() 841 if self.stream.current.type != "colon": 842 return node 843 next(self.stream) 844 args = [node] 845 846 if self.stream.current.type == "colon": 847 args.append(None) 848 elif self.stream.current.type not in ("rbracket", "comma"): 849 args.append(self.parse_expression()) 850 else: 851 args.append(None) 852 853 if self.stream.current.type == "colon": 854 next(self.stream) 855 if self.stream.current.type not in ("rbracket", "comma"): 856 args.append(self.parse_expression()) 857 else: 858 args.append(None) 859 else: 860 args.append(None) 861 862 return nodes.Slice(lineno=lineno, *args) # noqa: B026 863 864 def parse_call_args(self) -> t.Tuple: 865 token = self.stream.expect("lparen") 866 args = [] 867 kwargs = [] 868 dyn_args = None 869 dyn_kwargs = None 870 require_comma = False 871 872 def ensure(expr: bool) -> None: 873 if not expr: 874 self.fail("invalid syntax for function call expression", token.lineno) 875 876 while self.stream.current.type != "rparen": 877 if require_comma: 878 self.stream.expect("comma") 879 880 # support for trailing comma 881 if self.stream.current.type == "rparen": 882 break 883 884 if self.stream.current.type == "mul": 885 ensure(dyn_args is None and dyn_kwargs is None) 886 next(self.stream) 887 dyn_args = self.parse_expression() 888 elif self.stream.current.type == "pow": 889 ensure(dyn_kwargs is None) 890 next(self.stream) 891 dyn_kwargs = self.parse_expression() 892 else: 893 if ( 894 self.stream.current.type == "name" 895 and self.stream.look().type == "assign" 896 ): 897 # Parsing a kwarg 898 ensure(dyn_kwargs is None) 899 key = self.stream.current.value 900 self.stream.skip(2) 901 value = self.parse_expression() 902 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) 903 else: 904 # Parsing an arg 905 ensure(dyn_args is None and dyn_kwargs is None and not kwargs) 906 args.append(self.parse_expression()) 907 908 require_comma = True 909 910 self.stream.expect("rparen") 911 return args, kwargs, dyn_args, dyn_kwargs 912 913 def parse_call(self, node: nodes.Expr) -> nodes.Call: 914 # The lparen will be expected in parse_call_args, but the lineno 915 # needs to be recorded before the stream is advanced. 916 token = self.stream.current 917 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 918 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) 919 920 def parse_filter( 921 self, node: t.Optional[nodes.Expr], start_inline: bool = False 922 ) -> t.Optional[nodes.Expr]: 923 while self.stream.current.type == "pipe" or start_inline: 924 if not start_inline: 925 next(self.stream) 926 token = self.stream.expect("name") 927 name = token.value 928 while self.stream.current.type == "dot": 929 next(self.stream) 930 name += "." + self.stream.expect("name").value 931 if self.stream.current.type == "lparen": 932 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 933 else: 934 args = [] 935 kwargs = [] 936 dyn_args = dyn_kwargs = None 937 node = nodes.Filter( 938 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno 939 ) 940 start_inline = False 941 return node 942 943 def parse_test(self, node: nodes.Expr) -> nodes.Expr: 944 token = next(self.stream) 945 if self.stream.current.test("name:not"): 946 next(self.stream) 947 negated = True 948 else: 949 negated = False 950 name = self.stream.expect("name").value 951 while self.stream.current.type == "dot": 952 next(self.stream) 953 name += "." + self.stream.expect("name").value 954 dyn_args = dyn_kwargs = None 955 kwargs = [] 956 if self.stream.current.type == "lparen": 957 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args() 958 elif self.stream.current.type in { 959 "name", 960 "string", 961 "integer", 962 "float", 963 "lparen", 964 "lbracket", 965 "lbrace", 966 } and not self.stream.current.test_any("name:else", "name:or", "name:and"): 967 if self.stream.current.test("name:is"): 968 self.fail("You cannot chain multiple tests with is") 969 arg_node = self.parse_primary() 970 arg_node = self.parse_postfix(arg_node) 971 args = [arg_node] 972 else: 973 args = [] 974 node = nodes.Test( 975 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno 976 ) 977 if negated: 978 node = nodes.Not(node, lineno=token.lineno) 979 return node 980 981 def subparse( 982 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None 983 ) -> t.List[nodes.Node]: 984 body: t.List[nodes.Node] = [] 985 data_buffer: t.List[nodes.Node] = [] 986 add_data = data_buffer.append 987 988 if end_tokens is not None: 989 self._end_token_stack.append(end_tokens) 990 991 def flush_data() -> None: 992 if data_buffer: 993 lineno = data_buffer[0].lineno 994 body.append(nodes.Output(data_buffer[:], lineno=lineno)) 995 del data_buffer[:] 996 997 try: 998 while self.stream: 999 token = self.stream.current 1000 if token.type == "data": 1001 if token.value: 1002 add_data(nodes.TemplateData(token.value, lineno=token.lineno)) 1003 next(self.stream) 1004 elif token.type == "variable_begin": 1005 next(self.stream) 1006 add_data(self.parse_tuple(with_condexpr=True)) 1007 self.stream.expect("variable_end") 1008 elif token.type == "block_begin": 1009 flush_data() 1010 next(self.stream) 1011 if end_tokens is not None and self.stream.current.test_any( 1012 *end_tokens 1013 ): 1014 return body 1015 rv = self.parse_statement() 1016 if isinstance(rv, list): 1017 body.extend(rv) 1018 else: 1019 body.append(rv) 1020 self.stream.expect("block_end") 1021 else: 1022 raise AssertionError("internal parsing error") 1023 1024 flush_data() 1025 finally: 1026 if end_tokens is not None: 1027 self._end_token_stack.pop() 1028 return body 1029 1030 def parse(self) -> nodes.Template: 1031 """Parse the whole template into a `Template` node.""" 1032 result = nodes.Template(self.subparse(), lineno=1) 1033 result.set_environment(self.environment) 1034 return result 1035