Coverage for /var/srv/projects/api.amasfac.comuna18.com/tmp/venv/lib/python3.9/site-packages/jinja2/parser.py: 7%
661 statements
« prev ^ index » next coverage.py v6.4.4, created at 2023-07-17 14:22 -0600
1"""Parse tokens from the lexer into nodes for the compiler."""
2import typing
3import typing as t
5from . import nodes
6from .exceptions import TemplateAssertionError
7from .exceptions import TemplateSyntaxError
8from .lexer import describe_token
9from .lexer import describe_token_expr
11if t.TYPE_CHECKING: 11 ↛ 12line 11 didn't jump to line 12, because the condition on line 11 was never true
12 import typing_extensions as te
13 from .environment import Environment
15_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
16_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
18_statement_keywords = frozenset(
19 [
20 "for",
21 "if",
22 "block",
23 "extends",
24 "print",
25 "macro",
26 "include",
27 "from",
28 "import",
29 "set",
30 "with",
31 "autoescape",
32 ]
33)
34_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
36_math_nodes: t.Dict[str, t.Type[nodes.Expr]] = {
37 "add": nodes.Add,
38 "sub": nodes.Sub,
39 "mul": nodes.Mul,
40 "div": nodes.Div,
41 "floordiv": nodes.FloorDiv,
42 "mod": nodes.Mod,
43}
46class Parser:
47 """This is the central parsing class Jinja uses. It's passed to
48 extensions and can be used to parse expressions or statements.
49 """
51 def __init__(
52 self,
53 environment: "Environment",
54 source: str,
55 name: t.Optional[str] = None,
56 filename: t.Optional[str] = None,
57 state: t.Optional[str] = None,
58 ) -> None:
59 self.environment = environment
60 self.stream = environment._tokenize(source, name, filename, state)
61 self.name = name
62 self.filename = filename
63 self.closed = False
64 self.extensions: t.Dict[
65 str, t.Callable[["Parser"], t.Union[nodes.Node, t.List[nodes.Node]]]
66 ] = {}
67 for extension in environment.iter_extensions():
68 for tag in extension.tags:
69 self.extensions[tag] = extension.parse
70 self._last_identifier = 0
71 self._tag_stack: t.List[str] = []
72 self._end_token_stack: t.List[t.Tuple[str, ...]] = []
74 def fail(
75 self,
76 msg: str,
77 lineno: t.Optional[int] = None,
78 exc: t.Type[TemplateSyntaxError] = TemplateSyntaxError,
79 ) -> "te.NoReturn":
80 """Convenience method that raises `exc` with the message, passed
81 line number or last line number as well as the current name and
82 filename.
83 """
84 if lineno is None:
85 lineno = self.stream.current.lineno
86 raise exc(msg, lineno, self.name, self.filename)
88 def _fail_ut_eof(
89 self,
90 name: t.Optional[str],
91 end_token_stack: t.List[t.Tuple[str, ...]],
92 lineno: t.Optional[int],
93 ) -> "te.NoReturn":
94 expected: t.Set[str] = set()
95 for exprs in end_token_stack:
96 expected.update(map(describe_token_expr, exprs))
97 if end_token_stack:
98 currently_looking: t.Optional[str] = " or ".join(
99 map(repr, map(describe_token_expr, end_token_stack[-1]))
100 )
101 else:
102 currently_looking = None
104 if name is None:
105 message = ["Unexpected end of template."]
106 else:
107 message = [f"Encountered unknown tag {name!r}."]
109 if currently_looking:
110 if name is not None and name in expected:
111 message.append(
112 "You probably made a nesting mistake. Jinja is expecting this tag,"
113 f" but currently looking for {currently_looking}."
114 )
115 else:
116 message.append(
117 f"Jinja was looking for the following tags: {currently_looking}."
118 )
120 if self._tag_stack:
121 message.append(
122 "The innermost block that needs to be closed is"
123 f" {self._tag_stack[-1]!r}."
124 )
126 self.fail(" ".join(message), lineno)
128 def fail_unknown_tag(
129 self, name: str, lineno: t.Optional[int] = None
130 ) -> "te.NoReturn":
131 """Called if the parser encounters an unknown tag. Tries to fail
132 with a human readable error message that could help to identify
133 the problem.
134 """
135 self._fail_ut_eof(name, self._end_token_stack, lineno)
137 def fail_eof(
138 self,
139 end_tokens: t.Optional[t.Tuple[str, ...]] = None,
140 lineno: t.Optional[int] = None,
141 ) -> "te.NoReturn":
142 """Like fail_unknown_tag but for end of template situations."""
143 stack = list(self._end_token_stack)
144 if end_tokens is not None:
145 stack.append(end_tokens)
146 self._fail_ut_eof(None, stack, lineno)
148 def is_tuple_end(
149 self, extra_end_rules: t.Optional[t.Tuple[str, ...]] = None
150 ) -> bool:
151 """Are we at the end of a tuple?"""
152 if self.stream.current.type in ("variable_end", "block_end", "rparen"):
153 return True
154 elif extra_end_rules is not None:
155 return self.stream.current.test_any(extra_end_rules) # type: ignore
156 return False
158 def free_identifier(self, lineno: t.Optional[int] = None) -> nodes.InternalName:
159 """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
160 self._last_identifier += 1
161 rv = object.__new__(nodes.InternalName)
162 nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
163 return rv
165 def parse_statement(self) -> t.Union[nodes.Node, t.List[nodes.Node]]:
166 """Parse a single statement."""
167 token = self.stream.current
168 if token.type != "name":
169 self.fail("tag name expected", token.lineno)
170 self._tag_stack.append(token.value)
171 pop_tag = True
172 try:
173 if token.value in _statement_keywords:
174 f = getattr(self, f"parse_{self.stream.current.value}")
175 return f() # type: ignore
176 if token.value == "call":
177 return self.parse_call_block()
178 if token.value == "filter":
179 return self.parse_filter_block()
180 ext = self.extensions.get(token.value)
181 if ext is not None:
182 return ext(self)
184 # did not work out, remove the token we pushed by accident
185 # from the stack so that the unknown tag fail function can
186 # produce a proper error message.
187 self._tag_stack.pop()
188 pop_tag = False
189 self.fail_unknown_tag(token.value, token.lineno)
190 finally:
191 if pop_tag:
192 self._tag_stack.pop()
194 def parse_statements(
195 self, end_tokens: t.Tuple[str, ...], drop_needle: bool = False
196 ) -> t.List[nodes.Node]:
197 """Parse multiple statements into a list until one of the end tokens
198 is reached. This is used to parse the body of statements as it also
199 parses template data if appropriate. The parser checks first if the
200 current token is a colon and skips it if there is one. Then it checks
201 for the block end and parses until if one of the `end_tokens` is
202 reached. Per default the active token in the stream at the end of
203 the call is the matched end token. If this is not wanted `drop_needle`
204 can be set to `True` and the end token is removed.
205 """
206 # the first token may be a colon for python compatibility
207 self.stream.skip_if("colon")
209 # in the future it would be possible to add whole code sections
210 # by adding some sort of end of statement token and parsing those here.
211 self.stream.expect("block_end")
212 result = self.subparse(end_tokens)
214 # we reached the end of the template too early, the subparser
215 # does not check for this, so we do that now
216 if self.stream.current.type == "eof":
217 self.fail_eof(end_tokens)
219 if drop_needle:
220 next(self.stream)
221 return result
223 def parse_set(self) -> t.Union[nodes.Assign, nodes.AssignBlock]:
224 """Parse an assign statement."""
225 lineno = next(self.stream).lineno
226 target = self.parse_assign_target(with_namespace=True)
227 if self.stream.skip_if("assign"):
228 expr = self.parse_tuple()
229 return nodes.Assign(target, expr, lineno=lineno)
230 filter_node = self.parse_filter(None)
231 body = self.parse_statements(("name:endset",), drop_needle=True)
232 return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
234 def parse_for(self) -> nodes.For:
235 """Parse a for loop."""
236 lineno = self.stream.expect("name:for").lineno
237 target = self.parse_assign_target(extra_end_rules=("name:in",))
238 self.stream.expect("name:in")
239 iter = self.parse_tuple(
240 with_condexpr=False, extra_end_rules=("name:recursive",)
241 )
242 test = None
243 if self.stream.skip_if("name:if"):
244 test = self.parse_expression()
245 recursive = self.stream.skip_if("name:recursive")
246 body = self.parse_statements(("name:endfor", "name:else"))
247 if next(self.stream).value == "endfor":
248 else_ = []
249 else:
250 else_ = self.parse_statements(("name:endfor",), drop_needle=True)
251 return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
253 def parse_if(self) -> nodes.If:
254 """Parse an if construct."""
255 node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
256 while True:
257 node.test = self.parse_tuple(with_condexpr=False)
258 node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
259 node.elif_ = []
260 node.else_ = []
261 token = next(self.stream)
262 if token.test("name:elif"):
263 node = nodes.If(lineno=self.stream.current.lineno)
264 result.elif_.append(node)
265 continue
266 elif token.test("name:else"):
267 result.else_ = self.parse_statements(("name:endif",), drop_needle=True)
268 break
269 return result
271 def parse_with(self) -> nodes.With:
272 node = nodes.With(lineno=next(self.stream).lineno)
273 targets: t.List[nodes.Expr] = []
274 values: t.List[nodes.Expr] = []
275 while self.stream.current.type != "block_end":
276 if targets:
277 self.stream.expect("comma")
278 target = self.parse_assign_target()
279 target.set_ctx("param")
280 targets.append(target)
281 self.stream.expect("assign")
282 values.append(self.parse_expression())
283 node.targets = targets
284 node.values = values
285 node.body = self.parse_statements(("name:endwith",), drop_needle=True)
286 return node
288 def parse_autoescape(self) -> nodes.Scope:
289 node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
290 node.options = [nodes.Keyword("autoescape", self.parse_expression())]
291 node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
292 return nodes.Scope([node])
294 def parse_block(self) -> nodes.Block:
295 node = nodes.Block(lineno=next(self.stream).lineno)
296 node.name = self.stream.expect("name").value
297 node.scoped = self.stream.skip_if("name:scoped")
298 node.required = self.stream.skip_if("name:required")
300 # common problem people encounter when switching from django
301 # to jinja. we do not support hyphens in block names, so let's
302 # raise a nicer error message in that case.
303 if self.stream.current.type == "sub":
304 self.fail(
305 "Block names in Jinja have to be valid Python identifiers and may not"
306 " contain hyphens, use an underscore instead."
307 )
309 node.body = self.parse_statements(("name:endblock",), drop_needle=True)
311 # enforce that required blocks only contain whitespace or comments
312 # by asserting that the body, if not empty, is just TemplateData nodes
313 # with whitespace data
314 if node.required and not all(
315 isinstance(child, nodes.TemplateData) and child.data.isspace()
316 for body in node.body
317 for child in body.nodes # type: ignore
318 ):
319 self.fail("Required blocks can only contain comments or whitespace")
321 self.stream.skip_if("name:" + node.name)
322 return node
324 def parse_extends(self) -> nodes.Extends:
325 node = nodes.Extends(lineno=next(self.stream).lineno)
326 node.template = self.parse_expression()
327 return node
329 def parse_import_context(
330 self, node: _ImportInclude, default: bool
331 ) -> _ImportInclude:
332 if self.stream.current.test_any(
333 "name:with", "name:without"
334 ) and self.stream.look().test("name:context"):
335 node.with_context = next(self.stream).value == "with"
336 self.stream.skip()
337 else:
338 node.with_context = default
339 return node
341 def parse_include(self) -> nodes.Include:
342 node = nodes.Include(lineno=next(self.stream).lineno)
343 node.template = self.parse_expression()
344 if self.stream.current.test("name:ignore") and self.stream.look().test(
345 "name:missing"
346 ):
347 node.ignore_missing = True
348 self.stream.skip(2)
349 else:
350 node.ignore_missing = False
351 return self.parse_import_context(node, True)
353 def parse_import(self) -> nodes.Import:
354 node = nodes.Import(lineno=next(self.stream).lineno)
355 node.template = self.parse_expression()
356 self.stream.expect("name:as")
357 node.target = self.parse_assign_target(name_only=True).name
358 return self.parse_import_context(node, False)
360 def parse_from(self) -> nodes.FromImport:
361 node = nodes.FromImport(lineno=next(self.stream).lineno)
362 node.template = self.parse_expression()
363 self.stream.expect("name:import")
364 node.names = []
366 def parse_context() -> bool:
367 if self.stream.current.value in {
368 "with",
369 "without",
370 } and self.stream.look().test("name:context"):
371 node.with_context = next(self.stream).value == "with"
372 self.stream.skip()
373 return True
374 return False
376 while True:
377 if node.names:
378 self.stream.expect("comma")
379 if self.stream.current.type == "name":
380 if parse_context():
381 break
382 target = self.parse_assign_target(name_only=True)
383 if target.name.startswith("_"):
384 self.fail(
385 "names starting with an underline can not be imported",
386 target.lineno,
387 exc=TemplateAssertionError,
388 )
389 if self.stream.skip_if("name:as"):
390 alias = self.parse_assign_target(name_only=True)
391 node.names.append((target.name, alias.name))
392 else:
393 node.names.append(target.name)
394 if parse_context() or self.stream.current.type != "comma":
395 break
396 else:
397 self.stream.expect("name")
398 if not hasattr(node, "with_context"):
399 node.with_context = False
400 return node
402 def parse_signature(self, node: _MacroCall) -> None:
403 args = node.args = []
404 defaults = node.defaults = []
405 self.stream.expect("lparen")
406 while self.stream.current.type != "rparen":
407 if args:
408 self.stream.expect("comma")
409 arg = self.parse_assign_target(name_only=True)
410 arg.set_ctx("param")
411 if self.stream.skip_if("assign"):
412 defaults.append(self.parse_expression())
413 elif defaults:
414 self.fail("non-default argument follows default argument")
415 args.append(arg)
416 self.stream.expect("rparen")
418 def parse_call_block(self) -> nodes.CallBlock:
419 node = nodes.CallBlock(lineno=next(self.stream).lineno)
420 if self.stream.current.type == "lparen":
421 self.parse_signature(node)
422 else:
423 node.args = []
424 node.defaults = []
426 call_node = self.parse_expression()
427 if not isinstance(call_node, nodes.Call):
428 self.fail("expected call", node.lineno)
429 node.call = call_node
430 node.body = self.parse_statements(("name:endcall",), drop_needle=True)
431 return node
433 def parse_filter_block(self) -> nodes.FilterBlock:
434 node = nodes.FilterBlock(lineno=next(self.stream).lineno)
435 node.filter = self.parse_filter(None, start_inline=True) # type: ignore
436 node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
437 return node
439 def parse_macro(self) -> nodes.Macro:
440 node = nodes.Macro(lineno=next(self.stream).lineno)
441 node.name = self.parse_assign_target(name_only=True).name
442 self.parse_signature(node)
443 node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
444 return node
446 def parse_print(self) -> nodes.Output:
447 node = nodes.Output(lineno=next(self.stream).lineno)
448 node.nodes = []
449 while self.stream.current.type != "block_end":
450 if node.nodes:
451 self.stream.expect("comma")
452 node.nodes.append(self.parse_expression())
453 return node
455 @typing.overload
456 def parse_assign_target(
457 self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
458 ) -> nodes.Name:
459 ...
461 @typing.overload
462 def parse_assign_target(
463 self,
464 with_tuple: bool = True,
465 name_only: bool = False,
466 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
467 with_namespace: bool = False,
468 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
469 ...
471 def parse_assign_target(
472 self,
473 with_tuple: bool = True,
474 name_only: bool = False,
475 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
476 with_namespace: bool = False,
477 ) -> t.Union[nodes.NSRef, nodes.Name, nodes.Tuple]:
478 """Parse an assignment target. As Jinja allows assignments to
479 tuples, this function can parse all allowed assignment targets. Per
480 default assignments to tuples are parsed, that can be disable however
481 by setting `with_tuple` to `False`. If only assignments to names are
482 wanted `name_only` can be set to `True`. The `extra_end_rules`
483 parameter is forwarded to the tuple parsing function. If
484 `with_namespace` is enabled, a namespace assignment may be parsed.
485 """
486 target: nodes.Expr
488 if with_namespace and self.stream.look().type == "dot":
489 token = self.stream.expect("name")
490 next(self.stream) # dot
491 attr = self.stream.expect("name")
492 target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
493 elif name_only:
494 token = self.stream.expect("name")
495 target = nodes.Name(token.value, "store", lineno=token.lineno)
496 else:
497 if with_tuple:
498 target = self.parse_tuple(
499 simplified=True, extra_end_rules=extra_end_rules
500 )
501 else:
502 target = self.parse_primary()
504 target.set_ctx("store")
506 if not target.can_assign():
507 self.fail(
508 f"can't assign to {type(target).__name__.lower()!r}", target.lineno
509 )
511 return target # type: ignore
513 def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
514 """Parse an expression. Per default all expressions are parsed, if
515 the optional `with_condexpr` parameter is set to `False` conditional
516 expressions are not parsed.
517 """
518 if with_condexpr:
519 return self.parse_condexpr()
520 return self.parse_or()
522 def parse_condexpr(self) -> nodes.Expr:
523 lineno = self.stream.current.lineno
524 expr1 = self.parse_or()
525 expr3: t.Optional[nodes.Expr]
527 while self.stream.skip_if("name:if"):
528 expr2 = self.parse_or()
529 if self.stream.skip_if("name:else"):
530 expr3 = self.parse_condexpr()
531 else:
532 expr3 = None
533 expr1 = nodes.CondExpr(expr2, expr1, expr3, lineno=lineno)
534 lineno = self.stream.current.lineno
535 return expr1
537 def parse_or(self) -> nodes.Expr:
538 lineno = self.stream.current.lineno
539 left = self.parse_and()
540 while self.stream.skip_if("name:or"):
541 right = self.parse_and()
542 left = nodes.Or(left, right, lineno=lineno)
543 lineno = self.stream.current.lineno
544 return left
546 def parse_and(self) -> nodes.Expr:
547 lineno = self.stream.current.lineno
548 left = self.parse_not()
549 while self.stream.skip_if("name:and"):
550 right = self.parse_not()
551 left = nodes.And(left, right, lineno=lineno)
552 lineno = self.stream.current.lineno
553 return left
555 def parse_not(self) -> nodes.Expr:
556 if self.stream.current.test("name:not"):
557 lineno = next(self.stream).lineno
558 return nodes.Not(self.parse_not(), lineno=lineno)
559 return self.parse_compare()
561 def parse_compare(self) -> nodes.Expr:
562 lineno = self.stream.current.lineno
563 expr = self.parse_math1()
564 ops = []
565 while True:
566 token_type = self.stream.current.type
567 if token_type in _compare_operators:
568 next(self.stream)
569 ops.append(nodes.Operand(token_type, self.parse_math1()))
570 elif self.stream.skip_if("name:in"):
571 ops.append(nodes.Operand("in", self.parse_math1()))
572 elif self.stream.current.test("name:not") and self.stream.look().test(
573 "name:in"
574 ):
575 self.stream.skip(2)
576 ops.append(nodes.Operand("notin", self.parse_math1()))
577 else:
578 break
579 lineno = self.stream.current.lineno
580 if not ops:
581 return expr
582 return nodes.Compare(expr, ops, lineno=lineno)
584 def parse_math1(self) -> nodes.Expr:
585 lineno = self.stream.current.lineno
586 left = self.parse_concat()
587 while self.stream.current.type in ("add", "sub"):
588 cls = _math_nodes[self.stream.current.type]
589 next(self.stream)
590 right = self.parse_concat()
591 left = cls(left, right, lineno=lineno)
592 lineno = self.stream.current.lineno
593 return left
595 def parse_concat(self) -> nodes.Expr:
596 lineno = self.stream.current.lineno
597 args = [self.parse_math2()]
598 while self.stream.current.type == "tilde":
599 next(self.stream)
600 args.append(self.parse_math2())
601 if len(args) == 1:
602 return args[0]
603 return nodes.Concat(args, lineno=lineno)
605 def parse_math2(self) -> nodes.Expr:
606 lineno = self.stream.current.lineno
607 left = self.parse_pow()
608 while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
609 cls = _math_nodes[self.stream.current.type]
610 next(self.stream)
611 right = self.parse_pow()
612 left = cls(left, right, lineno=lineno)
613 lineno = self.stream.current.lineno
614 return left
616 def parse_pow(self) -> nodes.Expr:
617 lineno = self.stream.current.lineno
618 left = self.parse_unary()
619 while self.stream.current.type == "pow":
620 next(self.stream)
621 right = self.parse_unary()
622 left = nodes.Pow(left, right, lineno=lineno)
623 lineno = self.stream.current.lineno
624 return left
626 def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
627 token_type = self.stream.current.type
628 lineno = self.stream.current.lineno
629 node: nodes.Expr
631 if token_type == "sub":
632 next(self.stream)
633 node = nodes.Neg(self.parse_unary(False), lineno=lineno)
634 elif token_type == "add":
635 next(self.stream)
636 node = nodes.Pos(self.parse_unary(False), lineno=lineno)
637 else:
638 node = self.parse_primary()
639 node = self.parse_postfix(node)
640 if with_filter:
641 node = self.parse_filter_expr(node)
642 return node
644 def parse_primary(self) -> nodes.Expr:
645 token = self.stream.current
646 node: nodes.Expr
647 if token.type == "name":
648 if token.value in ("true", "false", "True", "False"):
649 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
650 elif token.value in ("none", "None"):
651 node = nodes.Const(None, lineno=token.lineno)
652 else:
653 node = nodes.Name(token.value, "load", lineno=token.lineno)
654 next(self.stream)
655 elif token.type == "string":
656 next(self.stream)
657 buf = [token.value]
658 lineno = token.lineno
659 while self.stream.current.type == "string":
660 buf.append(self.stream.current.value)
661 next(self.stream)
662 node = nodes.Const("".join(buf), lineno=lineno)
663 elif token.type in ("integer", "float"):
664 next(self.stream)
665 node = nodes.Const(token.value, lineno=token.lineno)
666 elif token.type == "lparen":
667 next(self.stream)
668 node = self.parse_tuple(explicit_parentheses=True)
669 self.stream.expect("rparen")
670 elif token.type == "lbracket":
671 node = self.parse_list()
672 elif token.type == "lbrace":
673 node = self.parse_dict()
674 else:
675 self.fail(f"unexpected {describe_token(token)!r}", token.lineno)
676 return node
678 def parse_tuple(
679 self,
680 simplified: bool = False,
681 with_condexpr: bool = True,
682 extra_end_rules: t.Optional[t.Tuple[str, ...]] = None,
683 explicit_parentheses: bool = False,
684 ) -> t.Union[nodes.Tuple, nodes.Expr]:
685 """Works like `parse_expression` but if multiple expressions are
686 delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
687 This method could also return a regular expression instead of a tuple
688 if no commas where found.
690 The default parsing mode is a full tuple. If `simplified` is `True`
691 only names and literals are parsed. The `no_condexpr` parameter is
692 forwarded to :meth:`parse_expression`.
694 Because tuples do not require delimiters and may end in a bogus comma
695 an extra hint is needed that marks the end of a tuple. For example
696 for loops support tuples between `for` and `in`. In that case the
697 `extra_end_rules` is set to ``['name:in']``.
699 `explicit_parentheses` is true if the parsing was triggered by an
700 expression in parentheses. This is used to figure out if an empty
701 tuple is a valid expression or not.
702 """
703 lineno = self.stream.current.lineno
704 if simplified:
705 parse = self.parse_primary
706 elif with_condexpr:
707 parse = self.parse_expression
708 else:
710 def parse() -> nodes.Expr:
711 return self.parse_expression(with_condexpr=False)
713 args: t.List[nodes.Expr] = []
714 is_tuple = False
716 while True:
717 if args:
718 self.stream.expect("comma")
719 if self.is_tuple_end(extra_end_rules):
720 break
721 args.append(parse())
722 if self.stream.current.type == "comma":
723 is_tuple = True
724 else:
725 break
726 lineno = self.stream.current.lineno
728 if not is_tuple:
729 if args:
730 return args[0]
732 # if we don't have explicit parentheses, an empty tuple is
733 # not a valid expression. This would mean nothing (literally
734 # nothing) in the spot of an expression would be an empty
735 # tuple.
736 if not explicit_parentheses:
737 self.fail(
738 "Expected an expression,"
739 f" got {describe_token(self.stream.current)!r}"
740 )
742 return nodes.Tuple(args, "load", lineno=lineno)
744 def parse_list(self) -> nodes.List:
745 token = self.stream.expect("lbracket")
746 items: t.List[nodes.Expr] = []
747 while self.stream.current.type != "rbracket":
748 if items:
749 self.stream.expect("comma")
750 if self.stream.current.type == "rbracket":
751 break
752 items.append(self.parse_expression())
753 self.stream.expect("rbracket")
754 return nodes.List(items, lineno=token.lineno)
756 def parse_dict(self) -> nodes.Dict:
757 token = self.stream.expect("lbrace")
758 items: t.List[nodes.Pair] = []
759 while self.stream.current.type != "rbrace":
760 if items:
761 self.stream.expect("comma")
762 if self.stream.current.type == "rbrace":
763 break
764 key = self.parse_expression()
765 self.stream.expect("colon")
766 value = self.parse_expression()
767 items.append(nodes.Pair(key, value, lineno=key.lineno))
768 self.stream.expect("rbrace")
769 return nodes.Dict(items, lineno=token.lineno)
771 def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
772 while True:
773 token_type = self.stream.current.type
774 if token_type == "dot" or token_type == "lbracket":
775 node = self.parse_subscript(node)
776 # calls are valid both after postfix expressions (getattr
777 # and getitem) as well as filters and tests
778 elif token_type == "lparen":
779 node = self.parse_call(node)
780 else:
781 break
782 return node
784 def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
785 while True:
786 token_type = self.stream.current.type
787 if token_type == "pipe":
788 node = self.parse_filter(node) # type: ignore
789 elif token_type == "name" and self.stream.current.value == "is":
790 node = self.parse_test(node)
791 # calls are valid both after postfix expressions (getattr
792 # and getitem) as well as filters and tests
793 elif token_type == "lparen":
794 node = self.parse_call(node)
795 else:
796 break
797 return node
799 def parse_subscript(
800 self, node: nodes.Expr
801 ) -> t.Union[nodes.Getattr, nodes.Getitem]:
802 token = next(self.stream)
803 arg: nodes.Expr
805 if token.type == "dot":
806 attr_token = self.stream.current
807 next(self.stream)
808 if attr_token.type == "name":
809 return nodes.Getattr(
810 node, attr_token.value, "load", lineno=token.lineno
811 )
812 elif attr_token.type != "integer":
813 self.fail("expected name or number", attr_token.lineno)
814 arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
815 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
816 if token.type == "lbracket":
817 args: t.List[nodes.Expr] = []
818 while self.stream.current.type != "rbracket":
819 if args:
820 self.stream.expect("comma")
821 args.append(self.parse_subscribed())
822 self.stream.expect("rbracket")
823 if len(args) == 1:
824 arg = args[0]
825 else:
826 arg = nodes.Tuple(args, "load", lineno=token.lineno)
827 return nodes.Getitem(node, arg, "load", lineno=token.lineno)
828 self.fail("expected subscript expression", token.lineno)
830 def parse_subscribed(self) -> nodes.Expr:
831 lineno = self.stream.current.lineno
832 args: t.List[t.Optional[nodes.Expr]]
834 if self.stream.current.type == "colon":
835 next(self.stream)
836 args = [None]
837 else:
838 node = self.parse_expression()
839 if self.stream.current.type != "colon":
840 return node
841 next(self.stream)
842 args = [node]
844 if self.stream.current.type == "colon":
845 args.append(None)
846 elif self.stream.current.type not in ("rbracket", "comma"):
847 args.append(self.parse_expression())
848 else:
849 args.append(None)
851 if self.stream.current.type == "colon":
852 next(self.stream)
853 if self.stream.current.type not in ("rbracket", "comma"):
854 args.append(self.parse_expression())
855 else:
856 args.append(None)
857 else:
858 args.append(None)
860 return nodes.Slice(lineno=lineno, *args)
862 def parse_call_args(self) -> t.Tuple:
863 token = self.stream.expect("lparen")
864 args = []
865 kwargs = []
866 dyn_args = None
867 dyn_kwargs = None
868 require_comma = False
870 def ensure(expr: bool) -> None:
871 if not expr:
872 self.fail("invalid syntax for function call expression", token.lineno)
874 while self.stream.current.type != "rparen":
875 if require_comma:
876 self.stream.expect("comma")
878 # support for trailing comma
879 if self.stream.current.type == "rparen":
880 break
882 if self.stream.current.type == "mul":
883 ensure(dyn_args is None and dyn_kwargs is None)
884 next(self.stream)
885 dyn_args = self.parse_expression()
886 elif self.stream.current.type == "pow":
887 ensure(dyn_kwargs is None)
888 next(self.stream)
889 dyn_kwargs = self.parse_expression()
890 else:
891 if (
892 self.stream.current.type == "name"
893 and self.stream.look().type == "assign"
894 ):
895 # Parsing a kwarg
896 ensure(dyn_kwargs is None)
897 key = self.stream.current.value
898 self.stream.skip(2)
899 value = self.parse_expression()
900 kwargs.append(nodes.Keyword(key, value, lineno=value.lineno))
901 else:
902 # Parsing an arg
903 ensure(dyn_args is None and dyn_kwargs is None and not kwargs)
904 args.append(self.parse_expression())
906 require_comma = True
908 self.stream.expect("rparen")
909 return args, kwargs, dyn_args, dyn_kwargs
911 def parse_call(self, node: nodes.Expr) -> nodes.Call:
912 # The lparen will be expected in parse_call_args, but the lineno
913 # needs to be recorded before the stream is advanced.
914 token = self.stream.current
915 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
916 return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
918 def parse_filter(
919 self, node: t.Optional[nodes.Expr], start_inline: bool = False
920 ) -> t.Optional[nodes.Expr]:
921 while self.stream.current.type == "pipe" or start_inline:
922 if not start_inline:
923 next(self.stream)
924 token = self.stream.expect("name")
925 name = token.value
926 while self.stream.current.type == "dot":
927 next(self.stream)
928 name += "." + self.stream.expect("name").value
929 if self.stream.current.type == "lparen":
930 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
931 else:
932 args = []
933 kwargs = []
934 dyn_args = dyn_kwargs = None
935 node = nodes.Filter(
936 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
937 )
938 start_inline = False
939 return node
941 def parse_test(self, node: nodes.Expr) -> nodes.Expr:
942 token = next(self.stream)
943 if self.stream.current.test("name:not"):
944 next(self.stream)
945 negated = True
946 else:
947 negated = False
948 name = self.stream.expect("name").value
949 while self.stream.current.type == "dot":
950 next(self.stream)
951 name += "." + self.stream.expect("name").value
952 dyn_args = dyn_kwargs = None
953 kwargs = []
954 if self.stream.current.type == "lparen":
955 args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
956 elif self.stream.current.type in {
957 "name",
958 "string",
959 "integer",
960 "float",
961 "lparen",
962 "lbracket",
963 "lbrace",
964 } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
965 if self.stream.current.test("name:is"):
966 self.fail("You cannot chain multiple tests with is")
967 arg_node = self.parse_primary()
968 arg_node = self.parse_postfix(arg_node)
969 args = [arg_node]
970 else:
971 args = []
972 node = nodes.Test(
973 node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno
974 )
975 if negated:
976 node = nodes.Not(node, lineno=token.lineno)
977 return node
979 def subparse(
980 self, end_tokens: t.Optional[t.Tuple[str, ...]] = None
981 ) -> t.List[nodes.Node]:
982 body: t.List[nodes.Node] = []
983 data_buffer: t.List[nodes.Node] = []
984 add_data = data_buffer.append
986 if end_tokens is not None:
987 self._end_token_stack.append(end_tokens)
989 def flush_data() -> None:
990 if data_buffer:
991 lineno = data_buffer[0].lineno
992 body.append(nodes.Output(data_buffer[:], lineno=lineno))
993 del data_buffer[:]
995 try:
996 while self.stream:
997 token = self.stream.current
998 if token.type == "data":
999 if token.value:
1000 add_data(nodes.TemplateData(token.value, lineno=token.lineno))
1001 next(self.stream)
1002 elif token.type == "variable_begin":
1003 next(self.stream)
1004 add_data(self.parse_tuple(with_condexpr=True))
1005 self.stream.expect("variable_end")
1006 elif token.type == "block_begin":
1007 flush_data()
1008 next(self.stream)
1009 if end_tokens is not None and self.stream.current.test_any(
1010 *end_tokens
1011 ):
1012 return body
1013 rv = self.parse_statement()
1014 if isinstance(rv, list):
1015 body.extend(rv)
1016 else:
1017 body.append(rv)
1018 self.stream.expect("block_end")
1019 else:
1020 raise AssertionError("internal parsing error")
1022 flush_data()
1023 finally:
1024 if end_tokens is not None:
1025 self._end_token_stack.pop()
1026 return body
1028 def parse(self) -> nodes.Template:
1029 """Parse the whole template into a `Template` node."""
1030 result = nodes.Template(self.subparse(), lineno=1)
1031 result.set_environment(self.environment)
1032 return result