From 4943bb405cd4880dc14883331da8c05bbed4a8f2 Mon Sep 17 00:00:00 2001 From: Fabien Potencier Date: Fri, 5 Jun 2026 20:18:56 +0200 Subject: [PATCH 1/3] Track the source offset of each token --- CHANGELOG | 3 +- src/Environment.php | 8 ++-- src/Lexer.php | 78 ++++++++++++++++++++++++++------------- src/Token.php | 17 +++++++++ tests/LexerTest.php | 89 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 30 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 443f4f010..a45bdfb62 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,6 @@ -# 3.27.2 (2026-XX-XX) +# 3.28.0 (2026-XX-XX) + * Track the source offset of each token and expose it via `Token::getOffset()` * Fix nested `block()` calls to resolve against the overriding template when a block rendered through `block(name, template)` calls `parent()` * Stop reporting a skipped test in `IntegrationTestCase` when there is no legacy test to run * Make the `IntegrationTestCase` and `NodeTestCase` test helpers compatible with PHPUnit 11 diff --git a/src/Environment.php b/src/Environment.php index ba9e8a18d..c6b1d8964 100644 --- a/src/Environment.php +++ b/src/Environment.php @@ -43,11 +43,11 @@ use Twig\TokenParser\TokenParserInterface; */ class Environment { - public const VERSION = '3.27.2-DEV'; - public const VERSION_ID = 32702; + public const VERSION = '3.28.0-DEV'; + public const VERSION_ID = 32800; public const MAJOR_VERSION = 3; - public const MINOR_VERSION = 27; - public const RELEASE_VERSION = 2; + public const MINOR_VERSION = 28; + public const RELEASE_VERSION = 0; public const EXTRA_VERSION = 'DEV'; private $charset; diff --git a/src/Lexer.php b/src/Lexer.php index e65f5bedc..60c4815ba 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -62,6 +62,8 @@ class Lexer public const REGEX_INLINE_COMMENT = '/#[^\n]*/A'; public const PUNCTUATION = '()[]{}?:.,|'; + private const REGEX_RAW_INLINE_COMMENT = '/#[^\r\n]*/A'; + private const SPECIAL_CHARS = [ 'f' => "\f", 'n' => "\n", @@ -113,7 +115,7 @@ class Lexer '|'. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]* '|'. - preg_quote($this->options['tag_block'][1], '#').'\n?'. // %}\n? + preg_quote($this->options['tag_block'][1], '#').'(?:\r\n?|\n)?'. // %}(?:\r\n?|\n)? ') }Ax', @@ -143,7 +145,7 @@ class Lexer '|'. preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~#}[ \t\0\x0B]* '|'. - preg_quote($this->options['tag_comment'][1], '#').'\n?'. // #}\n? + preg_quote($this->options['tag_comment'][1], '#').'(?:\r\n?|\n)?'. // #}(?:\r\n?|\n)? ') }sx', @@ -187,7 +189,7 @@ class Lexer $this->initialize(); $this->source = $source; - $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode()); + $this->code = $source->getCode(); $this->cursor = 0; $this->lineno = 1; $this->end = \strlen($this->code); @@ -241,8 +243,9 @@ class Lexer { // if no matches are left we return the rest of the template as simple text token if ($this->position == \count($this->positions[0]) - 1) { - $this->pushToken(Token::TEXT_TYPE, substr($this->code, $this->cursor)); - $this->cursor = $this->end; + $text = substr($this->code, $this->cursor); + $this->pushToken(Token::TEXT_TYPE, $this->normalizeNewlines($text)); + $this->moveCursor($text); return; } @@ -270,15 +273,20 @@ class Lexer $text = rtrim($text, " \t\0\x0B"); } } - $this->pushToken(Token::TEXT_TYPE, $text); - $this->moveCursor($textContent.$position[0]); + $this->pushToken(Token::TEXT_TYPE, $this->normalizeNewlines($text)); + $this->moveCursor($textContent); switch ($this->positions[1][$this->position][0]) { case $this->options['tag_comment'][0]: + $this->moveCursor($position[0]); $this->lexComment(); break; case $this->options['tag_block'][0]: + $lineno = $this->lineno; + $cursor = $this->cursor; + $this->moveCursor($position[0]); + // raw data? if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, 0, $this->cursor)) { $this->moveCursor($match[0]); @@ -288,14 +296,17 @@ class Lexer $this->moveCursor($match[0]); $this->lineno = (int) $match[1]; } else { - $this->pushToken(Token::BLOCK_START_TYPE); + $this->pushToken(Token::BLOCK_START_TYPE, '', $cursor, $lineno); $this->pushState(self::STATE_BLOCK); $this->currentVarBlockLine = $this->lineno; } break; case $this->options['tag_variable'][0]: - $this->pushToken(Token::VAR_START_TYPE); + $lineno = $this->lineno; + $cursor = $this->cursor; + $this->moveCursor($position[0]); + $this->pushToken(Token::VAR_START_TYPE, '', $cursor, $lineno); $this->pushState(self::STATE_VAR); $this->currentVarBlockLine = $this->lineno; break; @@ -305,8 +316,7 @@ class Lexer private function lexBlock(): void { if (!$this->brackets && preg_match($this->regexes['lex_block'], $this->code, $match, 0, $this->cursor)) { - $this->pushToken(Token::BLOCK_END_TYPE); - $this->moveCursor($match[0]); + $this->pushClosingToken(Token::BLOCK_END_TYPE, $match[0]); $this->popState(); } else { $this->lexExpression(); @@ -316,8 +326,7 @@ class Lexer private function lexVar(): void { if (!$this->brackets && preg_match($this->regexes['lex_var'], $this->code, $match, 0, $this->cursor)) { - $this->pushToken(Token::VAR_END_TYPE); - $this->moveCursor($match[0]); + $this->pushClosingToken(Token::VAR_END_TYPE, $match[0]); $this->popState(); } else { $this->lexExpression(); @@ -358,11 +367,11 @@ class Lexer elseif (str_contains(self::PUNCTUATION, $this->code[$this->cursor])) { $this->checkBrackets($this->code[$this->cursor]); $this->pushToken(Token::PUNCTUATION_TYPE, $this->code[$this->cursor]); - ++$this->cursor; + $this->moveCursor($this->code[$this->cursor]); } // strings elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) { - $this->pushToken(Token::STRING_TYPE, $this->stripcslashes(substr($match[0], 1, -1), substr($match[0], 0, 1))); + $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($this->normalizeNewlines(substr($match[0], 1, -1)), substr($match[0], 0, 1))); $this->moveCursor($match[0]); } // opening double quoted string @@ -372,7 +381,7 @@ class Lexer $this->moveCursor($match[0]); } // inline comment - elseif (preg_match(self::REGEX_INLINE_COMMENT, $this->code, $match, 0, $this->cursor)) { + elseif (preg_match(self::REGEX_RAW_INLINE_COMMENT, $this->code, $match, 0, $this->cursor)) { $this->moveCursor($match[0]); } // unlexable @@ -444,6 +453,7 @@ class Lexer throw new SyntaxError('Unexpected end of file: Unclosed "verbatim" block.', $this->lineno, $this->source); } + $offset = $this->cursor; $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor); $this->moveCursor($text.$match[0][0]); @@ -459,7 +469,7 @@ class Lexer } } - $this->pushToken(Token::TEXT_TYPE, $text); + $this->pushToken(Token::TEXT_TYPE, $this->normalizeNewlines($text), $offset); } private function lexComment(): void @@ -479,7 +489,7 @@ class Lexer $this->moveCursor($match[0]); $this->pushState(self::STATE_INTERPOLATION); } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && '' !== $match[0]) { - $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($match[0], '"')); + $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($this->normalizeNewlines($match[0]), '"')); $this->moveCursor($match[0]); } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) { [$expect, $lineno] = array_pop($this->brackets); @@ -488,7 +498,7 @@ class Lexer } $this->popState(); - ++$this->cursor; + $this->moveCursor($match[0]); } else { // unlexable throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source); @@ -500,28 +510,46 @@ class Lexer $bracket = end($this->brackets); if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)) { array_pop($this->brackets); - $this->pushToken(Token::INTERPOLATION_END_TYPE); - $this->moveCursor($match[0]); + $this->pushClosingToken(Token::INTERPOLATION_END_TYPE, $match[0]); $this->popState(); } else { $this->lexExpression(); } } - private function pushToken($type, $value = ''): void + private function pushToken($type, $value = '', ?int $offset = null, ?int $lineno = null): void { // do not push empty text tokens if (Token::TEXT_TYPE === $type && '' === $value) { return; } - $this->tokens[] = new Token($type, $value, $this->lineno); + // by default the token starts at the current cursor; callers that + // emit a token after consuming it must pass an explicit offset + $this->tokens[] = new Token($type, $value, $lineno ?? $this->lineno, $offset ?? $this->cursor); } private function moveCursor($text): void { - $this->cursor += \strlen($text); - $this->lineno += substr_count($text, "\n"); + $length = \strlen($text); + $this->cursor += $length; + $this->lineno += substr_count($this->normalizeNewlines($text), "\n"); + } + + private function normalizeNewlines(string $text): string + { + return str_replace(["\r\n", "\r"], "\n", $text); + } + + private function pushClosingToken(int $type, string $match): void + { + $leadingWhitespaceLength = \strlen($match) - \strlen(ltrim($match)); + if ($leadingWhitespaceLength) { + $this->moveCursor(substr($match, 0, $leadingWhitespaceLength)); + } + + $this->pushToken($type); + $this->moveCursor(substr($match, $leadingWhitespaceLength)); } private function getOperatorRegex(): string diff --git a/src/Token.php b/src/Token.php index 823c77387..0c0e46a81 100644 --- a/src/Token.php +++ b/src/Token.php @@ -39,10 +39,14 @@ final class Token */ public const SPREAD_TYPE = 13; + /** + * @param non-negative-int|null $offset + */ public function __construct( private int $type, private $value, private int $lineno, + private ?int $offset = null, ) { if (self::ARROW_TYPE === $type) { trigger_deprecation('twig/twig', '3.21', 'The "%s" token type is deprecated, "arrow" is now an operator.', self::ARROW_TYPE); @@ -124,6 +128,19 @@ final class Token return $this->lineno; } + /** + * Returns the 0-based byte offset of the token in the source code. + * + * Returns null for tokens that are not tied to a source position (e.g. + * tokens synthesized by a token parser). + * + * @return non-negative-int|null + */ + public function getOffset(): ?int + { + return $this->offset; + } + /** * @deprecated since Twig 3.19 */ diff --git a/tests/LexerTest.php b/tests/LexerTest.php index 0b2ae755a..579aa7da4 100644 --- a/tests/LexerTest.php +++ b/tests/LexerTest.php @@ -734,4 +734,93 @@ bar yield ['{{ { a: 1 }}', '}']; yield ['{{ ([1] + 3)) }}', ')']; } + + public function testTokensCarryTheirSourceOffset() + { + $template = 'Hello {{ name }}!'; + + $lexer = new Lexer(new Environment(new ArrayLoader())); + $stream = $lexer->tokenize(new Source($template, 'index')); + + // [type, offset] pairs; offsets point at the start of each lexeme in the source + $expected = [ + [Token::TEXT_TYPE, 0], // "Hello " + [Token::VAR_START_TYPE, 6], // "{{" + [Token::NAME_TYPE, 9], // "name" + [Token::VAR_END_TYPE, 14], // "}}" + [Token::TEXT_TYPE, 16], // "!" + [Token::EOF_TYPE, 17], + ]; + + foreach ($expected as [$type, $offset]) { + $token = $stream->getCurrent(); + $this->assertTrue($token->test($type), \sprintf('Expected token "%s".', Token::typeToEnglish($type))); + $this->assertSame($offset, $token->getOffset()); + + if (!$stream->isEOF()) { + $stream->next(); + } + } + } + + public function testOffsetsAllowRecoveringTheRawExpressionSource() + { + $template = "Hello {{ name|upper ~ '!' }}"; + + $lexer = new Lexer(new Environment(new ArrayLoader())); + $stream = $lexer->tokenize(new Source($template, 'index')); + + $stream->expect(Token::TEXT_TYPE); + $start = $stream->expect(Token::VAR_START_TYPE)->getOffset(); + while (!$stream->test(Token::VAR_END_TYPE)) { + $stream->next(); + } + $end = $stream->getCurrent()->getOffset(); + + // slice the raw expression out of the original source, between "{{" and "}}" + $raw = trim(substr($template, $start + 2, $end - $start - 2)); + + $this->assertSame("name|upper ~ '!'", $raw); + } + + public function testOffsetsReferToTheOriginalSourceWhenLineEndingsAreNormalized() + { + $template = "Hello\r\n{{ name }}"; + + $lexer = new Lexer(new Environment(new ArrayLoader())); + $stream = $lexer->tokenize(new Source($template, 'index')); + + $stream->expect(Token::TEXT_TYPE); + $start = $stream->expect(Token::VAR_START_TYPE)->getOffset(); + $this->assertSame('{{', substr($template, $start, 2)); + + $name = $stream->expect(Token::NAME_TYPE); + $this->assertSame('name', substr($template, $name->getOffset(), 4)); + + $end = $stream->expect(Token::VAR_END_TYPE)->getOffset(); + $this->assertSame('name', trim(substr($template, $start + 2, $end - $start - 2))); + } + + public function testBlockTagDelimitersPointAtTheMarkers() + { + $template = '{% set x = 1 %}'; + + $lexer = new Lexer(new Environment(new ArrayLoader())); + $stream = $lexer->tokenize(new Source($template, 'index')); + + // the opening "{%" and the closing "%}" both point at the marker itself, + // not at the whitespace the closing regex also consumes + $this->assertSame(0, $stream->expect(Token::BLOCK_START_TYPE)->getOffset()); + $this->assertSame('{%', substr($template, 0, 2)); + while (!$stream->test(Token::BLOCK_END_TYPE)) { + $stream->next(); + } + $end = $stream->getCurrent()->getOffset(); + $this->assertSame('%}', substr($template, $end, 2)); + } + + public function testSyntheticTokensHaveNoOffset() + { + $this->assertNull((new Token(Token::NAME_TYPE, 'foo', 1))->getOffset()); + } } From a82782ac309da88bbf6c514f484ecfa760874819 Mon Sep 17 00:00:00 2001 From: Fabien Potencier Date: Fri, 5 Jun 2026 20:20:32 +0200 Subject: [PATCH 2/3] Report columns in syntax errors --- CHANGELOG | 1 + src/Error/Error.php | 34 +++++++++-- src/Lexer.php | 6 +- src/Source.php | 19 +++++++ src/TokenStream.php | 3 +- tests/ExpressionParserTest.php | 2 +- .../syntax_error_in_reused_template.test | 2 +- .../underscored_numbers_error.test | 2 +- ...on_not_supported_as_default_separator.test | 2 +- .../tags/macro/from_syntax_error.test | 2 +- .../tags/macro/import_syntax_error.test | 2 +- tests/LexerTest.php | 56 +++++++++++++++++-- tests/SourceTest.php | 38 +++++++++++++ 13 files changed, 149 insertions(+), 20 deletions(-) create mode 100644 tests/SourceTest.php diff --git a/CHANGELOG b/CHANGELOG index a45bdfb62..6cbe452cd 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,6 @@ # 3.28.0 (2026-XX-XX) + * Report the column number in syntax errors and expose it via `Error::getTemplateColumn()` * Track the source offset of each token and expose it via `Token::getOffset()` * Fix nested `block()` calls to resolve against the overriding template when a block rendered through `block(name, template)` calls `parent()` * Stop reporting a skipped test in `IntegrationTestCase` when there is no legacy test to run diff --git a/src/Error/Error.php b/src/Error/Error.php index 97ed2df99..8494d243f 100644 --- a/src/Error/Error.php +++ b/src/Error/Error.php @@ -36,6 +36,8 @@ use Twig\Template; class Error extends \Exception { private $lineno; + /** @var positive-int|null */ + private ?int $columnno; private $rawMessage; private ?Source $source; private string $phpFile; @@ -46,17 +48,19 @@ class Error extends \Exception * * By default, automatic guessing is enabled. * - * @param string $message The error message - * @param int $lineno The template line where the error occurred - * @param Source|null $source The source context where the error occurred + * @param string $message The error message + * @param int $lineno The template line where the error occurred + * @param Source|null $source The source context where the error occurred + * @param positive-int|null $columnno The template column where the error occurred */ - public function __construct(string $message, int $lineno = -1, ?Source $source = null, ?\Throwable $previous = null) + public function __construct(string $message, int $lineno = -1, ?Source $source = null, ?\Throwable $previous = null, ?int $columnno = null) { parent::__construct('', 0, $previous); $this->phpFile = $this->getFile(); $this->phpLine = $this->getLine(); $this->lineno = $lineno; + $this->columnno = $columnno; $this->source = $source; $this->rawMessage = $message; $this->updateRepr(); @@ -78,6 +82,25 @@ class Error extends \Exception $this->updateRepr(); } + /** + * Returns the 1-based column where the error occurred, or null if unknown. + * + * @return positive-int|null + */ + public function getTemplateColumn(): ?int + { + return $this->columnno; + } + + /** + * @param positive-int|null $columnno + */ + public function setTemplateColumn(?int $columnno): void + { + $this->columnno = $columnno; + $this->updateRepr(); + } + public function getSourceContext(): ?Source { return $this->source; @@ -127,6 +150,9 @@ class Error extends \Exception } if ($this->lineno > 0) { $this->message .= \sprintf(' at line %d', $this->lineno); + if (null !== $this->columnno) { + $this->message .= \sprintf(' column %d', $this->columnno); + } } if ($punctuation) { $this->message .= $punctuation; diff --git a/src/Lexer.php b/src/Lexer.php index 60c4815ba..d79258a92 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -386,7 +386,7 @@ class Lexer } // unlexable else { - throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source); + throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source, columnno: $this->source->getColumn($this->cursor)); } } @@ -501,7 +501,7 @@ class Lexer $this->moveCursor($match[0]); } else { // unlexable - throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source); + throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source, columnno: $this->source->getColumn($this->cursor)); } } @@ -608,7 +608,7 @@ class Lexer } elseif (\in_array($code, $this->closingBrackets, true)) { // closing bracket if (!$this->brackets) { - throw new SyntaxError(\sprintf('Unexpected "%s".', $code), $this->lineno, $this->source); + throw new SyntaxError(\sprintf('Unexpected "%s".', $code), $this->lineno, $this->source, columnno: $this->source->getColumn($this->cursor)); } [$expect, $lineno] = array_pop($this->brackets); diff --git a/src/Source.php b/src/Source.php index 0f626b62d..0c97f255a 100644 --- a/src/Source.php +++ b/src/Source.php @@ -44,4 +44,23 @@ final class Source { return $this->path; } + + /** + * Returns the 1-based column for a 0-based byte offset in the source code. + * + * A negative offset means the position is unknown and yields null. + * + * @return positive-int|null + */ + public function getColumn(int $offset): ?int + { + if ($offset < 0) { + return null; + } + + $before = str_replace(["\r\n", "\r"], "\n", substr($this->code, 0, $offset)); + $lineStart = strrpos($before, "\n"); + + return false === $lineStart ? \strlen($before) + 1 : \strlen($before) - $lineStart; + } } diff --git a/src/TokenStream.php b/src/TokenStream.php index 7ee7539f1..2586750fa 100644 --- a/src/TokenStream.php +++ b/src/TokenStream.php @@ -83,7 +83,8 @@ final class TokenStream $token->getValue() ? \sprintf(' of value "%s"', $token->getValue()) : '', Token::typeToEnglish($type), $value ? \sprintf(' with value "%s"', $value) : ''), $line, - $this->source + $this->source, + columnno: $this->source->getColumn($token->getOffset() ?? -1), ); } $this->next(); diff --git a/tests/ExpressionParserTest.php b/tests/ExpressionParserTest.php index 7bf164d34..b48f8d613 100644 --- a/tests/ExpressionParserTest.php +++ b/tests/ExpressionParserTest.php @@ -414,7 +414,7 @@ class ExpressionParserTest extends TestCase $parser = new Parser($env); $this->expectException(SyntaxError::class); - $this->expectExceptionMessage('An argument must be a name. Unexpected token "string" of value "a" ("name" expected) in "index" at line 1.'); + $this->expectExceptionMessage('An argument must be a name. Unexpected token "string" of value "a" ("name" expected) in "index" at line 1 column 14.'); $parser->parse($env->tokenize(new Source('{% macro foo("a") %}{% endmacro %}', 'index'))); } diff --git a/tests/Fixtures/exceptions/syntax_error_in_reused_template.test b/tests/Fixtures/exceptions/syntax_error_in_reused_template.test index 0ee19cc62..bdb0ff37a 100644 --- a/tests/Fixtures/exceptions/syntax_error_in_reused_template.test +++ b/tests/Fixtures/exceptions/syntax_error_in_reused_template.test @@ -7,4 +7,4 @@ Exception for syntax error in reused template {% do node.data 5 %} {% endblock %} --EXCEPTION-- -Twig\Error\SyntaxError: Unexpected token "number" of value "5" ("end of statement block" expected) in "foo.twig" at line 3. +Twig\Error\SyntaxError: Unexpected token "number" of value "5" ("end of statement block" expected) in "foo.twig" at line 3 column 21. diff --git a/tests/Fixtures/expressions/underscored_numbers_error.test b/tests/Fixtures/expressions/underscored_numbers_error.test index 839d606f1..26e71c012 100644 --- a/tests/Fixtures/expressions/underscored_numbers_error.test +++ b/tests/Fixtures/expressions/underscored_numbers_error.test @@ -5,4 +5,4 @@ Twig does not allow to use 2 underscored between digits in numbers --DATA-- return [] --EXCEPTION-- -Twig\Error\SyntaxError: Unexpected token "name" of value "__2" ("end of print statement" expected) in "index.twig" at line 2. +Twig\Error\SyntaxError: Unexpected token "name" of value "__2" ("end of print statement" expected) in "index.twig" at line 2 column 5. diff --git a/tests/Fixtures/tags/macro/colon_not_supported_as_default_separator.test b/tests/Fixtures/tags/macro/colon_not_supported_as_default_separator.test index 6b0bb25fb..0c8c510ac 100644 --- a/tests/Fixtures/tags/macro/colon_not_supported_as_default_separator.test +++ b/tests/Fixtures/tags/macro/colon_not_supported_as_default_separator.test @@ -7,4 +7,4 @@ --DATA-- return [] --EXCEPTION-- -Twig\Error\SyntaxError: Arguments must be separated by a comma. Unexpected token "punctuation" of value ":" ("punctuation" expected with value ",") in "index.twig" at line 2. +Twig\Error\SyntaxError: Arguments must be separated by a comma. Unexpected token "punctuation" of value ":" ("punctuation" expected with value ",") in "index.twig" at line 2 column 18. diff --git a/tests/Fixtures/tags/macro/from_syntax_error.test b/tests/Fixtures/tags/macro/from_syntax_error.test index 6223cfe94..b5ba0e640 100644 --- a/tests/Fixtures/tags/macro/from_syntax_error.test +++ b/tests/Fixtures/tags/macro/from_syntax_error.test @@ -5,4 +5,4 @@ --DATA-- return [] --EXCEPTION-- -Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "import") in "index.twig" at line 2. +Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "import") in "index.twig" at line 2 column 22. diff --git a/tests/Fixtures/tags/macro/import_syntax_error.test b/tests/Fixtures/tags/macro/import_syntax_error.test index b9817f0ee..e53f8de0c 100644 --- a/tests/Fixtures/tags/macro/import_syntax_error.test +++ b/tests/Fixtures/tags/macro/import_syntax_error.test @@ -7,4 +7,4 @@ --DATA-- return [] --EXCEPTION-- -Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "as") in "index.twig" at line 2. +Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "as") in "index.twig" at line 2 column 24. diff --git a/tests/LexerTest.php b/tests/LexerTest.php index 579aa7da4..d75769c9f 100644 --- a/tests/LexerTest.php +++ b/tests/LexerTest.php @@ -717,22 +717,22 @@ bar * @dataProvider getTemplateForUnexpectedBracketInExpression */ #[DataProvider('getTemplateForUnexpectedBracketInExpression')] - public function testUnexpectedBracketInExpression(string $template, string $bracket) + public function testUnexpectedBracketInExpression(string $template, string $bracket, int $column) { $lexer = new Lexer(new Environment(new ArrayLoader())); $this->expectException(SyntaxError::class); - $this->expectExceptionMessage(\sprintf('Unexpected "%s" in "index" at line 1.', $bracket)); + $this->expectExceptionMessage(\sprintf('Unexpected "%s" in "index" at line 1 column %d.', $bracket, $column)); $lexer->tokenize(new Source($template, 'index')); } public static function getTemplateForUnexpectedBracketInExpression() { - yield ['{{ 1 + 3) }}', ')']; - yield ['{{ obj] }}', ']']; - yield ['{{ { a: 1 }}', '}']; - yield ['{{ ([1] + 3)) }}', ')']; + yield ['{{ 1 + 3) }}', ')', 9]; + yield ['{{ obj] }}', ']', 7]; + yield ['{{ { a: 1 }}', '}', 12]; + yield ['{{ ([1] + 3)) }}', ')', 13]; } public function testTokensCarryTheirSourceOffset() @@ -819,8 +819,52 @@ bar $this->assertSame('%}', substr($template, $end, 2)); } + public function testClosingDelimiterLineMatchesTheMarkerLine() + { + $template = "{% from 'forms.twig'\n %}"; + $env = new Environment(new ArrayLoader()); + + try { + $env->parse($env->tokenize(new Source($template, 'index'))); + $this->fail('A SyntaxError should have been thrown.'); + } catch (SyntaxError $e) { + $this->assertSame(2, $e->getTemplateLine()); + $this->assertSame(3, $e->getTemplateColumn()); + $this->assertStringEndsWith('at line 2 column 3.', $e->getMessage()); + } + } + public function testSyntheticTokensHaveNoOffset() { $this->assertNull((new Token(Token::NAME_TYPE, 'foo', 1))->getOffset()); } + + public function testSyntaxErrorReportsTheColumn() + { + $lexer = new Lexer(new Environment(new ArrayLoader())); + + try { + $lexer->tokenize(new Source("{{ 1 + 3) }}\n{{ ok }}", 'index')); + $this->fail('A SyntaxError should have been thrown.'); + } catch (SyntaxError $e) { + $this->assertSame(1, $e->getTemplateLine()); + $this->assertSame(9, $e->getTemplateColumn()); + $this->assertStringEndsWith('at line 1 column 9.', $e->getMessage()); + } + } + + public function testSyntaxErrorColumnUsesOriginalSourceOffsets() + { + $template = "x\r\n{{ 1__2 }}"; + $env = new Environment(new ArrayLoader()); + + try { + $env->parse($env->tokenize(new Source($template, 'index'))); + $this->fail('A SyntaxError should have been thrown.'); + } catch (SyntaxError $e) { + $this->assertSame(2, $e->getTemplateLine()); + $this->assertSame(5, $e->getTemplateColumn()); + $this->assertStringEndsWith('at line 2 column 5.', $e->getMessage()); + } + } } diff --git a/tests/SourceTest.php b/tests/SourceTest.php new file mode 100644 index 000000000..3064f2cc9 --- /dev/null +++ b/tests/SourceTest.php @@ -0,0 +1,38 @@ +assertSame(1, $source->getColumn(0)); + $this->assertSame(3, $source->getColumn(2)); + + // a "\n" closes the line; the next character starts a new line at column 1 + $this->assertSame(1, $source->getColumn(4)); + $this->assertSame(4, $source->getColumn(7)); + + // "\r\n" and "\r" also close the line + $this->assertSame(1, (new Source("foo\r\nbar", 'index'))->getColumn(5)); + $this->assertSame(1, (new Source("foo\rbar", 'index'))->getColumn(4)); + + // an unknown offset yields null + $this->assertNull($source->getColumn(-1)); + } +} From 3868bac531632682160b8e5885b16eba8a137a5a Mon Sep 17 00:00:00 2001 From: Fabien Potencier Date: Fri, 5 Jun 2026 20:33:28 +0200 Subject: [PATCH 3/3] Avoid allocating a normalized copy when counting newlines without carriage returns --- src/Lexer.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Lexer.php b/src/Lexer.php index d79258a92..1c482d161 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -531,9 +531,10 @@ class Lexer private function moveCursor($text): void { - $length = \strlen($text); - $this->cursor += $length; - $this->lineno += substr_count($this->normalizeNewlines($text), "\n"); + $this->cursor += \strlen($text); + // count "\r\n" and "\r" as a single newline without allocating a + // normalized copy when the chunk has no carriage return (common case) + $this->lineno += str_contains($text, "\r") ? substr_count($this->normalizeNewlines($text), "\n") : substr_count($text, "\n"); } private function normalizeNewlines(string $text): string