feature #4834 Track the source offset of each token and expose it in syntax errors (fabpot)

This PR was squashed before being merged into the 3.x branch.

Discussion
----------

Track the source offset of each token and expose it in syntax errors

Commits
-------

3868bac531 Avoid allocating a normalized copy when counting newlines without carriage returns
a82782ac30 Report columns in syntax errors
4943bb405c Track the source offset of each token

CHANGELOG

@@ -1,5 +1,7 @@
-# 3.27.2 (2026-XX-XX)
+# 3.28.0 (2026-XX-XX)
 
+ * Report the column number in syntax errors and expose it via `Error::getTemplateColumn()`
+ * Track the source offset of each token and expose it via `Token::getOffset()`
  * Fix nested `block()` calls to resolve against the overriding template when a block rendered through `block(name, template)` calls `parent()`
  * Stop reporting a skipped test in `IntegrationTestCase` when there is no legacy test to run
  * Make the `IntegrationTestCase` and `NodeTestCase` test helpers compatible with PHPUnit 11

src/Environment.php

@@ -43,11 +43,11 @@ use Twig\TokenParser\TokenParserInterface;
  */
 class Environment
 {
-    public const VERSION = '3.27.2-DEV';
-    public const VERSION_ID = 32702;
+    public const VERSION = '3.28.0-DEV';
+    public const VERSION_ID = 32800;
     public const MAJOR_VERSION = 3;
-    public const MINOR_VERSION = 27;
-    public const RELEASE_VERSION = 2;
+    public const MINOR_VERSION = 28;
+    public const RELEASE_VERSION = 0;
     public const EXTRA_VERSION = 'DEV';
 
     private $charset;

src/Error/Error.php

@@ -36,6 +36,8 @@ use Twig\Template;
 class Error extends \Exception
 {
     private $lineno;
+    /** @var positive-int|null */
+    private ?int $columnno;
     private $rawMessage;
     private ?Source $source;
     private string $phpFile;
@@ -46,17 +48,19 @@ class Error extends \Exception
      *
      * By default, automatic guessing is enabled.
      *
-     * @param string      $message The error message
-     * @param int         $lineno  The template line where the error occurred
-     * @param Source|null $source  The source context where the error occurred
+     * @param string            $message  The error message
+     * @param int               $lineno   The template line where the error occurred
+     * @param Source|null       $source   The source context where the error occurred
+     * @param positive-int|null $columnno The template column where the error occurred
      */
-    public function __construct(string $message, int $lineno = -1, ?Source $source = null, ?\Throwable $previous = null)
+    public function __construct(string $message, int $lineno = -1, ?Source $source = null, ?\Throwable $previous = null, ?int $columnno = null)
     {
         parent::__construct('', 0, $previous);
 
         $this->phpFile = $this->getFile();
         $this->phpLine = $this->getLine();
         $this->lineno = $lineno;
+        $this->columnno = $columnno;
         $this->source = $source;
         $this->rawMessage = $message;
         $this->updateRepr();
@@ -78,6 +82,25 @@ class Error extends \Exception
         $this->updateRepr();
     }
 
+    /**
+     * Returns the 1-based column where the error occurred, or null if unknown.
+     *
+     * @return positive-int|null
+     */
+    public function getTemplateColumn(): ?int
+    {
+        return $this->columnno;
+    }
+
+    /**
+     * @param positive-int|null $columnno
+     */
+    public function setTemplateColumn(?int $columnno): void
+    {
+        $this->columnno = $columnno;
+        $this->updateRepr();
+    }
+
     public function getSourceContext(): ?Source
     {
         return $this->source;
@@ -127,6 +150,9 @@ class Error extends \Exception
         }
         if ($this->lineno > 0) {
             $this->message .= \sprintf(' at line %d', $this->lineno);
+            if (null !== $this->columnno) {
+                $this->message .= \sprintf(' column %d', $this->columnno);
+            }
         }
         if ($punctuation) {
             $this->message .= $punctuation;

src/Lexer.php

@@ -62,6 +62,8 @@ class Lexer
     public const REGEX_INLINE_COMMENT = '/#[^\n]*/A';
     public const PUNCTUATION = '()[]{}?:.,|';
 
+    private const REGEX_RAW_INLINE_COMMENT = '/#[^\r\n]*/A';
+
     private const SPECIAL_CHARS = [
         'f' => "\f",
         'n' => "\n",
@@ -113,7 +115,7 @@ class Lexer
                     '|'.
                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_block'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~%}[ \t\0\x0B]*
                     '|'.
-                    preg_quote($this->options['tag_block'][1], '#').'\n?'. // %}\n?
+                    preg_quote($this->options['tag_block'][1], '#').'(?:\r\n?|\n)?'. // %}(?:\r\n?|\n)?
                 ')
             }Ax',
 
@@ -143,7 +145,7 @@ class Lexer
                     '|'.
                     preg_quote($this->options['whitespace_line_trim'].$this->options['tag_comment'][1], '#').'['.$this->options['whitespace_line_chars'].']*'. // ~#}[ \t\0\x0B]*
                     '|'.
-                    preg_quote($this->options['tag_comment'][1], '#').'\n?'. // #}\n?
+                    preg_quote($this->options['tag_comment'][1], '#').'(?:\r\n?|\n)?'. // #}(?:\r\n?|\n)?
                 ')
             }sx',
 
@@ -187,7 +189,7 @@ class Lexer
         $this->initialize();
 
         $this->source = $source;
-        $this->code = str_replace(["\r\n", "\r"], "\n", $source->getCode());
+        $this->code = $source->getCode();
         $this->cursor = 0;
         $this->lineno = 1;
         $this->end = \strlen($this->code);
@@ -241,8 +243,9 @@ class Lexer
     {
         // if no matches are left we return the rest of the template as simple text token
         if ($this->position == \count($this->positions[0]) - 1) {
-            $this->pushToken(Token::TEXT_TYPE, substr($this->code, $this->cursor));
-            $this->cursor = $this->end;
+            $text = substr($this->code, $this->cursor);
+            $this->pushToken(Token::TEXT_TYPE, $this->normalizeNewlines($text));
+            $this->moveCursor($text);
 
             return;
         }
@@ -270,15 +273,20 @@ class Lexer
                 $text = rtrim($text, " \t\0\x0B");
             }
         }
-        $this->pushToken(Token::TEXT_TYPE, $text);
-        $this->moveCursor($textContent.$position[0]);
+        $this->pushToken(Token::TEXT_TYPE, $this->normalizeNewlines($text));
+        $this->moveCursor($textContent);
 
         switch ($this->positions[1][$this->position][0]) {
             case $this->options['tag_comment'][0]:
+                $this->moveCursor($position[0]);
                 $this->lexComment();
                 break;
 
             case $this->options['tag_block'][0]:
+                $lineno = $this->lineno;
+                $cursor = $this->cursor;
+                $this->moveCursor($position[0]);
+
                 // raw data?
                 if (preg_match($this->regexes['lex_block_raw'], $this->code, $match, 0, $this->cursor)) {
                     $this->moveCursor($match[0]);
@@ -288,14 +296,17 @@ class Lexer
                     $this->moveCursor($match[0]);
                     $this->lineno = (int) $match[1];
                 } else {
-                    $this->pushToken(Token::BLOCK_START_TYPE);
+                    $this->pushToken(Token::BLOCK_START_TYPE, '', $cursor, $lineno);
                     $this->pushState(self::STATE_BLOCK);
                     $this->currentVarBlockLine = $this->lineno;
                 }
                 break;
 
             case $this->options['tag_variable'][0]:
-                $this->pushToken(Token::VAR_START_TYPE);
+                $lineno = $this->lineno;
+                $cursor = $this->cursor;
+                $this->moveCursor($position[0]);
+                $this->pushToken(Token::VAR_START_TYPE, '', $cursor, $lineno);
                 $this->pushState(self::STATE_VAR);
                 $this->currentVarBlockLine = $this->lineno;
                 break;
@@ -305,8 +316,7 @@ class Lexer
     private function lexBlock(): void
     {
         if (!$this->brackets && preg_match($this->regexes['lex_block'], $this->code, $match, 0, $this->cursor)) {
-            $this->pushToken(Token::BLOCK_END_TYPE);
-            $this->moveCursor($match[0]);
+            $this->pushClosingToken(Token::BLOCK_END_TYPE, $match[0]);
             $this->popState();
         } else {
             $this->lexExpression();
@@ -316,8 +326,7 @@ class Lexer
     private function lexVar(): void
     {
         if (!$this->brackets && preg_match($this->regexes['lex_var'], $this->code, $match, 0, $this->cursor)) {
-            $this->pushToken(Token::VAR_END_TYPE);
-            $this->moveCursor($match[0]);
+            $this->pushClosingToken(Token::VAR_END_TYPE, $match[0]);
             $this->popState();
         } else {
             $this->lexExpression();
@@ -358,11 +367,11 @@ class Lexer
         elseif (str_contains(self::PUNCTUATION, $this->code[$this->cursor])) {
             $this->checkBrackets($this->code[$this->cursor]);
             $this->pushToken(Token::PUNCTUATION_TYPE, $this->code[$this->cursor]);
-            ++$this->cursor;
+            $this->moveCursor($this->code[$this->cursor]);
         }
         // strings
         elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
-            $this->pushToken(Token::STRING_TYPE, $this->stripcslashes(substr($match[0], 1, -1), substr($match[0], 0, 1)));
+            $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($this->normalizeNewlines(substr($match[0], 1, -1)), substr($match[0], 0, 1)));
             $this->moveCursor($match[0]);
         }
         // opening double quoted string
@@ -372,12 +381,12 @@ class Lexer
             $this->moveCursor($match[0]);
         }
         // inline comment
-        elseif (preg_match(self::REGEX_INLINE_COMMENT, $this->code, $match, 0, $this->cursor)) {
+        elseif (preg_match(self::REGEX_RAW_INLINE_COMMENT, $this->code, $match, 0, $this->cursor)) {
             $this->moveCursor($match[0]);
         }
         // unlexable
         else {
-            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
+            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source, columnno: $this->source->getColumn($this->cursor));
         }
     }
 
@@ -444,6 +453,7 @@ class Lexer
             throw new SyntaxError('Unexpected end of file: Unclosed "verbatim" block.', $this->lineno, $this->source);
         }
 
+        $offset = $this->cursor;
         $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor);
         $this->moveCursor($text.$match[0][0]);
 
@@ -459,7 +469,7 @@ class Lexer
             }
         }
 
-        $this->pushToken(Token::TEXT_TYPE, $text);
+        $this->pushToken(Token::TEXT_TYPE, $this->normalizeNewlines($text), $offset);
     }
 
     private function lexComment(): void
@@ -479,7 +489,7 @@ class Lexer
             $this->moveCursor($match[0]);
             $this->pushState(self::STATE_INTERPOLATION);
         } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && '' !== $match[0]) {
-            $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($match[0], '"'));
+            $this->pushToken(Token::STRING_TYPE, $this->stripcslashes($this->normalizeNewlines($match[0]), '"'));
             $this->moveCursor($match[0]);
         } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
             [$expect, $lineno] = array_pop($this->brackets);
@@ -488,10 +498,10 @@ class Lexer
             }
 
             $this->popState();
-            ++$this->cursor;
+            $this->moveCursor($match[0]);
         } else {
             // unlexable
-            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
+            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source, columnno: $this->source->getColumn($this->cursor));
         }
     }
 
@@ -500,28 +510,47 @@ class Lexer
         $bracket = end($this->brackets);
         if ($this->options['interpolation'][0] === $bracket[0] && preg_match($this->regexes['interpolation_end'], $this->code, $match, 0, $this->cursor)) {
             array_pop($this->brackets);
-            $this->pushToken(Token::INTERPOLATION_END_TYPE);
-            $this->moveCursor($match[0]);
+            $this->pushClosingToken(Token::INTERPOLATION_END_TYPE, $match[0]);
             $this->popState();
         } else {
             $this->lexExpression();
         }
     }
 
-    private function pushToken($type, $value = ''): void
+    private function pushToken($type, $value = '', ?int $offset = null, ?int $lineno = null): void
     {
         // do not push empty text tokens
         if (Token::TEXT_TYPE === $type && '' === $value) {
             return;
         }
 
-        $this->tokens[] = new Token($type, $value, $this->lineno);
+        // by default the token starts at the current cursor; callers that
+        // emit a token after consuming it must pass an explicit offset
+        $this->tokens[] = new Token($type, $value, $lineno ?? $this->lineno, $offset ?? $this->cursor);
     }
 
     private function moveCursor($text): void
     {
         $this->cursor += \strlen($text);
-        $this->lineno += substr_count($text, "\n");
+        // count "\r\n" and "\r" as a single newline without allocating a
+        // normalized copy when the chunk has no carriage return (common case)
+        $this->lineno += str_contains($text, "\r") ? substr_count($this->normalizeNewlines($text), "\n") : substr_count($text, "\n");
+    }
+
+    private function normalizeNewlines(string $text): string
+    {
+        return str_replace(["\r\n", "\r"], "\n", $text);
+    }
+
+    private function pushClosingToken(int $type, string $match): void
+    {
+        $leadingWhitespaceLength = \strlen($match) - \strlen(ltrim($match));
+        if ($leadingWhitespaceLength) {
+            $this->moveCursor(substr($match, 0, $leadingWhitespaceLength));
+        }
+
+        $this->pushToken($type);
+        $this->moveCursor(substr($match, $leadingWhitespaceLength));
     }
 
     private function getOperatorRegex(): string
@@ -580,7 +609,7 @@ class Lexer
         } elseif (\in_array($code, $this->closingBrackets, true)) {
             // closing bracket
             if (!$this->brackets) {
-                throw new SyntaxError(\sprintf('Unexpected "%s".', $code), $this->lineno, $this->source);
+                throw new SyntaxError(\sprintf('Unexpected "%s".', $code), $this->lineno, $this->source, columnno: $this->source->getColumn($this->cursor));
             }
 
             [$expect, $lineno] = array_pop($this->brackets);

src/Source.php

@@ -44,4 +44,23 @@ final class Source
     {
         return $this->path;
     }
+
+    /**
+     * Returns the 1-based column for a 0-based byte offset in the source code.
+     *
+     * A negative offset means the position is unknown and yields null.
+     *
+     * @return positive-int|null
+     */
+    public function getColumn(int $offset): ?int
+    {
+        if ($offset < 0) {
+            return null;
+        }
+
+        $before = str_replace(["\r\n", "\r"], "\n", substr($this->code, 0, $offset));
+        $lineStart = strrpos($before, "\n");
+
+        return false === $lineStart ? \strlen($before) + 1 : \strlen($before) - $lineStart;
+    }
 }

src/Token.php

@@ -39,10 +39,14 @@ final class Token
      */
     public const SPREAD_TYPE = 13;
 
+    /**
+     * @param non-negative-int|null $offset
+     */
     public function __construct(
         private int $type,
         private $value,
         private int $lineno,
+        private ?int $offset = null,
     ) {
         if (self::ARROW_TYPE === $type) {
             trigger_deprecation('twig/twig', '3.21', 'The "%s" token type is deprecated, "arrow" is now an operator.', self::ARROW_TYPE);
@@ -124,6 +128,19 @@ final class Token
         return $this->lineno;
     }
 
+    /**
+     * Returns the 0-based byte offset of the token in the source code.
+     *
+     * Returns null for tokens that are not tied to a source position (e.g.
+     * tokens synthesized by a token parser).
+     *
+     * @return non-negative-int|null
+     */
+    public function getOffset(): ?int
+    {
+        return $this->offset;
+    }
+
     /**
      * @deprecated since Twig 3.19
      */

src/TokenStream.php

@@ -83,7 +83,8 @@ final class TokenStream
                 $token->getValue() ? \sprintf(' of value "%s"', $token->getValue()) : '',
                 Token::typeToEnglish($type), $value ? \sprintf(' with value "%s"', $value) : ''),
                 $line,
-                $this->source
+                $this->source,
+                columnno: $this->source->getColumn($token->getOffset() ?? -1),
             );
         }
         $this->next();

tests/ExpressionParserTest.php

@@ -414,7 +414,7 @@ class ExpressionParserTest extends TestCase
         $parser = new Parser($env);
 
         $this->expectException(SyntaxError::class);
-        $this->expectExceptionMessage('An argument must be a name. Unexpected token "string" of value "a" ("name" expected) in "index" at line 1.');
+        $this->expectExceptionMessage('An argument must be a name. Unexpected token "string" of value "a" ("name" expected) in "index" at line 1 column 14.');
 
         $parser->parse($env->tokenize(new Source('{% macro foo("a") %}{% endmacro %}', 'index')));
     }

tests/Fixtures/exceptions/syntax_error_in_reused_template.test

@@ -7,4 +7,4 @@ Exception for syntax error in reused template
     {% do node.data 5 %}
 {% endblock %}
 --EXCEPTION--
-Twig\Error\SyntaxError: Unexpected token "number" of value "5" ("end of statement block" expected) in "foo.twig" at line 3.
+Twig\Error\SyntaxError: Unexpected token "number" of value "5" ("end of statement block" expected) in "foo.twig" at line 3 column 21.

tests/Fixtures/expressions/underscored_numbers_error.test

@@ -5,4 +5,4 @@ Twig does not allow to use 2 underscored between digits in numbers
 --DATA--
 return []
 --EXCEPTION--
-Twig\Error\SyntaxError: Unexpected token "name" of value "__2" ("end of print statement" expected) in "index.twig" at line 2.
+Twig\Error\SyntaxError: Unexpected token "name" of value "__2" ("end of print statement" expected) in "index.twig" at line 2 column 5.

tests/Fixtures/tags/macro/colon_not_supported_as_default_separator.test

@@ -7,4 +7,4 @@
 --DATA--
 return []
 --EXCEPTION--
-Twig\Error\SyntaxError: Arguments must be separated by a comma. Unexpected token "punctuation" of value ":" ("punctuation" expected with value ",") in "index.twig" at line 2.
+Twig\Error\SyntaxError: Arguments must be separated by a comma. Unexpected token "punctuation" of value ":" ("punctuation" expected with value ",") in "index.twig" at line 2 column 18.

tests/Fixtures/tags/macro/from_syntax_error.test

@@ -5,4 +5,4 @@
 --DATA--
 return []
 --EXCEPTION--
-Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "import") in "index.twig" at line 2.
+Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "import") in "index.twig" at line 2 column 22.

tests/Fixtures/tags/macro/import_syntax_error.test

@@ -7,4 +7,4 @@
 --DATA--
 return []
 --EXCEPTION--
-Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "as") in "index.twig" at line 2.
+Twig\Error\SyntaxError: Unexpected token "end of statement block" ("name" expected with value "as") in "index.twig" at line 2 column 24.

tests/LexerTest.php

@@ -717,21 +717,154 @@ bar
      * @dataProvider getTemplateForUnexpectedBracketInExpression
      */
     #[DataProvider('getTemplateForUnexpectedBracketInExpression')]
-    public function testUnexpectedBracketInExpression(string $template, string $bracket)
+    public function testUnexpectedBracketInExpression(string $template, string $bracket, int $column)
     {
         $lexer = new Lexer(new Environment(new ArrayLoader()));
 
         $this->expectException(SyntaxError::class);
-        $this->expectExceptionMessage(\sprintf('Unexpected "%s" in "index" at line 1.', $bracket));
+        $this->expectExceptionMessage(\sprintf('Unexpected "%s" in "index" at line 1 column %d.', $bracket, $column));
 
         $lexer->tokenize(new Source($template, 'index'));
     }
 
     public static function getTemplateForUnexpectedBracketInExpression()
     {
-        yield ['{{ 1 + 3) }}', ')'];
-        yield ['{{ obj] }}', ']'];
-        yield ['{{ { a: 1 }}', '}'];
-        yield ['{{ ([1] + 3)) }}', ')'];
+        yield ['{{ 1 + 3) }}', ')', 9];
+        yield ['{{ obj] }}', ']', 7];
+        yield ['{{ { a: 1 }}', '}', 12];
+        yield ['{{ ([1] + 3)) }}', ')', 13];
+    }
+
+    public function testTokensCarryTheirSourceOffset()
+    {
+        $template = 'Hello {{ name }}!';
+
+        $lexer = new Lexer(new Environment(new ArrayLoader()));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+
+        // [type, offset] pairs; offsets point at the start of each lexeme in the source
+        $expected = [
+            [Token::TEXT_TYPE, 0],   // "Hello "
+            [Token::VAR_START_TYPE, 6],   // "{{"
+            [Token::NAME_TYPE, 9],   // "name"
+            [Token::VAR_END_TYPE, 14],  // "}}"
+            [Token::TEXT_TYPE, 16],  // "!"
+            [Token::EOF_TYPE, 17],
+        ];
+
+        foreach ($expected as [$type, $offset]) {
+            $token = $stream->getCurrent();
+            $this->assertTrue($token->test($type), \sprintf('Expected token "%s".', Token::typeToEnglish($type)));
+            $this->assertSame($offset, $token->getOffset());
+
+            if (!$stream->isEOF()) {
+                $stream->next();
+            }
+        }
+    }
+
+    public function testOffsetsAllowRecoveringTheRawExpressionSource()
+    {
+        $template = "Hello {{ name|upper ~ '!' }}";
+
+        $lexer = new Lexer(new Environment(new ArrayLoader()));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+
+        $stream->expect(Token::TEXT_TYPE);
+        $start = $stream->expect(Token::VAR_START_TYPE)->getOffset();
+        while (!$stream->test(Token::VAR_END_TYPE)) {
+            $stream->next();
+        }
+        $end = $stream->getCurrent()->getOffset();
+
+        // slice the raw expression out of the original source, between "{{" and "}}"
+        $raw = trim(substr($template, $start + 2, $end - $start - 2));
+
+        $this->assertSame("name|upper ~ '!'", $raw);
+    }
+
+    public function testOffsetsReferToTheOriginalSourceWhenLineEndingsAreNormalized()
+    {
+        $template = "Hello\r\n{{ name }}";
+
+        $lexer = new Lexer(new Environment(new ArrayLoader()));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+
+        $stream->expect(Token::TEXT_TYPE);
+        $start = $stream->expect(Token::VAR_START_TYPE)->getOffset();
+        $this->assertSame('{{', substr($template, $start, 2));
+
+        $name = $stream->expect(Token::NAME_TYPE);
+        $this->assertSame('name', substr($template, $name->getOffset(), 4));
+
+        $end = $stream->expect(Token::VAR_END_TYPE)->getOffset();
+        $this->assertSame('name', trim(substr($template, $start + 2, $end - $start - 2)));
+    }
+
+    public function testBlockTagDelimitersPointAtTheMarkers()
+    {
+        $template = '{% set x = 1 %}';
+
+        $lexer = new Lexer(new Environment(new ArrayLoader()));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+
+        // the opening "{%" and the closing "%}" both point at the marker itself,
+        // not at the whitespace the closing regex also consumes
+        $this->assertSame(0, $stream->expect(Token::BLOCK_START_TYPE)->getOffset());
+        $this->assertSame('{%', substr($template, 0, 2));
+        while (!$stream->test(Token::BLOCK_END_TYPE)) {
+            $stream->next();
+        }
+        $end = $stream->getCurrent()->getOffset();
+        $this->assertSame('%}', substr($template, $end, 2));
+    }
+
+    public function testClosingDelimiterLineMatchesTheMarkerLine()
+    {
+        $template = "{% from 'forms.twig'\n  %}";
+        $env = new Environment(new ArrayLoader());
+
+        try {
+            $env->parse($env->tokenize(new Source($template, 'index')));
+            $this->fail('A SyntaxError should have been thrown.');
+        } catch (SyntaxError $e) {
+            $this->assertSame(2, $e->getTemplateLine());
+            $this->assertSame(3, $e->getTemplateColumn());
+            $this->assertStringEndsWith('at line 2 column 3.', $e->getMessage());
+        }
+    }
+
+    public function testSyntheticTokensHaveNoOffset()
+    {
+        $this->assertNull((new Token(Token::NAME_TYPE, 'foo', 1))->getOffset());
+    }
+
+    public function testSyntaxErrorReportsTheColumn()
+    {
+        $lexer = new Lexer(new Environment(new ArrayLoader()));
+
+        try {
+            $lexer->tokenize(new Source("{{ 1 + 3) }}\n{{ ok }}", 'index'));
+            $this->fail('A SyntaxError should have been thrown.');
+        } catch (SyntaxError $e) {
+            $this->assertSame(1, $e->getTemplateLine());
+            $this->assertSame(9, $e->getTemplateColumn());
+            $this->assertStringEndsWith('at line 1 column 9.', $e->getMessage());
+        }
+    }
+
+    public function testSyntaxErrorColumnUsesOriginalSourceOffsets()
+    {
+        $template = "x\r\n{{ 1__2 }}";
+        $env = new Environment(new ArrayLoader());
+
+        try {
+            $env->parse($env->tokenize(new Source($template, 'index')));
+            $this->fail('A SyntaxError should have been thrown.');
+        } catch (SyntaxError $e) {
+            $this->assertSame(2, $e->getTemplateLine());
+            $this->assertSame(5, $e->getTemplateColumn());
+            $this->assertStringEndsWith('at line 2 column 5.', $e->getMessage());
+        }
     }
 }

tests/SourceTest.php

@@ -0,0 +1,38 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Twig\Tests;
+
+use PHPUnit\Framework\TestCase;
+use Twig\Source;
+
+class SourceTest extends TestCase
+{
+    public function testGetColumn()
+    {
+        $source = new Source("foo\nbarbaz\nqux", 'index');
+
+        // first line: column is the 1-based offset
+        $this->assertSame(1, $source->getColumn(0));
+        $this->assertSame(3, $source->getColumn(2));
+
+        // a "\n" closes the line; the next character starts a new line at column 1
+        $this->assertSame(1, $source->getColumn(4));
+        $this->assertSame(4, $source->getColumn(7));
+
+        // "\r\n" and "\r" also close the line
+        $this->assertSame(1, (new Source("foo\r\nbar", 'index'))->getColumn(5));
+        $this->assertSame(1, (new Source("foo\rbar", 'index'))->getColumn(4));
+
+        // an unknown offset yields null
+        $this->assertNull($source->getColumn(-1));
+    }
+}