Merge branch '3.x' into 4.x

* 3.x:
  fix version numbers for deprecations
  Require "erusev/parsedown": "dev-master as 1.x-dev"
  Support underscores in number literals
  Remove deprecate usage of AssignNameExpression in For Node
  Update coding standards

doc/coding_standards.rst

@@ -88,17 +88,17 @@ standards:
     [1, 2, 3]
     {'name': 'Fabien'}
 
-* Do not put any spaces before and after ``=`` in macro argument declarations:
+* Put exactly one space before and after ``=`` in macro argument declarations:
 
   .. code-block:: twig
 
-    {% macro html_input(class="input") %}
+    {% macro html_input(class = "input") %}
 
-* Put exactly one space after the ``:`` sign in macro argument declarations:
+* Put exactly one space after the ``:`` sign when using named arguments:
 
   .. code-block:: twig
 
-    {% macro html_input(class: "input") %}
+    {{ html_input(class: "input") }}
 
 * Use snake case for all variable names (provided by the application and
   created in templates), function/filter/test names, argument names and named

doc/templates.rst

@@ -598,7 +598,8 @@ exist:
 
 * ``42`` / ``42.23``: Integers and floating point numbers are created by
   writing the number down. If a dot is present the number is a float,
-  otherwise an integer.
+  otherwise an integer. Underscores can be used as digits separator to 
+  improve readability (``-3_141.592_65`` is equivalent to ``-3141.59265``).
 
 * ``["first_name", "last_name"]``: Sequences are defined by a sequence of expressions
   separated by a comma (``,``) and wrapped with squared brackets (``[]``).
@@ -1120,4 +1121,4 @@ Twig can be extended. If you want to create your own extensions, read the
 .. _`Modern Twig`:                          https://marketplace.visualstudio.com/items?itemName=Stanislav.vscode-twig
 .. _`Twig Language Server`:                 https://github.com/kaermorchen/twig-language-server/tree/master/packages/language-server
 .. _`Twiggy`:                               https://marketplace.visualstudio.com/items?itemName=moetelo.twiggy
-.. _`PHP spaceship operator documentation`: https://www.php.net/manual/en/language.operators.comparison.php
+.. _`PHP spaceship operator documentation`: https://www.php.net/manual/en/language.operators.comparison.php

extra/markdown-extra/composer.json

@@ -20,7 +20,8 @@
         "twig/twig": "^3.13|^4.0"
     },
     "require-dev": {
-        "erusev/parsedown": "^1.7",
+        "symfony/phpunit-bridge": "^6.4|^7.0",
+        "erusev/parsedown": "dev-master as 1.x-dev",
         "league/commonmark": "^1.0|^2.0",
         "league/html-to-markdown": "^4.8|^5.0",
         "michelf/php-markdown": "^1.8|^2.0"

phpstan-baseline.neon

@@ -11,3 +11,9 @@ parameters:
 			identifier: arguments.count
 			count: 1
 			path: src/Node/IncludeNode.php
+
+		- # Adding 0 to the string representation of a number is valid and what we want here
+			message: '#^Binary operation "\+" between 0 and string results in an error\.$#'
+			identifier: binaryOp.invalid
+			count: 1
+			path: src/Lexer.php

src/Lexer.php

@@ -59,8 +59,16 @@ class Lexer
     public const STATE_INTERPOLATION = 4;
 
     public const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
-    public const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?([Ee][\+\-][0-9]+)?/A';
     public const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
+
+    public const REGEX_NUMBER = '/(?(DEFINE)
+        (?<LNUM>[0-9]+(_[0-9]+)*)               # Integers (with underscores)   123_456
+        (?<FRAC>\.(?&LNUM))                     # Fractional part               .456
+        (?<EXPONENT>[eE][+-]?(?&LNUM))          # Exponent part                 E+10
+        (?<DNUM>(?&LNUM)(?:(?&FRAC))?)          # Decimal number                123_456.456
+    )(?:(?&DNUM)(?:(?&EXPONENT))?)              #                               123_456.456E+10
+    /Ax';
+    
     public const REGEX_DQ_STRING_DELIM = '/"/A';
     public const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
     public const REGEX_INLINE_COMMENT = '/#[^\n]*/A';
@@ -360,11 +368,7 @@ class Lexer
         }
         // numbers
         elseif (preg_match(self::REGEX_NUMBER, $this->code, $match, 0, $this->cursor)) {
-            $number = (float) $match[0];  // floats
-            if (ctype_digit($match[0]) && $number <= \PHP_INT_MAX) {
-                $number = (int) $match[0]; // integers lower than the maximum
-            }
-            $this->pushToken(Token::NUMBER_TYPE, $number);
+            $this->pushToken(Token::NUMBER_TYPE, 0 + str_replace('_', '', $match[0]));
             $this->moveCursor($match[0]);
         }
         // punctuation

tests/Fixtures/expressions/underscored_numbers.test

@@ -0,0 +1,24 @@
+--TEST--
+Twig compile numbers literals with underscores correctly
+--TEMPLATE--
+{{ 0_0 is same as 0 ? 'ok' : 'ko' }}
+{{ 1_23 is same as 123 ? 'ok' : 'ko' }}
+{{ 12_3 is same as 123 ? 'ok' : 'ko' }}
+{{ 1_2_3 is same as 123 ? 'ok' : 'ko' }}
+{{ -1_2 is same as -12 ? 'ok' : 'ko' }}
+{{ 1_2.3_4 is same as 12.34 ? 'ok' : 'ko' }}
+{{ -1_2.3_4 is same as -12.34 ? 'ok' : 'ko' }}
+{{ 1.2_3e-4 is same as 1.23e-4 ? 'ok' : 'ko' }}
+{{ -1.2_3e+4 is same as -1.23e+4 ? 'ok' : 'ko' }}
+--DATA--
+return []
+--EXPECT--
+ok
+ok
+ok
+ok
+ok
+ok
+ok
+ok
+ok

tests/Fixtures/expressions/underscored_numbers_error.test

@@ -0,0 +1,8 @@
+--TEST--
+Twig does not allow to use 2 underscored between digits in numbers
+--TEMPLATE--
+{{ 1__2 }}
+--DATA--
+return []
+--EXCEPTION--
+Twig\Error\SyntaxError: Unexpected token "name" of value "__2" ("end of print statement" expected) in "index.twig" at line 2.