feature #2996 Add "filter", "map", and "reduce" filters (fabpot)

This PR was squashed before being merged into the 1.x branch (closes #2996).

Discussion
----------

Add "filter", "map", and "reduce" filters

This PR adds support for 3 new filters: `filter`, `map`, and `reduce`. They take an arrow function as an argument (a PHP closure).

I have restricted the usage of arrow functions as much as possible as it makes no sense to support them everywhere. So, for now, they are only accepted as arguments to filters (using them as arguments to function is not supported but could be easily added if we have a use case).

The syntax is the following: `(x) => x + 3` where `(x)` is the list of arguments, `=>` starts the body, and the body is any Twig expression. Within the arrow function, the context is also available: `(x) => x + offset` works if `offset` is defined in the current context.

~~These new filters will allow us to deprecate the `if` support on the `for` tag, which does not work well with the `loop` variable:~~

```twig

{% set sizes = {xs: 34, s: 36, m: 38, l: 40, xl: 42} %}

{# before #}
{% for name, size in sizes if size < 38 %}
    {{ name }} = {{ size }}
    {% loop.last ? 'LAST' %} {# <--- works with this PR #}
{% endfor %}

{# after #}
{% for name, size in sizes|filter(size => size < 38) %}
    {{ name }} = {{ size }}
    {{ loop.last ? 'LAST' }}
{% endfor %}
```

This closes #2785

Commits
-------

5c15f897 added the key to the map and filter filters
13274bbf added support for iterators
175041e0 removed fn in front of arrow functions
dc277635 changed arrow syntax
b30bce11 added "filter", "map", and "reduce" filters

CHANGELOG

@@ -1,5 +1,6 @@
 * 1.41.0 (2019-XX-XX)
 
+ * added "filter", "map", and "reduce" filters (and support for arrow functions)
  * fixed partial output leak when a PHP fatal error occurs
  * optimized context access on PHP 7.4
 

doc/filters/filter.rst

@@ -0,0 +1,47 @@
+``filter``
+=========
+
+.. versionadded:: 1.41
+    The ``filter`` filter was added in Twig 1.41.
+
+The ``filter`` filter filters elements of a sequence or a mapping using an arrow
+function. The arrow function receives the value of the sequence or mapping:
+
+.. code-block:: twig
+
+    {% set sizes = [34, 36, 38, 40, 42] %}
+
+    {% for v in sizes|filter(v => v > 38) -%}
+        {{ v }}
+    {% endfor %}
+    {# output 40 42 #}
+
+    {% set sizes = {
+        xs: 34,
+        s:  36,
+        m:  38,
+        l:  40,
+        xl: 42,
+    } %}
+
+    {% for k, v in sizes|filter(v => v > 38) -%}
+        {{ k }} = {{ v }}
+    {% endfor %}
+    {# output l = 40 xl = 42 #}
+
+The arrow function also receives the key as a second argument:
+
+.. code-block:: twig
+
+    {% for k, v in sizes|filter((v, k) => v > 38 and k != "xl") -%}
+        {{ k }} = {{ v }}
+    {% endfor %}
+    {# output l = 40 #}
+
+Note that the arrow function has access to the current context.
+
+Arguments
+---------
+
+* ``array``: The sequence or mapping
+* ``arrow``: The arrow function

doc/filters/index.rst

@@ -12,6 +12,7 @@ Filters
     date_modify
     default
     escape
+    filter
     first
     format
     join
@@ -20,10 +21,12 @@ Filters
     last
     length
     lower
+    map
     merge
     nl2br
     number_format
     raw
+    reduce
     replace
     reverse
     round

doc/filters/join.rst

@@ -19,7 +19,7 @@ define it with the optional first parameter:
 
     {{ [1, 2, 3]|join('|') }}
     {# outputs 1|2|3 #}
-    
+
 A second parameter can also be provided that will be the separator used between
 the last two items of the sequence:
 
@@ -27,7 +27,7 @@ the last two items of the sequence:
 
     {{ [1, 2, 3]|join(', ', ' and ') }}
     {# outputs 1, 2 and 3 #}
-    
+
 Arguments
 ---------
 

doc/filters/map.rst

@@ -0,0 +1,38 @@
+``map``
+=======
+
+.. versionadded:: 1.41
+    The ``map`` filter was added in Twig 1.41.
+
+The ``map`` filter applies an arrow function to the elements of a sequence or a
+mapping. The arrow function receives the value of the sequence or mapping:
+
+.. code-block:: twig
+
+    {% set people = [
+        {first: "Bob", last: "Smith"},
+        {first: "Alice", last: "Dupond"},
+    ] %}
+
+    {{ people|map(p => "#{p.first} #{p.last}")|join(', ') }}
+    {# outputs Bob Smith, Alice Dupond #}
+
+The arrow function also receives the key as a second argument:
+
+.. code-block:: twig
+
+    {% set people = {
+        "Bob": "Smith",
+        "Alice": "Dupond",
+    } %}
+
+    {{ people|map((first, last) => "#{first} #{last}")|join(', ') }}
+    {# outputs Bob Smith, Alice Dupond #}
+
+Note that the arrow function has access to the current context.
+
+Arguments
+---------
+
+* ``array``: The sequence or mapping
+* ``arrow``: The arrow function

doc/filters/reduce.rst

@@ -0,0 +1,33 @@
+``reduce``
+=========
+
+.. versionadded:: 1.41
+    The ``reduce`` filter was added in Twig 1.41.
+
+The ``reduce`` filter iteratively reduces a sequence or a mapping to a single
+value using an arrow function, so as to reduce it to a single value. The arrow
+function receives the return value of the previous iteration and the current
+value of the sequence or mapping:
+
+.. code-block:: twig
+
+    {% set numbers = [1, 2, 3] %}
+
+    {{ numbers|reduce((carry, v) => carry + v) }}
+    {# output 6 #}
+
+The ``reduce`` filter takes an ``initial`` value as a second argument:
+
+.. code-block:: twig
+
+    {{ numbers|reduce((carry, v) => carry + v, 10) }}
+    {# output 16 #}
+
+Note that the arrow function has access to the current context.
+
+Arguments
+---------
+
+* ``array``: The sequence or mapping
+* ``arrow``: The arrow function
+* ``initial``: The initial value

src/ExpressionParser.php

@@ -14,6 +14,7 @@ namespace Twig;
 
 use Twig\Error\SyntaxError;
 use Twig\Node\Expression\ArrayExpression;
+use Twig\Node\Expression\ArrowFunctionExpression;
 use Twig\Node\Expression\AssignNameExpression;
 use Twig\Node\Expression\Binary\ConcatBinary;
 use Twig\Node\Expression\BlockReferenceExpression;
@@ -68,8 +69,12 @@ class ExpressionParser
         }
     }
 
-    public function parseExpression($precedence = 0)
+    public function parseExpression($precedence = 0, $allowArrow = false)
     {
+        if ($allowArrow && $arrow = $this->parseArrow()) {
+            return $arrow;
+        }
+
         $expr = $this->getPrimary();
         $token = $this->parser->getCurrentToken();
         while ($this->isBinary($token) && $this->binaryOperators[$token->getValue()]['precedence'] >= $precedence) {
@@ -98,6 +103,64 @@ class ExpressionParser
         return $expr;
     }
 
+    /**
+     * @return ArrowFunctionExpression|null
+     */
+    private function parseArrow()
+    {
+        $stream = $this->parser->getStream();
+
+        // short array syntax (one argument, no parentheses)?
+        if ($stream->look(1)->test(Token::ARROW_TYPE)) {
+            $line = $stream->getCurrent()->getLine();
+            $token = $stream->expect(Token::NAME_TYPE);
+            $names = [new AssignNameExpression($token->getValue(), $token->getLine())];
+            $stream->expect(Token::ARROW_TYPE);
+
+            return new ArrowFunctionExpression($this->parseExpression(0), new Node($names), $line);
+        }
+
+        // first, determine if we are parsing an arrow function by finding => (long form)
+        $i = 0;
+        if (!$stream->look($i)->test(Token::PUNCTUATION_TYPE, '(')) {
+            return null;
+        }
+        ++$i;
+        while (true) {
+            // variable name
+            ++$i;
+            if (!$stream->look($i)->test(Token::PUNCTUATION_TYPE, ',')) {
+                break;
+            }
+            ++$i;
+        }
+        if (!$stream->look($i)->test(Token::PUNCTUATION_TYPE, ')')) {
+            return null;
+        }
+        ++$i;
+        if (!$stream->look($i)->test(Token::ARROW_TYPE)) {
+            return null;
+        }
+
+        // yes, let's parse it properly
+        $token = $stream->expect(Token::PUNCTUATION_TYPE, '(');
+        $line = $token->getLine();
+
+        $names = [];
+        while (true) {
+            $token = $stream->expect(Token::NAME_TYPE);
+            $names[] = new AssignNameExpression($token->getValue(), $token->getLine());
+
+            if (!$stream->nextIf(Token::PUNCTUATION_TYPE, ',')) {
+                break;
+            }
+        }
+        $stream->expect(Token::PUNCTUATION_TYPE, ')');
+        $stream->expect(Token::ARROW_TYPE);
+
+        return new ArrowFunctionExpression($this->parseExpression(0), new Node($names), $line);
+    }
+
     protected function getPrimary()
     {
         $token = $this->parser->getCurrentToken();
@@ -499,7 +562,7 @@ class ExpressionParser
             if (!$this->parser->getStream()->test(Token::PUNCTUATION_TYPE, '(')) {
                 $arguments = new Node();
             } else {
-                $arguments = $this->parseArguments(true);
+                $arguments = $this->parseArguments(true, false, true);
             }
 
             $class = $this->getFilterNodeClass($name->getAttribute('value'), $token->getLine());
@@ -526,7 +589,7 @@ class ExpressionParser
      *
      * @throws SyntaxError
      */
-    public function parseArguments($namedArguments = false, $definition = false)
+    public function parseArguments($namedArguments = false, $definition = false, $allowArrow = false)
     {
         $args = [];
         $stream = $this->parser->getStream();
@@ -541,7 +604,7 @@ class ExpressionParser
                 $token = $stream->expect(Token::NAME_TYPE, null, 'An argument must be a name');
                 $value = new NameExpression($token->getValue(), $this->parser->getCurrentToken()->getLine());
             } else {
-                $value = $this->parseExpression();
+                $value = $this->parseExpression(0, $allowArrow);
             }
 
             $name = null;
@@ -558,7 +621,7 @@ class ExpressionParser
                         throw new SyntaxError(sprintf('A default value for an argument must be a constant (a boolean, a string, a number, or an array).'), $token->getLine(), $stream->getSourceContext());
                     }
                 } else {
-                    $value = $this->parseExpression();
+                    $value = $this->parseExpression(0, $allowArrow);
                 }
             }
 

src/Extension/CoreExtension.php

@@ -194,6 +194,9 @@ class CoreExtension extends AbstractExtension
             new TwigFilter('sort', 'twig_sort_filter'),
             new TwigFilter('merge', 'twig_array_merge'),
             new TwigFilter('batch', 'twig_array_batch'),
+            new TwigFilter('filter', 'twig_array_filter'),
+            new TwigFilter('map', 'twig_array_map'),
+            new TwigFilter('reduce', 'twig_array_reduce'),
 
             // string/array filters
             new TwigFilter('reverse', 'twig_reverse_filter', ['needs_environment' => true]),
@@ -1682,4 +1685,36 @@ function twig_array_batch($items, $size, $fill = null, $preserveKeys = true)
 
     return $result;
 }
+
+function twig_array_filter($array, $arrow)
+{
+    if (\is_array($array)) {
+        if (\PHP_VERSION_ID >= 50600) {
+            return array_filter($array, $arrow, \ARRAY_FILTER_USE_BOTH);
+        }
+
+        return array_filter($array, $arrow);
+    }
+
+    return new \CallbackFilterIterator($array, $arrow);
+}
+
+function twig_array_map($array, $arrow)
+{
+    $r = [];
+    foreach ($array as $k => $v) {
+        $r[$k] = $arrow($v, $k);
+    }
+
+    return $r;
+}
+
+function twig_array_reduce($array, $arrow, $initial = null)
+{
+    if (!\is_array($array)) {
+        $array = iterator_to_array($array);
+    }
+
+    return array_reduce($array, $arrow, $initial);
+}
 }

src/Lexer.php

@@ -333,8 +333,13 @@ class Lexer implements \Twig_LexerInterface
             }
         }
 
+        // arrow function
+        if ('=' === $this->code[$this->cursor] && '>' === $this->code[$this->cursor + 1]) {
+            $this->pushToken(Token::ARROW_TYPE, '=>');
+            $this->moveCursor('=>');
+        }
         // operators
-        if (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
+        elseif (preg_match($this->regexes['operator'], $this->code, $match, 0, $this->cursor)) {
             $this->pushToken(Token::OPERATOR_TYPE, preg_replace('/\s+/', ' ', $match[0]));
             $this->moveCursor($match[0]);
         }

src/Node/Expression/ArrowFunctionExpression.php

@@ -0,0 +1,64 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Twig\Node\Expression;
+
+use Twig\Compiler;
+use Twig\Node\Node;
+
+/**
+ * Represents an arrow function.
+ *
+ * @author Fabien Potencier <fabien@symfony.com>
+ */
+class ArrowFunctionExpression extends AbstractExpression
+{
+    public function __construct(AbstractExpression $expr, Node $names, $lineno, $tag = null)
+    {
+        parent::__construct(['expr' => $expr, 'names' => $names], [], $lineno, $tag);
+    }
+
+    public function compile(Compiler $compiler)
+    {
+        $compiler
+            ->addDebugInfo($this)
+            ->raw('function (')
+        ;
+        foreach ($this->getNode('names') as $i => $name) {
+            if ($i) {
+                $compiler->raw(', ');
+            }
+
+            $compiler
+                ->raw('$__')
+                ->raw($name->getAttribute('name'))
+                ->raw('__')
+            ;
+        }
+        $compiler
+            ->raw(') use ($context) { ')
+        ;
+        foreach ($this->getNode('names') as $name) {
+            $compiler
+                ->raw('$context["')
+                ->raw($name->getAttribute('name'))
+                ->raw('"] = $__')
+                ->raw($name->getAttribute('name'))
+                ->raw('__; ')
+            ;
+        }
+        $compiler
+            ->raw('return ')
+            ->subcompile($this->getNode('expr'))
+            ->raw('; }')
+        ;
+    }
+}

src/Token.php

@@ -38,6 +38,7 @@ class Token
     const PUNCTUATION_TYPE = 9;
     const INTERPOLATION_START_TYPE = 10;
     const INTERPOLATION_END_TYPE = 11;
+    const ARROW_TYPE = 12;
 
     /**
      * @param int    $type   The type of the token
@@ -157,6 +158,9 @@ class Token
             case self::INTERPOLATION_END_TYPE:
                 $name = 'INTERPOLATION_END_TYPE';
                 break;
+            case self::ARROW_TYPE:
+                $name = 'ARROW_TYPE';
+                break;
             default:
                 throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type));
         }
@@ -200,6 +204,8 @@ class Token
                 return 'begin of string interpolation';
             case self::INTERPOLATION_END_TYPE:
                 return 'end of string interpolation';
+            case self::ARROW_TYPE:
+                return 'arrow function';
             default:
                 throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type));
         }

test/Twig/Tests/Fixtures/expressions/not_arrow_fn.test

@@ -0,0 +1,8 @@
+--TEST--
+A string in parentheses cannot be confused with an arrow function
+--TEMPLATE--
+{{ ["foo", "bar"]|join(("f")) }}
+--DATA--
+return []
+--EXPECT--
+foofbar

test/Twig/Tests/Fixtures/filters/filter.test

@@ -0,0 +1,36 @@
+--TEST--
+"filter" filter
+--TEMPLATE--
+{% set offset = 3 %}
+
+{% for k, v in [1, 5, 3, 4, 5]|filter((v) => v > offset) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in {a: 1, b: 2, c: 5, d: 8}|filter(v => v > offset) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in [1, 5, 3, 4, 5]|filter(v => v > offset) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in it|filter((v) => v > offset) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+--DATA--
+return ['it' => new \ArrayIterator(['a' => 1, 'b' => 2, 'c' => 5, 'd' => 8])]
+--EXPECT--
+1 = 5
+3 = 4
+4 = 5
+
+c = 5
+d = 8
+
+1 = 5
+3 = 4
+4 = 5
+
+c = 5
+d = 8

test/Twig/Tests/Fixtures/filters/filter_php_56.test

@@ -0,0 +1,20 @@
+--TEST--
+"filter" filter (PHP 5.6 required)
+--CONDITION--
+version_compare(phpversion(), '5.6.0', '>=')
+--TEMPLATE--
+{% set offset = 3 %}
+
+{% for k, v in {a: 1, b: 2, c: 5, d: 8}|filter((v, k) => (v > offset) and (k != "d")) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in it|filter((v, k) => (v > offset) and (k != "d")) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+--DATA--
+return ['it' => new \ArrayIterator(['a' => 1, 'b' => 2, 'c' => 5, 'd' => 8])]
+--EXPECT--
+c = 5
+
+c = 5

test/Twig/Tests/Fixtures/filters/map.test

@@ -0,0 +1,41 @@
+--TEST--
+"map" filter
+--TEMPLATE--
+{% set offset = 3 %}
+
+{% for k, v in [1, 2]|map((item) => item + 2 ) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in {a: 1, b: 2}|map((item) => item ~ "*" ) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in {a: 1, b: 2}|map((item, k) => item ~ "*" ~ k ) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in [1, 2]|map(item => item + 2 ) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+
+{% for k, v in it|map(item => item + 2 ) -%}
+    {{ k }} = {{ v }}
+{% endfor %}
+--DATA--
+return ['it' => new \ArrayIterator([1, 2])]
+--EXPECT--
+0 = 3
+1 = 4
+
+a = 1*
+b = 2*
+
+a = 1*a
+b = 2*b
+
+0 = 3
+1 = 4
+
+0 = 3
+1 = 4

test/Twig/Tests/Fixtures/filters/reduce.test

@@ -0,0 +1,14 @@
+--TEST--
+"reduce" filter
+--TEMPLATE--
+{% set offset = 3 %}
+
+{{ [1, -1, 4]|reduce((carry, item) => carry + item + offset, 10) }}
+
+{{ it|reduce((carry, item) => carry + item + offset, 10) }}
+--DATA--
+return ['it' => new \ArrayIterator([1, -1, 4])]
+--EXPECT--
+23
+
+23