Add needs_charset option for filters and functions

CHANGELOG

@@ -1,5 +1,6 @@
 # 3.10.0 (2024-XX-XX)
 
+ * Add `needs_charset` option for filters and functions
  * Extract the escaping logic from the `EscapingExtension` class to a new
    `EscapingRuntime` class.
 

doc/advanced.rst

@@ -175,6 +175,17 @@ The ``\Twig\TwigFilter`` class takes an array of options as its last argument::
 
     $filter = new \Twig\TwigFilter('rot13', 'str_rot13', $options);
 
+Charset-aware Filters
+~~~~~~~~~~~~~~~~~~~~~
+
+If you want to access the default charset in your filter, set the
+``needs_charset`` option to ``true``; Twig will pass the default charset as the
+first argument to the filter call::
+
+    $filter = new \Twig\TwigFilter('rot13', function (string $charset, $string) {
+        return str_rot13($string);
+    }, ['needs_charset' => true]);
+
 Environment-aware Filters
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 

src/Extension/CoreExtension.php

@@ -199,10 +199,10 @@ final class CoreExtension extends AbstractExtension
             new TwigFilter('convert_encoding', [self::class, 'convertEncoding']),
 
             // string filters
-            new TwigFilter('title', [self::class, 'titleCase'], ['needs_environment' => true]),
-            new TwigFilter('capitalize', [self::class, 'capitalize'], ['needs_environment' => true]),
-            new TwigFilter('upper', [self::class, 'upper'], ['needs_environment' => true]),
-            new TwigFilter('lower', [self::class, 'lower'], ['needs_environment' => true]),
+            new TwigFilter('title', [self::class, 'titleCase'], ['needs_charset' => true]),
+            new TwigFilter('capitalize', [self::class, 'capitalize'], ['needs_charset' => true]),
+            new TwigFilter('upper', [self::class, 'upper'], ['needs_charset' => true]),
+            new TwigFilter('lower', [self::class, 'lower'], ['needs_charset' => true]),
             new TwigFilter('striptags', [self::class, 'striptags']),
             new TwigFilter('trim', [self::class, 'trim']),
             new TwigFilter('nl2br', [self::class, 'nl2br'], ['pre_escape' => 'html', 'is_safe' => ['html']]),
@@ -210,7 +210,7 @@ final class CoreExtension extends AbstractExtension
 
             // array helpers
             new TwigFilter('join', [self::class, 'join']),
-            new TwigFilter('split', [self::class, 'split'], ['needs_environment' => true]),
+            new TwigFilter('split', [self::class, 'split'], ['needs_charset' => true]),
             new TwigFilter('sort', [self::class, 'sort'], ['needs_environment' => true]),
             new TwigFilter('merge', [self::class, 'merge']),
             new TwigFilter('batch', [self::class, 'batch']),
@@ -220,11 +220,11 @@ final class CoreExtension extends AbstractExtension
             new TwigFilter('reduce', [self::class, 'reduce'], ['needs_environment' => true]),
 
             // string/array filters
-            new TwigFilter('reverse', [self::class, 'reverse'], ['needs_environment' => true]),
-            new TwigFilter('length', [self::class, 'length'], ['needs_environment' => true]),
-            new TwigFilter('slice', [self::class, 'slice'], ['needs_environment' => true]),
-            new TwigFilter('first', [self::class, 'first'], ['needs_environment' => true]),
-            new TwigFilter('last', [self::class, 'last'], ['needs_environment' => true]),
+            new TwigFilter('reverse', [self::class, 'reverse'], ['needs_charset' => true]),
+            new TwigFilter('length', [self::class, 'length'], ['needs_charset' => true]),
+            new TwigFilter('slice', [self::class, 'slice'], ['needs_charset' => true]),
+            new TwigFilter('first', [self::class, 'first'], ['needs_charset' => true]),
+            new TwigFilter('last', [self::class, 'last'], ['needs_charset' => true]),
 
             // iteration and runtime
             new TwigFilter('default', [self::class, 'default'], ['node_class' => DefaultFilter::class]),
@@ -240,7 +240,7 @@ final class CoreExtension extends AbstractExtension
             new TwigFunction('range', 'range'),
             new TwigFunction('constant', [self::class, 'constant']),
             new TwigFunction('cycle', [self::class, 'cycle']),
-            new TwigFunction('random', [self::class, 'random'], ['needs_environment' => true]),
+            new TwigFunction('random', [self::class, 'random'], ['needs_charset' => true]),
             new TwigFunction('date', [self::class, 'convertDate'], ['needs_environment' => true]),
             new TwigFunction('include', [self::class, 'include'], ['needs_environment' => true, 'needs_context' => true, 'is_safe' => ['all']]),
             new TwigFunction('source', [self::class, 'source'], ['needs_environment' => true, 'is_safe' => ['all']]),
@@ -350,7 +350,7 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function random(Environment $env, $values = null, $max = null)
+    public static function random(string $charset, $values = null, $max = null)
     {
         if (null === $values) {
             return null === $max ? mt_rand() : mt_rand(0, (int) $max);
@@ -377,8 +377,6 @@ final class CoreExtension extends AbstractExtension
                 return '';
             }
 
-            $charset = $env->getCharset();
-
             if ('UTF-8' !== $charset) {
                 $values = self::convertEncoding($values, 'UTF-8', $charset);
             }
@@ -654,7 +652,7 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function slice(Environment $env, $item, $start, $length = null, $preserveKeys = false)
+    public static function slice(string $charset, $item, $start, $length = null, $preserveKeys = false)
     {
         if ($item instanceof \Traversable) {
             while ($item instanceof \IteratorAggregate) {
@@ -676,7 +674,7 @@ final class CoreExtension extends AbstractExtension
             return \array_slice($item, $start, $length, $preserveKeys);
         }
 
-        return mb_substr((string) $item, $start, $length, $env->getCharset());
+        return mb_substr((string) $item, $start, $length, $charset);
     }
 
     /**
@@ -688,9 +686,9 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function first(Environment $env, $item)
+    public static function first(string $charset, $item)
     {
-        $elements = self::slice($env, $item, 0, 1, false);
+        $elements = self::slice($charset, $item, 0, 1, false);
 
         return \is_string($elements) ? $elements : current($elements);
     }
@@ -704,9 +702,9 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function last(Environment $env, $item)
+    public static function last(string $charset, $item)
     {
-        $elements = self::slice($env, $item, -1, 1, false);
+        $elements = self::slice($charset, $item, -1, 1, false);
 
         return \is_string($elements) ? $elements : current($elements);
     }
@@ -775,7 +773,7 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function split(Environment $env, $value, $delimiter, $limit = null): array
+    public static function split(string $charset, $value, $delimiter, $limit = null): array
     {
         $value = $value ?? '';
 
@@ -787,14 +785,14 @@ final class CoreExtension extends AbstractExtension
             return preg_split('/(?<!^)(?!$)/u', $value);
         }
 
-        $length = mb_strlen($value, $env->getCharset());
+        $length = mb_strlen($value, $charset);
         if ($length < $limit) {
             return [$value];
         }
 
         $r = [];
         for ($i = 0; $i < $length; $i += $limit) {
-            $r[] = mb_substr($value, $i, $limit, $env->getCharset());
+            $r[] = mb_substr($value, $i, $limit, $charset);
         }
 
         return $r;
@@ -868,7 +866,7 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function reverse(Environment $env, $item, $preserveKeys = false)
+    public static function reverse(string $charset, $item, $preserveKeys = false)
     {
         if ($item instanceof \Traversable) {
             return array_reverse(iterator_to_array($item), $preserveKeys);
@@ -880,8 +878,6 @@ final class CoreExtension extends AbstractExtension
 
         $string = (string) $item;
 
-        $charset = $env->getCharset();
-
         if ('UTF-8' !== $charset) {
             $string = self::convertEncoding($string, 'UTF-8', $charset);
         }
@@ -1125,14 +1121,14 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function length(Environment $env, $thing): int
+    public static function length(string $charset, $thing): int
     {
         if (null === $thing) {
             return 0;
         }
 
         if (\is_scalar($thing)) {
-            return mb_strlen($thing, $env->getCharset());
+            return mb_strlen($thing, $charset);
         }
 
         if ($thing instanceof \Countable || \is_array($thing) || $thing instanceof \SimpleXMLElement) {
@@ -1144,7 +1140,7 @@ final class CoreExtension extends AbstractExtension
         }
 
         if (method_exists($thing, '__toString')) {
-            return mb_strlen((string) $thing, $env->getCharset());
+            return mb_strlen((string) $thing, $charset);
         }
 
         return 1;
@@ -1157,9 +1153,9 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function upper(Environment $env, $string): string
+    public static function upper(string $charset, $string): string
     {
-        return mb_strtoupper($string ?? '', $env->getCharset());
+        return mb_strtoupper($string ?? '', $charset);
     }
 
     /**
@@ -1169,9 +1165,9 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function lower(Environment $env, $string): string
+    public static function lower(string $charset, $string): string
     {
-        return mb_strtolower($string ?? '', $env->getCharset());
+        return mb_strtolower($string ?? '', $charset);
     }
 
     /**
@@ -1194,9 +1190,9 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function titleCase(Environment $env, $string): string
+    public static function titleCase(string $charset, $string): string
     {
-        return mb_convert_case($string ?? '', \MB_CASE_TITLE, $env->getCharset());
+        return mb_convert_case($string ?? '', \MB_CASE_TITLE, $charset);
     }
 
     /**
@@ -1206,10 +1202,8 @@ final class CoreExtension extends AbstractExtension
      *
      * @internal
      */
-    public static function capitalize(Environment $env, $string): string
+    public static function capitalize(string $charset, $string): string
     {
-        $charset = $env->getCharset();
-
         return mb_strtoupper(mb_substr($string ?? '', 0, 1, $charset), $charset).mb_strtolower(mb_substr($string ?? '', 1, null, $charset), $charset);
     }
 

src/Node/Expression/CallExpression.php

@@ -61,7 +61,15 @@ abstract class CallExpression extends AbstractExpression
 
         $first = true;
 
+        if ($this->hasAttribute('needs_charset') && $this->getAttribute('needs_charset')) {
+            $compiler->raw('$this->env->getCharset()');
+            $first = false;
+        }
+
         if ($this->hasAttribute('needs_environment') && $this->getAttribute('needs_environment')) {
+            if (!$first) {
+                $compiler->raw(', ');
+            }
             $compiler->raw('$this->env');
             $first = false;
         }
@@ -245,6 +253,9 @@ abstract class CallExpression extends AbstractExpression
         if ($this->hasNode('node')) {
             array_shift($parameters);
         }
+        if ($this->hasAttribute('needs_charset') && $this->getAttribute('needs_charset')) {
+            array_shift($parameters);
+        }
         if ($this->hasAttribute('needs_environment') && $this->getAttribute('needs_environment')) {
             array_shift($parameters);
         }

src/Node/Expression/FilterExpression.php

@@ -29,6 +29,7 @@ class FilterExpression extends CallExpression
 
         $this->setAttribute('name', $name);
         $this->setAttribute('type', 'filter');
+        $this->setAttribute('needs_charset', $filter->needsCharset());
         $this->setAttribute('needs_environment', $filter->needsEnvironment());
         $this->setAttribute('needs_context', $filter->needsContext());
         $this->setAttribute('arguments', $filter->getArguments());

src/Node/Expression/FunctionExpression.php

@@ -29,6 +29,7 @@ class FunctionExpression extends CallExpression
 
         $this->setAttribute('name', $name);
         $this->setAttribute('type', 'function');
+        $this->setAttribute('needs_charset', $function->needsCharset());
         $this->setAttribute('needs_environment', $function->needsEnvironment());
         $this->setAttribute('needs_context', $function->needsContext());
         $this->setAttribute('arguments', $function->getArguments());

src/Resources/core.php

@@ -31,7 +31,7 @@ function twig_random(Environment $env, $values = null, $max = null)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::random($env, $values, $max);
+    return CoreExtension::random($env->getCharset(), $values, $max);
 }
 
 /**
@@ -141,7 +141,7 @@ function twig_slice(Environment $env, $item, $start, $length = null, $preserveKe
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::slice($env, $item, $start, $length, $preserveKeys);
+    return CoreExtension::slice($env->getCharset(), $item, $start, $length, $preserveKeys);
 }
 
 /**
@@ -152,7 +152,7 @@ function twig_first(Environment $env, $item)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::first($env, $item);
+    return CoreExtension::first($env->getCharset(), $item);
 }
 
 /**
@@ -163,7 +163,7 @@ function twig_last(Environment $env, $item)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::last($env, $item);
+    return CoreExtension::last($env->getCharset(), $item);
 }
 
 /**
@@ -185,7 +185,7 @@ function twig_split_filter(Environment $env, $value, $delimiter, $limit = null)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::split($env, $value, $delimiter, $limit);
+    return CoreExtension::split($env->getCharset(), $value, $delimiter, $limit);
 }
 
 /**
@@ -207,7 +207,7 @@ function twig_reverse_filter(Environment $env, $item, $preserveKeys = false)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::reverse($env, $item, $preserveKeys);
+    return CoreExtension::reverse($env->getCharset(), $item, $preserveKeys);
 }
 
 /**
@@ -284,7 +284,7 @@ function twig_length_filter(Environment $env, $thing)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::length($env, $thing);
+    return CoreExtension::length($env->getCharset(), $thing);
 }
 
 /**
@@ -295,7 +295,7 @@ function twig_upper_filter(Environment $env, $string)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::upper($env, $string);
+    return CoreExtension::upper($env->getCharset(), $string);
 }
 
 /**
@@ -306,7 +306,7 @@ function twig_lower_filter(Environment $env, $string)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::lower($env, $string);
+    return CoreExtension::lower($env->getCharset(), $string);
 }
 
 /**
@@ -328,7 +328,7 @@ function twig_title_string_filter(Environment $env, $string)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::titleCase($env, $string);
+    return CoreExtension::titleCase($env->getCharset(), $string);
 }
 
 /**
@@ -339,7 +339,7 @@ function twig_capitalize_string_filter(Environment $env, $string)
 {
     trigger_deprecation('twig/twig', '3.9', 'Using the internal "%s" function is deprecated.', __FUNCTION__);
 
-    return CoreExtension::capitalize($env, $string);
+    return CoreExtension::capitalize($env->getCharset(), $string);
 }
 
 /**

src/TwigFilter.php

@@ -38,6 +38,7 @@ final class TwigFilter
         $this->options = array_merge([
             'needs_environment' => false,
             'needs_context' => false,
+            'needs_charset' => false,
             'is_variadic' => false,
             'is_safe' => null,
             'is_safe_callback' => null,
@@ -79,6 +80,11 @@ final class TwigFilter
         return $this->arguments;
     }
 
+    public function needsCharset(): bool
+    {
+        return $this->options['needs_charset'];
+    }
+
     public function needsEnvironment(): bool
     {
         return $this->options['needs_environment'];

src/TwigFunction.php

@@ -38,6 +38,7 @@ final class TwigFunction
         $this->options = array_merge([
             'needs_environment' => false,
             'needs_context' => false,
+            'needs_charset' => false,
             'is_variadic' => false,
             'is_safe' => null,
             'is_safe_callback' => null,
@@ -77,6 +78,11 @@ final class TwigFunction
         return $this->arguments;
     }
 
+    public function needsCharset(): bool
+    {
+        return $this->options['needs_charset'];
+    }
+
     public function needsEnvironment(): bool
     {
         return $this->options['needs_environment'];

tests/Extension/CoreTest.php

@@ -12,10 +12,8 @@ namespace Twig\Tests\Extension;
  */
 
 use PHPUnit\Framework\TestCase;
-use Twig\Environment;
 use Twig\Error\RuntimeError;
 use Twig\Extension\CoreExtension;
-use Twig\Loader\LoaderInterface;
 
 class CoreTest extends TestCase
 {
@@ -24,10 +22,8 @@ class CoreTest extends TestCase
      */
     public function testRandomFunction(array $expectedInArray, $value1, $value2 = null)
     {
-        $env = new Environment($this->createMock(LoaderInterface::class));
-
         for ($i = 0; $i < 100; ++$i) {
-            $this->assertTrue(\in_array(CoreExtension::random($env, $value1, $value2), $expectedInArray, true)); // assertContains() would not consider the type
+            $this->assertTrue(\in_array(CoreExtension::random('UTF-8', $value1, $value2), $expectedInArray, true)); // assertContains() would not consider the type
         }
     }
 
@@ -85,45 +81,38 @@ class CoreTest extends TestCase
         $max = mt_getrandmax();
 
         for ($i = 0; $i < 100; ++$i) {
-            $val = CoreExtension::random(new Environment($this->createMock(LoaderInterface::class)));
+            $val = CoreExtension::random('UTF-8');
             $this->assertTrue(\is_int($val) && $val >= 0 && $val <= $max);
         }
     }
 
     public function testRandomFunctionReturnsAsIs()
     {
-        $this->assertSame('', CoreExtension::random(new Environment($this->createMock(LoaderInterface::class)), ''));
-        $this->assertSame('', CoreExtension::random(new Environment($this->createMock(LoaderInterface::class), ['charset' => null]), ''));
+        $this->assertSame('', CoreExtension::random('UTF-8', ''));
 
         $instance = new \stdClass();
-        $this->assertSame($instance, CoreExtension::random(new Environment($this->createMock(LoaderInterface::class)), $instance));
+        $this->assertSame($instance, CoreExtension::random('UTF-8', $instance));
     }
 
     public function testRandomFunctionOfEmptyArrayThrowsException()
     {
         $this->expectException(RuntimeError::class);
-        CoreExtension::random(new Environment($this->createMock(LoaderInterface::class)), []);
+        CoreExtension::random('UTF-8', []);
     }
 
     public function testRandomFunctionOnNonUTF8String()
     {
-        $twig = new Environment($this->createMock(LoaderInterface::class));
-        $twig->setCharset('ISO-8859-1');
-
         $text = iconv('UTF-8', 'ISO-8859-1', 'Äé');
         for ($i = 0; $i < 30; ++$i) {
-            $rand = CoreExtension::random($twig, $text);
+            $rand = CoreExtension::random('ISO-8859-1', $text);
             $this->assertTrue(\in_array(iconv('ISO-8859-1', 'UTF-8', $rand), ['Ä', 'é'], true));
         }
     }
 
     public function testReverseFilterOnNonUTF8String()
     {
-        $twig = new Environment($this->createMock(LoaderInterface::class));
-        $twig->setCharset('ISO-8859-1');
-
         $input = iconv('UTF-8', 'ISO-8859-1', 'Äé');
-        $output = iconv('ISO-8859-1', 'UTF-8', CoreExtension::reverse($twig, $input));
+        $output = iconv('ISO-8859-1', 'UTF-8', CoreExtension::reverse('ISO-8859-1', $input));
 
         $this->assertEquals($output, 'éÄ');
     }
@@ -133,8 +122,7 @@ class CoreTest extends TestCase
      */
     public function testTwigFirst($expected, $input)
     {
-        $twig = new Environment($this->createMock(LoaderInterface::class));
-        $this->assertSame($expected, CoreExtension::first($twig, $input));
+        $this->assertSame($expected, CoreExtension::first('UTF-8', $input));
     }
 
     public function provideTwigFirstCases()
@@ -155,8 +143,7 @@ class CoreTest extends TestCase
      */
     public function testTwigLast($expected, $input)
     {
-        $twig = new Environment($this->createMock(LoaderInterface::class));
-        $this->assertSame($expected, CoreExtension::last($twig, $input));
+        $this->assertSame($expected, CoreExtension::last('UTF-8', $input));
     }
 
     public function provideTwigLastCases()
@@ -228,8 +215,7 @@ class CoreTest extends TestCase
      */
     public function testSliceFilter($expected, $input, $start, $length = null, $preserveKeys = false)
     {
-        $twig = new Environment($this->createMock(LoaderInterface::class));
-        $this->assertSame($expected, CoreExtension::slice($twig, $input, $start, $length, $preserveKeys));
+        $this->assertSame($expected, CoreExtension::slice('UTF-8', $input, $start, $length, $preserveKeys));
     }
 
     public function provideSliceFilterCases()

tests/Node/Expression/FilterTest.php

@@ -69,7 +69,7 @@ class FilterTest extends NodeTestCase
         $node = $this->createFilter($expr, 'upper');
         $node = $this->createFilter($node, 'number_format', [new ConstantExpression(2, 1), new ConstantExpression('.', 1), new ConstantExpression(',', 1)]);
 
-        $tests[] = [$node, 'Twig\Extension\CoreExtension::formatNumber($this->env, Twig\Extension\CoreExtension::upper($this->env, "foo"), 2, ".", ",")'];
+        $tests[] = [$node, 'Twig\Extension\CoreExtension::formatNumber($this->env, Twig\Extension\CoreExtension::upper($this->env->getCharset(), "foo"), 2, ".", ",")'];
 
         // named arguments
         $date = new ConstantExpression(0, 1);
@@ -91,11 +91,11 @@ class FilterTest extends NodeTestCase
         $node = $this->createFilter($string, 'reverse', [
             'preserve_keys' => new ConstantExpression(true, 1),
         ]);
-        $tests[] = [$node, 'Twig\Extension\CoreExtension::reverse($this->env, "abc", true)'];
+        $tests[] = [$node, 'Twig\Extension\CoreExtension::reverse($this->env->getCharset(), "abc", true)'];
         $node = $this->createFilter($string, 'reverse', [
             'preserveKeys' => new ConstantExpression(true, 1),
         ]);
-        $tests[] = [$node, 'Twig\Extension\CoreExtension::reverse($this->env, "abc", true)'];
+        $tests[] = [$node, 'Twig\Extension\CoreExtension::reverse($this->env->getCharset(), "abc", true)'];
 
         // filter as an anonymous function
         $node = $this->createFilter(new ConstantExpression('foo', 1), 'anonymous');