Merge remote branch 'markstory/whitespace'

* markstory/whitespace:
  Updating documentation for whitespace trim tags.
  Making trim tags consume all whitespace, both horizontal and vertical.
  Adding additional test suggested by nikic.
  Making a capturing group non capturing, as the group isn't used.
  Moving newline trimming into the regexp used to match end of tags.
  Applying changes suggested by nikic to simplify how end of comments are processed.
  Fixing some whitespace.
  Fixing typo.
  Adding a bit of documentation for whitespace trimming.
  Adding whitespace trimming support to other tag types (comments, variables) Refactoring how whitespace trimming is done, and moving it into a separate option that is combined with other tag types. Adding more tests. Refs #284
  Fixing use of non-existent method name.
  Adding a test case for mixing {%- and %} tags together.
  Adding a test that uses tabs. Added a test for trailing tabs and -%} Refs #284
  Implementing {%- which trims non-newline whitespace preceding it. Refs

doc/templates.rst

@@ -162,6 +162,36 @@ Use the ``spaceless`` tag to remove whitespace between HTML tags:
 
     {# output will be <div><strong>foo</strong></div> #}
 
+In addition to the spaceless tag you can also control whitespace on a per tag 
+level.  By using the whitespace control modifier on your tags you can trim
+leading and or trailing whitespace from any tag type:
+
+.. code-block:: jinja
+
+    {% set value = 'no spaces' %}
+    {#- No leading/trailing whitespace -#}
+    {%- if true -%}
+        {{- value -}}
+    {%- endif -%}
+
+    {# output 'spaces' #}
+
+The above sample shows the default whitespace control modifier, and how you can
+use it to remove whitespace around tags.  Trimming space will consume all whitespace
+for that side of the tag.  It is possible to use whitespace trimming on one side
+of a tag:
+
+.. code-block:: jinja
+
+    {% set value = 'no spaces' %}
+    <li>    {{- value }}    </li>
+
+    {# outputs '<li>value    </li>' #}
+
+..versionadded:: 1.1
+
+    Tag level whitespace control was added in 1.1
+
 Escaping
 --------
 

lib/Twig/Lexer.php

@@ -48,6 +48,7 @@ class Twig_Lexer implements Twig_LexerInterface
             'tag_comment'  => array('{#', '#}'),
             'tag_block'    => array('{%', '%}'),
             'tag_variable' => array('{{', '}}'),
+            'whitespace_trim' => '-'
         ), $options);
     }
 
@@ -110,17 +111,21 @@ class Twig_Lexer implements Twig_LexerInterface
     protected function lexData()
     {
         $pos = $this->end;
-        if (false !== ($tmpPos = strpos($this->code, $this->options['tag_comment'][0], $this->cursor))  && $tmpPos < $pos) {
+        $append = '';
+
+        // Find the first token after the cursor.
+        foreach (array('tag_comment', 'tag_variable', 'tag_block') as $type) {
+            $tmpPos = strpos($this->code, $this->options[$type][0], $this->cursor);
+            if (false !== $tmpPos && $tmpPos < $pos) {
+                $trimBlock = false;
+                $append = '';
                 $pos = $tmpPos;
-            $token = $this->options['tag_comment'][0];
+                $token = $this->options[$type][0];
+                if (strpos($this->code, $this->options['whitespace_trim'], $pos) === ($pos + strlen($token))) {
+                    $trimBlock = true;
+                    $append = $this->options['whitespace_trim'];
                 }
-        if (false !== ($tmpPos = strpos($this->code, $this->options['tag_variable'][0], $this->cursor)) && $tmpPos < $pos) {
-            $pos = $tmpPos;
-            $token = $this->options['tag_variable'][0];
             }
-        if (false !== ($tmpPos = strpos($this->code, $this->options['tag_block'][0], $this->cursor))    && $tmpPos < $pos) {
-            $pos = $tmpPos;
-            $token = $this->options['tag_block'][0];
         }
 
         // if no matches are left we return the rest of the template as simple text token
@@ -131,24 +136,24 @@ class Twig_Lexer implements Twig_LexerInterface
         }
         
         // push the template text first
-        $text = substr($this->code, $this->cursor, $pos - $this->cursor);
+        $text = $textContent = substr($this->code, $this->cursor, $pos - $this->cursor);
+        if (true === $trimBlock) {
+            $text = rtrim($text);
+        }
         $this->pushToken(Twig_Token::TEXT_TYPE, $text);
-        $this->moveCursor($text.$token);
+        $this->moveCursor($textContent . $token . $append);
 
         switch ($token) {
             case $this->options['tag_comment'][0]:
-                if (false === $pos = strpos($this->code, $this->options['tag_comment'][1], $this->cursor)) {
+                $commentEndRegex = '/.*?(?:' . preg_quote($this->options['whitespace_trim'], '/')
+                                   . preg_quote($this->options['tag_comment'][1], '/') . '\s*|'
+                                   . preg_quote($this->options['tag_comment'][1], '/') . ')\n?/As';
+
+                if (!preg_match($commentEndRegex, $this->code, $match, null, $this->cursor)) {
                     throw new Twig_Error_Syntax('unclosed comment', $this->lineno, $this->filename);
                 }
 
-                $this->moveCursor(substr($this->code, $this->cursor, $pos - $this->cursor) . $this->options['tag_comment'][1]);
+                $this->moveCursor($match[0]);
-
-                // mimics the behavior of PHP by removing the newline that follows instructions if present
-                if ("\n" === substr($this->code, $this->cursor, 1)) {
-                    ++$this->cursor;
-                    ++$this->lineno;
-                }
-
                 break;
 
             case $this->options['tag_block'][0]:
@@ -172,16 +177,13 @@ class Twig_Lexer implements Twig_LexerInterface
 
     protected function lexBlock()
     {
-        if (empty($this->brackets) && preg_match('/\s*'.preg_quote($this->options['tag_block'][1], '/').'/A', $this->code, $match, null, $this->cursor)) {
+        $trimTag = preg_quote($this->options['whitespace_trim'] . $this->options['tag_block'][1], '/');
+        $endTag = preg_quote($this->options['tag_block'][1], '/');
+
+        if (empty($this->brackets) && preg_match('/\s*(?:' . $trimTag . '\s*|\s*' . $endTag . ')\n?/A', $this->code, $match, null, $this->cursor)) {
             $this->pushToken(Twig_Token::BLOCK_END_TYPE);
             $this->moveCursor($match[0]);
             $this->state = self::STATE_DATA;
-
-            // mimics the behavior of PHP by removing the newline that follows instructions if present
-            if ("\n" === substr($this->code, $this->cursor, 1)) {
-                ++$this->cursor;
-                ++$this->lineno;
-            }
         }
         else {
             $this->lexExpression();
@@ -190,7 +192,10 @@ class Twig_Lexer implements Twig_LexerInterface
 
     protected function lexVar()
     {
-        if (empty($this->brackets) && preg_match('/\s*'.preg_quote($this->options['tag_variable'][1], '/').'/A', $this->code, $match, null, $this->cursor)) {
+        $trimTag = preg_quote($this->options['whitespace_trim'] . $this->options['tag_variable'][1], '/');
+        $endTag = preg_quote($this->options['tag_variable'][1], '/');
+        
+        if (empty($this->brackets) && preg_match('/\s*' . $trimTag . '\s*|\s*' . $endTag . '/A', $this->code, $match, null, $this->cursor)) {
             $this->pushToken(Twig_Token::VAR_END_TYPE);
             $this->moveCursor($match[0]);
             $this->state = self::STATE_DATA;

test/Twig/Tests/Fixtures/tags/trim_block.test

@@ -0,0 +1,74 @@
+--TEST--
+Whitespace trimming on tags.
+--TEMPLATE--
+{{ 5 * '{#-'|length }}
+{{ '{{-'|length * 5 + '{%-'|length }}
+
+Trim on control tag:
+{% for i in range(1, 9) -%}
+	{{ i }}
+{%- endfor %}
+
+
+Trim on output tag:
+{% for i in range(1, 9) %}
+	{{- i -}}
+{% endfor %}
+
+
+Trim comments:
+      
+{#- Invisible -#}
+       
+After the comment.
+
+Trim leading space:
+{% if leading %}
+
+		{{- leading }}
+{% endif %}
+
+{%- if leading %}
+	{{- leading }}
+
+{%- endif %}
+
+
+Trim trailing space:
+{% if trailing -%}          
+	{{ trailing -}}
+
+{% endif -%}
+
+Combined:
+
+{%- if both -%}
+<ul>
+	<li>    {{- both -}}   </li>
+</ul>
+
+{%- endif -%}
+
+end
+--DATA--
+return array('leading' => 'leading space', 'trailing' => 'trailing space', 'both' => 'both')
+--EXPECT--
+15
+18
+
+Trim on control tag:
+123456789
+
+Trim on output tag:
+123456789
+
+Trim comments:After the comment.
+
+Trim leading space:
+leading space
+leading space
+
+Trim trailing space:
+trailing spaceCombined:<ul>
+	<li>both</li>
+</ul>end