#30 - Improved control for UTF8 chars

src/Egulias/EmailValidator/EmailLexer.php

@@ -143,7 +143,6 @@ class EmailLexer extends AbstractLexer
             '\r\n',
             '::',
             '\s+?',
-            '[\x10-\x1F]+',
             '.',
             );
     }
@@ -155,7 +154,7 @@ class EmailLexer extends AbstractLexer
      */
     protected function getNonCatchablePatterns()
     {
-        return array('[\x7f-\xff]+');
+        return array('[\xA0-\xff]+');
     }
 
     /**
@@ -167,16 +166,20 @@ class EmailLexer extends AbstractLexer
      */
     protected function getType(&$value)
     {
-
         if ($this->isNullType($value)) {
             return self::C_NUL;
         }
 
-        if (isset($this->charValue[$value])) {
+        if ($this->isValid($value)) {
             return $this->charValue[$value];
         }
 
-        if ($this->isInvalid($value)) {
+        if ($this->isUTF8Invalid($value)) {
+            $this->hasInvalidTokens = true;
+            return self::INVALID;
+        }
+
+        if ($this->isASCIIInvalid($value)) {
             $this->hasInvalidTokens = true;
             return self::INVALID;
         }
@@ -184,8 +187,18 @@ class EmailLexer extends AbstractLexer
         return  self::GENERIC;
     }
 
+    protected function isValid($value)
+    {
+        if (isset($this->charValue[$value])) {
+            return true;
+        }
+
+        return false;
+    }
+
     /**
-     * @param string $value
+     * @param $value
+     * @return bool
      */
     protected function isNullType($value)
     {
@@ -197,15 +210,25 @@ class EmailLexer extends AbstractLexer
     }
 
     /**
-     * @param string $value
+     * @param $value
+     * @return bool
      */
-    protected function isInvalid($value)
+    protected function isASCIIInvalid($value)
     {
-        if (preg_match('/[\x10-\x1F\x{0001}-\x{000F}\x{0080}-\x{009F}]+/', $value)) {
+        if (isset($this->invalidASCII[ord($value)])) {
             return true;
         }
 
-        if (isset($this->invalidASCII[ord($value)])) {
+        return false;
+    }
+
+    /**
+     * @param $value
+     * @return bool
+     */
+    protected function isUTF8Invalid($value)
+    {
+        if (preg_match('/\p{Cc}+/u', $value)) {
             return true;
         }
 

src/Egulias/EmailValidator/EmailParser.php

@@ -29,7 +29,8 @@ class EmailParser
     }
 
     /**
-     * @param string $str
+     * @param $str
+     * @return array
      */
     public function parse($str)
     {
@@ -39,15 +40,16 @@ class EmailParser
             throw new \InvalidArgumentException('ERR_NOLOCALPART');
         }
 
-        if ($this->lexer->hasInvalidTokens()) {
-            throw new \InvalidArgumentException('ERR_INVALID_ATEXT');
-        }
 
         $this->localPartParser->parse($str);
         $this->domainPartParser->parse($str);
 
         $this->setParts($str);
 
+        if ($this->lexer->hasInvalidTokens()) {
+            throw new \InvalidArgumentException('ERR_INVALID_ATEXT');
+        }
+
         return array('local' => $this->localPart, 'domain' => $this->domainPart);
     }
 

tests/egulias/Tests/EmailValidator/EmailLexerTest.php

@@ -54,8 +54,8 @@ class EmailLexerTests extends \PHPUnit_Framework_TestCase
     {
         $chars = array();
         for ($i = 0; $i < 0x100; ++$i) {
-            $c = $this->utf8_chr($i);
-            if (preg_match('/[\x{0001}-\x{000F}\x{0080}-\x{009F}]/u', $c)) {
+            $c = $this->utf8Chr($i);
+            if (preg_match('/(?=\p{Cc})(?=[^\t\n\n\r])/u', $c) && !preg_match('/\x{0000}/u', $c)) {
                 $chars[] = array($c);
             }
         }
@@ -63,7 +63,8 @@ class EmailLexerTests extends \PHPUnit_Framework_TestCase
         return $chars;
     }
 
-    protected function utf8_chr($code_point) {
+    protected function utf8Chr($code_point)
+    {
 
         if ($code_point < 0 || 0x10FFFF < $code_point || (0xD800 <= $code_point && $code_point <= 0xDFFF)) {
             return '';
@@ -72,16 +73,16 @@ class EmailLexerTests extends \PHPUnit_Framework_TestCase
         if ($code_point < 0x80) {
             $hex[0] = $code_point;
             $ret = chr($hex[0]);
-        } else if ($code_point < 0x800) {
+        } elseif ($code_point < 0x800) {
             $hex[0] = 0x1C0 | $code_point >> 6;
             $hex[1] = 0x80  | $code_point & 0x3F;
             $ret = chr($hex[0]).chr($hex[1]);
-        } else if ($code_point < 0x10000) {
+        } elseif ($code_point < 0x10000) {
             $hex[0] = 0xE0 | $code_point >> 12;
             $hex[1] = 0x80 | $code_point >> 6 & 0x3F;
             $hex[2] = 0x80 | $code_point & 0x3F;
             $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]);
-        } else  {
+        } else {
             $hex[0] = 0xF0 | $code_point >> 18;
             $hex[1] = 0x80 | $code_point >> 12 & 0x3F;
             $hex[2] = 0x80 | $code_point >> 6 & 0x3F;

tests/egulias/Tests/EmailValidator/EmailValidatorTest.php

@@ -26,7 +26,7 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
         $this->assertTrue($this->validator->isValid($email));
     }
 
-    public function testInvalidUTF8Emails()
+    public function testInvalidUTF8Email()
     {
         $validator = new EmailValidator;
         $email     = "\x80\x81\x82@\x83\x84\x85.\x86\x87\x88";