Merge pull request #38 from egulias/improved-utf8

#30 - Improved utf8

composer.json

@@ -15,7 +15,7 @@
   },
   "require":      {
     "php": ">= 5.3.3",
-    "doctrine/lexer": "~1.0"
+    "doctrine/lexer": "dev-master"
   },
   "require-dev" :   {
     "satooshi/php-coveralls": "dev-master"

composer.lock

@@ -4,26 +4,31 @@
         "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "hash": "9e9dff0cc08c7292600453e681201e13",
+    "hash": "7c82c4ea93186526be106a60ea69f6f1",
     "packages": [
         {
             "name": "doctrine/lexer",
-            "version": "v1.0",
+            "version": "dev-master",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/lexer.git",
-                "reference": "2f708a85bb3aab5d99dab8be435abd73e0b18acb"
+                "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/lexer/zipball/2f708a85bb3aab5d99dab8be435abd73e0b18acb",
+                "url": "https://api.github.com/repos/doctrine/lexer/zipball/83893c552fd2045dd78aef794c31e694c37c0b8c",
-                "reference": "2f708a85bb3aab5d99dab8be435abd73e0b18acb",
+                "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c",
                 "shasum": ""
             },
             "require": {
                 "php": ">=5.3.2"
             },
             "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.0.x-dev"
+                }
+            },
             "autoload": {
                 "psr-0": {
                     "Doctrine\\Common\\Lexer\\": "lib/"
@@ -34,20 +39,17 @@
                 "MIT"
             ],
             "authors": [
-                {
-                    "name": "Guilherme Blanco",
-                    "email": "guilhermeblanco@gmail.com",
-                    "homepage": "http://www.instaclick.com"
-                },
                 {
                     "name": "Roman Borschel",
                     "email": "roman@code-factory.org"
                 },
+                {
+                    "name": "Guilherme Blanco",
+                    "email": "guilhermeblanco@gmail.com"
+                },
                 {
                     "name": "Johannes Schmitt",
-                    "email": "schmittjoh@gmail.com",
+                    "email": "schmittjoh@gmail.com"
-                    "homepage": "http://jmsyst.com",
-                    "role": "Developer of wrapped JMSSerializerBundle"
                 }
             ],
             "description": "Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers.",
@@ -56,7 +58,7 @@
                 "lexer",
                 "parser"
             ],
-            "time": "2013-01-12 18:59:04"
+            "time": "2014-09-09 13:34:57"
         }
     ],
     "packages-dev": [
@@ -196,12 +198,12 @@
             "source": {
                 "type": "git",
                 "url": "https://github.com/satooshi/php-coveralls.git",
-                "reference": "94389a0ebdb64857d6899b5e0254dffa99e5aa96"
+                "reference": "2fbf803803d179ab1082807308a67bbd5a760c70"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/satooshi/php-coveralls/zipball/94389a0ebdb64857d6899b5e0254dffa99e5aa96",
+                "url": "https://api.github.com/repos/satooshi/php-coveralls/zipball/2fbf803803d179ab1082807308a67bbd5a760c70",
-                "reference": "94389a0ebdb64857d6899b5e0254dffa99e5aa96",
+                "reference": "2fbf803803d179ab1082807308a67bbd5a760c70",
                 "shasum": ""
             },
             "require": {
@@ -263,21 +265,21 @@
                 "github",
                 "test"
             ],
-            "time": "2014-07-09 10:45:38"
+            "time": "2014-11-11 15:35:34"
         },
         {
             "name": "symfony/config",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
             "target-dir": "Symfony/Component/Config",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/Config.git",
-                "reference": "080eabdc256c1d7a3a7cf6296271edb68eb1ab2b"
+                "reference": "0316364bfebc8b080077c731a99f189341476bd7"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Config/zipball/080eabdc256c1d7a3a7cf6296271edb68eb1ab2b",
+                "url": "https://api.github.com/repos/symfony/Config/zipball/0316364bfebc8b080077c731a99f189341476bd7",
-                "reference": "080eabdc256c1d7a3a7cf6296271edb68eb1ab2b",
+                "reference": "0316364bfebc8b080077c731a99f189341476bd7",
                 "shasum": ""
             },
             "require": {
@@ -311,21 +313,21 @@
             ],
             "description": "Symfony Config Component",
             "homepage": "http://symfony.com",
-            "time": "2014-08-31 03:22:04"
+            "time": "2014-09-23 05:25:11"
         },
         {
             "name": "symfony/console",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
             "target-dir": "Symfony/Component/Console",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/Console.git",
-                "reference": "748beed2a1e73179c3f5154d33fe6ae100c1aeb1"
+                "reference": "6f177fca24200a5b97aef5ce7a5c98124a0f0db0"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Console/zipball/748beed2a1e73179c3f5154d33fe6ae100c1aeb1",
+                "url": "https://api.github.com/repos/symfony/Console/zipball/6f177fca24200a5b97aef5ce7a5c98124a0f0db0",
-                "reference": "748beed2a1e73179c3f5154d33fe6ae100c1aeb1",
+                "reference": "6f177fca24200a5b97aef5ce7a5c98124a0f0db0",
                 "shasum": ""
             },
             "require": {
@@ -366,21 +368,21 @@
             ],
             "description": "Symfony Console Component",
             "homepage": "http://symfony.com",
-            "time": "2014-08-14 16:10:54"
+            "time": "2014-10-05 13:57:04"
         },
         {
             "name": "symfony/event-dispatcher",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
             "target-dir": "Symfony/Component/EventDispatcher",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/EventDispatcher.git",
-                "reference": "8faf5cc7e80fde74a650a36e60d32ce3c3e0457b"
+                "reference": "804eb28dbbfba9ffdab21fe2066744906cea2212"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/8faf5cc7e80fde74a650a36e60d32ce3c3e0457b",
+                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/804eb28dbbfba9ffdab21fe2066744906cea2212",
-                "reference": "8faf5cc7e80fde74a650a36e60d32ce3c3e0457b",
+                "reference": "804eb28dbbfba9ffdab21fe2066744906cea2212",
                 "shasum": ""
             },
             "require": {
@@ -389,7 +391,7 @@
             "require-dev": {
                 "psr/log": "~1.0",
                 "symfony/config": "~2.0",
-                "symfony/dependency-injection": "~2.0",
+                "symfony/dependency-injection": "~2.0,<2.6.0",
                 "symfony/stopwatch": "~2.2"
             },
             "suggest": {
@@ -423,21 +425,21 @@
             ],
             "description": "Symfony EventDispatcher Component",
             "homepage": "http://symfony.com",
-            "time": "2014-07-28 13:20:46"
+            "time": "2014-10-01 15:43:05"
         },
         {
             "name": "symfony/filesystem",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
             "target-dir": "Symfony/Component/Filesystem",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/Filesystem.git",
-                "reference": "a765efd199e02ff4001c115c318e219030be9364"
+                "reference": "4e62fab0060a826561c78b665925b37c870c45f5"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Filesystem/zipball/a765efd199e02ff4001c115c318e219030be9364",
+                "url": "https://api.github.com/repos/symfony/Filesystem/zipball/4e62fab0060a826561c78b665925b37c870c45f5",
-                "reference": "a765efd199e02ff4001c115c318e219030be9364",
+                "reference": "4e62fab0060a826561c78b665925b37c870c45f5",
                 "shasum": ""
             },
             "require": {
@@ -470,21 +472,21 @@
             ],
             "description": "Symfony Filesystem Component",
             "homepage": "http://symfony.com",
-            "time": "2014-09-03 09:00:14"
+            "time": "2014-09-22 09:14:18"
         },
         {
             "name": "symfony/stopwatch",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
             "target-dir": "Symfony/Component/Stopwatch",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/Stopwatch.git",
-                "reference": "22ab4f76cdeefd38b00022a6be5709190a2fd046"
+                "reference": "9f8a33a24f2378c0ec5f372a8d50b2d43069c050"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Stopwatch/zipball/22ab4f76cdeefd38b00022a6be5709190a2fd046",
+                "url": "https://api.github.com/repos/symfony/Stopwatch/zipball/9f8a33a24f2378c0ec5f372a8d50b2d43069c050",
-                "reference": "22ab4f76cdeefd38b00022a6be5709190a2fd046",
+                "reference": "9f8a33a24f2378c0ec5f372a8d50b2d43069c050",
                 "shasum": ""
             },
             "require": {
@@ -517,21 +519,21 @@
             ],
             "description": "Symfony Stopwatch Component",
             "homepage": "http://symfony.com",
-            "time": "2014-08-14 16:10:54"
+            "time": "2014-09-22 09:14:18"
         },
         {
             "name": "symfony/yaml",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
             "target-dir": "Symfony/Component/Yaml",
             "source": {
                 "type": "git",
                 "url": "https://github.com/symfony/Yaml.git",
-                "reference": "01a7695bcfb013d0a15c6757e15aae120342986f"
+                "reference": "2d9f527449cabfa8543dd7fa3a466d6ae83d6726"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Yaml/zipball/01a7695bcfb013d0a15c6757e15aae120342986f",
+                "url": "https://api.github.com/repos/symfony/Yaml/zipball/2d9f527449cabfa8543dd7fa3a466d6ae83d6726",
-                "reference": "01a7695bcfb013d0a15c6757e15aae120342986f",
+                "reference": "2d9f527449cabfa8543dd7fa3a466d6ae83d6726",
                 "shasum": ""
             },
             "require": {
@@ -564,21 +566,18 @@
             ],
             "description": "Symfony Yaml Component",
             "homepage": "http://symfony.com",
-            "time": "2014-08-31 03:22:04"
+            "time": "2014-10-01 05:50:18"
         }
     ],
-    "aliases": [
+    "aliases": [],
-
-    ],
     "minimum-stability": "stable",
     "stability-flags": {
+        "doctrine/lexer": 20,
         "satooshi/php-coveralls": 20
     },
     "prefer-stable": false,
     "platform": {
         "php": ">= 5.3.3"
     },
-    "platform-dev": [
+    "platform-dev": []
-
-    ]
 }

src/Egulias/EmailValidator/EmailLexer.php

@@ -142,8 +142,7 @@ class EmailLexer extends AbstractLexer
             '[0-9]+',
             '\r\n',
             '::',
-            '\s+',
+            '\s+?',
-            '[\x10-\x1F]+',
             '.',
             );
     }
@@ -155,7 +154,7 @@ class EmailLexer extends AbstractLexer
      */
     protected function getNonCatchablePatterns()
     {
-        return array('[\x7f-\xff]+');
+        return array('[\xA0-\xff]+');
     }
 
     /**
@@ -167,16 +166,20 @@ class EmailLexer extends AbstractLexer
      */
     protected function getType(&$value)
     {
-
         if ($this->isNullType($value)) {
             return self::C_NUL;
         }
 
-        if (isset($this->charValue[$value])) {
+        if ($this->isValid($value)) {
             return $this->charValue[$value];
         }
 
-        if ($this->isInvalid($value)) {
+        if ($this->isUTF8Invalid($value)) {
+            $this->hasInvalidTokens = true;
+            return self::INVALID;
+        }
+
+        if ($this->isASCIIInvalid($value)) {
             $this->hasInvalidTokens = true;
             return self::INVALID;
         }
@@ -184,8 +187,18 @@ class EmailLexer extends AbstractLexer
         return  self::GENERIC;
     }
 
+    protected function isValid($value)
+    {
+        if (isset($this->charValue[$value])) {
+            return true;
+        }
+
+        return false;
+    }
+
     /**
-     * @param string $value
+     * @param $value
+     * @return bool
      */
     protected function isNullType($value)
     {
@@ -197,18 +210,33 @@ class EmailLexer extends AbstractLexer
     }
 
     /**
-     * @param string $value
+     * @param $value
+     * @return bool
      */
-    protected function isInvalid($value)
+    protected function isASCIIInvalid($value)
     {
-        if (preg_match('/[\x10-\x1F]+/', $value)) {
-            return true;
-        }
-
         if (isset($this->invalidASCII[ord($value)])) {
             return true;
         }
 
         return false;
     }
+
+    /**
+     * @param $value
+     * @return bool
+     */
+    protected function isUTF8Invalid($value)
+    {
+        if (preg_match('/\p{Cc}+/u', $value)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    protected function getModifiers()
+    {
+        return 'iu';
+    }
 }

src/Egulias/EmailValidator/EmailParser.php

@@ -29,7 +29,8 @@ class EmailParser
     }
 
     /**
-     * @param string $str
+     * @param $str
+     * @return array
      */
     public function parse($str)
     {
@@ -39,15 +40,16 @@ class EmailParser
             throw new \InvalidArgumentException('ERR_NOLOCALPART');
         }
 
-        if ($this->lexer->hasInvalidTokens()) {
-            throw new \InvalidArgumentException('ERR_INVALID_ATEXT');
-        }
 
         $this->localPartParser->parse($str);
         $this->domainPartParser->parse($str);
 
         $this->setParts($str);
 
+        if ($this->lexer->hasInvalidTokens()) {
+            throw new \InvalidArgumentException('ERR_INVALID_ATEXT');
+        }
+
         return array('local' => $this->localPart, 'domain' => $this->domainPart);
     }
 

src/Egulias/EmailValidator/EmailValidator.php

@@ -84,7 +84,7 @@ class EmailValidator
             return false;
         }
 
-        $dns = false;
+        $dns = true;
         if ($checkDNS) {
             $dns = $this->checkDNS();
         }
@@ -95,12 +95,7 @@ class EmailValidator
             return false;
         }
 
-        return ($strict) ? $this->checkStrict($dns) : true;
+        return !$strict || (!$this->hasWarnings() && $dns);
-    }
-
-    private function checkStrict($dns)
-    {
-         return !($this->hasWarnings() && !$dns);
     }
 
     /**

src/Egulias/EmailValidator/Parser/LocalPart.php

@@ -6,7 +6,6 @@ use Egulias\EmailValidator\EmailLexer;
 use Egulias\EmailValidator\EmailValidator;
 use \InvalidArgumentException;
 
-
 class LocalPart extends Parser
 {
     public function parse($localPart)

tests/egulias/Tests/EmailValidator/EmailLexerTest.php

@@ -26,6 +26,73 @@ class EmailLexerTests extends \PHPUnit_Framework_TestCase
         $this->assertEquals($token, $lexer->token['type']);
     }
 
+    public function testLexerParsesMultipleSpaces()
+    {
+        $lexer = new EmailLexer();
+        $lexer->setInput('  ');
+        $lexer->moveNext();
+        $lexer->moveNext();
+        $this->assertEquals(EmailLexer::S_SP, $lexer->token['type']);
+        $lexer->moveNext();
+        $this->assertEquals(EmailLexer::S_SP, $lexer->token['type']);
+    }
+
+    /**
+     * @dataProvider invalidUTF8CharsProvider
+     */
+    public function testLexerParsesInvalidUTF8($char)
+    {
+        $lexer = new EmailLexer();
+        $lexer->setInput($char);
+        $lexer->moveNext();
+        $lexer->moveNext();
+
+        $this->assertEquals(EmailLexer::INVALID, $lexer->token['type']);
+    }
+
+    public function invalidUTF8CharsProvider()
+    {
+        $chars = array();
+        for ($i = 0; $i < 0x100; ++$i) {
+            $c = $this->utf8Chr($i);
+            if (preg_match('/(?=\p{Cc})(?=[^\t\n\n\r])/u', $c) && !preg_match('/\x{0000}/u', $c)) {
+                $chars[] = array($c);
+            }
+        }
+
+        return $chars;
+    }
+
+    protected function utf8Chr($code_point)
+    {
+
+        if ($code_point < 0 || 0x10FFFF < $code_point || (0xD800 <= $code_point && $code_point <= 0xDFFF)) {
+            return '';
+        }
+
+        if ($code_point < 0x80) {
+            $hex[0] = $code_point;
+            $ret = chr($hex[0]);
+        } elseif ($code_point < 0x800) {
+            $hex[0] = 0x1C0 | $code_point >> 6;
+            $hex[1] = 0x80  | $code_point & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]);
+        } elseif ($code_point < 0x10000) {
+            $hex[0] = 0xE0 | $code_point >> 12;
+            $hex[1] = 0x80 | $code_point >> 6 & 0x3F;
+            $hex[2] = 0x80 | $code_point & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]);
+        } else {
+            $hex[0] = 0xF0 | $code_point >> 18;
+            $hex[1] = 0x80 | $code_point >> 12 & 0x3F;
+            $hex[2] = 0x80 | $code_point >> 6 & 0x3F;
+            $hex[3] = 0x80 | $code_point  & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]).chr($hex[3]);
+        }
+
+        return $ret;
+    }
+
     public function testLexerForTab()
     {
         $lexer = new EmailLexer();

tests/egulias/Tests/EmailValidator/EmailValidatorTest.php

@@ -26,6 +26,14 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
         $this->assertTrue($this->validator->isValid($email));
     }
 
+    public function testInvalidUTF8Email()
+    {
+        $validator = new EmailValidator;
+        $email     = "\x80\x81\x82@\x83\x84\x85.\x86\x87\x88";
+
+        $this->assertFalse($validator->isValid($email));
+    }
+
     public function getValidEmails()
     {
         return array(
@@ -62,6 +70,8 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
     {
         return array(
 
+            array('user  name@example.com'),
+            array('user   name@example.com'),
             array('example.@example.co.uk'),
             array('example@example@example.co.uk'),
             array('(test_exampel@example.fr)'),
@@ -150,6 +160,16 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
         $this->assertEquals($warnings, $this->validator->getWarnings());
     }
 
+    /**
+     * @dataProvider getInvalidEmailsWithWarnings
+     */
+    public function testInvalidEmailsWithDnsCheckAndStrictMode($warnings, $email)
+    {
+        $this->assertFalse($this->validator->isValid($email, true, true));
+
+        $this->assertEquals($warnings, $this->validator->getWarnings());
+    }
+
     public function getInvalidEmailsWithWarnings()
     {
         return array(
@@ -301,14 +321,15 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
                 'parttoolonglocalparttoolonglocalparttoolonglocalparttoolonglocalparttoolonglocalparttoolonglocalpart'.
                 'toolonglocalparttoolonglocalparttoolonglocalparttoolonglocalpar'
             ),
+            array(
+                array(
+                    EmailValidator::DNSWARN_NO_RECORD,
+                ),
+                'test@test'
+            ),
         );
     }
 
-    public function testInvalidEmailsWithDNSAndStrict()
-    {
-        $this->assertFalse($this->validator->isValid('test@test', true, true));
-    }
-
     public function testInvalidEmailsWithStrict()
     {
         $this->assertFalse($this->validator->isValid('"test"@test', false, true));