Merge pull request #38 from egulias/improved-utf8

#30 - Improved utf8
#30 - Improved control for UTF8 chars
2026-08-31 12:40:28 +00:00 · 2014-11-29 10:09:14 +01:00 · 2014-11-29 10:05:19 +01:00 · 2014-11-17 01:23:05 +01:00 · 2014-11-17 01:18:11 +01:00 · 2014-11-16 19:15:26 +01:00
8 changed files with 197 additions and 86 deletions
@@ -15,7 +15,7 @@
  },
  "require":      {
    "php": ">= 5.3.3",
-    "doctrine/lexer": "~1.0"
+    "doctrine/lexer": "dev-master"
  },
  "require-dev" :   {
    "satooshi/php-coveralls": "dev-master"
@@ -4,26 +4,31 @@
        "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
        "This file is @generated automatically"
    ],
-    "hash": "9e9dff0cc08c7292600453e681201e13",
+    "hash": "7c82c4ea93186526be106a60ea69f6f1",
    "packages": [
        {
            "name": "doctrine/lexer",
-            "version": "v1.0",
+            "version": "dev-master",
            "source": {
                "type": "git",
                "url": "https://github.com/doctrine/lexer.git",
-                "reference": "2f708a85bb3aab5d99dab8be435abd73e0b18acb"
+                "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/lexer/zipball/2f708a85bb3aab5d99dab8be435abd73e0b18acb",
-                "reference": "2f708a85bb3aab5d99dab8be435abd73e0b18acb",
+                "url": "https://api.github.com/repos/doctrine/lexer/zipball/83893c552fd2045dd78aef794c31e694c37c0b8c",
+                "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c",
                "shasum": ""
            },
            "require": {
                "php": ">=5.3.2"
            },
            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.0.x-dev"
+                }
+            },
            "autoload": {
                "psr-0": {
                    "Doctrine\\Common\\Lexer\\": "lib/"
@@ -34,20 +39,17 @@
                "MIT"
            ],
            "authors": [
-                {
-                    "name": "Guilherme Blanco",
-                    "email": "guilhermeblanco@gmail.com",
-                    "homepage": "http://www.instaclick.com"
-                },
                {
                    "name": "Roman Borschel",
                    "email": "roman@code-factory.org"
                },
+                {
+                    "name": "Guilherme Blanco",
+                    "email": "guilhermeblanco@gmail.com"
+                },
                {
                    "name": "Johannes Schmitt",
-                    "email": "schmittjoh@gmail.com",
-                    "homepage": "http://jmsyst.com",
-                    "role": "Developer of wrapped JMSSerializerBundle"
+                    "email": "schmittjoh@gmail.com"
                }
            ],
            "description": "Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers.",
@@ -56,7 +58,7 @@
                "lexer",
                "parser"
            ],
-            "time": "2013-01-12 18:59:04"
+            "time": "2014-09-09 13:34:57"
        }
    ],
    "packages-dev": [
@@ -196,12 +198,12 @@
            "source": {
                "type": "git",
                "url": "https://github.com/satooshi/php-coveralls.git",
-                "reference": "94389a0ebdb64857d6899b5e0254dffa99e5aa96"
+                "reference": "2fbf803803d179ab1082807308a67bbd5a760c70"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/satooshi/php-coveralls/zipball/94389a0ebdb64857d6899b5e0254dffa99e5aa96",
-                "reference": "94389a0ebdb64857d6899b5e0254dffa99e5aa96",
+                "url": "https://api.github.com/repos/satooshi/php-coveralls/zipball/2fbf803803d179ab1082807308a67bbd5a760c70",
+                "reference": "2fbf803803d179ab1082807308a67bbd5a760c70",
                "shasum": ""
            },
            "require": {
@@ -263,21 +265,21 @@
                "github",
                "test"
            ],
-            "time": "2014-07-09 10:45:38"
+            "time": "2014-11-11 15:35:34"
        },
        {
            "name": "symfony/config",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Config",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Config.git",
-                "reference": "080eabdc256c1d7a3a7cf6296271edb68eb1ab2b"
+                "reference": "0316364bfebc8b080077c731a99f189341476bd7"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Config/zipball/080eabdc256c1d7a3a7cf6296271edb68eb1ab2b",
-                "reference": "080eabdc256c1d7a3a7cf6296271edb68eb1ab2b",
+                "url": "https://api.github.com/repos/symfony/Config/zipball/0316364bfebc8b080077c731a99f189341476bd7",
+                "reference": "0316364bfebc8b080077c731a99f189341476bd7",
                "shasum": ""
            },
            "require": {
@@ -311,21 +313,21 @@
            ],
            "description": "Symfony Config Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-31 03:22:04"
+            "time": "2014-09-23 05:25:11"
        },
        {
            "name": "symfony/console",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Console",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Console.git",
-                "reference": "748beed2a1e73179c3f5154d33fe6ae100c1aeb1"
+                "reference": "6f177fca24200a5b97aef5ce7a5c98124a0f0db0"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Console/zipball/748beed2a1e73179c3f5154d33fe6ae100c1aeb1",
-                "reference": "748beed2a1e73179c3f5154d33fe6ae100c1aeb1",
+                "url": "https://api.github.com/repos/symfony/Console/zipball/6f177fca24200a5b97aef5ce7a5c98124a0f0db0",
+                "reference": "6f177fca24200a5b97aef5ce7a5c98124a0f0db0",
                "shasum": ""
            },
            "require": {
@@ -366,21 +368,21 @@
            ],
            "description": "Symfony Console Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-14 16:10:54"
+            "time": "2014-10-05 13:57:04"
        },
        {
            "name": "symfony/event-dispatcher",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/EventDispatcher",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/EventDispatcher.git",
-                "reference": "8faf5cc7e80fde74a650a36e60d32ce3c3e0457b"
+                "reference": "804eb28dbbfba9ffdab21fe2066744906cea2212"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/8faf5cc7e80fde74a650a36e60d32ce3c3e0457b",
-                "reference": "8faf5cc7e80fde74a650a36e60d32ce3c3e0457b",
+                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/804eb28dbbfba9ffdab21fe2066744906cea2212",
+                "reference": "804eb28dbbfba9ffdab21fe2066744906cea2212",
                "shasum": ""
            },
            "require": {
@@ -389,7 +391,7 @@
            "require-dev": {
                "psr/log": "~1.0",
                "symfony/config": "~2.0",
-                "symfony/dependency-injection": "~2.0",
+                "symfony/dependency-injection": "~2.0,<2.6.0",
                "symfony/stopwatch": "~2.2"
            },
            "suggest": {
@@ -423,21 +425,21 @@
            ],
            "description": "Symfony EventDispatcher Component",
            "homepage": "http://symfony.com",
-            "time": "2014-07-28 13:20:46"
+            "time": "2014-10-01 15:43:05"
        },
        {
            "name": "symfony/filesystem",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Filesystem",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Filesystem.git",
-                "reference": "a765efd199e02ff4001c115c318e219030be9364"
+                "reference": "4e62fab0060a826561c78b665925b37c870c45f5"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Filesystem/zipball/a765efd199e02ff4001c115c318e219030be9364",
-                "reference": "a765efd199e02ff4001c115c318e219030be9364",
+                "url": "https://api.github.com/repos/symfony/Filesystem/zipball/4e62fab0060a826561c78b665925b37c870c45f5",
+                "reference": "4e62fab0060a826561c78b665925b37c870c45f5",
                "shasum": ""
            },
            "require": {
@@ -470,21 +472,21 @@
            ],
            "description": "Symfony Filesystem Component",
            "homepage": "http://symfony.com",
-            "time": "2014-09-03 09:00:14"
+            "time": "2014-09-22 09:14:18"
        },
        {
            "name": "symfony/stopwatch",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Stopwatch",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Stopwatch.git",
-                "reference": "22ab4f76cdeefd38b00022a6be5709190a2fd046"
+                "reference": "9f8a33a24f2378c0ec5f372a8d50b2d43069c050"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Stopwatch/zipball/22ab4f76cdeefd38b00022a6be5709190a2fd046",
-                "reference": "22ab4f76cdeefd38b00022a6be5709190a2fd046",
+                "url": "https://api.github.com/repos/symfony/Stopwatch/zipball/9f8a33a24f2378c0ec5f372a8d50b2d43069c050",
+                "reference": "9f8a33a24f2378c0ec5f372a8d50b2d43069c050",
                "shasum": ""
            },
            "require": {
@@ -517,21 +519,21 @@
            ],
            "description": "Symfony Stopwatch Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-14 16:10:54"
+            "time": "2014-09-22 09:14:18"
        },
        {
            "name": "symfony/yaml",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Yaml",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Yaml.git",
-                "reference": "01a7695bcfb013d0a15c6757e15aae120342986f"
+                "reference": "2d9f527449cabfa8543dd7fa3a466d6ae83d6726"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Yaml/zipball/01a7695bcfb013d0a15c6757e15aae120342986f",
-                "reference": "01a7695bcfb013d0a15c6757e15aae120342986f",
+                "url": "https://api.github.com/repos/symfony/Yaml/zipball/2d9f527449cabfa8543dd7fa3a466d6ae83d6726",
+                "reference": "2d9f527449cabfa8543dd7fa3a466d6ae83d6726",
                "shasum": ""
            },
            "require": {
@@ -564,21 +566,18 @@
            ],
            "description": "Symfony Yaml Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-31 03:22:04"
+            "time": "2014-10-01 05:50:18"
        }
    ],
-    "aliases": [
-
-    ],
+    "aliases": [],
    "minimum-stability": "stable",
    "stability-flags": {
+        "doctrine/lexer": 20,
        "satooshi/php-coveralls": 20
    },
    "prefer-stable": false,
    "platform": {
        "php": ">= 5.3.3"
    },
-    "platform-dev": [
-
-    ]
+    "platform-dev": []
 }
@@ -142,8 +142,7 @@ class EmailLexer extends AbstractLexer
            '[0-9]+',
            '\r\n',
            '::',
-            '\s+',
-            '[\x10-\x1F]+',
+            '\s+?',
            '.',
            );
    }
@@ -155,7 +154,7 @@ class EmailLexer extends AbstractLexer
     */
    protected function getNonCatchablePatterns()
    {
-        return array('[\x7f-\xff]+');
+        return array('[\xA0-\xff]+');
    }

    /**
@@ -167,16 +166,20 @@ class EmailLexer extends AbstractLexer
     */
    protected function getType(&$value)
    {
-
        if ($this->isNullType($value)) {
            return self::C_NUL;
        }

-        if (isset($this->charValue[$value])) {
+        if ($this->isValid($value)) {
            return $this->charValue[$value];
        }

-        if ($this->isInvalid($value)) {
+        if ($this->isUTF8Invalid($value)) {
+            $this->hasInvalidTokens = true;
+            return self::INVALID;
+        }
+
+        if ($this->isASCIIInvalid($value)) {
            $this->hasInvalidTokens = true;
            return self::INVALID;
        }
@@ -184,8 +187,18 @@ class EmailLexer extends AbstractLexer
        return  self::GENERIC;
    }

+    protected function isValid($value)
+    {
+        if (isset($this->charValue[$value])) {
+            return true;
+        }
+
+        return false;
+    }
+
    /**
-     * @param string $value
+     * @param $value
+     * @return bool
     */
    protected function isNullType($value)
    {
@@ -197,18 +210,33 @@ class EmailLexer extends AbstractLexer
    }

    /**
-     * @param string $value
+     * @param $value
+     * @return bool
     */
-    protected function isInvalid($value)
+    protected function isASCIIInvalid($value)
    {
-        if (preg_match('/[\x10-\x1F]+/', $value)) {
-            return true;
-        }
-
        if (isset($this->invalidASCII[ord($value)])) {
            return true;
        }

        return false;
    }
+
+    /**
+     * @param $value
+     * @return bool
+     */
+    protected function isUTF8Invalid($value)
+    {
+        if (preg_match('/\p{Cc}+/u', $value)) {
+            return true;
+        }
+
+        return false;
+    }
+
+    protected function getModifiers()
+    {
+        return 'iu';
+    }
 }
@@ -29,7 +29,8 @@ class EmailParser
    }

    /**
-     * @param string $str
+     * @param $str
+     * @return array
     */
    public function parse($str)
    {
@@ -39,15 +40,16 @@ class EmailParser
            throw new \InvalidArgumentException('ERR_NOLOCALPART');
        }

-        if ($this->lexer->hasInvalidTokens()) {
-            throw new \InvalidArgumentException('ERR_INVALID_ATEXT');
-        }

        $this->localPartParser->parse($str);
        $this->domainPartParser->parse($str);

        $this->setParts($str);

+        if ($this->lexer->hasInvalidTokens()) {
+            throw new \InvalidArgumentException('ERR_INVALID_ATEXT');
+        }
+
        return array('local' => $this->localPart, 'domain' => $this->domainPart);
    }

@@ -84,7 +84,7 @@ class EmailValidator
            return false;
        }

-        $dns = false;
+        $dns = true;
        if ($checkDNS) {
            $dns = $this->checkDNS();
        }
@@ -95,12 +95,7 @@ class EmailValidator
            return false;
        }

-        return ($strict) ? $this->checkStrict($dns) : true;
-    }
-
-    private function checkStrict($dns)
-    {
-         return !($this->hasWarnings() && !$dns);
+        return !$strict || (!$this->hasWarnings() && $dns);
    }

    /**
@@ -6,7 +6,6 @@ use Egulias\EmailValidator\EmailLexer;
 use Egulias\EmailValidator\EmailValidator;
 use \InvalidArgumentException;

-
 class LocalPart extends Parser
 {
    public function parse($localPart)
@@ -26,6 +26,73 @@ class EmailLexerTests extends \PHPUnit_Framework_TestCase
        $this->assertEquals($token, $lexer->token['type']);
    }

+    public function testLexerParsesMultipleSpaces()
+    {
+        $lexer = new EmailLexer();
+        $lexer->setInput('  ');
+        $lexer->moveNext();
+        $lexer->moveNext();
+        $this->assertEquals(EmailLexer::S_SP, $lexer->token['type']);
+        $lexer->moveNext();
+        $this->assertEquals(EmailLexer::S_SP, $lexer->token['type']);
+    }
+
+    /**
+     * @dataProvider invalidUTF8CharsProvider
+     */
+    public function testLexerParsesInvalidUTF8($char)
+    {
+        $lexer = new EmailLexer();
+        $lexer->setInput($char);
+        $lexer->moveNext();
+        $lexer->moveNext();
+
+        $this->assertEquals(EmailLexer::INVALID, $lexer->token['type']);
+    }
+
+    public function invalidUTF8CharsProvider()
+    {
+        $chars = array();
+        for ($i = 0; $i < 0x100; ++$i) {
+            $c = $this->utf8Chr($i);
+            if (preg_match('/(?=\p{Cc})(?=[^\t\n\n\r])/u', $c) && !preg_match('/\x{0000}/u', $c)) {
+                $chars[] = array($c);
+            }
+        }
+
+        return $chars;
+    }
+
+    protected function utf8Chr($code_point)
+    {
+
+        if ($code_point < 0 || 0x10FFFF < $code_point || (0xD800 <= $code_point && $code_point <= 0xDFFF)) {
+            return '';
+        }
+
+        if ($code_point < 0x80) {
+            $hex[0] = $code_point;
+            $ret = chr($hex[0]);
+        } elseif ($code_point < 0x800) {
+            $hex[0] = 0x1C0 | $code_point >> 6;
+            $hex[1] = 0x80  | $code_point & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]);
+        } elseif ($code_point < 0x10000) {
+            $hex[0] = 0xE0 | $code_point >> 12;
+            $hex[1] = 0x80 | $code_point >> 6 & 0x3F;
+            $hex[2] = 0x80 | $code_point & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]);
+        } else {
+            $hex[0] = 0xF0 | $code_point >> 18;
+            $hex[1] = 0x80 | $code_point >> 12 & 0x3F;
+            $hex[2] = 0x80 | $code_point >> 6 & 0x3F;
+            $hex[3] = 0x80 | $code_point  & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]).chr($hex[3]);
+        }
+
+        return $ret;
+    }
+
    public function testLexerForTab()
    {
        $lexer = new EmailLexer();
@@ -26,6 +26,14 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
        $this->assertTrue($this->validator->isValid($email));
    }

+    public function testInvalidUTF8Email()
+    {
+        $validator = new EmailValidator;
+        $email     = "\x80\x81\x82@\x83\x84\x85.\x86\x87\x88";
+
+        $this->assertFalse($validator->isValid($email));
+    }
+
    public function getValidEmails()
    {
        return array(
@@ -62,6 +70,8 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
    {
        return array(

+            array('user  name@example.com'),
+            array('user   name@example.com'),
            array('example.@example.co.uk'),
            array('example@example@example.co.uk'),
            array('(test_exampel@example.fr)'),
@@ -150,6 +160,16 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
        $this->assertEquals($warnings, $this->validator->getWarnings());
    }

+    /**
+     * @dataProvider getInvalidEmailsWithWarnings
+     */
+    public function testInvalidEmailsWithDnsCheckAndStrictMode($warnings, $email)
+    {
+        $this->assertFalse($this->validator->isValid($email, true, true));
+
+        $this->assertEquals($warnings, $this->validator->getWarnings());
+    }
+
    public function getInvalidEmailsWithWarnings()
    {
        return array(
@@ -301,14 +321,15 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
                'parttoolonglocalparttoolonglocalparttoolonglocalparttoolonglocalparttoolonglocalparttoolonglocalpart'.
                'toolonglocalparttoolonglocalparttoolonglocalparttoolonglocalpar'
            ),
+            array(
+                array(
+                    EmailValidator::DNSWARN_NO_RECORD,
+                ),
+                'test@test'
+            ),
        );
    }

-    public function testInvalidEmailsWithDNSAndStrict()
-    {
-        $this->assertFalse($this->validator->isValid('test@test', true, true));
-    }
-
    public function testInvalidEmailsWithStrict()
    {
        $this->assertFalse($this->validator->isValid('"test"@test', false, true));
Author	SHA1	Message	Date
Eduardo Gulias Davis	7a64ea18af	Merge pull request #38 from egulias/improved-utf8 #30 - Improved utf8	2014-11-29 10:09:14 +01:00
Eduardo Gulias Davis	9103f4f99b	#30 - Improved control for UTF8 chars	2014-11-29 10:05:19 +01:00
Eduardo Gulias Davis	3f623e9006	Merge pull request #37 from egulias/multiple_spaces #36 - Multiple spaces in local part	2014-11-17 01:23:05 +01:00
Eduardo Gulias Davis	833eb65135	#30 - [WIP] - improve utf-8 control	2014-11-17 01:18:11 +01:00
Eduardo Gulias Davis	e61481fefb	#36 - Multiple spaces in local part	2014-11-16 19:15:26 +01:00
Eduardo Gulias Davis	518f80a0ff	Merge pull request #35 from craue/fix-strict-check make strict check work again	2014-11-06 09:59:44 +01:00
Christian Raue	2ec7babda6	fixed strict check	2014-11-05 01:09:55 +01:00
Christian Raue	05367768d6	improved tests for strict checking	2014-11-04 14:01:13 +01:00