#30 - [WIP] - improve utf-8 control

2026-08-27 18:48:40 +00:00 · 2014-11-17 01:18:11 +01:00
parent e61481fefb
commit 833eb65135
5 changed files with 124 additions and 57 deletions
@@ -15,7 +15,7 @@
  },
  "require":      {
    "php": ">= 5.3.3",
-    "doctrine/lexer": "~1.0"
+    "doctrine/lexer": "dev-master"
  },
  "require-dev" :   {
    "satooshi/php-coveralls": "dev-master"
@@ -4,26 +4,31 @@
        "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
        "This file is @generated automatically"
    ],
-    "hash": "9e9dff0cc08c7292600453e681201e13",
+    "hash": "7c82c4ea93186526be106a60ea69f6f1",
    "packages": [
        {
            "name": "doctrine/lexer",
-            "version": "v1.0",
+            "version": "dev-master",
            "source": {
                "type": "git",
                "url": "https://github.com/doctrine/lexer.git",
-                "reference": "2f708a85bb3aab5d99dab8be435abd73e0b18acb"
+                "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/lexer/zipball/2f708a85bb3aab5d99dab8be435abd73e0b18acb",
-                "reference": "2f708a85bb3aab5d99dab8be435abd73e0b18acb",
+                "url": "https://api.github.com/repos/doctrine/lexer/zipball/83893c552fd2045dd78aef794c31e694c37c0b8c",
+                "reference": "83893c552fd2045dd78aef794c31e694c37c0b8c",
                "shasum": ""
            },
            "require": {
                "php": ">=5.3.2"
            },
            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.0.x-dev"
+                }
+            },
            "autoload": {
                "psr-0": {
                    "Doctrine\\Common\\Lexer\\": "lib/"
@@ -34,20 +39,17 @@
                "MIT"
            ],
            "authors": [
-                {
-                    "name": "Guilherme Blanco",
-                    "email": "guilhermeblanco@gmail.com",
-                    "homepage": "http://www.instaclick.com"
-                },
                {
                    "name": "Roman Borschel",
                    "email": "roman@code-factory.org"
                },
+                {
+                    "name": "Guilherme Blanco",
+                    "email": "guilhermeblanco@gmail.com"
+                },
                {
                    "name": "Johannes Schmitt",
-                    "email": "schmittjoh@gmail.com",
-                    "homepage": "http://jmsyst.com",
-                    "role": "Developer of wrapped JMSSerializerBundle"
+                    "email": "schmittjoh@gmail.com"
                }
            ],
            "description": "Base library for a lexer that can be used in Top-Down, Recursive Descent Parsers.",
@@ -56,7 +58,7 @@
                "lexer",
                "parser"
            ],
-            "time": "2013-01-12 18:59:04"
+            "time": "2014-09-09 13:34:57"
        }
    ],
    "packages-dev": [
@@ -196,12 +198,12 @@
            "source": {
                "type": "git",
                "url": "https://github.com/satooshi/php-coveralls.git",
-                "reference": "94389a0ebdb64857d6899b5e0254dffa99e5aa96"
+                "reference": "2fbf803803d179ab1082807308a67bbd5a760c70"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/satooshi/php-coveralls/zipball/94389a0ebdb64857d6899b5e0254dffa99e5aa96",
-                "reference": "94389a0ebdb64857d6899b5e0254dffa99e5aa96",
+                "url": "https://api.github.com/repos/satooshi/php-coveralls/zipball/2fbf803803d179ab1082807308a67bbd5a760c70",
+                "reference": "2fbf803803d179ab1082807308a67bbd5a760c70",
                "shasum": ""
            },
            "require": {
@@ -263,21 +265,21 @@
                "github",
                "test"
            ],
-            "time": "2014-07-09 10:45:38"
+            "time": "2014-11-11 15:35:34"
        },
        {
            "name": "symfony/config",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Config",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Config.git",
-                "reference": "080eabdc256c1d7a3a7cf6296271edb68eb1ab2b"
+                "reference": "0316364bfebc8b080077c731a99f189341476bd7"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Config/zipball/080eabdc256c1d7a3a7cf6296271edb68eb1ab2b",
-                "reference": "080eabdc256c1d7a3a7cf6296271edb68eb1ab2b",
+                "url": "https://api.github.com/repos/symfony/Config/zipball/0316364bfebc8b080077c731a99f189341476bd7",
+                "reference": "0316364bfebc8b080077c731a99f189341476bd7",
                "shasum": ""
            },
            "require": {
@@ -311,21 +313,21 @@
            ],
            "description": "Symfony Config Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-31 03:22:04"
+            "time": "2014-09-23 05:25:11"
        },
        {
            "name": "symfony/console",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Console",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Console.git",
-                "reference": "748beed2a1e73179c3f5154d33fe6ae100c1aeb1"
+                "reference": "6f177fca24200a5b97aef5ce7a5c98124a0f0db0"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Console/zipball/748beed2a1e73179c3f5154d33fe6ae100c1aeb1",
-                "reference": "748beed2a1e73179c3f5154d33fe6ae100c1aeb1",
+                "url": "https://api.github.com/repos/symfony/Console/zipball/6f177fca24200a5b97aef5ce7a5c98124a0f0db0",
+                "reference": "6f177fca24200a5b97aef5ce7a5c98124a0f0db0",
                "shasum": ""
            },
            "require": {
@@ -366,21 +368,21 @@
            ],
            "description": "Symfony Console Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-14 16:10:54"
+            "time": "2014-10-05 13:57:04"
        },
        {
            "name": "symfony/event-dispatcher",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/EventDispatcher",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/EventDispatcher.git",
-                "reference": "8faf5cc7e80fde74a650a36e60d32ce3c3e0457b"
+                "reference": "804eb28dbbfba9ffdab21fe2066744906cea2212"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/8faf5cc7e80fde74a650a36e60d32ce3c3e0457b",
-                "reference": "8faf5cc7e80fde74a650a36e60d32ce3c3e0457b",
+                "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/804eb28dbbfba9ffdab21fe2066744906cea2212",
+                "reference": "804eb28dbbfba9ffdab21fe2066744906cea2212",
                "shasum": ""
            },
            "require": {
@@ -389,7 +391,7 @@
            "require-dev": {
                "psr/log": "~1.0",
                "symfony/config": "~2.0",
-                "symfony/dependency-injection": "~2.0",
+                "symfony/dependency-injection": "~2.0,<2.6.0",
                "symfony/stopwatch": "~2.2"
            },
            "suggest": {
@@ -423,21 +425,21 @@
            ],
            "description": "Symfony EventDispatcher Component",
            "homepage": "http://symfony.com",
-            "time": "2014-07-28 13:20:46"
+            "time": "2014-10-01 15:43:05"
        },
        {
            "name": "symfony/filesystem",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Filesystem",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Filesystem.git",
-                "reference": "a765efd199e02ff4001c115c318e219030be9364"
+                "reference": "4e62fab0060a826561c78b665925b37c870c45f5"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Filesystem/zipball/a765efd199e02ff4001c115c318e219030be9364",
-                "reference": "a765efd199e02ff4001c115c318e219030be9364",
+                "url": "https://api.github.com/repos/symfony/Filesystem/zipball/4e62fab0060a826561c78b665925b37c870c45f5",
+                "reference": "4e62fab0060a826561c78b665925b37c870c45f5",
                "shasum": ""
            },
            "require": {
@@ -470,21 +472,21 @@
            ],
            "description": "Symfony Filesystem Component",
            "homepage": "http://symfony.com",
-            "time": "2014-09-03 09:00:14"
+            "time": "2014-09-22 09:14:18"
        },
        {
            "name": "symfony/stopwatch",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Stopwatch",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Stopwatch.git",
-                "reference": "22ab4f76cdeefd38b00022a6be5709190a2fd046"
+                "reference": "9f8a33a24f2378c0ec5f372a8d50b2d43069c050"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Stopwatch/zipball/22ab4f76cdeefd38b00022a6be5709190a2fd046",
-                "reference": "22ab4f76cdeefd38b00022a6be5709190a2fd046",
+                "url": "https://api.github.com/repos/symfony/Stopwatch/zipball/9f8a33a24f2378c0ec5f372a8d50b2d43069c050",
+                "reference": "9f8a33a24f2378c0ec5f372a8d50b2d43069c050",
                "shasum": ""
            },
            "require": {
@@ -517,21 +519,21 @@
            ],
            "description": "Symfony Stopwatch Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-14 16:10:54"
+            "time": "2014-09-22 09:14:18"
        },
        {
            "name": "symfony/yaml",
-            "version": "v2.5.4",
+            "version": "v2.5.6",
            "target-dir": "Symfony/Component/Yaml",
            "source": {
                "type": "git",
                "url": "https://github.com/symfony/Yaml.git",
-                "reference": "01a7695bcfb013d0a15c6757e15aae120342986f"
+                "reference": "2d9f527449cabfa8543dd7fa3a466d6ae83d6726"
            },
            "dist": {
                "type": "zip",
-                "url": "https://api.github.com/repos/symfony/Yaml/zipball/01a7695bcfb013d0a15c6757e15aae120342986f",
-                "reference": "01a7695bcfb013d0a15c6757e15aae120342986f",
+                "url": "https://api.github.com/repos/symfony/Yaml/zipball/2d9f527449cabfa8543dd7fa3a466d6ae83d6726",
+                "reference": "2d9f527449cabfa8543dd7fa3a466d6ae83d6726",
                "shasum": ""
            },
            "require": {
@@ -564,21 +566,18 @@
            ],
            "description": "Symfony Yaml Component",
            "homepage": "http://symfony.com",
-            "time": "2014-08-31 03:22:04"
+            "time": "2014-10-01 05:50:18"
        }
    ],
-    "aliases": [
-
-    ],
+    "aliases": [],
    "minimum-stability": "stable",
    "stability-flags": {
+        "doctrine/lexer": 20,
        "satooshi/php-coveralls": 20
    },
    "prefer-stable": false,
    "platform": {
        "php": ">= 5.3.3"
    },
-    "platform-dev": [
-
-    ]
+    "platform-dev": []
 }
@@ -201,7 +201,7 @@ class EmailLexer extends AbstractLexer
     */
    protected function isInvalid($value)
    {
-        if (preg_match('/[\x10-\x1F]+/', $value)) {
+        if (preg_match('/[\x10-\x1F\x{0001}-\x{000F}\x{0080}-\x{009F}]+/', $value)) {
            return true;
        }

@@ -211,4 +211,9 @@ class EmailLexer extends AbstractLexer

        return false;
    }
+
+    protected function getModifiers()
+    {
+        return 'iu';
+    }
 }
@@ -37,6 +37,61 @@ class EmailLexerTests extends \PHPUnit_Framework_TestCase
        $this->assertEquals(EmailLexer::S_SP, $lexer->token['type']);
    }

+    /**
+     * @dataProvider invalidUTF8CharsProvider
+     */
+    public function testLexerParsesInvalidUTF8($char)
+    {
+        $lexer = new EmailLexer();
+        $lexer->setInput($char);
+        $lexer->moveNext();
+        $lexer->moveNext();
+
+        $this->assertEquals(EmailLexer::INVALID, $lexer->token['type']);
+    }
+
+    public function invalidUTF8CharsProvider()
+    {
+        $chars = array();
+        for ($i = 0; $i < 0x100; ++$i) {
+            $c = $this->utf8_chr($i);
+            if (preg_match('/[\x{0001}-\x{000F}\x{0080}-\x{009F}]/u', $c)) {
+                $chars[] = array($c);
+            }
+        }
+
+        return $chars;
+    }
+
+    protected function utf8_chr($code_point) {
+
+        if ($code_point < 0 || 0x10FFFF < $code_point || (0xD800 <= $code_point && $code_point <= 0xDFFF)) {
+            return '';
+        }
+
+        if ($code_point < 0x80) {
+            $hex[0] = $code_point;
+            $ret = chr($hex[0]);
+        } else if ($code_point < 0x800) {
+            $hex[0] = 0x1C0 | $code_point >> 6;
+            $hex[1] = 0x80  | $code_point & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]);
+        } else if ($code_point < 0x10000) {
+            $hex[0] = 0xE0 | $code_point >> 12;
+            $hex[1] = 0x80 | $code_point >> 6 & 0x3F;
+            $hex[2] = 0x80 | $code_point & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]);
+        } else  {
+            $hex[0] = 0xF0 | $code_point >> 18;
+            $hex[1] = 0x80 | $code_point >> 12 & 0x3F;
+            $hex[2] = 0x80 | $code_point >> 6 & 0x3F;
+            $hex[3] = 0x80 | $code_point  & 0x3F;
+            $ret = chr($hex[0]).chr($hex[1]).chr($hex[2]).chr($hex[3]);
+        }
+
+        return $ret;
+    }
+
    public function testLexerForTab()
    {
        $lexer = new EmailLexer();
@@ -26,6 +26,14 @@ class EmailValidatorTest extends \PHPUnit_Framework_TestCase
        $this->assertTrue($this->validator->isValid($email));
    }

+    public function testInvalidUTF8Emails()
+    {
+        $validator = new EmailValidator;
+        $email     = "\x80\x81\x82@\x83\x84\x85.\x86\x87\x88";
+
+        $this->assertFalse($validator->isValid($email));
+    }
+
    public function getValidEmails()
    {
        return array(