url: update IDNA handling

Remove custom tests for invalid IDNA domains in url-idna.js in favor of
the more comprehensive official set.

PR-URL: https://github.com/nodejs/node/pull/13362
Refs: https://github.com/whatwg/url/pull/309
Refs: https://github.com/w3c/web-platform-tests/pull/5976
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Daijiro Wachi <daijiro.wachi@gmail.com>

2 files changed, 368 insertions, 220 deletions

diff --git a/test/fixtures/url-idna.js b/test/fixtures/url-idna.js
index cbfe702e93..4b8f5a48cc 100644
--- a/test/fixtures/url-idna.js
+++ b/test/fixtures/url-idna.js
@@ -1,223 +1,215 @@
 'use strict';
 
 // Credit for list: http://www.i18nguy.com/markup/idna-examples.html
-module.exports = {
-  valid: [
-    { ascii: 'xn--mgbaal8b0b9b2b.icom.museum',
-      unicode: 'افغانستا.icom.museum'
-    },
-    {
-      ascii: 'xn--lgbbat1ad8j.icom.museum',
-      unicode: 'الجزائر.icom.museum'
-    },
-    {
-      ascii: 'xn--sterreich-z7a.icom.museum',
-      unicode: 'österreich.icom.museum'
-    },
-    {
-      ascii: 'xn--54b6eqazv8bc7e.icom.museum',
-      unicode: 'বাংলাদেশ.icom.museum'
-    },
-    {
-      ascii: 'xn--80abmy0agn7e.icom.museum',
-      unicode: 'беларусь.icom.museum'
-    },
-    {
-      ascii: 'xn--belgi-rsa.icom.museum',
-      unicode: 'belgië.icom.museum'
-    },
-    {
-      ascii: 'xn--80abgvm6a7d2b.icom.museum',
-      unicode: 'българия.icom.museum'
-    },
-    {
-      ascii: 'xn--mgbfqim.icom.museum',
-      unicode: 'تشادر.icom.museum'
-    },
-    {
-      ascii: 'xn--fiqs8s.icom.museum',
-      unicode: '中国.icom.museum'
-    },
-    {
-      ascii: 'xn--mgbu4chg.icom.museum',
-      unicode: 'القمر.icom.museum'
-    },
-    {
-      ascii: 'xn--vxakcego.icom.museum',
-      unicode: 'κυπρος.icom.museum'
-    },
-    {
-      ascii: 'xn--eskrepublika-ebb62d.icom.museum',
-      unicode: 'českárepublika.icom.museum'
-    },
-    {
-      ascii: 'xn--wgbh1c.icom.museum',
-      unicode: 'مصر.icom.museum'
-    },
-    {
-      ascii: 'xn--hxakic4aa.icom.museum',
-      unicode: 'ελλάδα.icom.museum'
-    },
-    {
-      ascii: 'xn--magyarorszg-t7a.icom.museum',
-      unicode: 'magyarország.icom.museum'
-    },
-    {
-      ascii: 'xn--sland-ysa.icom.museum',
-      unicode: 'ísland.icom.museum'
-    },
-    {
-      ascii: 'xn--h2brj9c.icom.museum',
-      unicode: 'भारत.icom.museum'
-    },
-    {
-      ascii: 'xn--mgba3a4fra.icom.museum',
-      unicode: 'ايران.icom.museum'
-    },
-    {
-      ascii: 'xn--ire-9la.icom.museum',
-      unicode: 'éire.icom.museum'
-    },
-    {
-      ascii: 'xn--4dbklr2c8d.xn--4dbrk0ce.museum',
-      unicode: 'איקו״ם.ישראל.museum'
-    },
-    {
-      ascii: 'xn--wgv71a.icom.museum',
-      unicode: '日本.icom.museum'
-    },
-    {
-      ascii: 'xn--igbhzh7gpa.icom.museum',
-      unicode: 'الأردن.icom.museum'
-    },
-    {
-      ascii: 'xn--80aaa0a6awh12ed.icom.museum',
-      unicode: 'қазақстан.icom.museum'
-    },
-    {
-      ascii: 'xn--3e0b707e.icom.museum',
-      unicode: '한국.icom.museum'
-    },
-    {
-      ascii: 'xn--80afmksoji0fc.icom.museum',
-      unicode: 'кыргызстан.icom.museum'
-    },
-    {
-      ascii: 'xn--q7ce6a.icom.museum',
-      unicode: 'ລາວ.icom.museum'
-    },
-    {
-      ascii: 'xn--mgbb7fjb.icom.museum',
-      unicode: 'لبنان.icom.museum'
-    },
-    {
-      ascii: 'xn--80aaldqjmmi6x.icom.museum',
-      unicode: 'македонија.icom.museum'
-    },
-    {
-      ascii: 'xn--mgbah1a3hjkrd.icom.museum',
-      unicode: 'موريتانيا.icom.museum'
-    },
-    {
-      ascii: 'xn--mxico-bsa.icom.museum',
-      unicode: 'méxico.icom.museum'
-    },
-    {
-      ascii: 'xn--c1aqabffc0aq.icom.museum',
-      unicode: 'монголулс.icom.museum'
-    },
-    {
-      ascii: 'xn--mgbc0a9azcg.icom.museum',
-      unicode: 'المغرب.icom.museum'
-    },
-    {
-      ascii: 'xn--l2bey1c2b.icom.museum',
-      unicode: 'नेपाल.icom.museum'
-    },
-    {
-      ascii: 'xn--mgb9awbf.icom.museum',
-      unicode: 'عمان.icom.museum'
-    },
-    {
-      ascii: 'xn--wgbl6a.icom.museum',
-      unicode: 'قطر.icom.museum'
-    },
-    {
-      ascii: 'xn--romnia-yta.icom.museum',
-      unicode: 'românia.icom.museum'
-    },
-    {
-      ascii: 'xn--h1alffa9f.xn--h1aegh.museum',
-      unicode: 'россия.иком.museum'
-    },
-    {
-      ascii: 'xn--80aaabm1ab4blmeec9e7n.xn--h1aegh.museum',
-      unicode: 'србијаицрнагора.иком.museum'
-    },
-    {
-      ascii: 'xn--xkc2al3hye2a.icom.museum',
-      unicode: 'இலங்கை.icom.museum'
-    },
-    {
-      ascii: 'xn--espaa-rta.icom.museum',
-      unicode: 'españa.icom.museum'
-    },
-    {
-      ascii: 'xn--o3cw4h.icom.museum',
-      unicode: 'ไทย.icom.museum'
-    },
-    {
-      ascii: 'xn--pgbs0dh.icom.museum',
-      unicode: 'تونس.icom.museum'
-    },
-    {
-      ascii: 'xn--trkiye-3ya.icom.museum',
-      unicode: 'türkiye.icom.museum'
-    },
-    {
-      ascii: 'xn--80aaxgrpt.icom.museum',
-      unicode: 'украина.icom.museum'
-    },
-    {
-      ascii: 'xn--vitnam-jk8b.icom.museum',
-      unicode: 'việtnam.icom.museum'
-    },
-    // long label
-    {
-      ascii: `${'a'.repeat(64)}.com`,
-      unicode: `${'a'.repeat(64)}.com`,
-    },
-    // long URL
-    {
-      ascii: `${`${'a'.repeat(64)}.`.repeat(4)}com`,
-      unicode: `${`${'a'.repeat(64)}.`.repeat(4)}com`
-    },
-    // URLs with hyphen
-    {
-      ascii: 'r4---sn-a5mlrn7s.gevideo.com',
-      unicode: 'r4---sn-a5mlrn7s.gevideo.com'
-    },
-    {
-      ascii: '-sn-a5mlrn7s.gevideo.com',
-      unicode: '-sn-a5mlrn7s.gevideo.com'
-    },
-    {
-      ascii: 'sn-a5mlrn7s-.gevideo.com',
-      unicode: 'sn-a5mlrn7s-.gevideo.com'
-    },
-    {
-      ascii: '-sn-a5mlrn7s-.gevideo.com',
-      unicode: '-sn-a5mlrn7s-.gevideo.com'
-    },
-    {
-      ascii: '-sn--a5mlrn7s-.gevideo.com',
-      unicode: '-sn--a5mlrn7s-.gevideo.com'
-    }
-  ],
-  invalid: [
-    // invalid character
-    '\ufffd.com',
-    // invalid bi-directional character
-    'تشادرlatin.icom.museum'
-  ]
-}
+module.exports = [
+  { ascii: 'xn--mgbaal8b0b9b2b.icom.museum',
+    unicode: 'افغانستا.icom.museum'
+  },
+  {
+    ascii: 'xn--lgbbat1ad8j.icom.museum',
+    unicode: 'الجزائر.icom.museum'
+  },
+  {
+    ascii: 'xn--sterreich-z7a.icom.museum',
+    unicode: 'österreich.icom.museum'
+  },
+  {
+    ascii: 'xn--54b6eqazv8bc7e.icom.museum',
+    unicode: 'বাংলাদেশ.icom.museum'
+  },
+  {
+    ascii: 'xn--80abmy0agn7e.icom.museum',
+    unicode: 'беларусь.icom.museum'
+  },
+  {
+    ascii: 'xn--belgi-rsa.icom.museum',
+    unicode: 'belgië.icom.museum'
+  },
+  {
+    ascii: 'xn--80abgvm6a7d2b.icom.museum',
+    unicode: 'българия.icom.museum'
+  },
+  {
+    ascii: 'xn--mgbfqim.icom.museum',
+    unicode: 'تشادر.icom.museum'
+  },
+  {
+    ascii: 'xn--fiqs8s.icom.museum',
+    unicode: '中国.icom.museum'
+  },
+  {
+    ascii: 'xn--mgbu4chg.icom.museum',
+    unicode: 'القمر.icom.museum'
+  },
+  {
+    ascii: 'xn--vxakcego.icom.museum',
+    unicode: 'κυπρος.icom.museum'
+  },
+  {
+    ascii: 'xn--eskrepublika-ebb62d.icom.museum',
+    unicode: 'českárepublika.icom.museum'
+  },
+  {
+    ascii: 'xn--wgbh1c.icom.museum',
+    unicode: 'مصر.icom.museum'
+  },
+  {
+    ascii: 'xn--hxakic4aa.icom.museum',
+    unicode: 'ελλάδα.icom.museum'
+  },
+  {
+    ascii: 'xn--magyarorszg-t7a.icom.museum',
+    unicode: 'magyarország.icom.museum'
+  },
+  {
+    ascii: 'xn--sland-ysa.icom.museum',
+    unicode: 'ísland.icom.museum'
+  },
+  {
+    ascii: 'xn--h2brj9c.icom.museum',
+    unicode: 'भारत.icom.museum'
+  },
+  {
+    ascii: 'xn--mgba3a4fra.icom.museum',
+    unicode: 'ايران.icom.museum'
+  },
+  {
+    ascii: 'xn--ire-9la.icom.museum',
+    unicode: 'éire.icom.museum'
+  },
+  {
+    ascii: 'xn--4dbklr2c8d.xn--4dbrk0ce.museum',
+    unicode: 'איקו״ם.ישראל.museum'
+  },
+  {
+    ascii: 'xn--wgv71a.icom.museum',
+    unicode: '日本.icom.museum'
+  },
+  {
+    ascii: 'xn--igbhzh7gpa.icom.museum',
+    unicode: 'الأردن.icom.museum'
+  },
+  {
+    ascii: 'xn--80aaa0a6awh12ed.icom.museum',
+    unicode: 'қазақстан.icom.museum'
+  },
+  {
+    ascii: 'xn--3e0b707e.icom.museum',
+    unicode: '한국.icom.museum'
+  },
+  {
+    ascii: 'xn--80afmksoji0fc.icom.museum',
+    unicode: 'кыргызстан.icom.museum'
+  },
+  {
+    ascii: 'xn--q7ce6a.icom.museum',
+    unicode: 'ລາວ.icom.museum'
+  },
+  {
+    ascii: 'xn--mgbb7fjb.icom.museum',
+    unicode: 'لبنان.icom.museum'
+  },
+  {
+    ascii: 'xn--80aaldqjmmi6x.icom.museum',
+    unicode: 'македонија.icom.museum'
+  },
+  {
+    ascii: 'xn--mgbah1a3hjkrd.icom.museum',
+    unicode: 'موريتانيا.icom.museum'
+  },
+  {
+    ascii: 'xn--mxico-bsa.icom.museum',
+    unicode: 'méxico.icom.museum'
+  },
+  {
+    ascii: 'xn--c1aqabffc0aq.icom.museum',
+    unicode: 'монголулс.icom.museum'
+  },
+  {
+    ascii: 'xn--mgbc0a9azcg.icom.museum',
+    unicode: 'المغرب.icom.museum'
+  },
+  {
+    ascii: 'xn--l2bey1c2b.icom.museum',
+    unicode: 'नेपाल.icom.museum'
+  },
+  {
+    ascii: 'xn--mgb9awbf.icom.museum',
+    unicode: 'عمان.icom.museum'
+  },
+  {
+    ascii: 'xn--wgbl6a.icom.museum',
+    unicode: 'قطر.icom.museum'
+  },
+  {
+    ascii: 'xn--romnia-yta.icom.museum',
+    unicode: 'românia.icom.museum'
+  },
+  {
+    ascii: 'xn--h1alffa9f.xn--h1aegh.museum',
+    unicode: 'россия.иком.museum'
+  },
+  {
+    ascii: 'xn--80aaabm1ab4blmeec9e7n.xn--h1aegh.museum',
+    unicode: 'србијаицрнагора.иком.museum'
+  },
+  {
+    ascii: 'xn--xkc2al3hye2a.icom.museum',
+    unicode: 'இலங்கை.icom.museum'
+  },
+  {
+    ascii: 'xn--espaa-rta.icom.museum',
+    unicode: 'españa.icom.museum'
+  },
+  {
+    ascii: 'xn--o3cw4h.icom.museum',
+    unicode: 'ไทย.icom.museum'
+  },
+  {
+    ascii: 'xn--pgbs0dh.icom.museum',
+    unicode: 'تونس.icom.museum'
+  },
+  {
+    ascii: 'xn--trkiye-3ya.icom.museum',
+    unicode: 'türkiye.icom.museum'
+  },
+  {
+    ascii: 'xn--80aaxgrpt.icom.museum',
+    unicode: 'украина.icom.museum'
+  },
+  {
+    ascii: 'xn--vitnam-jk8b.icom.museum',
+    unicode: 'việtnam.icom.museum'
+  },
+  // long label
+  {
+    ascii: `${'a'.repeat(64)}.com`,
+    unicode: `${'a'.repeat(64)}.com`,
+  },
+  // long URL
+  {
+    ascii: `${`${'a'.repeat(64)}.`.repeat(4)}com`,
+    unicode: `${`${'a'.repeat(64)}.`.repeat(4)}com`
+  },
+  // URLs with hyphen
+  {
+    ascii: 'r4---sn-a5mlrn7s.gevideo.com',
+    unicode: 'r4---sn-a5mlrn7s.gevideo.com'
+  },
+  {
+    ascii: '-sn-a5mlrn7s.gevideo.com',
+    unicode: '-sn-a5mlrn7s.gevideo.com'
+  },
+  {
+    ascii: 'sn-a5mlrn7s-.gevideo.com',
+    unicode: 'sn-a5mlrn7s-.gevideo.com'
+  },
+  {
+    ascii: '-sn-a5mlrn7s-.gevideo.com',
+    unicode: '-sn-a5mlrn7s-.gevideo.com'
+  },
+  {
+    ascii: '-sn--a5mlrn7s-.gevideo.com',
+    unicode: '-sn--a5mlrn7s-.gevideo.com'
+  }
+];
diff --git a/test/fixtures/url-toascii.js b/test/fixtures/url-toascii.js
new file mode 100644
index 0000000000..ea5e0f22ba
--- /dev/null
+++ b/test/fixtures/url-toascii.js
@@ -0,0 +1,156 @@
+'use strict';
+
+/* WPT Refs:
+   https://github.com/w3c/web-platform-tests/blob/4839a0a804/url/toascii.json
+   License: http://www.w3.org/Consortium/Legal/2008/04-testsuite-copyright.html
+*/
+module.exports =
+[
+  "This resource is focused on highlighting issues with UTS #46 ToASCII",
+  {
+    "comment": "Label with hyphens in 3rd and 4th position",
+    "input": "aa--",
+    "output": "aa--"
+  },
+  {
+    "input": "a†--",
+    "output": "xn--a---kp0a"
+  },
+  {
+    "input": "ab--c",
+    "output": "ab--c"
+  },
+  {
+    "comment": "Label with leading hyphen",
+    "input": "-x",
+    "output": "-x"
+  },
+  {
+    "input": "-†",
+    "output": "xn----xhn"
+  },
+  {
+    "input": "-x.xn--nxa",
+    "output": "-x.xn--nxa"
+  },
+  {
+    "input": "-x.β",
+    "output": "-x.xn--nxa"
+  },
+  {
+    "comment": "Label with trailing hyphen",
+    "input": "x-.xn--nxa",
+    "output": "x-.xn--nxa"
+  },
+  {
+    "input": "x-.β",
+    "output": "x-.xn--nxa"
+  },
+  {
+    "comment": "Empty labels",
+    "input": "x..xn--nxa",
+    "output": "x..xn--nxa"
+  },
+  {
+    "input": "x..β",
+    "output": "x..xn--nxa"
+  },
+  {
+    "comment": "Invalid Punycode",
+    "input": "xn--a",
+    "output": null
+  },
+  {
+    "input": "xn--a.xn--nxa",
+    "output": null
+  },
+  {
+    "input": "xn--a.β",
+    "output": null
+  },
+  {
+    "comment": "Valid Punycode",
+    "input": "xn--nxa.xn--nxa",
+    "output": "xn--nxa.xn--nxa"
+  },
+  {
+    "comment": "Mixed",
+    "input": "xn--nxa.β",
+    "output": "xn--nxa.xn--nxa"
+  },
+  {
+    "input": "ab--c.xn--nxa",
+    "output": "ab--c.xn--nxa"
+  },
+  {
+    "input": "ab--c.β",
+    "output": "ab--c.xn--nxa"
+  },
+  {
+    "comment": "CheckJoiners is true",
+    "input": "\u200D.example",
+    "output": null
+  },
+  {
+    "input": "xn--1ug.example",
+    "output": null
+  },
+  {
+    "comment": "CheckBidi is true",
+    "input": "يa",
+    "output": null
+  },
+  {
+    "input": "xn--a-yoc",
+    "output": null
+  },
+  {
+    "comment": "processing_option is Nontransitional_Processing",
+    "input": "ශ්‍රී",
+    "output": "xn--10cl1a0b660p"
+  },
+  {
+    "input": "نامه‌ای",
+    "output": "xn--mgba3gch31f060k"
+  },
+  {
+    "comment": "U+FFFD",
+    "input": "\uFFFD.com",
+    "output": null
+  },
+  {
+    "comment": "U+FFFD character encoded in Punycode",
+    "input": "xn--zn7c.com",
+    "output": null
+  },
+  {
+    "comment": "Label longer than 63 code points",
+    "input": "x01234567890123456789012345678901234567890123456789012345678901x",
+    "output": "x01234567890123456789012345678901234567890123456789012345678901x"
+  },
+  {
+    "input": "x01234567890123456789012345678901234567890123456789012345678901†",
+    "output": "xn--x01234567890123456789012345678901234567890123456789012345678901-6963b"
+  },
+  {
+    "input": "x01234567890123456789012345678901234567890123456789012345678901x.xn--nxa",
+    "output": "x01234567890123456789012345678901234567890123456789012345678901x.xn--nxa"
+  },
+  {
+    "input": "x01234567890123456789012345678901234567890123456789012345678901x.β",
+    "output": "x01234567890123456789012345678901234567890123456789012345678901x.xn--nxa"
+  },
+  {
+    "comment": "Domain excluding TLD longer than 253 code points",
+    "input": "01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.0123456789012345678901234567890123456789012345678.x",
+    "output": "01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.0123456789012345678901234567890123456789012345678.x"
+  },
+  {
+    "input": "01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.0123456789012345678901234567890123456789012345678.xn--nxa",
+    "output": "01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.0123456789012345678901234567890123456789012345678.xn--nxa"
+  },
+  {
+    "input": "01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.0123456789012345678901234567890123456789012345678.β",
+    "output": "01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.01234567890123456789012345678901234567890123456789.0123456789012345678901234567890123456789012345678.xn--nxa"
+  }
+]