Merge pull request #676 from djc/idna2008

Implement support for reporting errors on invalid IDNA2008 characters 
Includes changes from Optimize IDNA tables #677

Author: valenting

idna/src/make_uts46_mapping_table.py

@@ -78,6 +78,12 @@ def rust_slice(s):
             unicode_str = u''.join(char(c) for c in fields[2].strip().split(' '))
         elif mapping == "Deviation":
             unicode_str = u''
+
+    if len(fields) > 3:
+        assert fields[3].strip() in ('NV8', 'XV8'), fields[3]
+        assert mapping == 'Valid', mapping
+        mapping = 'DisallowedIdna2008'
+
     ranges.append((first, last, mapping, unicode_str))
 
 def mergeable_key(r):
@@ -86,7 +92,7 @@ def mergeable_key(r):
     # These types have associated data, so we should not merge them.
     if mapping in ('Mapped', 'Deviation', 'DisallowedStd3Mapped'):
         return r
-    assert mapping in ('Valid', 'Ignored', 'Disallowed', 'DisallowedStd3Valid')
+    assert mapping in ('Valid', 'Ignored', 'Disallowed', 'DisallowedStd3Valid', 'DisallowedIdna2008')
     return mapping
 
 grouped_ranges = itertools.groupby(ranges, key=mergeable_key)
@@ -116,11 +122,7 @@ def mergeable_key(r):
         # Assert we're seeing the surrogate case here.
         assert last_char == 0xd7ff
         assert next_char == 0xe000
-    first = group[0][0]
-    last = group[-1][1]
-    mapping = group[0][2]
-    unicode_str = group[0][3]
-    optimized_ranges.append((first, last, mapping, unicode_str))
+    optimized_ranges.append((group[0][0], group[-1][1]) + group[0][2:])
 
 def is_single_char_range(r):
     (first, last, _, _) = r
@@ -148,30 +150,22 @@ def merge_single_char_ranges(ranges):
 
 optimized_ranges = list(merge_single_char_ranges(optimized_ranges))
 
-
-print("static TABLE: &[Range] = &[")
-
-for ranges in optimized_ranges:
-    first = ranges[0][0]
-    last = ranges[-1][1]
-    print("    Range { from: '%s', to: '%s', }," % (escape_char(char(first)),
-                                                            escape_char(char(last))))
-
-print("];\n")
-
-print("static INDEX_TABLE: &[u16] = &[")
-
 SINGLE_MARKER = 1 << 15
 
+print("static TABLE: &[(char, u16)] = &[")
+
 offset = 0
 for ranges in optimized_ranges:
     assert offset < SINGLE_MARKER
 
     block_len = len(ranges)
     single = SINGLE_MARKER if block_len == 1 else 0
-    print("    %s," % (offset | single))
+    index = offset | single
     offset += block_len
 
+    start = escape_char(char(ranges[0][0]))
+    print("    ('%s', %s)," % (start, index))
+
 print("];\n")
 
 print("static MAPPING_TABLE: &[Mapping] = &[")

idna/src/uts46.rs

@@ -11,7 +11,6 @@
 
 use self::Mapping::*;
 use crate::punycode;
-use std::cmp::Ordering::{Equal, Greater, Less};
 use std::{error::Error as StdError, fmt};
 use unicode_bidi::{bidi_class, BidiClass};
 use unicode_normalization::char::is_combining_mark;
@@ -48,38 +47,26 @@ enum Mapping {
     Disallowed,
     DisallowedStd3Valid,
     DisallowedStd3Mapped(StringTableSlice),
-}
-
-struct Range {
-    from: char,
-    to: char,
+    DisallowedIdna2008,
 }
 
 fn find_char(codepoint: char) -> &'static Mapping {
-    let r = TABLE.binary_search_by(|ref range| {
-        if codepoint > range.to {
-            Less
-        } else if codepoint < range.from {
-            Greater
-        } else {
-            Equal
-        }
-    });
-    r.ok()
-        .map(|i| {
-            const SINGLE_MARKER: u16 = 1 << 15;
+    let idx = match TABLE.binary_search_by_key(&codepoint, |&val| val.0) {
+        Ok(idx) => idx,
+        Err(idx) => idx - 1,
+    };
 
-            let x = INDEX_TABLE[i];
-            let single = (x & SINGLE_MARKER) != 0;
-            let offset = !SINGLE_MARKER & x;
+    const SINGLE_MARKER: u16 = 1 << 15;
 
-            if single {
-                &MAPPING_TABLE[offset as usize]
-            } else {
-                &MAPPING_TABLE[(offset + (codepoint as u16 - TABLE[i].from as u16)) as usize]
-            }
-        })
-        .unwrap()
+    let (base, x) = TABLE[idx];
+    let single = (x & SINGLE_MARKER) != 0;
+    let offset = !SINGLE_MARKER & x;
+
+    if single {
+        &MAPPING_TABLE[offset as usize]
+    } else {
+        &MAPPING_TABLE[(offset + (codepoint as u16 - base as u16)) as usize]
+    }
 }
 
 struct Mapper<'a> {
@@ -140,6 +127,12 @@ impl<'a> Iterator for Mapper<'a> {
                     self.slice = Some(decode_slice(slice).chars());
                     continue;
                 }
+                Mapping::DisallowedIdna2008 => {
+                    if self.config.use_idna_2008_rules {
+                        self.errors.disallowed_in_idna_2008 = true;
+                    }
+                    codepoint
+                }
             });
         }
     }
@@ -310,7 +303,7 @@ fn check_validity(label: &str, config: Config, errors: &mut Errors) {
 
     // V6: Check against Mapping Table
     if label.chars().any(|c| match *find_char(c) {
-        Mapping::Valid => false,
+        Mapping::Valid | Mapping::DisallowedIdna2008 => false,
         Mapping::Deviation(_) => config.transitional_processing,
         Mapping::DisallowedStd3Valid => config.use_std3_ascii_rules,
         _ => true,
@@ -510,6 +503,7 @@ pub struct Config {
     transitional_processing: bool,
     verify_dns_length: bool,
     check_hyphens: bool,
+    use_idna_2008_rules: bool,
 }
 
 /// The defaults are that of https://url.spec.whatwg.org/#idna
@@ -524,6 +518,7 @@ impl Default for Config {
 
             // Only use for to_ascii, not to_unicode
             verify_dns_length: false,
+            use_idna_2008_rules: false,
         }
     }
 }
@@ -553,6 +548,12 @@ impl Config {
         self
     }
 
+    #[inline]
+    pub fn use_idna_2008_rules(mut self, value: bool) -> Self {
+        self.use_idna_2008_rules = value;
+        self
+    }
+
     /// http://www.unicode.org/reports/tr46/#ToASCII
     pub fn to_ascii(self, domain: &str) -> Result<String, Errors> {
         let mut result = String::new();
@@ -599,6 +600,7 @@ pub struct Errors {
     disallowed_character: bool,
     too_long_for_dns: bool,
     too_short_for_dns: bool,
+    disallowed_in_idna_2008: bool,
 }
 
 impl Errors {
@@ -615,6 +617,7 @@ impl Errors {
             disallowed_character,
             too_long_for_dns,
             too_short_for_dns,
+            disallowed_in_idna_2008,
         } = *self;
         punycode
             || check_hyphens
@@ -627,6 +630,7 @@ impl Errors {
             || disallowed_character
             || too_long_for_dns
             || too_short_for_dns
+            || disallowed_in_idna_2008
     }
 }
 
@@ -644,6 +648,7 @@ impl fmt::Debug for Errors {
             disallowed_character,
             too_long_for_dns,
             too_short_for_dns,
+            disallowed_in_idna_2008,
         } = *self;
 
         let fields = [
@@ -661,6 +666,7 @@ impl fmt::Debug for Errors {
             ("disallowed_character", disallowed_character),
             ("too_long_for_dns", too_long_for_dns),
             ("too_short_for_dns", too_short_for_dns),
+            ("disallowed_in_idna_2008", disallowed_in_idna_2008),
         ];
 
         let mut empty = true;