Apply ICU patches for ICU tickets 6175 (ISO-2022 and ...

third_party/icu38/source/test/testdata/conversion.txt

 //*******************************************************************************
 //
-//   Copyright (C) 2003-2007, International Business Machines
+//   Copyright (C) 2003-2008, International Business Machines
 //   Corporation and others.  All Rights Reserved.
 //
 //   file name:  conversion.txt
 //   encoding:   US-ASCII
 //   tab size:   8 (not used)
 //   indentation:4
 //
 //   created on: 2003jul15
 //   created by: Markus W. Scherer
 //
@@ skipping 141 matching lines @@
           :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 },
           :int{1}, :int{0}, "", "&C", :bin{""}
         }
         {
           "ISO-2022-CN-EXT",
           :bin{ 4f1b4f4f1b242b494f1b4f4f4f4f1b204f },
           "O\\x1BOOO\u492bO\\x1B O",
           :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 },
           :int{1}, :int{0}, "", "&C", :bin{""}
         }
+        // Test ticket 5691: HZ with illegal tilde sequences.
+        {
+          "HZ",
+          :bin{ 417e20427e21437e80447e7b41417e207e41427e7f41437e7d5a },
+          "A\\x7E B\\x7E!C\\x7E\\x80D\u4eae\\x7E\\x20\\x7E\u8c05\\x7E\\x7F\u64a9Z",
+          :intvector{ 0,1,1,1,1,2,3,4,4,4,4,5,6,7,7,7,7,7,7,7,7,9,                          // SBCS
+                      12,14,14,14,14,14,14,14,14,16,16,16,16,17,19,19,19,19,19,19,19,19,21, // DBCS
+                      25 },                                                                 // SBCS
+          :int{1}, :int{0}, "", "&C", :bin{""}
+        }
         // Test ticket 5691: Example from Peter Edberg.
         {
           "ISO-2022-JP",
           :bin{ 1b244230212f7e742630801b284a621b2458631b2842648061 },
           "\u4e9c\ufffd\u7199\ufffdb\ufffd$Xcd\ufffda",
           :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 },
           :int{1}, :int{0}, "", "?", :bin{""}
         }
+        // Test bug 6071 (2:1 Unicode:charset SBCS mapping).
+        {
+          "*test1bmp",
+          :bin{ 050008 },
+          "e@uv",
+          :intvector{ 0,1,2,2 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e
+        {
+          "HZ",
+          :bin{ 7e7b21212120217e217f772100007e217e7e7d207e7e807e0a2b },
+          "\u3000\ufffd\u3013\ufffd\u9ccc\ufffd\ufffd\u3013 ~\ufffd+",
+          :intvector{ 2,4,6,8,10,12,14,15,19,20,22,25 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
         // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and
         // using the Shift-JIS table for JIS X 0208 (ticket #5797)
         {
           "ISO-2022-JP",
           :bin{ 1b284a7d7e801b2442306c20217f7e21202160217f22202225227f5f211b2842 },
           "}\u203e\ufffd\u4e00\ufffd\ufffd\ufffd\ufffd\xf7\ufffd\ufffd\u25b2\ufffd\u6f3e",
           :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 },
           :int{1}, :int{1}, "", "?", :bin{""}
         }
         // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets()
@@ skipping 123 matching lines @@
           :int{1}, :int{1}, "", ".", :bin{""}
         }
         // more complicated example
         {
           "ISO-2022-KR",
           :bin{ 411b242943420e4141affe0f43 },
           "AB\uc88b%XAF%XFEC",
           :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 },
           :int{1}, :int{1}, "", "&", :bin{""}
         }
+        // empty segment (using substitution and stop)
+        {
+          "ISO-2022-KR",
+          :bin{ 1b242943610e0f620d0a },
+          "a\uFFFDb\u000D\u000A",
+          :intvector{ 4, 6, 7, 8, 9 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "ISO-2022-KR",
+          :bin{ 1b242943610e0f620d0a },
+          "a",
+          :intvector{ 4 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"0f"}
+        }
 
         // ISO-2022-JP
 
         // truncated, partial escape sequence
         {
           "ISO-2022-JP",
           :bin{ 1b }, "", :intvector{},
           :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
         }
         {
@@ skipping 30 matching lines @@
           "JIS7",
           :bin{ 41420e41420f4142 }, "AB\uff81\uff82AB", :intvector{ 0, 1, 3, 4, 6, 7 },
           :int{1}, :int{1}, "", ".", :bin{""}
         }
         // JIS8 with Katakana
         {
           "JIS8",
           :bin{ 41c15c1b284a5cc242 }, "A\uff81\\\xa5\uff82B", :intvector{ 0, 1, 2, 6, 7, 8 },
           :int{1}, :int{1}, "", ".", :bin{""}
         }
+        // empty segment (using substitution and stop)
+        {
+          "ISO-2022-JP",
+          :bin{ 61621b24421b284263640d0a },
+          "ab\uFFFDcd\u000D\u000A",
+          :intvector{ 0, 1, 5, 8, 9, 10, 11 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "ISO-2022-JP",
+          :bin{ 61621b24421b284263640d0a },
+          "ab",
+          :intvector{ 0, 1 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"1b2842"}
+        }
 
         // ISO-2022-CN
 
         // truncated, partial escape sequence
         {
           "ISO_2022,locale=zh,version=1",
           :bin{ 1b }, "", :intvector{},
           :int{1}, :int{1}, "truncated", ".", :bin{ 1b }
         }
         {
@@ skipping 50 matching lines @@
           "ISO-2022-CN-EXT",
           :bin{ 411b4e2121 }, "\x41", :intvector{ 0 },
           :int{1}, :int{1}, "illesc", ".", :bin{ 1b }
         }
         // G3 designator: recognized, but not supported for -CN (only for -CN-EXT)
         {
           "ISO-2022-CN",
           :bin{ 411b242b491b4f2121 }, "\x41", :intvector{ 0 },
           :int{1}, :int{1}, "unsuppesc", ".", :bin{ 1b242b49 }
         }
+        // empty segment 1 (using substitution and stop)
+        {
+          "ISO-2022-CN",
+          :bin{ 611b242941620e0f1b242a481b4e6a65630d0a },
+          "ab\uFFFD\u994Cc\u000D\u000A",
+          :intvector{ 0, 5, 7, 14, 16, 17, 18 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "ISO-2022-CN",
+          :bin{ 611b242941620e0f1b242a481b4e6a65630d0a },
+          "ab",
+          :intvector{ 0, 5 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"0f"}
+        }
+        // empty segment 2 (using substitution and stop)
+        {
+          "ISO-2022-CN",
+          :bin{ 611b242941620e1b24294768640f630d0a },
+          "ab\uFFFD\u5F70c\u000D\u000A",
+          :intvector{ 0, 5, 7, 11, 14, 15, 16 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "ISO-2022-CN",
+          :bin{ 611b242941620e1b24294768640f630d0a },
+          "ab",
+          :intvector{ 0, 5 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"1b242947"}
+        }
 
         // ISO-2022 SBCS
         // [U_ENABLE_GENERIC_ISO_2022]
         // The _generic_ ISO-2022 converter is disabled starting 2003-dec-03 (ICU 2.8).
         // For details see the icu mailing list from 2003-dec-01 and the ucnv2022.c file.
         // Language-specific variants of ISO-2022 continue to be available as listed below.
         //{
         //  "ISO_2022",
         //  :bin{ 0008090a0d1a1c1f203f415c7d7e7f },
         //  "\x00\x08\t\n\r\x1a\x1c\x1f ?A\\}~\x7f",
         //  :intvector{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 },
         //  :int{1}, :int{1}, "", ".", :bin{""}
         //}
 
+        // HZ-GB-2312
+
+        // empty segment 1 (using substitution and stop)
+        {
+          "HZ-GB-2312",
+          :bin{ 61627e7b7e7d6364 },
+          "ab\uFFFDcd",
+          :intvector{ 0, 1, 4, 6, 7 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "HZ-GB-2312",
+          :bin{ 61627e7b7e7d63640d0a },
+          "ab",
+          :intvector{ 0, 1 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"}
+        }
+        // empty segment 2 & legal redundant switches (using substitution and stop)
+        {
+          "HZ-GB-2312",
+          :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d },
+          "ab\u4E0D\u7A7A\uFFFD\u4E00cdef\uFFFD",
+          :intvector{ 0, 1, 4, 6, 10, 12, 16, 17, 20, 21, 24 },
+          :int{1}, :int{1}, "", "?", :bin{""}
+        }
+        {
+          "HZ-GB-2312",
+          :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d },
+          "ab\u4E0D\u7A7A",
+          :intvector{ 0, 1, 4, 6 },
+          :int{1}, :int{1}, "illesc", ".", :bin{"7e7b"}
+        }
+
         // DBCS-only extensions
         {
           "ibm-970",
           :bin{ 617eece9b2eb },
           "\x61\x7e\u4e00\ub000",
           :intvector{ 0, 1, 2, 4 },
           :int{1}, :int{1}, "", "?", :bin{""}
         }
 
         {
@@ skipping 151 matching lines @@
           :intvector{ 0, 2 },
           :int{1}, :int{0}, "", "?", :bin{""}
         }
         { // Verify gb18030 enumeration
           "gb18030",
           :bin{ 8130D2398130D3308136A5318136A532 },
           "\u0450\u0452\u200F\u2011",
           :intvector{ 0, 4, 8, 12 },
           :int{1}, :int{0}, "", "?", :bin{""}
         }
+        // Test iso-2022-jp-2 miscellaneous symbols
+        {
+          "iso-2022-jp-2",
+          :bin{ 1b242843224f224e1b2842 },
+          "\u260E\u260F",
+          :intvector{ 4, 6 },
+          :int{1}, :int{0}, "", ".", :bin{""}
+        }
       }
     }
 
     // --------------------------------------------------------------------- ***
 
     fromUnicode {
       Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" }
       Cases {
+        // Test bug 6071 (1:2 Unicode:charset SBCS mapping).
+        {
+          "*test1bmp",
+          "e@t",
+          :bin{       05000709 },
+          :intvector{ 0,1,2,2 },
+          :int{1}, :int{0}, "", "?", ""
+        }
         // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and
         // using the Shift-JIS table for JIS X 0208 (ticket #5797)
         {
           "ISO-2022-JP",
           "\u203e\xa5\u4e00\ufa10\u6f3e\u0391",
           :bin{       1b284a7e5c1b2442306c222e5f2126211b2842 },
           :intvector{ 0,0,0,0,1,2,2,2,2,2,3,3,4,4,5,5,5,5,5 },
           :int{1}, :int{0}, "", "?=\u3013", ""  // U+3013 Geta Mark converts to 222e
         }
         // Verify that mappings that would result in byte values outside 20..7F (for SBCS)
@@ skipping 787 matching lines @@
         {
           "ibm-25546",
           "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac01\uffe6]",
           "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac02\uffe7-\U0010ffff]",
           :int{0}
         }
 
         // versions of ISO-2022-JP
         {
           "ISO-2022-JP",
-          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u203e\uff61-\uff9f\u4e00\u4e01\uffe5]",
-          "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\ufa0e-\ufa2d\uffe6-\U0010ffff]",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2015\u203e\u4e00\u4e01\uffe5]",
+          "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u2014\u301c\u4e02\u4e27-\u4e29\u4fe0\u663b\u9eb5\ufa0e-\ufa2d\uff61-\uff9f\uffe4\uffe6-\U0010ffff]",
           :int{0}
-        }   
+        }
         {
           "ISO-2022-JP-2",
-          "[\x00-\x0d\x10-\x1a\x1c-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\uff61-\uff9f\u4e00-\u4e05\uffe6]",
-          "[\x0e\x0f\x1b\uffe7-\U0010ffff]",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uffe6]",
+          "[\x0e\x0f\x1b\uff61-\uff9f\uffe4\uffe7-\U0010ffff]",
           :int{0}
         }
+        {
+          "JIS7",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uff61-\uff9f\uffe6]",
+          "[\x0e\x0f\x1b\uffe4\uffe7-\U0010ffff]",
+          :int{0}
+        }
+        // with fallbacks
+        {
+          "ISO-2022-JP",
+          "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2014\u2015\u203e\u301c\u4e00\u4e01\u4fe0\u9eb5\uff61-\uff9f\uffe5]",
+          "[\x0e\x0f\x1b\xa6\u0100-\u0113\u0385-\u038a\u4e02\u4e27-\u4e29\u663b\ufa0e-\ufa2d\uffe4\uffe6-\U0010ffff]",
+          :int{1}
+        }
 
         // versions of ISO-2022-CN
         {
           "ISO-2022-CN",
           "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00\u4e01\u9f98\ufe6b]",
           "[\x0e\x0f\x1b\u4e29\uffe6-\U0010ffff]",
           :int{0}
         }
         {
           "ISO-2022-CN-EXT",
           "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]",
           "[\x0e\x0f\x1b\U00020001\U00020002\U0002a6d7-\U0010ffff]",
           :int{0}
         }
 
+        // HZ
+        {
+          "HZ",
+          "[\u0410-\u044f\u4e00\u4e01\u4e03]",
+          "[\u4e02\u4e04-\u4e06\uac00-\ud7ff]",
+          :int{0}
+        }
+        
+        // LMBCS
+        {
+          "LMBCS",
+          "[\x00-\U0010ffff]",
+          "[]",
+          :int{0}
+        }
+
         // DBCS-only
         {
           "ibm-971",
           "[\xa1\xa4\uac01\ub000]",
           "[\x00-\x9f\u2015]",
           :int{0}
         }
 
         {
           "ibm-16684",
@@ skipping 16 matching lines @@
           "*test3",
           "[\x05\x0b\xc0\u20ac\U00023456\U00101234"
               "{\U00101234\U00050005\U00060006}{\U00101234\U00050005}{\U00101234\U00060006}{\xc4\xc4\U00101234\x05}]",
           "[\x06\x0e\U00034567\U000febcd{\U00101234\U00070007}]",
           :int{0}
         }
       }
     }
   }
 }