OLD | NEW |
---|---|
1 // Copyright 2014 the V8 project authors. All rights reserved. | 1 // Copyright 2014 the V8 project authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 #ifdef V8_I18N_SUPPORT | 6 #ifdef V8_I18N_SUPPORT |
7 #include "src/runtime/runtime-utils.h" | 7 #include "src/runtime/runtime-utils.h" |
8 | 8 |
9 #include "src/api.h" | 9 #include "src/api.h" |
10 #include "src/api-natives.h" | 10 #include "src/api-natives.h" |
(...skipping 11 matching lines...) Expand all Loading... | |
22 #include "unicode/dcfmtsym.h" | 22 #include "unicode/dcfmtsym.h" |
23 #include "unicode/decimfmt.h" | 23 #include "unicode/decimfmt.h" |
24 #include "unicode/dtfmtsym.h" | 24 #include "unicode/dtfmtsym.h" |
25 #include "unicode/dtptngen.h" | 25 #include "unicode/dtptngen.h" |
26 #include "unicode/locid.h" | 26 #include "unicode/locid.h" |
27 #include "unicode/numfmt.h" | 27 #include "unicode/numfmt.h" |
28 #include "unicode/numsys.h" | 28 #include "unicode/numsys.h" |
29 #include "unicode/rbbi.h" | 29 #include "unicode/rbbi.h" |
30 #include "unicode/smpdtfmt.h" | 30 #include "unicode/smpdtfmt.h" |
31 #include "unicode/timezone.h" | 31 #include "unicode/timezone.h" |
32 #include "unicode/translit.h" | |
32 #include "unicode/uchar.h" | 33 #include "unicode/uchar.h" |
33 #include "unicode/ucol.h" | 34 #include "unicode/ucol.h" |
34 #include "unicode/ucurr.h" | 35 #include "unicode/ucurr.h" |
35 #include "unicode/uloc.h" | 36 #include "unicode/uloc.h" |
37 #include "unicode/unistr.h" | |
36 #include "unicode/unum.h" | 38 #include "unicode/unum.h" |
37 #include "unicode/uversion.h" | 39 #include "unicode/uversion.h" |
38 | 40 |
39 | 41 |
40 namespace v8 { | 42 namespace v8 { |
41 namespace internal { | 43 namespace internal { |
42 | 44 |
43 RUNTIME_FUNCTION(Runtime_CanonicalizeLanguageTag) { | 45 RUNTIME_FUNCTION(Runtime_CanonicalizeLanguageTag) { |
44 HandleScope scope(isolate); | 46 HandleScope scope(isolate); |
45 Factory* factory = isolate->factory(); | 47 Factory* factory = isolate->factory(); |
(...skipping 696 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
742 } else if (status >= UBRK_WORD_LETTER && status < UBRK_WORD_LETTER_LIMIT) { | 744 } else if (status >= UBRK_WORD_LETTER && status < UBRK_WORD_LETTER_LIMIT) { |
743 return *isolate->factory()->NewStringFromStaticChars("letter"); | 745 return *isolate->factory()->NewStringFromStaticChars("letter"); |
744 } else if (status >= UBRK_WORD_KANA && status < UBRK_WORD_KANA_LIMIT) { | 746 } else if (status >= UBRK_WORD_KANA && status < UBRK_WORD_KANA_LIMIT) { |
745 return *isolate->factory()->NewStringFromStaticChars("kana"); | 747 return *isolate->factory()->NewStringFromStaticChars("kana"); |
746 } else if (status >= UBRK_WORD_IDEO && status < UBRK_WORD_IDEO_LIMIT) { | 748 } else if (status >= UBRK_WORD_IDEO && status < UBRK_WORD_IDEO_LIMIT) { |
747 return *isolate->factory()->NewStringFromStaticChars("ideo"); | 749 return *isolate->factory()->NewStringFromStaticChars("ideo"); |
748 } else { | 750 } else { |
749 return *isolate->factory()->NewStringFromStaticChars("unknown"); | 751 return *isolate->factory()->NewStringFromStaticChars("unknown"); |
750 } | 752 } |
751 } | 753 } |
754 | |
755 namespace { | |
756 void ConvertCaseWithTransliterator(icu::UnicodeString* input, | |
757 const char* transliterator_id) { | |
758 UErrorCode status = U_ZERO_ERROR; | |
759 base::SmartPointer<icu::Transliterator> translit( | |
760 icu::Transliterator::createInstance( | |
761 icu::UnicodeString(transliterator_id, -1, US_INV), UTRANS_FORWARD, | |
762 status)); | |
763 if (U_FAILURE(status)) return; | |
764 translit->transliterate(*input); | |
765 } | |
766 | |
767 MUST_USE_RESULT Object* LocaleConvertCase(Handle<String> s, Isolate* isolate, | |
768 bool is_to_upper, const char* lang) { | |
769 int32_t src_length = s->length(); | |
770 const UChar* src = nullptr; | |
771 | |
772 base::SmartArrayPointer<uc16> sap; | |
773 if (s->IsOneByteRepresentationUnderneath()) { | |
774 sap = s->ToWideCString(); | |
775 src = reinterpret_cast<const UChar*>(sap.get()); | |
776 } | |
777 | |
778 // Greek uppercasing has to be done via transliteration. | |
779 // TODO(jshin): Drop this special-casing once ICU's regular case conversion | |
780 // API supports Greek uppercasing. See | |
781 // http://bugs.icu-project.org/trac/ticket/10582 . | |
782 // ICU's C API for transliteration is nasty and we just use C++ API. | |
783 if (V8_UNLIKELY(!strncmp(lang, "el", 2) && is_to_upper)) { | |
784 icu::UnicodeString converted; | |
785 { | |
786 DisallowHeapAllocation no_gc; | |
787 String::FlatContent flat = s->GetFlatContent(); | |
Yang
2016/04/27 08:06:45
I don't see the string being flattened anywhere le
jungshik at Google
2016/04/28 10:50:09
Thank you for pointing this out. Actually, two of
Yang
2016/04/28 12:52:26
I did check the description of fastCopyFrom. In ei
| |
788 if (src == nullptr) { | |
789 DCHECK(flat.IsTwoByte()); | |
790 src = reinterpret_cast<const UChar*>(flat.ToUC16Vector().start()); | |
791 } | |
792 // Starts with the source string and will be replaced by the converted | |
793 // result. | |
794 converted.fastCopyFrom(icu::UnicodeString(false, src, src_length)); | |
795 ConvertCaseWithTransliterator(&converted, "el-Upper"); | |
796 } | |
797 Handle<String> result; | |
798 ASSIGN_RETURN_FAILURE_ON_EXCEPTION( | |
799 isolate, result, | |
800 isolate->factory()->NewStringFromTwoByte(Vector<const uint16_t>( | |
Yang
2016/04/27 08:06:45
This means that any case conversion with "problema
jungshik at Google
2016/04/28 10:50:10
That's a good point. If toLocale{L,U}Case(<no argu
| |
801 reinterpret_cast<const uint16_t*>(converted.getBuffer()), | |
802 converted.length()))); | |
803 return *result; | |
804 } | |
805 | |
806 auto case_converter = is_to_upper ? u_strToUpper : u_strToLower; | |
807 | |
808 int32_t dest_length = src_length; | |
809 UErrorCode error; | |
810 Handle<SeqTwoByteString> result; | |
811 | |
812 // This is not a real loop. It'll be executed only once (no overflow) or | |
813 // twice (overflow). | |
814 for (int i = 0; i < 2; ++i) { | |
815 result = | |
816 isolate->factory()->NewRawTwoByteString(dest_length).ToHandleChecked(); | |
817 DisallowHeapAllocation no_gc; | |
818 String::FlatContent flat = s->GetFlatContent(); | |
Yang
2016/04/27 08:06:45
We need to make sure s is flattened at this point
jungshik at Google
2016/04/28 10:50:09
Yup. See the comment above to the same question.
| |
819 // For OneByteString, |src| is already obtained with |sap| outside the loop. | |
820 if (flat.IsTwoByte()) | |
Yang
2016/04/27 08:06:45
add brackets here.
jungshik at Google
2016/04/28 10:50:10
Done.
| |
821 src = reinterpret_cast<const UChar*>(flat.ToUC16Vector().start()); | |
822 error = U_ZERO_ERROR; | |
823 dest_length = case_converter(reinterpret_cast<UChar*>(result->GetChars()), | |
824 dest_length, src, src_length, lang, &error); | |
825 if (error != U_BUFFER_OVERFLOW_ERROR) break; | |
826 } | |
827 | |
828 // In most cases, the output will fill the destination buffer completely | |
829 // leading to an unterminated string (U_STRING_NOT_TERMINATED_WARNING). | |
830 // Only in rare cases, it'll be shorter than the destination buffer and | |
831 // |result| has to be truncated. | |
832 DCHECK(U_SUCCESS(error)); | |
833 // dest_length == result->length() | |
Yang
2016/04/27 08:06:45
Can we make this a DCHECK?
jungshik at Google
2016/04/28 10:50:10
Done.
| |
834 if (V8_LIKELY(error == U_STRING_NOT_TERMINATED_WARNING)) return *result; | |
835 if (U_SUCCESS(error)) { | |
836 // dest_length < result->length() | |
Yang
2016/04/27 08:06:45
Also make this a DCHECK.
jungshik at Google
2016/04/28 10:50:10
Done.
| |
837 return *Handle<SeqTwoByteString>::cast( | |
838 SeqString::Truncate(result, dest_length)); | |
839 } | |
840 return *s; | |
841 } | |
842 | |
843 inline bool IsASCIIUpper(uint16_t ch) { return ch >= 'A' && ch <= 'Z'; } | |
844 | |
845 inline uint16_t ToLatin1Lower(uint16_t ch) { | |
846 return ch | | |
847 (((ch >= 'A' && ch <= 'Z') || (ch >= 0xC0 && ch <= 0xDE && ch != 0xD7)) | |
848 << 5); | |
849 } | |
850 | |
851 inline uint16_t ToASCIIUpper(uint16_t ch) { | |
852 return ch & ~((ch >= 'a' && ch <= 'z') << 5); | |
853 } | |
854 | |
855 // Does not work for U+00DF (sharp-s), U+00B5 (micron), U+00FF. | |
856 inline uint16_t ToLatin1Upper(uint16_t ch) { | |
857 DCHECK(ch != 0xDF && ch != 0xB5 && ch != 0xFF); | |
858 return ch & | |
859 ~(((ch >= 'a' && ch <= 'z') || (((ch & 0xE0) == 0xE0) && ch != 0xE7)) | |
860 << 5); | |
861 } | |
862 | |
863 template <typename Char> | |
864 bool ToUpperFastASCII(const Vector<const Char>& src, | |
865 Handle<SeqOneByteString> result) { | |
866 // Do a faster loop for the case where all the characters are ASCII. | |
867 uint16_t ored = 0; | |
868 int32_t index = 0; | |
869 for (auto it = src.begin(); it != src.end(); ++it) { | |
870 uint16_t ch = static_cast<uint16_t>(*it); | |
871 ored |= ch; | |
872 result->SeqOneByteStringSet(index++, ToASCIIUpper(ch)); | |
873 } | |
874 return !(ored & ~0x7F); | |
875 } | |
876 | |
877 const uint16_t sharp_s = 0xDF; | |
878 | |
879 template <typename Char> | |
880 bool ToUpperOneByte(const Vector<const Char>& src, | |
881 Handle<SeqOneByteString> result, int* sharp_s_count) { | |
882 // Still pretty-fast path for the input with non-ASCII Latin-1 characters. | |
883 | |
884 // There are two special cases. | |
885 // 1. U+00B5 and U+00FF are mapped to a character beyond U+00FF. | |
886 // 2. Lower case sharp-S converts to "SS" (two characters) | |
887 *sharp_s_count = 0; | |
888 int32_t index = 0; | |
889 for (auto it = src.begin(); it != src.end(); ++it) { | |
890 uint16_t ch = static_cast<uint16_t>(*it); | |
891 if (V8_UNLIKELY(ch == sharp_s)) { | |
892 ++(*sharp_s_count); | |
893 continue; | |
894 } | |
895 if (V8_UNLIKELY(ch == 0xB5 || ch == 0xFF)) { | |
896 // Since this upper-cased character does not fit in an 8-bit string, we | |
897 // need to take the 16-bit path. | |
898 return false; | |
899 } | |
900 result->SeqOneByteStringSet(index++, ToLatin1Upper(ch)); | |
901 } | |
902 | |
903 return true; | |
904 } | |
905 | |
906 template <typename Char> | |
907 void ToUpperWithSharpS(const Vector<const Char>& src, | |
908 Handle<SeqOneByteString> result) { | |
909 int32_t dest_index = 0; | |
910 for (auto it = src.begin(); it != src.end(); ++it) { | |
911 uint16_t ch = static_cast<uint16_t>(*it); | |
912 if (ch == sharp_s) { | |
913 result->SeqOneByteStringSet(dest_index++, 'S'); | |
914 result->SeqOneByteStringSet(dest_index++, 'S'); | |
915 } else { | |
916 result->SeqOneByteStringSet(dest_index++, ToLatin1Upper(ch)); | |
917 } | |
918 } | |
919 } | |
920 | |
921 } // namespace | |
922 | |
923 RUNTIME_FUNCTION(Runtime_StringToLowerCaseI18N) { | |
924 HandleScope scope(isolate); | |
925 DCHECK_EQ(args.length(), 1); | |
926 CONVERT_ARG_HANDLE_CHECKED(String, s, 0); | |
927 | |
928 int length = s->length(); | |
929 s = String::Flatten(s); | |
930 // First scan the string for uppercase and non-ASCII characters: | |
931 if (s->HasOnlyOneByteChars()) { | |
932 unsigned first_index_to_lower = length; | |
933 for (int index = 0; index < length; ++index) { | |
934 // Blink specializes this path for one-byte strings, so it | |
935 // does not need to do a generic get, but can do the equivalent | |
936 // of SeqOneByteStringGet. | |
937 uint16_t ch = s->Get(index); | |
938 if (V8_UNLIKELY(IsASCIIUpper(ch) || ch & ~0x7F)) { | |
939 first_index_to_lower = index; | |
940 break; | |
941 } | |
942 } | |
943 | |
944 // Nothing to do if the string is all ASCII with no uppercase. | |
945 if (first_index_to_lower == length) return *s; | |
946 | |
947 // We depend here on the invariant that the length of a Latin1 | |
948 // string is invariant under ToLowerCase, and the result always | |
949 // fits in the Latin1 range in the *root locale*. It does not hold | |
950 // for ToUpperCase even in the root locale. | |
951 Handle<SeqOneByteString> result; | |
952 ASSIGN_RETURN_FAILURE_ON_EXCEPTION( | |
953 isolate, result, isolate->factory()->NewRawOneByteString(length)); | |
954 | |
955 DisallowHeapAllocation no_gc; | |
956 String::FlatContent flat = s->GetFlatContent(); | |
957 if (flat.IsOneByte()) { | |
958 const uint8_t* src = flat.ToOneByteVector().start(); | |
959 CopyChars(result->GetChars(), src, first_index_to_lower); | |
960 for (int index = first_index_to_lower; index < length; ++index) { | |
961 uint16_t ch = static_cast<uint16_t>(src[index]); | |
962 result->SeqOneByteStringSet(index, ToLatin1Lower(ch)); | |
963 } | |
964 } else { | |
965 const uint16_t* src = flat.ToUC16Vector().start(); | |
966 CopyChars(result->GetChars(), src, first_index_to_lower); | |
967 for (int index = first_index_to_lower; index < length; ++index) { | |
968 uint16_t ch = src[index]; | |
969 result->SeqOneByteStringSet(index, ToLatin1Lower(ch)); | |
970 } | |
971 } | |
972 | |
973 return *result; | |
974 } | |
975 | |
976 // Blink had an additional case here for ASCII 2-byte strings, but | |
977 // that is subsumed by the above code (assuming there isn't a false | |
978 // negative for HasOnlyOneByteChars). | |
979 | |
980 // Do a slower implementation for cases that include non-ASCII characters. | |
981 return LocaleConvertCase(s, isolate, false, ""); | |
982 } | |
983 | |
984 RUNTIME_FUNCTION(Runtime_StringToUpperCaseI18N) { | |
985 HandleScope scope(isolate); | |
986 DCHECK_EQ(args.length(), 1); | |
987 CONVERT_ARG_HANDLE_CHECKED(String, s, 0); | |
988 | |
989 // This function could be optimized for no-op cases the way lowercase | |
990 // counterpart is, but in empirical testing, few actual calls to upper() | |
991 // are no-ops. So, it wouldn't be worth the extra time for pre-scanning. | |
992 | |
993 int32_t length = s->length(); | |
994 s = String::Flatten(s); | |
995 | |
996 if (s->HasOnlyOneByteChars()) { | |
997 Handle<SeqOneByteString> result; | |
998 ASSIGN_RETURN_FAILURE_ON_EXCEPTION( | |
999 isolate, result, isolate->factory()->NewRawOneByteString(length)); | |
1000 | |
1001 int sharp_s_count; | |
1002 bool is_result_single_byte; | |
1003 { | |
1004 DisallowHeapAllocation no_gc; | |
1005 String::FlatContent flat = s->GetFlatContent(); | |
1006 // If it was ok to slow down ASCII-only input slightly, ToUpperFastASCII | |
1007 // could be removed because ToUpperOneByte is pretty fast now (it | |
1008 // does not call ICU API any more.). | |
1009 if (flat.IsOneByte()) { | |
1010 Vector<const uint8_t> src = flat.ToOneByteVector(); | |
1011 if (ToUpperFastASCII(src, result)) return *result; | |
1012 is_result_single_byte = ToUpperOneByte(src, result, &sharp_s_count); | |
1013 } else { | |
1014 DCHECK(flat.IsTwoByte()); | |
1015 Vector<const uint16_t> src = flat.ToUC16Vector(); | |
1016 if (ToUpperFastASCII(src, result)) return *result; | |
1017 is_result_single_byte = ToUpperOneByte(src, result, &sharp_s_count); | |
1018 } | |
1019 } | |
1020 | |
1021 // Go to the full Unicode path if there are characters whose uppercase | |
1022 // is beyond the Latin-1 range (cannot be represented in OneByteString). | |
1023 if (V8_UNLIKELY(!is_result_single_byte)) | |
Yang
2016/04/27 08:06:45
add brackets please.
jungshik at Google
2016/04/28 10:50:10
Done.
| |
1024 return LocaleConvertCase(s, isolate, true, ""); | |
1025 | |
1026 if (sharp_s_count == 0) return *result; | |
1027 | |
1028 // We have sharp_s_count sharp-s characters, but the result is still | |
1029 // in the Latin-1 range. | |
1030 ASSIGN_RETURN_FAILURE_ON_EXCEPTION( | |
1031 isolate, result, | |
1032 isolate->factory()->NewRawOneByteString(length + sharp_s_count)); | |
1033 DisallowHeapAllocation no_gc; | |
1034 String::FlatContent flat = s->GetFlatContent(); | |
1035 if (flat.IsOneByte()) | |
Yang
2016/04/27 08:06:45
brackets
jungshik at Google
2016/04/28 10:50:10
Done.
| |
1036 ToUpperWithSharpS(flat.ToOneByteVector(), result); | |
1037 else | |
1038 ToUpperWithSharpS(flat.ToUC16Vector(), result); | |
1039 | |
1040 return *result; | |
1041 } | |
1042 | |
1043 return LocaleConvertCase(s, isolate, true, ""); | |
1044 } | |
1045 | |
1046 RUNTIME_FUNCTION(Runtime_StringLocaleConvertCase) { | |
1047 HandleScope scope(isolate); | |
1048 DCHECK_EQ(args.length(), 3); | |
1049 CONVERT_ARG_HANDLE_CHECKED(String, s, 0); | |
1050 CONVERT_BOOLEAN_ARG_CHECKED(is_upper, 1); | |
1051 CONVERT_ARG_HANDLE_CHECKED(SeqOneByteString, lang, 2); | |
1052 | |
1053 // All the languages requiring special handling ("az", "el", "lt", "tr") | |
1054 // have a 2-letter language code. | |
1055 DCHECK(lang->length() == 2); | |
1056 uint8_t lang_str[3]; | |
1057 memcpy(lang_str, lang->GetChars(), 2); | |
1058 lang_str[2] = 0; | |
1059 return LocaleConvertCase(s, isolate, is_upper, | |
1060 reinterpret_cast<const char*>(lang_str)); | |
1061 } | |
1062 | |
752 } // namespace internal | 1063 } // namespace internal |
753 } // namespace v8 | 1064 } // namespace v8 |
754 | 1065 |
755 #endif // V8_I18N_SUPPORT | 1066 #endif // V8_I18N_SUPPORT |
OLD | NEW |