1 // Copyright 2009 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/strings/utf_string_conversion_utils.h"
6
7 #include "base/third_party/icu/icu_utf.h"
8 #include "build/build_config.h"
9
10 namespace base {
11
12 // CountUnicodeCharacters ------------------------------------------------------
13
CountUnicodeCharacters(std::string_view text,size_t limit)14 std::optional<size_t> CountUnicodeCharacters(std::string_view text,
15 size_t limit) {
16 base_icu::UChar32 unused = 0;
17 size_t count = 0;
18 for (size_t index = 0; count < limit && index < text.size();
19 ++count, ++index) {
20 if (!ReadUnicodeCharacter(text.data(), text.size(), &index, &unused)) {
21 return std::nullopt;
22 }
23 }
24 return count;
25 }
26
27 // ReadUnicodeCharacter --------------------------------------------------------
28
ReadUnicodeCharacter(const char * src,size_t src_len,size_t * char_index,base_icu::UChar32 * code_point_out)29 bool ReadUnicodeCharacter(const char* src,
30 size_t src_len,
31 size_t* char_index,
32 base_icu::UChar32* code_point_out) {
33 base_icu::UChar32 code_point;
34 CBU8_NEXT(reinterpret_cast<const uint8_t*>(src), *char_index, src_len,
35 code_point);
36 *code_point_out = code_point;
37
38 // The ICU macro above moves to the next char, we want to point to the last
39 // char consumed.
40 (*char_index)--;
41
42 // Validate the decoded value.
43 return IsValidCodepoint(code_point);
44 }
45
ReadUnicodeCharacter(const char16_t * src,size_t src_len,size_t * char_index,base_icu::UChar32 * code_point)46 bool ReadUnicodeCharacter(const char16_t* src,
47 size_t src_len,
48 size_t* char_index,
49 base_icu::UChar32* code_point) {
50 if (CBU16_IS_SURROGATE(src[*char_index])) {
51 if (!CBU16_IS_SURROGATE_LEAD(src[*char_index]) || !src_len ||
52 *char_index >= src_len - 1 || !CBU16_IS_TRAIL(src[*char_index + 1])) {
53 // Invalid surrogate pair.
54 return false;
55 }
56
57 // Valid surrogate pair.
58 *code_point = CBU16_GET_SUPPLEMENTARY(src[*char_index],
59 src[*char_index + 1]);
60 (*char_index)++;
61 } else {
62 // Not a surrogate, just one 16-bit word.
63 *code_point = src[*char_index];
64 }
65
66 return IsValidCodepoint(*code_point);
67 }
68
69 #if defined(WCHAR_T_IS_32_BIT)
ReadUnicodeCharacter(const wchar_t * src,size_t src_len,size_t * char_index,base_icu::UChar32 * code_point)70 bool ReadUnicodeCharacter(const wchar_t* src,
71 size_t src_len,
72 size_t* char_index,
73 base_icu::UChar32* code_point) {
74 // Conversion is easy since the source is 32-bit.
75 *code_point = static_cast<base_icu::UChar32>(src[*char_index]);
76
77 // Validate the value.
78 return IsValidCodepoint(*code_point);
79 }
80 #endif // defined(WCHAR_T_IS_32_BIT)
81
82 // WriteUnicodeCharacter -------------------------------------------------------
83
WriteUnicodeCharacter(base_icu::UChar32 code_point,std::string * output)84 size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
85 std::string* output) {
86 if (code_point >= 0 && code_point <= 0x7f) {
87 // Fast path the common case of one byte.
88 output->push_back(static_cast<char>(code_point));
89 return 1;
90 }
91
92 // CBU8_APPEND_UNSAFE can append up to 4 bytes.
93 size_t char_offset = output->length();
94 size_t original_char_offset = char_offset;
95 output->resize(char_offset + CBU8_MAX_LENGTH);
96
97 CBU8_APPEND_UNSAFE(reinterpret_cast<uint8_t*>(output->data()), char_offset,
98 code_point);
99
100 // CBU8_APPEND_UNSAFE will advance our pointer past the inserted character, so
101 // it will represent the new length of the string.
102 output->resize(char_offset);
103 return char_offset - original_char_offset;
104 }
105
WriteUnicodeCharacter(base_icu::UChar32 code_point,std::u16string * output)106 size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
107 std::u16string* output) {
108 if (CBU16_LENGTH(code_point) == 1) {
109 // The code point is in the Basic Multilingual Plane (BMP).
110 output->push_back(static_cast<char16_t>(code_point));
111 return 1;
112 }
113 // Non-BMP characters use a double-character encoding.
114 size_t char_offset = output->length();
115 output->resize(char_offset + CBU16_MAX_LENGTH);
116 CBU16_APPEND_UNSAFE(&(*output)[0], char_offset, code_point);
117 return CBU16_MAX_LENGTH;
118 }
119
120 // Generalized Unicode converter -----------------------------------------------
121
122 template<typename CHAR>
PrepareForUTF8Output(const CHAR * src,size_t src_len,std::string * output)123 void PrepareForUTF8Output(const CHAR* src,
124 size_t src_len,
125 std::string* output) {
126 output->clear();
127 if (src_len == 0)
128 return;
129 if (src[0] < 0x80) {
130 // Assume that the entire input will be ASCII.
131 output->reserve(src_len);
132 } else {
133 // Assume that the entire input is non-ASCII and will have 3 bytes per char.
134 output->reserve(src_len * 3);
135 }
136 }
137
138 // Instantiate versions we know callers will need.
139 #if !BUILDFLAG(IS_WIN)
140 // wchar_t and char16_t are the same thing on Windows.
141 template void PrepareForUTF8Output(const wchar_t*, size_t, std::string*);
142 #endif
143 template void PrepareForUTF8Output(const char16_t*, size_t, std::string*);
144
145 template<typename STRING>
PrepareForUTF16Or32Output(const char * src,size_t src_len,STRING * output)146 void PrepareForUTF16Or32Output(const char* src,
147 size_t src_len,
148 STRING* output) {
149 output->clear();
150 if (src_len == 0)
151 return;
152 if (static_cast<unsigned char>(src[0]) < 0x80) {
153 // Assume the input is all ASCII, which means 1:1 correspondence.
154 output->reserve(src_len);
155 } else {
156 // Otherwise assume that the UTF-8 sequences will have 2 bytes for each
157 // character.
158 output->reserve(src_len / 2);
159 }
160 }
161
162 // Instantiate versions we know callers will need.
163 #if !BUILDFLAG(IS_WIN)
164 // std::wstring and std::u16string are the same thing on Windows.
165 template void PrepareForUTF16Or32Output(const char*, size_t, std::wstring*);
166 #endif
167 template void PrepareForUTF16Or32Output(const char*, size_t, std::u16string*);
168
169 } // namespace base
170