1 // Copyright 2012 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/i18n/icu_string_conversions.h"
6
7 #include <stddef.h>
8 #include <stdint.h>
9
10 #include <memory>
11 #include <string_view>
12
13 #include "base/check.h"
14 #include "base/notreached.h"
15 #include "base/strings/string_util.h"
16 #include "base/strings/utf_string_conversions.h"
17 #include "third_party/icu/source/common/unicode/normalizer2.h"
18 #include "third_party/icu/source/common/unicode/ucnv.h"
19 #include "third_party/icu/source/common/unicode/ucnv_cb.h"
20 #include "third_party/icu/source/common/unicode/ucnv_err.h"
21 #include "third_party/icu/source/common/unicode/ustring.h"
22
23 namespace base {
24
25 namespace {
26 // ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
27 // in source/common/ucnv_err.c.
28
29 // Copyright (c) 1995-2006 International Business Machines Corporation
30 // and others
31 //
32 // All rights reserved.
33 //
34
35 // Permission is hereby granted, free of charge, to any person obtaining a
36 // copy of this software and associated documentation files (the "Software"),
37 // to deal in the Software without restriction, including without limitation
38 // the rights to use, copy, modify, merge, publish, distribute, and/or
39 // sell copies of the Software, and to permit persons to whom the Software
40 // is furnished to do so, provided that the above copyright notice(s) and
41 // this permission notice appear in all copies of the Software and that
42 // both the above copyright notice(s) and this permission notice appear in
43 // supporting documentation.
44 //
45 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
46 // EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
47 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
48 // OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
49 // INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
50 // OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
51 // OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
52 // OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
53 // OR PERFORMANCE OF THIS SOFTWARE.
54 //
55 // Except as contained in this notice, the name of a copyright holder
56 // shall not be used in advertising or otherwise to promote the sale, use
57 // or other dealings in this Software without prior written authorization
58 // of the copyright holder.
59
60 // ___________________________________________________________________________
61 //
62 // All trademarks and registered trademarks mentioned herein are the property
63 // of their respective owners.
64
ToUnicodeCallbackSubstitute(const void * context,UConverterToUnicodeArgs * to_args,const char * code_units,int32_t length,UConverterCallbackReason reason,UErrorCode * err)65 void ToUnicodeCallbackSubstitute(const void* context,
66 UConverterToUnicodeArgs *to_args,
67 const char* code_units,
68 int32_t length,
69 UConverterCallbackReason reason,
70 UErrorCode * err) {
71 static const UChar kReplacementChar = 0xFFFD;
72 if (reason <= UCNV_IRREGULAR) {
73 if (context == nullptr ||
74 (*(reinterpret_cast<const char*>(context)) == 'i' &&
75 reason == UCNV_UNASSIGNED)) {
76 *err = U_ZERO_ERROR;
77 ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
78 }
79 // else the caller must have set the error code accordingly.
80 }
81 // else ignore the reset, close and clone calls.
82 }
83
ConvertFromUTF16(UConverter * converter,std::u16string_view src,OnStringConversionError::Type on_error,std::string * encoded)84 bool ConvertFromUTF16(UConverter* converter,
85 std::u16string_view src,
86 OnStringConversionError::Type on_error,
87 std::string* encoded) {
88 int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(
89 src.length(), ucnv_getMaxCharSize(converter));
90 encoded->resize(encoded_max_length);
91
92 UErrorCode status = U_ZERO_ERROR;
93
94 // Setup our error handler.
95 switch (on_error) {
96 case OnStringConversionError::FAIL:
97 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, nullptr,
98 nullptr, nullptr, &status);
99 break;
100 case OnStringConversionError::SKIP:
101 case OnStringConversionError::SUBSTITUTE:
102 ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, nullptr,
103 nullptr, nullptr, &status);
104 break;
105 }
106
107 // ucnv_fromUChars returns size not including terminating null
108 int actual_size =
109 ucnv_fromUChars(converter, &(*encoded)[0], encoded_max_length, src.data(),
110 src.length(), &status);
111 encoded->resize(actual_size);
112 ucnv_close(converter);
113 if (U_SUCCESS(status))
114 return true;
115 encoded->clear(); // Make sure the output is empty on error.
116 return false;
117 }
118
119 // Set up our error handler for ToUTF-16 converters
SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,UConverter * converter,UErrorCode * status)120 void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
121 UConverter* converter, UErrorCode* status) {
122 switch (on_error) {
123 case OnStringConversionError::FAIL:
124 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, nullptr, nullptr,
125 nullptr, status);
126 break;
127 case OnStringConversionError::SKIP:
128 ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, nullptr, nullptr,
129 nullptr, status);
130 break;
131 case OnStringConversionError::SUBSTITUTE:
132 ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, nullptr,
133 nullptr, nullptr, status);
134 break;
135 }
136 }
137
138 } // namespace
139
140 // Codepage <-> Wide/UTF-16 ---------------------------------------------------
141
UTF16ToCodepage(std::u16string_view utf16,const char * codepage_name,OnStringConversionError::Type on_error,std::string * encoded)142 bool UTF16ToCodepage(std::u16string_view utf16,
143 const char* codepage_name,
144 OnStringConversionError::Type on_error,
145 std::string* encoded) {
146 encoded->clear();
147
148 UErrorCode status = U_ZERO_ERROR;
149 UConverter* converter = ucnv_open(codepage_name, &status);
150 if (!U_SUCCESS(status))
151 return false;
152
153 return ConvertFromUTF16(converter, utf16, on_error, encoded);
154 }
155
CodepageToUTF16(std::string_view encoded,const char * codepage_name,OnStringConversionError::Type on_error,std::u16string * utf16)156 bool CodepageToUTF16(std::string_view encoded,
157 const char* codepage_name,
158 OnStringConversionError::Type on_error,
159 std::u16string* utf16) {
160 utf16->clear();
161
162 UErrorCode status = U_ZERO_ERROR;
163 UConverter* converter = ucnv_open(codepage_name, &status);
164 if (!U_SUCCESS(status))
165 return false;
166
167 // Even in the worst case, the maximum length in 2-byte units of UTF-16
168 // output would be at most the same as the number of bytes in input. There
169 // is no single-byte encoding in which a character is mapped to a
170 // non-BMP character requiring two 2-byte units.
171 //
172 // Moreover, non-BMP characters in legacy multibyte encodings
173 // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
174 // BOCU and SCSU, but we don't care about them.
175 size_t uchar_max_length = encoded.length() + 1;
176
177 SetUpErrorHandlerForToUChars(on_error, converter, &status);
178 std::unique_ptr<char16_t[]> buffer(new char16_t[uchar_max_length]);
179 int actual_size = ucnv_toUChars(
180 converter, buffer.get(), static_cast<int>(uchar_max_length),
181 encoded.data(), static_cast<int>(encoded.length()), &status);
182 ucnv_close(converter);
183 if (!U_SUCCESS(status)) {
184 utf16->clear(); // Make sure the output is empty on error.
185 return false;
186 }
187
188 utf16->assign(buffer.get(), actual_size);
189 return true;
190 }
191
ConvertToUtf8AndNormalize(std::string_view text,const std::string & charset,std::string * result)192 bool ConvertToUtf8AndNormalize(std::string_view text,
193 const std::string& charset,
194 std::string* result) {
195 result->clear();
196 std::u16string utf16;
197 if (!CodepageToUTF16(text, charset.c_str(), OnStringConversionError::FAIL,
198 &utf16))
199 return false;
200
201 UErrorCode status = U_ZERO_ERROR;
202 const icu::Normalizer2* normalizer = icu::Normalizer2::getNFCInstance(status);
203 DCHECK(U_SUCCESS(status));
204 if (U_FAILURE(status))
205 return false;
206 int32_t utf16_length = static_cast<int32_t>(utf16.length());
207 icu::UnicodeString normalized(utf16.data(), utf16_length);
208 int32_t normalized_prefix_length =
209 normalizer->spanQuickCheckYes(normalized, status);
210 if (normalized_prefix_length < utf16_length) {
211 icu::UnicodeString un_normalized(normalized, normalized_prefix_length);
212 normalized.truncate(normalized_prefix_length);
213 normalizer->normalizeSecondAndAppend(normalized, un_normalized, status);
214 }
215 if (U_FAILURE(status))
216 return false;
217 normalized.toUTF8String(*result);
218 return true;
219 }
220
221 } // namespace base
222