1 // Copyright 2011 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #ifndef BASE_STRINGS_UTF_STRING_CONVERSION_UTILS_H_
6 #define BASE_STRINGS_UTF_STRING_CONVERSION_UTILS_H_
7
8 // Low-level UTF handling functions. Most code will want to use the functions
9 // in utf_string_conversions.h
10
11 #include <stddef.h>
12 #include <stdint.h>
13
14 #include <limits>
15 #include <optional>
16 #include <string>
17 #include <string_view>
18
19 #include "base/base_export.h"
20 #include "base/third_party/icu/icu_utf.h"
21 #include "build/build_config.h"
22
23 namespace base {
24
IsValidCodepoint(base_icu::UChar32 code_point)25 inline bool IsValidCodepoint(base_icu::UChar32 code_point) {
26 // Excludes code points that are not Unicode scalar values, i.e.
27 // surrogate code points ([0xD800, 0xDFFF]). Additionally, excludes
28 // code points larger than 0x10FFFF (the highest codepoint allowed).
29 // Non-characters and unassigned code points are allowed.
30 // https://unicode.org/glossary/#unicode_scalar_value
31 return (code_point >= 0 && code_point < 0xD800) ||
32 (code_point >= 0xE000 && code_point <= 0x10FFFF);
33 }
34
IsValidCharacter(base_icu::UChar32 code_point)35 inline bool IsValidCharacter(base_icu::UChar32 code_point) {
36 // Excludes non-characters (U+FDD0..U+FDEF, and all code points
37 // ending in 0xFFFE or 0xFFFF) from the set of valid code points.
38 // https://unicode.org/faq/private_use.html#nonchar1
39 return (code_point >= 0 && code_point < 0xD800) ||
40 (code_point >= 0xE000 && code_point < 0xFDD0) ||
41 (code_point > 0xFDEF && code_point <= 0x10FFFF &&
42 (code_point & 0xFFFE) != 0xFFFE);
43 }
44
45 // CountUnicodeCharacters ------------------------------------------------------
46
47 // Returns the number of Unicode characters in `text`, up to the supplied
48 // `limit`, if `text` contains valid UTF-8. Returns `nullopt` otherwise.
49 BASE_EXPORT std::optional<size_t> CountUnicodeCharacters(
50 std::string_view text,
51 size_t limit = std::numeric_limits<size_t>::max());
52
53 // ReadUnicodeCharacter --------------------------------------------------------
54
55 // Reads a UTF-8 stream, placing the next code point into the given output
56 // |*code_point|. |src| represents the entire string to read, and |*char_index|
57 // is the character offset within the string to start reading at. |*char_index|
58 // will be updated to index the last character read, such that incrementing it
59 // (as in a for loop) will take the reader to the next character.
60 //
61 // Returns true on success. On false, |*code_point| will be invalid.
62 BASE_EXPORT bool ReadUnicodeCharacter(const char* src,
63 size_t src_len,
64 size_t* char_index,
65 base_icu::UChar32* code_point_out);
66
67 // Reads a UTF-16 character. The usage is the same as the 8-bit version above.
68 BASE_EXPORT bool ReadUnicodeCharacter(const char16_t* src,
69 size_t src_len,
70 size_t* char_index,
71 base_icu::UChar32* code_point);
72
73 #if defined(WCHAR_T_IS_32_BIT)
74 // Reads UTF-32 character. The usage is the same as the 8-bit version above.
75 BASE_EXPORT bool ReadUnicodeCharacter(const wchar_t* src,
76 size_t src_len,
77 size_t* char_index,
78 base_icu::UChar32* code_point);
79 #endif // defined(WCHAR_T_IS_32_BIT)
80
81 // WriteUnicodeCharacter -------------------------------------------------------
82
83 // Appends a UTF-8 character to the given 8-bit string. Returns the number of
84 // bytes written.
85 BASE_EXPORT size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
86 std::string* output);
87
88 // Appends the given code point as a UTF-16 character to the given 16-bit
89 // string. Returns the number of 16-bit values written.
90 BASE_EXPORT size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
91 std::u16string* output);
92
93 #if defined(WCHAR_T_IS_32_BIT)
94 // Appends the given UTF-32 character to the given 32-bit string. Returns the
95 // number of 32-bit values written.
WriteUnicodeCharacter(base_icu::UChar32 code_point,std::wstring * output)96 inline size_t WriteUnicodeCharacter(base_icu::UChar32 code_point,
97 std::wstring* output) {
98 // This is the easy case, just append the character.
99 output->push_back(static_cast<wchar_t>(code_point));
100 return 1;
101 }
102 #endif // defined(WCHAR_T_IS_32_BIT)
103
104 // Generalized Unicode converter -----------------------------------------------
105
106 // Guesses the length of the output in UTF-8 in bytes, clears that output
107 // string, and reserves that amount of space. We assume that the input
108 // character types are unsigned, which will be true for UTF-16 and -32 on our
109 // systems.
110 template<typename CHAR>
111 void PrepareForUTF8Output(const CHAR* src, size_t src_len, std::string* output);
112
113 // Prepares an output buffer (containing either UTF-16 or -32 data) given some
114 // UTF-8 input that will be converted to it. See PrepareForUTF8Output().
115 template<typename STRING>
116 void PrepareForUTF16Or32Output(const char* src, size_t src_len, STRING* output);
117
118 } // namespace base
119
120 #endif // BASE_STRINGS_UTF_STRING_CONVERSION_UTILS_H_
121