xref: /aosp_15_r20/external/libtextclassifier/native/utils/utf8/unilib-common.cc (revision 993b0882672172b81d12fad7a7ac0c3e5c824a12)

/*
 * Copyright (C) 2018 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "utils/utf8/unilib-common.h"

#include <algorithm>

namespace libtextclassifier3 {
namespace {

#define ARRAYSIZE(a) sizeof(a) / sizeof(*a)

// Derived from http://www.unicode.org/Public/UNIDATA/UnicodeData.txt
// grep -E "Ps" UnicodeData.txt | \
//   sed -rne "s/^([0-9A-Z]{4});.*(PAREN|BRACKET|BRAKCET|BRACE).*/0x\1, /p"
// IMPORTANT: entries with the same offsets in kOpeningBrackets and
//            kClosingBrackets must be counterparts.
constexpr char32 kOpeningBrackets[] = {
    0x0028, 0x005B, 0x007B, 0x0F3C, 0x2045, 0x207D, 0x208D, 0x2329, 0x2768,
    0x276A, 0x276C, 0x2770, 0x2772, 0x2774, 0x27E6, 0x27E8, 0x27EA, 0x27EC,
    0x27EE, 0x2983, 0x2985, 0x2987, 0x2989, 0x298B, 0x298D, 0x298F, 0x2991,
    0x2993, 0x2995, 0x2997, 0x29FC, 0x2E22, 0x2E24, 0x2E26, 0x2E28, 0x3008,
    0x300A, 0x300C, 0x300E, 0x3010, 0x3014, 0x3016, 0x3018, 0x301A, 0xFD3F,
    0xFE17, 0xFE35, 0xFE37, 0xFE39, 0xFE3B, 0xFE3D, 0xFE3F, 0xFE41, 0xFE43,
    0xFE47, 0xFE59, 0xFE5B, 0xFE5D, 0xFF08, 0xFF3B, 0xFF5B, 0xFF5F, 0xFF62};
constexpr int kNumOpeningBrackets = ARRAYSIZE(kOpeningBrackets);

// grep -E "Pe" UnicodeData.txt | \
//   sed -rne "s/^([0-9A-Z]{4});.*(PAREN|BRACKET|BRAKCET|BRACE).*/0x\1, /p"
constexpr char32 kClosingBrackets[] = {
    0x0029, 0x005D, 0x007D, 0x0F3D, 0x2046, 0x207E, 0x208E, 0x232A, 0x2769,
    0x276B, 0x276D, 0x2771, 0x2773, 0x2775, 0x27E7, 0x27E9, 0x27EB, 0x27ED,
    0x27EF, 0x2984, 0x2986, 0x2988, 0x298A, 0x298C, 0x298E, 0x2990, 0x2992,
    0x2994, 0x2996, 0x2998, 0x29FD, 0x2E23, 0x2E25, 0x2E27, 0x2E29, 0x3009,
    0x300B, 0x300D, 0x300F, 0x3011, 0x3015, 0x3017, 0x3019, 0x301B, 0xFD3E,
    0xFE18, 0xFE36, 0xFE38, 0xFE3A, 0xFE3C, 0xFE3E, 0xFE40, 0xFE42, 0xFE44,
    0xFE48, 0xFE5A, 0xFE5C, 0xFE5E, 0xFF09, 0xFF3D, 0xFF5D, 0xFF60, 0xFF63};
constexpr int kNumClosingBrackets = ARRAYSIZE(kClosingBrackets);

// grep -E "WS" UnicodeData.txt | sed -re "s/([0-9A-Z]+);.*/0x\1, /"
constexpr char32 kWhitespaces[] = {
    0x0009,  0x000A,  0x000B,  0x000C,  0x000D,  0x0020,  0x0085,  0x00A0,
    0x1680,  0x2000,  0x2001,  0x2002,  0x2003,  0x2004,  0x2005,  0x2006,
    0x2007,  0x2008,  0x2009,  0x200A,  0x2028,  0x2029,  0x202F,  0x205F,
    0x21C7,  0x21C8,  0x21C9,  0x21CA,  0x21F6,  0x2B31,  0x2B84,  0x2B85,
    0x2B86,  0x2B87,  0x2B94,  0x3000,  0x4DCC,  0x10344, 0x10347, 0x1DA0A,
    0x1DA0B, 0x1DA0C, 0x1DA0D, 0x1DA0E, 0x1DA0F, 0x1DA10, 0x1F4F0, 0x1F500,
    0x1F501, 0x1F502, 0x1F503, 0x1F504, 0x1F5D8, 0x1F5DE};
constexpr int kNumWhitespaces = ARRAYSIZE(kWhitespaces);

// https://en.wikipedia.org/wiki/Bidirectional_text
constexpr char32 kBidirectional[] = {0x061C, 0x200E, 0x200F, 0x202A,
                                     0x202B, 0x202C, 0x202D, 0x202E,
                                     0x2066, 0x2067, 0x2068, 0x2069};
constexpr int kNumBidirectional = ARRAYSIZE(kBidirectional);

// grep -E "Nd" UnicodeData.txt | sed -re "s/([0-9A-Z]+);.*/0x\1, /"
// As the name suggests, these ranges are always 10 codepoints long, so we just
// store the end of the range.
constexpr char32 kDecimalDigitRangesEnd[] = {
    0x0039,  0x0669,  0x06f9,  0x07c9,  0x096f,  0x09ef,  0x0a6f,  0x0aef,
    0x0b6f,  0x0bef,  0x0c6f,  0x0cef,  0x0d6f,  0x0def,  0x0e59,  0x0ed9,
    0x0f29,  0x1049,  0x1099,  0x17e9,  0x1819,  0x194f,  0x19d9,  0x1a89,
    0x1a99,  0x1b59,  0x1bb9,  0x1c49,  0x1c59,  0xa629,  0xa8d9,  0xa909,
    0xa9d9,  0xa9f9,  0xaa59,  0xabf9,  0xff19,  0x104a9, 0x1106f, 0x110f9,
    0x1113f, 0x111d9, 0x112f9, 0x11459, 0x114d9, 0x11659, 0x116c9, 0x11739,
    0x118e9, 0x11c59, 0x11d59, 0x16a69, 0x16b59, 0x1d7ff};
constexpr int kNumDecimalDigitRangesEnd = ARRAYSIZE(kDecimalDigitRangesEnd);

// Visual source: https://en.wikipedia.org/wiki/Latin_script_in_Unicode
// Source https://unicode-search.net/unicode-namesearch.pl?term=letter
// clang-format off
// grep "LATIN " latters.txt | grep -v "TAG LATIN" | grep -v "SQUARED LATIN" | grep -v "CIRCLED LATIN" | grep -v "PARENTHESIZED LATIN" | cut -d'  ' -f1 | cut -d'+' -f2 | sed -re "s/([0-9A-Z]+).*/0x\1, /" | tr -d "\n" NOLINT
// clang-format on
constexpr char32 kLatinLettersRangesStart[] = {0x0041, 0x0061, 0x00C0, 0x00D8,
                                               0x00F8, 0x1D00, 0x2C60, 0xAB30,
                                               0xFF21, 0xFF41};
constexpr int kNumLatinLettersRangesStart = ARRAYSIZE(kLatinLettersRangesStart);
constexpr char32 kLatinLettersRangesEnd[] = {0x005A, 0x007A, 0x00D6, 0x00F7,
                                             0x02A8, 0x1EFF, 0xA7B7, 0xAB64,
                                             0xFF3A, 0xFF5A};
constexpr int kNumLatinLettersRangesEnd = ARRAYSIZE(kLatinLettersRangesEnd);

// Source https://unicode-search.net/unicode-namesearch.pl?term=letter
constexpr char32 kArabicLettersRangesStart[] = {
    0x0620, 0x0641, 0x066E, 0x06EE, 0x0750, 0x08A0, 0xFB50, 0xFDFA, 0xFE80};
constexpr int kNumArabicLettersRangesStart =
    ARRAYSIZE(kArabicLettersRangesStart);
constexpr char32 kArabicLettersRangesEnd[] = {
    0x063F, 0x064A, 0x06D5, 0x06FF, 0x077F, 0x08BD, 0xFBFF, 0xFDFB, 0xFEF4};
constexpr int kNumArabicLettersRangesEnd = ARRAYSIZE(kArabicLettersRangesEnd);

// Source https://unicode-search.net/unicode-namesearch.pl?term=letter
constexpr char32 kCyrillicLettersRangesStart[] = {0x0400, 0x1C80, 0x2DE0,
                                                  0xA640, 0xA674, 0xA680};
constexpr int kNumCyrillicLettersRangesStart =
    ARRAYSIZE(kCyrillicLettersRangesStart);
constexpr char32 kCyrillicLettersRangesEnd[] = {0x052F, 0x1C88, 0x2DFF,
                                                0xA66E, 0xA67B, 0xA69F};
constexpr int kNumCyrillicLettersRangesEnd =
    ARRAYSIZE(kCyrillicLettersRangesEnd);

constexpr char32 kChineseLettersRangesStart[] = {
    0x4E00,  0xF900,  0x2F800, 0xFE30,  0x3400,
    0x20000, 0x2A700, 0x2B740, 0x2B820, 0x2CEB0};
constexpr int kNumChineseLettersRangesStart =
    ARRAYSIZE(kChineseLettersRangesStart);
constexpr char32 kChineseLettersRangesEnd[] = {
    0x9FFF,  0xFAFF,  0x2FA1F, 0xFE4F,  0x4DBF,
    0x2A6DF, 0x2B73F, 0x2B81F, 0x2CEAF, 0x2EBEF};
constexpr int kNumChineseLettersRangesEnd = ARRAYSIZE(kChineseLettersRangesEnd);

// Source https://unicode-search.net/unicode-namesearch.pl?term=letter
// Hiragana and Katakana
constexpr char32 kJapaneseLettersRangesStart[] = {0x3041, 0x30A1, 0x31F0,
                                                  0xFF66};
constexpr int kNumJapaneseLettersRangesStart =
    ARRAYSIZE(kJapaneseLettersRangesStart);
constexpr char32 kJapaneseLettersRangesEnd[] = {0x3096, 0x30FA, 0x31FF, 0xFF9D};
constexpr int kNumJapaneseLettersRangesEnd =
    ARRAYSIZE(kJapaneseLettersRangesEnd);

// Source https://unicode-search.net/unicode-namesearch.pl?term=letter
// Hangul
constexpr char32 kKoreanLettersRangesStart[] = {0x3131, 0xFFA1};
constexpr int kNumKoreanLettersRangesStart =
    ARRAYSIZE(kKoreanLettersRangesStart);
constexpr char32 kKoreanLettersRangesEnd[] = {0x318E, 0xFFDC};
constexpr int kNumKoreanLettersRangesEnd = ARRAYSIZE(kKoreanLettersRangesEnd);

// Source https://unicode-search.net/unicode-namesearch.pl?term=letter
constexpr char32 kThaiLettersRangesStart[] = {0x0E01};
constexpr int kNumThaiLettersRangesStart = ARRAYSIZE(kThaiLettersRangesStart);
constexpr char32 kThaiLettersRangesEnd[] = {0x0E2E};
constexpr int kNumThaiLettersRangesEnd = ARRAYSIZE(kThaiLettersRangesEnd);

// grep -E ";P.;" UnicodeData.txt | sed -re "s/([0-9A-Z]+);.*/0x\1, /"
constexpr char32 kPunctuationRangesStart[] = {
    0x0021,  0x0025,  0x002c,  0x003a,  0x003f,  0x005b,  0x005f,  0x007b,
    0x007d,  0x00a1,  0x00a7,  0x00ab,  0x00b6,  0x00bb,  0x00bf,  0x037e,
    0x0387,  0x055a,  0x0589,  0x05be,  0x05c0,  0x05c3,  0x05c6,  0x05f3,
    0x0609,  0x060c,  0x061b,  0x061e,  0x066a,  0x06d4,  0x0700,  0x07f7,
    0x0830,  0x085e,  0x0964,  0x0970,  0x09fd,  0x0a76,  0x0af0,  0x0c77,
    0x0c84,  0x0df4,  0x0e4f,  0x0e5a,  0x0f04,  0x0f14,  0x0f3a,  0x0f85,
    0x0fd0,  0x0fd9,  0x104a,  0x10fb,  0x1360,  0x1400,  0x166e,  0x169b,
    0x16eb,  0x1735,  0x17d4,  0x17d8,  0x1800,  0x1944,  0x1a1e,  0x1aa0,
    0x1aa8,  0x1b5a,  0x1bfc,  0x1c3b,  0x1c7e,  0x1cc0,  0x1cd3,  0x2010,
    0x2030,  0x2045,  0x2053,  0x207d,  0x208d,  0x2308,  0x2329,  0x2768,
    0x27c5,  0x27e6,  0x2983,  0x29d8,  0x29fc,  0x2cf9,  0x2cfe,  0x2d70,
    0x2e00,  0x2e30,  0x3001,  0x3008,  0x3014,  0x3030,  0x303d,  0x30a0,
    0x30fb,  0xa4fe,  0xa60d,  0xa673,  0xa67e,  0xa6f2,  0xa874,  0xa8ce,
    0xa8f8,  0xa8fc,  0xa92e,  0xa95f,  0xa9c1,  0xa9de,  0xaa5c,  0xaade,
    0xaaf0,  0xabeb,  0xfd3e,  0xfe10,  0xfe30,  0xfe54,  0xfe63,  0xfe68,
    0xfe6a,  0xff01,  0xff05,  0xff0c,  0xff1a,  0xff1f,  0xff3b,  0xff3f,
    0xff5b,  0xff5d,  0xff5f,  0x10100, 0x1039f, 0x103d0, 0x1056f, 0x10857,
    0x1091f, 0x1093f, 0x10a50, 0x10a7f, 0x10af0, 0x10b39, 0x10b99, 0x10f55,
    0x11047, 0x110bb, 0x110be, 0x11140, 0x11174, 0x111c5, 0x111cd, 0x111db,
    0x111dd, 0x11238, 0x112a9, 0x1144b, 0x1145b, 0x1145d, 0x114c6, 0x115c1,
    0x11641, 0x11660, 0x1173c, 0x1183b, 0x119e2, 0x11a3f, 0x11a9a, 0x11a9e,
    0x11c41, 0x11c70, 0x11ef7, 0x11fff, 0x12470, 0x16a6e, 0x16af5, 0x16b37,
    0x16b44, 0x16e97, 0x16fe2, 0x1bc9f, 0x1da87, 0x1e95e};
constexpr int kNumPunctuationRangesStart = ARRAYSIZE(kPunctuationRangesStart);
constexpr char32 kPunctuationRangesEnd[] = {
    0x0023,  0x002a,  0x002f,  0x003b,  0x0040,  0x005d,  0x005f,  0x007b,
    0x007d,  0x00a1,  0x00a7,  0x00ab,  0x00b7,  0x00bb,  0x00bf,  0x037e,
    0x0387,  0x055f,  0x058a,  0x05be,  0x05c0,  0x05c3,  0x05c6,  0x05f4,
    0x060a,  0x060d,  0x061b,  0x061f,  0x066d,  0x06d4,  0x070d,  0x07f9,
    0x083e,  0x085e,  0x0965,  0x0970,  0x09fd,  0x0a76,  0x0af0,  0x0c77,
    0x0c84,  0x0df4,  0x0e4f,  0x0e5b,  0x0f12,  0x0f14,  0x0f3d,  0x0f85,
    0x0fd4,  0x0fda,  0x104f,  0x10fb,  0x1368,  0x1400,  0x166e,  0x169c,
    0x16ed,  0x1736,  0x17d6,  0x17da,  0x180a,  0x1945,  0x1a1f,  0x1aa6,
    0x1aad,  0x1b60,  0x1bff,  0x1c3f,  0x1c7f,  0x1cc7,  0x1cd3,  0x2027,
    0x2043,  0x2051,  0x205e,  0x207e,  0x208e,  0x230b,  0x232a,  0x2775,
    0x27c6,  0x27ef,  0x2998,  0x29db,  0x29fd,  0x2cfc,  0x2cff,  0x2d70,
    0x2e2e,  0x2e4f,  0x3003,  0x3011,  0x301f,  0x3030,  0x303d,  0x30a0,
    0x30fb,  0xa4ff,  0xa60f,  0xa673,  0xa67e,  0xa6f7,  0xa877,  0xa8cf,
    0xa8fa,  0xa8fc,  0xa92f,  0xa95f,  0xa9cd,  0xa9df,  0xaa5f,  0xaadf,
    0xaaf1,  0xabeb,  0xfd3f,  0xfe19,  0xfe52,  0xfe61,  0xfe63,  0xfe68,
    0xfe6b,  0xff03,  0xff0a,  0xff0f,  0xff1b,  0xff20,  0xff3d,  0xff3f,
    0xff5b,  0xff5d,  0xff65,  0x10102, 0x1039f, 0x103d0, 0x1056f, 0x10857,
    0x1091f, 0x1093f, 0x10a58, 0x10a7f, 0x10af6, 0x10b3f, 0x10b9c, 0x10f59,
    0x1104d, 0x110bc, 0x110c1, 0x11143, 0x11175, 0x111c8, 0x111cd, 0x111db,
    0x111df, 0x1123d, 0x112a9, 0x1144f, 0x1145b, 0x1145d, 0x114c6, 0x115d7,
    0x11643, 0x1166c, 0x1173e, 0x1183b, 0x119e2, 0x11a46, 0x11a9c, 0x11aa2,
    0x11c45, 0x11c71, 0x11ef8, 0x11fff, 0x12474, 0x16a6f, 0x16af5, 0x16b3b,
    0x16b44, 0x16e9a, 0x16fe2, 0x1bc9f, 0x1da8b, 0x1e95f};
constexpr int kNumPunctuationRangesEnd = ARRAYSIZE(kPunctuationRangesEnd);

// grep -E "Lu" UnicodeData.txt | sed -re "s/([0-9A-Z]+);.*/0x\1, /"
// There are three common ways in which upper/lower case codepoint ranges
// were introduced: one offs, dense ranges, and ranges that alternate between
// lower and upper case. For the sake of keeping out binary size down, we
// treat each independently.
constexpr char32 kUpperSingles[] = {
    0x01b8, 0x01bc, 0x01c4, 0x01c7, 0x01ca, 0x01f1, 0x0376, 0x037f,
    0x03cf, 0x03f4, 0x03fa, 0x10c7, 0x10cd, 0x2102, 0x2107, 0x2115,
    0x2145, 0x2183, 0x2c72, 0x2c75, 0x2cf2, 0xa7b6};
constexpr int kNumUpperSingles = ARRAYSIZE(kUpperSingles);
constexpr char32 kUpperRanges1Start[] = {
    0x0041, 0x00c0, 0x00d8, 0x0181, 0x018a, 0x018e, 0x0193, 0x0196,
    0x019c, 0x019f, 0x01b2, 0x01f7, 0x023a, 0x023d, 0x0244, 0x0389,
    0x0392, 0x03a3, 0x03d2, 0x03fd, 0x0531, 0x10a0, 0x13a0, 0x1f08,
    0x1f18, 0x1f28, 0x1f38, 0x1f48, 0x1f68, 0x1fb8, 0x1fc8, 0x1fd8,
    0x1fe8, 0x1ff8, 0x210b, 0x2110, 0x2119, 0x212b, 0x2130, 0x213e,
    0x2c00, 0x2c63, 0x2c6e, 0x2c7e, 0xa7ab, 0xa7b0};
constexpr int kNumUpperRanges1Start = ARRAYSIZE(kUpperRanges1Start);
constexpr char32 kUpperRanges1End[] = {
    0x005a, 0x00d6, 0x00de, 0x0182, 0x018b, 0x0191, 0x0194, 0x0198,
    0x019d, 0x01a0, 0x01b3, 0x01f8, 0x023b, 0x023e, 0x0246, 0x038a,
    0x03a1, 0x03ab, 0x03d4, 0x042f, 0x0556, 0x10c5, 0x13f5, 0x1f0f,
    0x1f1d, 0x1f2f, 0x1f3f, 0x1f4d, 0x1f6f, 0x1fbb, 0x1fcb, 0x1fdb,
    0x1fec, 0x1ffb, 0x210d, 0x2112, 0x211d, 0x212d, 0x2133, 0x213f,
    0x2c2e, 0x2c64, 0x2c70, 0x2c80, 0xa7ae, 0xa7b4};
constexpr int kNumUpperRanges1End = ARRAYSIZE(kUpperRanges1End);
constexpr char32 kUpperRanges2Start[] = {
    0x0100, 0x0139, 0x014a, 0x0179, 0x0184, 0x0187, 0x01a2, 0x01a7, 0x01ac,
    0x01af, 0x01b5, 0x01cd, 0x01de, 0x01f4, 0x01fa, 0x0241, 0x0248, 0x0370,
    0x0386, 0x038c, 0x038f, 0x03d8, 0x03f7, 0x0460, 0x048a, 0x04c1, 0x04d0,
    0x1e00, 0x1e9e, 0x1f59, 0x2124, 0x2c60, 0x2c67, 0x2c82, 0x2ceb, 0xa640,
    0xa680, 0xa722, 0xa732, 0xa779, 0xa77e, 0xa78b, 0xa790, 0xa796};
constexpr int kNumUpperRanges2Start = ARRAYSIZE(kUpperRanges2Start);
constexpr char32 kUpperRanges2End[] = {
    0x0136, 0x0147, 0x0178, 0x017d, 0x0186, 0x0189, 0x01a6, 0x01a9, 0x01ae,
    0x01b1, 0x01b7, 0x01db, 0x01ee, 0x01f6, 0x0232, 0x0243, 0x024e, 0x0372,
    0x0388, 0x038e, 0x0391, 0x03ee, 0x03f9, 0x0480, 0x04c0, 0x04cd, 0x052e,
    0x1e94, 0x1efe, 0x1f5f, 0x212a, 0x2c62, 0x2c6d, 0x2ce2, 0x2ced, 0xa66c,
    0xa69a, 0xa72e, 0xa76e, 0xa77d, 0xa786, 0xa78d, 0xa792, 0xa7aa};
constexpr int kNumUpperRanges2End = ARRAYSIZE(kUpperRanges2End);

// grep -E "Ll" UnicodeData.txt | sed -re "s/([0-9A-Z]+);.*/0x\1, /"
constexpr char32 kLowerSingles[] = {
    0x00b5, 0x0188, 0x0192, 0x0195, 0x019e, 0x01b0, 0x01c6, 0x01c9,
    0x01f0, 0x023c, 0x0242, 0x0377, 0x0390, 0x03f5, 0x03f8, 0x1fbe,
    0x210a, 0x2113, 0x212f, 0x2134, 0x2139, 0x214e, 0x2184, 0x2c61,
    0x2ce4, 0x2cf3, 0x2d27, 0x2d2d, 0xa7af, 0xa7c3, 0xa7fa, 0x1d7cb};
constexpr int kNumLowerSingles = ARRAYSIZE(kLowerSingles);
constexpr char32 kLowerRanges1Start[] = {
    0x0061,  0x00df,  0x00f8,  0x017f,  0x018c,  0x0199,  0x01b9,  0x01bd,
    0x0234,  0x023f,  0x0250,  0x0295,  0x037b,  0x03ac,  0x03d0,  0x03d5,
    0x03f0,  0x03fb,  0x0430,  0x0560,  0x10d0,  0x10fd,  0x13f8,  0x1c80,
    0x1d00,  0x1d6b,  0x1d79,  0x1e96,  0x1f00,  0x1f10,  0x1f20,  0x1f30,
    0x1f40,  0x1f50,  0x1f60,  0x1f70,  0x1f80,  0x1f90,  0x1fa0,  0x1fb0,
    0x1fb6,  0x1fc2,  0x1fc6,  0x1fd0,  0x1fd6,  0x1fe0,  0x1ff2,  0x1ff6,
    0x210e,  0x213c,  0x2146,  0x2c30,  0x2c65,  0x2c77,  0x2d00,  0xa730,
    0xa772,  0xa794,  0xab30,  0xab60,  0xab70,  0xfb00,  0xfb13,  0xff41,
    0x10428, 0x104d8, 0x10cc0, 0x118c0, 0x16e60, 0x1d41a, 0x1d44e, 0x1d456,
    0x1d482, 0x1d4b6, 0x1d4be, 0x1d4c5, 0x1d4ea, 0x1d51e, 0x1d552, 0x1d586,
    0x1d5ba, 0x1d5ee, 0x1d622, 0x1d656, 0x1d68a, 0x1d6c2, 0x1d6dc, 0x1d6fc,
    0x1d716, 0x1d736, 0x1d750, 0x1d770, 0x1d78a, 0x1d7aa, 0x1d7c4, 0x1e922};
constexpr int kNumLowerRanges1Start = ARRAYSIZE(kLowerRanges1Start);
constexpr char32 kLowerRanges1End[] = {
    0x007a,  0x00f6,  0x00ff,  0x0180,  0x018d,  0x019b,  0x01ba,  0x01bf,
    0x0239,  0x0240,  0x0293,  0x02af,  0x037d,  0x03ce,  0x03d1,  0x03d7,
    0x03f3,  0x03fc,  0x045f,  0x0588,  0x10fa,  0x10ff,  0x13fd,  0x1c88,
    0x1d2b,  0x1d77,  0x1d9a,  0x1e9d,  0x1f07,  0x1f15,  0x1f27,  0x1f37,
    0x1f45,  0x1f57,  0x1f67,  0x1f7d,  0x1f87,  0x1f97,  0x1fa7,  0x1fb4,
    0x1fb7,  0x1fc4,  0x1fc7,  0x1fd3,  0x1fd7,  0x1fe7,  0x1ff4,  0x1ff7,
    0x210f,  0x213d,  0x2149,  0x2c5e,  0x2c66,  0x2c7b,  0x2d25,  0xa731,
    0xa778,  0xa795,  0xab5a,  0xab67,  0xabbf,  0xfb06,  0xfb17,  0xff5a,
    0x1044f, 0x104fb, 0x10cf2, 0x118df, 0x16e7f, 0x1d433, 0x1d454, 0x1d467,
    0x1d49b, 0x1d4b9, 0x1d4c3, 0x1d4cf, 0x1d503, 0x1d537, 0x1d56b, 0x1d59f,
    0x1d5d3, 0x1d607, 0x1d63b, 0x1d66f, 0x1d6a5, 0x1d6da, 0x1d6e1, 0x1d714,
    0x1d71b, 0x1d74e, 0x1d755, 0x1d788, 0x1d78f, 0x1d7c2, 0x1d7c9, 0x1e943};
constexpr int kNumLowerRanges1End = ARRAYSIZE(kLowerRanges1End);
constexpr char32 kLowerRanges2Start[] = {
    0x0101, 0x0138, 0x0149, 0x017a, 0x0183, 0x01a1, 0x01a8, 0x01ab,
    0x01b4, 0x01cc, 0x01dd, 0x01f3, 0x01f9, 0x0247, 0x0371, 0x03d9,
    0x0461, 0x048b, 0x04c2, 0x04cf, 0x1e01, 0x1e9f, 0x2c68, 0x2c71,
    0x2c74, 0x2c81, 0x2cec, 0xa641, 0xa681, 0xa723, 0xa733, 0xa77a,
    0xa77f, 0xa78c, 0xa791, 0xa797, 0xa7b5, 0x1d4bb};
constexpr int kNumLowerRanges2Start = ARRAYSIZE(kLowerRanges2Start);
constexpr char32 kLowerRanges2End[] = {
    0x0137, 0x0148, 0x0177, 0x017e, 0x0185, 0x01a5, 0x01aa, 0x01ad,
    0x01b6, 0x01dc, 0x01ef, 0x01f5, 0x0233, 0x024f, 0x0373, 0x03ef,
    0x0481, 0x04bf, 0x04ce, 0x052f, 0x1e95, 0x1eff, 0x2c6c, 0x2c73,
    0x2c76, 0x2ce3, 0x2cee, 0xa66d, 0xa69b, 0xa72f, 0xa771, 0xa77c,
    0xa787, 0xa78e, 0xa793, 0xa7a9, 0xa7bf, 0x1d4bd};
constexpr int kNumLowerRanges2End = ARRAYSIZE(kLowerRanges2End);

// grep -E "Lu" UnicodeData.txt | \
//   sed -rne "s/^([0-9A-Z]+);.*;([0-9A-Z]+);$/(0x\1, 0x\2), /p"
// We have two strategies for mapping from upper to lower case. We have single
// character lookups that do not follow a pattern, and ranges for which there
// is a constant codepoint shift.
// Note that these ranges ignore anything that's not an upper case character,
// so when applied to a non-uppercase character the result is incorrect.
constexpr int kToLowerSingles[] = {
    0x0130, 0x0178, 0x0181, 0x0186, 0x018b, 0x018e, 0x018f, 0x0190, 0x0191,
    0x0194, 0x0196, 0x0197, 0x0198, 0x019c, 0x019d, 0x019f, 0x01a6, 0x01a9,
    0x01ae, 0x01b7, 0x01f6, 0x01f7, 0x0220, 0x023a, 0x023d, 0x023e, 0x0243,
    0x0244, 0x0245, 0x037f, 0x0386, 0x038c, 0x03cf, 0x03f4, 0x03f9, 0x04c0,
    0x1e9e, 0x1fec, 0x2126, 0x212a, 0x212b, 0x2132, 0x2183, 0x2c60, 0x2c62,
    0x2c63, 0x2c64, 0x2c6d, 0x2c6e, 0x2c6f, 0x2c70, 0xa77d, 0xa78d, 0xa7aa,
    0xa7ab, 0xa7ac, 0xa7ad, 0xa7ae, 0xa7b0, 0xa7b1, 0xa7b2, 0xa7b3};
constexpr int kNumToLowerSingles = ARRAYSIZE(kToLowerSingles);
constexpr int kToLowerSinglesOffsets[] = {
    -199,   -121,   210,    206,    1,      79,     202,    203,    1,
    207,    211,    209,    1,      211,    213,    214,    218,    218,
    218,    219,    -97,    -56,    -130,   10795,  -163,   10792,  -195,
    69,     71,     116,    38,     64,     8,      -60,    -7,     15,
    -7615,  -7,     -7517,  -8383,  -8262,  28,     1,      1,      -10743,
    -3814,  -10727, -10780, -10749, -10783, -10782, -35332, -42280, -42308,
    -42319, -42315, -42305, -42308, -42258, -42282, -42261, 928};
constexpr int kNumToLowerSinglesOffsets = ARRAYSIZE(kToLowerSinglesOffsets);
constexpr int kToUpperSingles[] = {
    0x00b5, 0x00ff, 0x0131, 0x017f, 0x0180, 0x0195, 0x0199, 0x019a, 0x019e,
    0x01bf, 0x01dd, 0x01f3, 0x0250, 0x0251, 0x0252, 0x0253, 0x0254, 0x0259,
    0x025b, 0x025c, 0x0260, 0x0261, 0x0263, 0x0265, 0x0266, 0x0268, 0x0269,
    0x026a, 0x026b, 0x026c, 0x026f, 0x0271, 0x0272, 0x0275, 0x027d, 0x0280,
    0x0282, 0x0283, 0x0287, 0x0288, 0x0289, 0x028c, 0x0292, 0x029d, 0x029e,
    0x03ac, 0x03c2, 0x03cc, 0x03d0, 0x03d1, 0x03d5, 0x03d6, 0x03d7, 0x03f0,
    0x03f1, 0x03f2, 0x03f3, 0x03f5, 0x04cf, 0x1c80, 0x1c81, 0x1c82, 0x1c85,
    0x1c86, 0x1c87, 0x1c88, 0x1d79, 0x1d7d, 0x1d8e, 0x1e9b, 0x1fb3, 0x1fbe,
    0x1fc3, 0x1fe5, 0x1ff3, 0x214e, 0x2184, 0x2c61, 0x2c65, 0x2c66, 0xa794,
    0xab53};
constexpr int kNumToUpperSingles = ARRAYSIZE(kToUpperSingles);
constexpr int kToUpperSinglesOffsets[] = {
    743,   121,   -232,  -300,  195,   97,    -1,    163,   130,    56,
    -79,   -2,    10783, 10780, 10782, -210,  -206,  -202,  -203,   42319,
    -205,  42315, -207,  42280, 42308, -209,  -211,  42308, 10743,  42305,
    -211,  10749, -213,  -214,  10727, -218,  42307, -218,  42282,  -218,
    -69,   -71,   -219,  42261, 42258, -38,   -31,   -64,   -62,    -57,
    -47,   -54,   -8,    -86,   -80,   7,     -116,  -96,   -15,    -6254,
    -6253, -6244, -6243, -6236, -6181, 35266, 35332, 3814,  35384,  -59,
    9,     -7205, 9,     7,     9,     -28,   -1,    -1,    -10795, -10792,
    48,    -928};
constexpr int kNumToUpperSinglesOffsets = ARRAYSIZE(kToUpperSinglesOffsets);
constexpr int kToLowerRangesStart[] = {
    0x0041, 0x0100, 0x0189, 0x01a0, 0x01b1, 0x01b3, 0x0388,  0x038e,  0x0391,
    0x03d8, 0x03fd, 0x0400, 0x0410, 0x0460, 0x0531, 0x10a0,  0x13a0,  0x13f0,
    0x1e00, 0x1f08, 0x1fba, 0x1fc8, 0x1fd8, 0x1fda, 0x1fe8,  0x1fea,  0x1ff8,
    0x1ffa, 0x2c00, 0x2c67, 0x2c7e, 0x2c80, 0xff21, 0x10400, 0x10c80, 0x118a0};
constexpr int kNumToLowerRangesStart = ARRAYSIZE(kToLowerRangesStart);
constexpr int kToLowerRangesEnd[] = {
    0x00de, 0x0187, 0x019f, 0x01af, 0x01b2, 0x0386, 0x038c,  0x038f,  0x03cf,
    0x03fa, 0x03ff, 0x040f, 0x042f, 0x052e, 0x0556, 0x10cd,  0x13ef,  0x13f5,
    0x1efe, 0x1fb9, 0x1fbb, 0x1fcb, 0x1fd9, 0x1fdb, 0x1fe9,  0x1fec,  0x1ff9,
    0x2183, 0x2c64, 0x2c75, 0x2c7f, 0xa7b6, 0xff3a, 0x104d3, 0x10cb2, 0x118bf};
constexpr int kNumToLowerRangesEnd = ARRAYSIZE(kToLowerRangesEnd);
constexpr int kToLowerRangesOffsets[] = {
    32, 1,    205,  1,    217,   1, 37,     63, 32,  1,   -130, 80,
    32, 1,    48,   7264, 38864, 8, 1,      -8, -74, -86, -8,   -100,
    -8, -112, -128, -126, 48,    1, -10815, 1,  32,  40,  64,   32};
constexpr int kNumToLowerRangesOffsets = ARRAYSIZE(kToLowerRangesOffsets);
constexpr int kToUpperRangesStart[] = {
    0x0061, 0x0101, 0x01c6, 0x01ce, 0x023f,  0x0242,  0x0256, 0x028a,
    0x0371, 0x037b, 0x03ad, 0x03b1, 0x03cd,  0x03d9,  0x0430, 0x0450,
    0x0461, 0x0561, 0x10d0, 0x13f8, 0x1c83,  0x1e01,  0x1f00, 0x1f70,
    0x1f72, 0x1f76, 0x1f78, 0x1f7a, 0x1f7c,  0x1f80,  0x2c30, 0x2c68,
    0x2d00, 0xa641, 0xab70, 0xff41, 0x10428, 0x10cc0, 0x118c0};
constexpr int kNumToUpperRangesStart = ARRAYSIZE(kToUpperRangesStart);
constexpr int kToUpperRangesEnd[] = {
    0x00fe, 0x01bd, 0x01cc, 0x023c, 0x0240,  0x024f,  0x0257, 0x028b,
    0x0377, 0x037d, 0x03af, 0x03cb, 0x03ce,  0x03fb,  0x044f, 0x045f,
    0x052f, 0x0586, 0x10ff, 0x13fd, 0x1c84,  0x1eff,  0x1f67, 0x1f71,
    0x1f75, 0x1f77, 0x1f79, 0x1f7b, 0x1f7d,  0x1fe1,  0x2c5e, 0x2cf3,
    0x2d2d, 0xa7c3, 0xabbf, 0xff5a, 0x104fb, 0x10cf2, 0x16e7f};
constexpr int kNumToUpperRangesEnd = ARRAYSIZE(kToUpperRangesEnd);
constexpr int kToUpperRangesOffsets[]{
    -32, -1,  -2,  -1, 10815, -1,   -205,  -217,  -1,     130, -37, -32, -63,
    -1,  -32, -80, -1, -48,   3008, -8,    -6242, -1,     8,   74,  86,  100,
    128, 112, 126, 8,  -48,   -1,   -7264, -1,    -38864, -32, -40, -64, -32};
constexpr int kNumToUpperRangesOffsets = ARRAYSIZE(kToUpperRangesOffsets);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=PERCENT
constexpr char32 kPercentages[] = {0x0025, 0x066A, 0xFE6A, 0xFF05};
constexpr int kNumPercentages = ARRAYSIZE(kPercentages);

// Source from https://unicode-search.net/unicode-namesearch.pl?term=SLASH
constexpr char32 kSlashes[] = {0x002f, 0x0337, 0x0338, 0x2044, 0x2215, 0xff0f};
constexpr int kNumSlashes = ARRAYSIZE(kSlashes);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=minus
constexpr char32 kMinuses[] = {0x002d, 0x02d7, 0x2212, 0xff0d};
constexpr int kNumMinuses = ARRAYSIZE(kMinuses);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=NUMBER%20SIGN
constexpr char32 kNumberSign[] = {0x0023, 0xfe5f, 0xff03};
constexpr int kNumNumberSign = ARRAYSIZE(kNumberSign);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=period
constexpr char32 kDots[] = {0x002e, 0xfe52, 0xff0e};
constexpr int kNumDots = ARRAYSIZE(kDots);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=Apostrophe
constexpr char32 kApostrophe[] = {0x0027, 0x02BC, 0x02EE, 0x055A,
                                  0x07F4, 0x07F5, 0xFF07};
constexpr int kNumApostrophe = ARRAYSIZE(kApostrophe);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=Quotation
constexpr char32 kQuotation[] = {
    0x0022, 0x00AB, 0x00BB, 0x2018, 0x2019, 0x201A, 0x201B, 0x201C,
    0x201D, 0x201E, 0x201F, 0x2039, 0x203A, 0x275B, 0x275C, 0x275D,
    0x275E, 0x276E, 0x276F, 0x2E42, 0x301D, 0x301E, 0x301F, 0xFF02};
constexpr int kNumQuotation = ARRAYSIZE(kQuotation);

// Source: https://unicode-search.net/unicode-namesearch.pl?term=ampersand
constexpr char32 kAmpersand[] = {0x0026, 0xFE60, 0xFF06, 0x1F674, 0x1F675};
constexpr int kNumAmpersand = ARRAYSIZE(kAmpersand);

#undef ARRAYSIZE

static_assert(kNumOpeningBrackets == kNumClosingBrackets,
              "mismatching number of opening and closing brackets");
static_assert(kNumLowerRanges1Start == kNumLowerRanges1End,
              "number of uppercase stride 1 range starts/ends doesn't match");
static_assert(kNumLowerRanges2Start == kNumLowerRanges2End,
              "number of uppercase stride 2 range starts/ends doesn't match");
static_assert(kNumUpperRanges1Start == kNumUpperRanges1End,
              "number of uppercase stride 1 range starts/ends doesn't match");
static_assert(kNumUpperRanges2Start == kNumUpperRanges2End,
              "number of uppercase stride 2 range starts/ends doesn't match");
static_assert(kNumToLowerSingles == kNumToLowerSinglesOffsets,
              "number of to lower singles and offsets doesn't match");
static_assert(kNumToLowerRangesStart == kNumToLowerRangesEnd,
              "mismatching number of range starts/ends for to lower ranges");
static_assert(kNumToLowerRangesStart == kNumToLowerRangesOffsets,
              "number of to lower ranges and offsets doesn't match");
static_assert(kNumToUpperSingles == kNumToUpperSinglesOffsets,
              "number of to upper singles and offsets doesn't match");
static_assert(kNumToUpperRangesStart == kNumToUpperRangesEnd,
              "mismatching number of range starts/ends for to upper ranges");
static_assert(kNumToUpperRangesStart == kNumToUpperRangesOffsets,
              "number of to upper ranges and offsets doesn't match");
static_assert(kNumPunctuationRangesStart == kNumPunctuationRangesEnd,
              "mismatch number of start/ends for punctuation ranges.");
static_assert(kNumLatinLettersRangesStart == kNumLatinLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");
static_assert(kNumArabicLettersRangesStart == kNumArabicLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");
static_assert(kNumCyrillicLettersRangesStart == kNumCyrillicLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");
static_assert(kNumChineseLettersRangesStart == kNumChineseLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");
static_assert(kNumJapaneseLettersRangesStart == kNumJapaneseLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");
static_assert(kNumKoreanLettersRangesStart == kNumKoreanLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");
static_assert(kNumThaiLettersRangesStart == kNumThaiLettersRangesEnd,
              "mismatch number of start/ends for letters ranges.");

constexpr int kNoMatch = -1;

// Returns the index of the element in the array that matched the given
// codepoint, or kNoMatch if the element didn't exist.
// The input array must be in sorted order.
int GetMatchIndex(const char32* array, int array_length, char32 c) {
  const char32* end = array + array_length;
  const auto find_it = std::lower_bound(array, end, c);
  if (find_it != end && *find_it == c) {
    return find_it - array;
  } else {
    return kNoMatch;
  }
}

// Returns the index of the range in the array that overlapped the given
// codepoint, or kNoMatch if no such range existed.
// The input array must be in sorted order.
int GetOverlappingRangeIndex(const char32* arr, int arr_length,
                             int range_length, char32 c) {
  const char32* end = arr + arr_length;
  const auto find_it = std::lower_bound(arr, end, c);
  if (find_it == end) {
    return kNoMatch;
  }
  // The end is inclusive, we so subtract one less than the range length.
  const char32 range_end = *find_it;
  const char32 range_start = range_end - (range_length - 1);
  if (c < range_start || range_end < c) {
    return kNoMatch;
  } else {
    return find_it - arr;
  }
}

// As above, but with explicit codepoint start and end indices for the range.
// The input array must be in sorted order.
int GetOverlappingRangeIndex(const char32* start_arr, const char32* end_arr,
                             int arr_length, int stride, char32 c) {
  const char32* end_arr_end = end_arr + arr_length;
  const auto find_it = std::lower_bound(end_arr, end_arr_end, c);
  if (find_it == end_arr_end) {
    return kNoMatch;
  }
  // Find the corresponding start.
  const int range_index = find_it - end_arr;
  const char32 range_start = start_arr[range_index];
  const char32 range_end = *find_it;
  if (c < range_start || range_end < c) {
    return kNoMatch;
  }
  if ((c - range_start) % stride == 0) {
    return range_index;
  } else {
    return kNoMatch;
  }
}

}  // anonymous namespace

bool IsOpeningBracket(char32 codepoint) {
  return GetMatchIndex(kOpeningBrackets, kNumOpeningBrackets, codepoint) >= 0;
}

bool IsClosingBracket(char32 codepoint) {
  return GetMatchIndex(kClosingBrackets, kNumClosingBrackets, codepoint) >= 0;
}

bool IsWhitespace(char32 codepoint) {
  return GetMatchIndex(kWhitespaces, kNumWhitespaces, codepoint) >= 0;
}

bool IsBidirectional(char32 codepoint) {
  return GetMatchIndex(kBidirectional, kNumBidirectional, codepoint) >= 0;
}

bool IsDigit(char32 codepoint) {
  return GetOverlappingRangeIndex(kDecimalDigitRangesEnd,
                                  kNumDecimalDigitRangesEnd,
                                  /*range_length=*/10, codepoint) >= 0;
}

bool IsLower(char32 codepoint) {
  if (GetMatchIndex(kLowerSingles, kNumLowerSingles, codepoint) >= 0) {
    return true;
  } else if (GetOverlappingRangeIndex(kLowerRanges1Start, kLowerRanges1End,
                                      kNumLowerRanges1Start, /*stride=*/1,
                                      codepoint) >= 0) {
    return true;
  } else if (GetOverlappingRangeIndex(kLowerRanges2Start, kLowerRanges2End,
                                      kNumLowerRanges2Start, /*stride=*/2,
                                      codepoint) >= 0) {
    return true;
  } else {
    return false;
  }
}

bool IsUpper(char32 codepoint) {
  if (GetMatchIndex(kUpperSingles, kNumUpperSingles, codepoint) >= 0) {
    return true;
  } else if (GetOverlappingRangeIndex(kUpperRanges1Start, kUpperRanges1End,
                                      kNumUpperRanges1Start, /*stride=*/1,
                                      codepoint) >= 0) {
    return true;
  } else if (GetOverlappingRangeIndex(kUpperRanges2Start, kUpperRanges2End,
                                      kNumUpperRanges2Start, /*stride=*/2,
                                      codepoint) >= 0) {
    return true;
  } else {
    return false;
  }
}

bool IsPunctuation(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kPunctuationRangesStart, kPunctuationRangesEnd,
              kNumPunctuationRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsPercentage(char32 codepoint) {
  return GetMatchIndex(kPercentages, kNumPercentages, codepoint) >= 0;
}

bool IsSlash(char32 codepoint) {
  return GetMatchIndex(kSlashes, kNumSlashes, codepoint) >= 0;
}

bool IsMinus(char32 codepoint) {
  return GetMatchIndex(kMinuses, kNumMinuses, codepoint) >= 0;
}

bool IsNumberSign(char32 codepoint) {
  return GetMatchIndex(kNumberSign, kNumNumberSign, codepoint) >= 0;
}

bool IsDot(char32 codepoint) {
  return GetMatchIndex(kDots, kNumDots, codepoint) >= 0;
}

bool IsApostrophe(char32 codepoint) {
  return GetMatchIndex(kApostrophe, kNumApostrophe, codepoint) >= 0;
}

bool IsQuotation(char32 codepoint) {
  return GetMatchIndex(kQuotation, kNumQuotation, codepoint) >= 0;
}

bool IsAmpersand(char32 codepoint) {
  return GetMatchIndex(kAmpersand, kNumAmpersand, codepoint) >= 0;
}

bool IsLatinLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kLatinLettersRangesStart, kLatinLettersRangesEnd,
              kNumLatinLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsArabicLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kArabicLettersRangesStart, kArabicLettersRangesEnd,
              kNumArabicLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsCyrillicLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kCyrillicLettersRangesStart, kCyrillicLettersRangesEnd,
              kNumCyrillicLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsChineseLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kChineseLettersRangesStart, kChineseLettersRangesEnd,
              kNumChineseLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsJapaneseLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kJapaneseLettersRangesStart, kJapaneseLettersRangesEnd,
              kNumJapaneseLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsKoreanLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kKoreanLettersRangesStart, kKoreanLettersRangesEnd,
              kNumKoreanLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsThaiLetter(char32 codepoint) {
  return (GetOverlappingRangeIndex(
              kThaiLettersRangesStart, kThaiLettersRangesEnd,
              kNumThaiLettersRangesStart, /*stride=*/1, codepoint) >= 0);
}

bool IsCJTletter(char32 codepoint) {
  return IsJapaneseLetter(codepoint) || IsChineseLetter(codepoint) ||
         IsThaiLetter(codepoint);
}

bool IsLetter(char32 codepoint) {
  return IsLatinLetter(codepoint) || IsArabicLetter(codepoint) ||
         IsCyrillicLetter(codepoint) || IsJapaneseLetter(codepoint) ||
         IsKoreanLetter(codepoint) || IsThaiLetter(codepoint) ||
         IsChineseLetter(codepoint);
}

char32 ToLower(char32 codepoint) {
  // Make sure we still produce output even if the method is called for a
  // codepoint that's not an uppercase character.
  if (!IsUpper(codepoint)) {
    return codepoint;
  }
  const int singles_idx =
      GetMatchIndex(kToLowerSingles, kNumToLowerSingles, codepoint);
  if (singles_idx >= 0) {
    return codepoint + kToLowerSinglesOffsets[singles_idx];
  }
  const int ranges_idx =
      GetOverlappingRangeIndex(kToLowerRangesStart, kToLowerRangesEnd,
                               kNumToLowerRangesStart, /*stride=*/1, codepoint);
  if (ranges_idx >= 0) {
    return codepoint + kToLowerRangesOffsets[ranges_idx];
  }
  return codepoint;
}

char32 ToUpper(char32 codepoint) {
  // Make sure we still produce output even if the method is called for a
  // codepoint that's not an uppercase character.
  if (!IsLower(codepoint)) {
    return codepoint;
  }
  const int singles_idx =
      GetMatchIndex(kToUpperSingles, kNumToUpperSingles, codepoint);
  if (singles_idx >= 0) {
    return codepoint + kToUpperSinglesOffsets[singles_idx];
  }
  const int ranges_idx =
      GetOverlappingRangeIndex(kToUpperRangesStart, kToUpperRangesEnd,
                               kNumToUpperRangesStart, /*stride=*/1, codepoint);
  if (ranges_idx >= 0) {
    return codepoint + kToUpperRangesOffsets[ranges_idx];
  }
  return codepoint;
}

char32 GetPairedBracket(char32 codepoint) {
  const int open_offset =
      GetMatchIndex(kOpeningBrackets, kNumOpeningBrackets, codepoint);
  if (open_offset >= 0) {
    return kClosingBrackets[open_offset];
  }
  const int close_offset =
      GetMatchIndex(kClosingBrackets, kNumClosingBrackets, codepoint);
  if (close_offset >= 0) {
    return kOpeningBrackets[close_offset];
  }
  return codepoint;
}

}  // namespace libtextclassifier3