1fcaf7f86SDimitry Andric // -*- C++ -*- 2fcaf7f86SDimitry Andric //===----------------------------------------------------------------------===// 3fcaf7f86SDimitry Andric // 4fcaf7f86SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5fcaf7f86SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 6fcaf7f86SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7fcaf7f86SDimitry Andric // 8fcaf7f86SDimitry Andric //===----------------------------------------------------------------------===// 9fcaf7f86SDimitry Andric 10fcaf7f86SDimitry Andric #ifndef _LIBCPP___FORMAT_UNICODE_H 11fcaf7f86SDimitry Andric #define _LIBCPP___FORMAT_UNICODE_H 12fcaf7f86SDimitry Andric 13fcaf7f86SDimitry Andric #include <__assert> 1406c3fb27SDimitry Andric #include <__bit/countl.h> 1506c3fb27SDimitry Andric #include <__concepts/same_as.h> 16fcaf7f86SDimitry Andric #include <__config> 17fcaf7f86SDimitry Andric #include <__format/extended_grapheme_cluster_table.h> 18*0fca6ea1SDimitry Andric #include <__format/indic_conjunct_break_table.h> 1906c3fb27SDimitry Andric #include <__iterator/concepts.h> 2006c3fb27SDimitry Andric #include <__iterator/readable_traits.h> // iter_value_t 21*0fca6ea1SDimitry Andric #include <__utility/unreachable.h> 2206c3fb27SDimitry Andric #include <string_view> 23fcaf7f86SDimitry Andric 24fcaf7f86SDimitry Andric #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER) 25fcaf7f86SDimitry Andric # pragma GCC system_header 26fcaf7f86SDimitry Andric #endif 27fcaf7f86SDimitry Andric 28fcaf7f86SDimitry Andric _LIBCPP_BEGIN_NAMESPACE_STD 29fcaf7f86SDimitry Andric 3006c3fb27SDimitry Andric #if _LIBCPP_STD_VER >= 20 31fcaf7f86SDimitry Andric 32bdd1243dSDimitry Andric namespace __unicode { 33bdd1243dSDimitry Andric 3406c3fb27SDimitry Andric // Helper struct for the result of a consume operation. 3506c3fb27SDimitry Andric // 3606c3fb27SDimitry Andric // The status value for a correct code point is 0. This allows a valid value to 3706c3fb27SDimitry Andric // be used without masking. 3806c3fb27SDimitry Andric // When the decoding fails it know the number of code units affected. For the 3906c3fb27SDimitry Andric // current use-cases that value is not needed, therefore it is not stored. 4006c3fb27SDimitry Andric // The escape routine needs the number of code units for both a valid and 4106c3fb27SDimitry Andric // invalid character and keeps track of it itself. Doing it in this result 4206c3fb27SDimitry Andric // unconditionally would give some overhead when the value is unneeded. 4306c3fb27SDimitry Andric struct __consume_result { 4406c3fb27SDimitry Andric // When __status == __ok it contains the decoded code point. 4506c3fb27SDimitry Andric // Else it contains the replacement character U+FFFD 4606c3fb27SDimitry Andric char32_t __code_point : 31; 47bdd1243dSDimitry Andric 4806c3fb27SDimitry Andric enum : char32_t { 4906c3fb27SDimitry Andric // Consumed a well-formed code point. 5006c3fb27SDimitry Andric __ok = 0, 5106c3fb27SDimitry Andric // Encountered invalid UTF-8 5206c3fb27SDimitry Andric __error = 1 5306c3fb27SDimitry Andric } __status : 1 {__ok}; 54bdd1243dSDimitry Andric }; 5506c3fb27SDimitry Andric static_assert(sizeof(__consume_result) == sizeof(char32_t)); 56bdd1243dSDimitry Andric 57fcaf7f86SDimitry Andric # ifndef _LIBCPP_HAS_NO_UNICODE 58fcaf7f86SDimitry Andric 59fcaf7f86SDimitry Andric /// Implements the grapheme cluster boundary rules 60fcaf7f86SDimitry Andric /// 61fcaf7f86SDimitry Andric /// These rules are used to implement format's width estimation as stated in 62fcaf7f86SDimitry Andric /// [format.string.std]/11 63fcaf7f86SDimitry Andric /// 64fcaf7f86SDimitry Andric /// The Standard refers to UAX \#29 for Unicode 12.0.0 65fcaf7f86SDimitry Andric /// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules 66fcaf7f86SDimitry Andric /// 67fcaf7f86SDimitry Andric /// The data tables used are 68fcaf7f86SDimitry Andric /// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt 69fcaf7f86SDimitry Andric /// https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt 70fcaf7f86SDimitry Andric /// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt (for testing only) 71fcaf7f86SDimitry Andric 72fcaf7f86SDimitry Andric inline constexpr char32_t __replacement_character = U'\ufffd'; 73fcaf7f86SDimitry Andric 7406c3fb27SDimitry Andric // The error of a consume operation. 7506c3fb27SDimitry Andric // 7606c3fb27SDimitry Andric // This sets the code point to the replacement character. This code point does 7706c3fb27SDimitry Andric // not participate in the grapheme clustering, so grapheme clustering code can 7806c3fb27SDimitry Andric // ignore the error status and always use the code point. 7906c3fb27SDimitry Andric inline constexpr __consume_result __consume_result_error{__replacement_character, __consume_result::__error}; 8006c3fb27SDimitry Andric 8106c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_high_surrogate(char32_t __value) { 8206c3fb27SDimitry Andric return __value >= 0xd800 && __value <= 0xdbff; 8306c3fb27SDimitry Andric } 8406c3fb27SDimitry Andric 8506c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_low_surrogate(char32_t __value) { 8606c3fb27SDimitry Andric return __value >= 0xdc00 && __value <= 0xdfff; 8706c3fb27SDimitry Andric } 8806c3fb27SDimitry Andric 8906c3fb27SDimitry Andric // https://www.unicode.org/glossary/#surrogate_code_point 9006c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_surrogate(char32_t __value) { 9106c3fb27SDimitry Andric return __value >= 0xd800 && __value <= 0xdfff; 9206c3fb27SDimitry Andric } 9306c3fb27SDimitry Andric 9406c3fb27SDimitry Andric // https://www.unicode.org/glossary/#code_point 9506c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_code_point(char32_t __value) { 9606c3fb27SDimitry Andric return __value <= 0x10ffff; 9706c3fb27SDimitry Andric } 9806c3fb27SDimitry Andric 9906c3fb27SDimitry Andric // https://www.unicode.org/glossary/#unicode_scalar_value 10006c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) { 10106c3fb27SDimitry Andric return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value); 10206c3fb27SDimitry Andric } 10306c3fb27SDimitry Andric 10406c3fb27SDimitry Andric template <contiguous_iterator _Iterator> 10506c3fb27SDimitry Andric requires same_as<iter_value_t<_Iterator>, char> 10606c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) { 107fcaf7f86SDimitry Andric do { 10806c3fb27SDimitry Andric if ((*__char & 0b1100'0000) != 0b1000'0000) 109fcaf7f86SDimitry Andric return false; 110fcaf7f86SDimitry Andric --__count; 111fcaf7f86SDimitry Andric ++__char; 112fcaf7f86SDimitry Andric } while (__count); 113fcaf7f86SDimitry Andric return true; 114fcaf7f86SDimitry Andric } 115fcaf7f86SDimitry Andric 116fcaf7f86SDimitry Andric /// Helper class to extract a code unit from a Unicode character range. 117fcaf7f86SDimitry Andric /// 118fcaf7f86SDimitry Andric /// The stored range is a view. There are multiple specialization for different 119fcaf7f86SDimitry Andric /// character types. 120fcaf7f86SDimitry Andric template <class _CharT> 121fcaf7f86SDimitry Andric class __code_point_view; 122fcaf7f86SDimitry Andric 123fcaf7f86SDimitry Andric /// UTF-8 specialization. 124fcaf7f86SDimitry Andric template <> 125fcaf7f86SDimitry Andric class __code_point_view<char> { 12606c3fb27SDimitry Andric using _Iterator = basic_string_view<char>::const_iterator; 12706c3fb27SDimitry Andric 128fcaf7f86SDimitry Andric public: 12906c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last) 130fcaf7f86SDimitry Andric : __first_(__first), __last_(__last) {} 131fcaf7f86SDimitry Andric 132fcaf7f86SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } 13306c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } 134fcaf7f86SDimitry Andric 13506c3fb27SDimitry Andric // https://www.unicode.org/versions/latest/ch03.pdf#G7404 13606c3fb27SDimitry Andric // Based on Table 3-7, Well-Formed UTF-8 Byte Sequences 13706c3fb27SDimitry Andric // 13806c3fb27SDimitry Andric // Code Points First Byte Second Byte Third Byte Fourth Byte Remarks 13906c3fb27SDimitry Andric // U+0000..U+007F 00..7F U+0000..U+007F 1 code unit range 14006c3fb27SDimitry Andric // C0..C1 80..BF invalid overlong encoding 14106c3fb27SDimitry Andric // U+0080..U+07FF C2..DF 80..BF U+0080..U+07FF 2 code unit range 14206c3fb27SDimitry Andric // E0 80..9F 80..BF invalid overlong encoding 14306c3fb27SDimitry Andric // U+0800..U+0FFF E0 A0..BF 80..BF U+0800..U+FFFF 3 code unit range 14406c3fb27SDimitry Andric // U+1000..U+CFFF E1..EC 80..BF 80..BF 14506c3fb27SDimitry Andric // U+D000..U+D7FF ED 80..9F 80..BF 14606c3fb27SDimitry Andric // U+D800..U+DFFF ED A0..BF 80..BF invalid encoding of surrogate code point 14706c3fb27SDimitry Andric // U+E000..U+FFFF EE..EF 80..BF 80..BF 14806c3fb27SDimitry Andric // F0 80..8F 80..BF 80..BF invalid overlong encoding 14906c3fb27SDimitry Andric // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF U+10000..U+10FFFF 4 code unit range 15006c3fb27SDimitry Andric // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF 15106c3fb27SDimitry Andric // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF 15206c3fb27SDimitry Andric // F4 90..BF 80..BF 80..BF U+110000.. invalid code point range 15306c3fb27SDimitry Andric // 15406c3fb27SDimitry Andric // Unlike other parsers, these invalid entries are tested after decoding. 15506c3fb27SDimitry Andric // - The parser always needs to consume these code units 15606c3fb27SDimitry Andric // - The code is optimized for well-formed UTF-8 15706c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { 1581db9f3b2SDimitry Andric _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); 159bdd1243dSDimitry Andric 160bdd1243dSDimitry Andric // Based on the number of leading 1 bits the number of code units in the 161bdd1243dSDimitry Andric // code point can be determined. See 162bdd1243dSDimitry Andric // https://en.wikipedia.org/wiki/UTF-8#Encoding 163bdd1243dSDimitry Andric switch (std::countl_one(static_cast<unsigned char>(*__first_))) { 164bdd1243dSDimitry Andric case 0: 16506c3fb27SDimitry Andric return {static_cast<unsigned char>(*__first_++)}; 166bdd1243dSDimitry Andric 16706c3fb27SDimitry Andric case 2: { 16806c3fb27SDimitry Andric if (__last_ - __first_ < 2 || !__unicode::__is_continuation(__first_ + 1, 1)) [[unlikely]] 169bdd1243dSDimitry Andric break; 170bdd1243dSDimitry Andric 171bdd1243dSDimitry Andric char32_t __value = static_cast<unsigned char>(*__first_++) & 0x1f; 172bdd1243dSDimitry Andric __value <<= 6; 173bdd1243dSDimitry Andric __value |= static_cast<unsigned char>(*__first_++) & 0x3f; 17406c3fb27SDimitry Andric 17506c3fb27SDimitry Andric // These values should be encoded in 1 UTF-8 code unit. 17606c3fb27SDimitry Andric if (__value < 0x0080) [[unlikely]] 17706c3fb27SDimitry Andric return __consume_result_error; 17806c3fb27SDimitry Andric 17906c3fb27SDimitry Andric return {__value}; 180bdd1243dSDimitry Andric } 18106c3fb27SDimitry Andric 18206c3fb27SDimitry Andric case 3: { 18306c3fb27SDimitry Andric if (__last_ - __first_ < 3 || !__unicode::__is_continuation(__first_ + 1, 2)) [[unlikely]] 184bdd1243dSDimitry Andric break; 185bdd1243dSDimitry Andric 186bdd1243dSDimitry Andric char32_t __value = static_cast<unsigned char>(*__first_++) & 0x0f; 187bdd1243dSDimitry Andric __value <<= 6; 188bdd1243dSDimitry Andric __value |= static_cast<unsigned char>(*__first_++) & 0x3f; 189bdd1243dSDimitry Andric __value <<= 6; 190bdd1243dSDimitry Andric __value |= static_cast<unsigned char>(*__first_++) & 0x3f; 19106c3fb27SDimitry Andric 19206c3fb27SDimitry Andric // These values should be encoded in 1 or 2 UTF-8 code units. 19306c3fb27SDimitry Andric if (__value < 0x0800) [[unlikely]] 19406c3fb27SDimitry Andric return __consume_result_error; 19506c3fb27SDimitry Andric 19606c3fb27SDimitry Andric // A surrogate value is always encoded in 3 UTF-8 code units. 19706c3fb27SDimitry Andric if (__unicode::__is_surrogate(__value)) [[unlikely]] 19806c3fb27SDimitry Andric return __consume_result_error; 19906c3fb27SDimitry Andric 20006c3fb27SDimitry Andric return {__value}; 201bdd1243dSDimitry Andric } 20206c3fb27SDimitry Andric 20306c3fb27SDimitry Andric case 4: { 20406c3fb27SDimitry Andric if (__last_ - __first_ < 4 || !__unicode::__is_continuation(__first_ + 1, 3)) [[unlikely]] 205bdd1243dSDimitry Andric break; 206bdd1243dSDimitry Andric 207bdd1243dSDimitry Andric char32_t __value = static_cast<unsigned char>(*__first_++) & 0x07; 208bdd1243dSDimitry Andric __value <<= 6; 209bdd1243dSDimitry Andric __value |= static_cast<unsigned char>(*__first_++) & 0x3f; 210bdd1243dSDimitry Andric __value <<= 6; 211bdd1243dSDimitry Andric __value |= static_cast<unsigned char>(*__first_++) & 0x3f; 212bdd1243dSDimitry Andric __value <<= 6; 213bdd1243dSDimitry Andric __value |= static_cast<unsigned char>(*__first_++) & 0x3f; 214bdd1243dSDimitry Andric 21506c3fb27SDimitry Andric // These values should be encoded in 1, 2, or 3 UTF-8 code units. 21606c3fb27SDimitry Andric if (__value < 0x10000) [[unlikely]] 21706c3fb27SDimitry Andric return __consume_result_error; 218bdd1243dSDimitry Andric 21906c3fb27SDimitry Andric // A value too large is always encoded in 4 UTF-8 code units. 22006c3fb27SDimitry Andric if (!__unicode::__is_code_point(__value)) [[unlikely]] 22106c3fb27SDimitry Andric return __consume_result_error; 22206c3fb27SDimitry Andric 22306c3fb27SDimitry Andric return {__value}; 224bdd1243dSDimitry Andric } 225bdd1243dSDimitry Andric } 226bdd1243dSDimitry Andric // An invalid number of leading ones can be garbage or a code unit in the 227bdd1243dSDimitry Andric // middle of a code point. By consuming one code unit the parser may get 228bdd1243dSDimitry Andric // "in sync" after a few code units. 22906c3fb27SDimitry Andric ++__first_; 23006c3fb27SDimitry Andric return __consume_result_error; 231bdd1243dSDimitry Andric } 232bdd1243dSDimitry Andric 233fcaf7f86SDimitry Andric private: 23406c3fb27SDimitry Andric _Iterator __first_; 23506c3fb27SDimitry Andric _Iterator __last_; 236fcaf7f86SDimitry Andric }; 237fcaf7f86SDimitry Andric 238bdd1243dSDimitry Andric # ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS 239bdd1243dSDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_high(wchar_t __value) { 240bdd1243dSDimitry Andric return __value >= 0xd800 && __value <= 0xdbff; 241bdd1243dSDimitry Andric } 242bdd1243dSDimitry Andric 243bdd1243dSDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_low(wchar_t __value) { 244bdd1243dSDimitry Andric return __value >= 0xdc00 && __value <= 0xdfff; 245bdd1243dSDimitry Andric } 246bdd1243dSDimitry Andric 247fcaf7f86SDimitry Andric /// This specialization depends on the size of wchar_t 248fcaf7f86SDimitry Andric /// - 2 UTF-16 (for example Windows and AIX) 249fcaf7f86SDimitry Andric /// - 4 UTF-32 (for example Linux) 250fcaf7f86SDimitry Andric template <> 251fcaf7f86SDimitry Andric class __code_point_view<wchar_t> { 25206c3fb27SDimitry Andric using _Iterator = typename basic_string_view<wchar_t>::const_iterator; 25306c3fb27SDimitry Andric 254fcaf7f86SDimitry Andric public: 255bdd1243dSDimitry Andric static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sizeof(wchar_t) has a not implemented value"); 256bdd1243dSDimitry Andric 25706c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last) 258fcaf7f86SDimitry Andric : __first_(__first), __last_(__last) {} 259fcaf7f86SDimitry Andric 26006c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } 261fcaf7f86SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } 262fcaf7f86SDimitry Andric 26306c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { 2641db9f3b2SDimitry Andric _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); 265fcaf7f86SDimitry Andric 26606c3fb27SDimitry Andric char32_t __value = static_cast<char32_t>(*__first_++); 267fcaf7f86SDimitry Andric if constexpr (sizeof(wchar_t) == 2) { 26806c3fb27SDimitry Andric if (__unicode::__is_low_surrogate(__value)) [[unlikely]] 26906c3fb27SDimitry Andric return __consume_result_error; 270fcaf7f86SDimitry Andric 27106c3fb27SDimitry Andric if (__unicode::__is_high_surrogate(__value)) { 27206c3fb27SDimitry Andric if (__first_ == __last_ || !__unicode::__is_low_surrogate(static_cast<char32_t>(*__first_))) [[unlikely]] 27306c3fb27SDimitry Andric return __consume_result_error; 27406c3fb27SDimitry Andric 27506c3fb27SDimitry Andric __value -= 0xd800; 27606c3fb27SDimitry Andric __value <<= 10; 27706c3fb27SDimitry Andric __value += static_cast<char32_t>(*__first_++) - 0xdc00; 27806c3fb27SDimitry Andric __value += 0x10000; 27906c3fb27SDimitry Andric 28006c3fb27SDimitry Andric if (!__unicode::__is_code_point(__value)) [[unlikely]] 28106c3fb27SDimitry Andric return __consume_result_error; 282fcaf7f86SDimitry Andric } 283fcaf7f86SDimitry Andric } else { 28406c3fb27SDimitry Andric if (!__unicode::__is_scalar_value(__value)) [[unlikely]] 28506c3fb27SDimitry Andric return __consume_result_error; 286fcaf7f86SDimitry Andric } 287fcaf7f86SDimitry Andric 28806c3fb27SDimitry Andric return {__value}; 289bdd1243dSDimitry Andric } 290bdd1243dSDimitry Andric 291fcaf7f86SDimitry Andric private: 29206c3fb27SDimitry Andric _Iterator __first_; 29306c3fb27SDimitry Andric _Iterator __last_; 294fcaf7f86SDimitry Andric }; 295bdd1243dSDimitry Andric # endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS 296fcaf7f86SDimitry Andric 297*0fca6ea1SDimitry Andric // State machine to implement the Extended Grapheme Cluster Boundary 298*0fca6ea1SDimitry Andric // 299*0fca6ea1SDimitry Andric // The exact rules may change between Unicode versions. 300*0fca6ea1SDimitry Andric // This implements the extended rules see 301*0fca6ea1SDimitry Andric // https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries 302*0fca6ea1SDimitry Andric class __extended_grapheme_cluster_break { 303*0fca6ea1SDimitry Andric using __EGC_property = __extended_grapheme_custer_property_boundary::__property; 304*0fca6ea1SDimitry Andric using __inCB_property = __indic_conjunct_break::__property; 305fcaf7f86SDimitry Andric 306*0fca6ea1SDimitry Andric public: 307*0fca6ea1SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point) 308*0fca6ea1SDimitry Andric : __prev_code_point_(__first_code_point), 309*0fca6ea1SDimitry Andric __prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) { 310*0fca6ea1SDimitry Andric // Initializes the active rule. 311*0fca6ea1SDimitry Andric if (__prev_property_ == __EGC_property::__Extended_Pictographic) 312*0fca6ea1SDimitry Andric __active_rule_ = __rule::__GB11_emoji; 313*0fca6ea1SDimitry Andric else if (__prev_property_ == __EGC_property::__Regional_Indicator) 314*0fca6ea1SDimitry Andric __active_rule_ = __rule::__GB12_GB13_regional_indicator; 315*0fca6ea1SDimitry Andric else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant) 316*0fca6ea1SDimitry Andric __active_rule_ = __rule::__GB9c_indic_conjunct_break; 317*0fca6ea1SDimitry Andric } 318fcaf7f86SDimitry Andric 319*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) { 320*0fca6ea1SDimitry Andric __EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point); 321*0fca6ea1SDimitry Andric bool __result = __evaluate(__next_code_point, __next_property); 322*0fca6ea1SDimitry Andric __prev_code_point_ = __next_code_point; 323*0fca6ea1SDimitry Andric __prev_property_ = __next_property; 324*0fca6ea1SDimitry Andric return __result; 325*0fca6ea1SDimitry Andric } 326fcaf7f86SDimitry Andric 327*0fca6ea1SDimitry Andric // The code point whose break propery are considered during the next 328*0fca6ea1SDimitry Andric // evaluation cyle. 329*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; } 330*0fca6ea1SDimitry Andric 331*0fca6ea1SDimitry Andric private: 332*0fca6ea1SDimitry Andric // The naming of the identifiers matches the Unicode standard. 333*0fca6ea1SDimitry Andric // NOLINTBEGIN(readability-identifier-naming) 334*0fca6ea1SDimitry Andric 335*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool 336*0fca6ea1SDimitry Andric __evaluate(char32_t __next_code_point, __EGC_property __next_property) { 337*0fca6ea1SDimitry Andric switch (__active_rule_) { 338*0fca6ea1SDimitry Andric case __rule::__none: 339*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 340*0fca6ea1SDimitry Andric case __rule::__GB9c_indic_conjunct_break: 341*0fca6ea1SDimitry Andric return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property); 342*0fca6ea1SDimitry Andric case __rule::__GB11_emoji: 343*0fca6ea1SDimitry Andric return __evaluate_GB11_emoji(__next_code_point, __next_property); 344*0fca6ea1SDimitry Andric case __rule::__GB12_GB13_regional_indicator: 345*0fca6ea1SDimitry Andric return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property); 346*0fca6ea1SDimitry Andric } 347*0fca6ea1SDimitry Andric __libcpp_unreachable(); 348*0fca6ea1SDimitry Andric } 349*0fca6ea1SDimitry Andric 350*0fca6ea1SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) { 351fcaf7f86SDimitry Andric // *** Break at the start and end of text, unless the text is empty. *** 352fcaf7f86SDimitry Andric 353*0fca6ea1SDimitry Andric _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); // GB1 354*0fca6ea1SDimitry Andric _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller"); // GB2 355fcaf7f86SDimitry Andric 356fcaf7f86SDimitry Andric // *** Do not break between a CR and LF. Otherwise, break before and after controls. *** 357*0fca6ea1SDimitry Andric if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) // GB3 358fcaf7f86SDimitry Andric return false; 359fcaf7f86SDimitry Andric 360*0fca6ea1SDimitry Andric if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR || 361*0fca6ea1SDimitry Andric __prev_property_ == __EGC_property::__LF) // GB4 362fcaf7f86SDimitry Andric return true; 363fcaf7f86SDimitry Andric 364*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR || 365*0fca6ea1SDimitry Andric __next_property == __EGC_property::__LF) // GB5 366fcaf7f86SDimitry Andric return true; 367fcaf7f86SDimitry Andric 368fcaf7f86SDimitry Andric // *** Do not break Hangul syllable sequences. *** 369*0fca6ea1SDimitry Andric if (__prev_property_ == __EGC_property::__L && 370*0fca6ea1SDimitry Andric (__next_property == __EGC_property::__L || __next_property == __EGC_property::__V || 371*0fca6ea1SDimitry Andric __next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) // GB6 372fcaf7f86SDimitry Andric return false; 373fcaf7f86SDimitry Andric 374*0fca6ea1SDimitry Andric if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) && 375*0fca6ea1SDimitry Andric (__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) // GB7 376fcaf7f86SDimitry Andric return false; 377fcaf7f86SDimitry Andric 378*0fca6ea1SDimitry Andric if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) && 379*0fca6ea1SDimitry Andric __next_property == __EGC_property::__T) // GB8 380fcaf7f86SDimitry Andric return false; 381fcaf7f86SDimitry Andric 382fcaf7f86SDimitry Andric // *** Do not break before extending characters or ZWJ. *** 383*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ) 384fcaf7f86SDimitry Andric return false; // GB9 385fcaf7f86SDimitry Andric 386fcaf7f86SDimitry Andric // *** Do not break before SpacingMarks, or after Prepend characters. *** 387*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__SpacingMark) // GB9a 388fcaf7f86SDimitry Andric return false; 389fcaf7f86SDimitry Andric 390*0fca6ea1SDimitry Andric if (__prev_property_ == __EGC_property::__Prepend) // GB9b 391fcaf7f86SDimitry Andric return false; 392fcaf7f86SDimitry Andric 393*0fca6ea1SDimitry Andric // *** Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. *** 394*0fca6ea1SDimitry Andric if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) { 395*0fca6ea1SDimitry Andric __active_rule_ = __rule::__GB9c_indic_conjunct_break; 396*0fca6ea1SDimitry Andric __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant; 397*0fca6ea1SDimitry Andric return true; 398*0fca6ea1SDimitry Andric } 399*0fca6ea1SDimitry Andric 400fcaf7f86SDimitry Andric // *** Do not break within emoji modifier sequences or emoji zwj sequences. *** 401*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Extended_Pictographic) { 402*0fca6ea1SDimitry Andric __active_rule_ = __rule::__GB11_emoji; 403*0fca6ea1SDimitry Andric __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic; 404*0fca6ea1SDimitry Andric return true; 405*0fca6ea1SDimitry Andric } 406fcaf7f86SDimitry Andric 407fcaf7f86SDimitry Andric // *** Do not break within emoji flag sequences *** 408fcaf7f86SDimitry Andric 409fcaf7f86SDimitry Andric // That is, do not break between regional indicator (RI) symbols if there 410fcaf7f86SDimitry Andric // is an odd number of RI characters before the break point. 411*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Regional_Indicator) { // GB12 + GB13 412*0fca6ea1SDimitry Andric __active_rule_ = __rule::__GB12_GB13_regional_indicator; 413*0fca6ea1SDimitry Andric return true; 414fcaf7f86SDimitry Andric } 415fcaf7f86SDimitry Andric 416fcaf7f86SDimitry Andric // *** Otherwise, break everywhere. *** 417fcaf7f86SDimitry Andric return true; // GB999 418fcaf7f86SDimitry Andric } 419fcaf7f86SDimitry Andric 420*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool 421*0fca6ea1SDimitry Andric __evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) { 422*0fca6ea1SDimitry Andric __inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point); 423*0fca6ea1SDimitry Andric if (__break == __inCB_property::__none) { 424*0fca6ea1SDimitry Andric __active_rule_ = __rule::__none; 425*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 426*0fca6ea1SDimitry Andric } 427*0fca6ea1SDimitry Andric 428*0fca6ea1SDimitry Andric switch (__GB9c_indic_conjunct_break_state_) { 429*0fca6ea1SDimitry Andric case __GB9c_indic_conjunct_break_state::__Consonant: 430*0fca6ea1SDimitry Andric if (__break == __inCB_property::__Extend) { 431*0fca6ea1SDimitry Andric return false; 432*0fca6ea1SDimitry Andric } 433*0fca6ea1SDimitry Andric if (__break == __inCB_property::__Linker) { 434*0fca6ea1SDimitry Andric __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker; 435*0fca6ea1SDimitry Andric return false; 436*0fca6ea1SDimitry Andric } 437*0fca6ea1SDimitry Andric __active_rule_ = __rule::__none; 438*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 439*0fca6ea1SDimitry Andric 440*0fca6ea1SDimitry Andric case __GB9c_indic_conjunct_break_state::__Linker: 441*0fca6ea1SDimitry Andric if (__break == __inCB_property::__Extend) { 442*0fca6ea1SDimitry Andric return false; 443*0fca6ea1SDimitry Andric } 444*0fca6ea1SDimitry Andric if (__break == __inCB_property::__Linker) { 445*0fca6ea1SDimitry Andric return false; 446*0fca6ea1SDimitry Andric } 447*0fca6ea1SDimitry Andric if (__break == __inCB_property::__Consonant) { 448*0fca6ea1SDimitry Andric __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant; 449*0fca6ea1SDimitry Andric return false; 450*0fca6ea1SDimitry Andric } 451*0fca6ea1SDimitry Andric __active_rule_ = __rule::__none; 452*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 453*0fca6ea1SDimitry Andric } 454*0fca6ea1SDimitry Andric __libcpp_unreachable(); 455*0fca6ea1SDimitry Andric } 456*0fca6ea1SDimitry Andric 457*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool 458*0fca6ea1SDimitry Andric __evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) { 459*0fca6ea1SDimitry Andric switch (__GB11_emoji_state_) { 460*0fca6ea1SDimitry Andric case __GB11_emoji_state::__Extended_Pictographic: 461*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Extend) { 462*0fca6ea1SDimitry Andric __GB11_emoji_state_ = __GB11_emoji_state::__Extend; 463*0fca6ea1SDimitry Andric return false; 464*0fca6ea1SDimitry Andric } 465*0fca6ea1SDimitry Andric [[fallthrough]]; 466*0fca6ea1SDimitry Andric case __GB11_emoji_state::__Extend: 467*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__ZWJ) { 468*0fca6ea1SDimitry Andric __GB11_emoji_state_ = __GB11_emoji_state::__ZWJ; 469*0fca6ea1SDimitry Andric return false; 470*0fca6ea1SDimitry Andric } 471*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Extend) 472*0fca6ea1SDimitry Andric return false; 473*0fca6ea1SDimitry Andric __active_rule_ = __rule::__none; 474*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 475*0fca6ea1SDimitry Andric 476*0fca6ea1SDimitry Andric case __GB11_emoji_state::__ZWJ: 477*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Extended_Pictographic) { 478*0fca6ea1SDimitry Andric __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic; 479*0fca6ea1SDimitry Andric return false; 480*0fca6ea1SDimitry Andric } 481*0fca6ea1SDimitry Andric __active_rule_ = __rule::__none; 482*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 483*0fca6ea1SDimitry Andric } 484*0fca6ea1SDimitry Andric __libcpp_unreachable(); 485*0fca6ea1SDimitry Andric } 486*0fca6ea1SDimitry Andric 487*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool 488*0fca6ea1SDimitry Andric __evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) { 489*0fca6ea1SDimitry Andric __active_rule_ = __rule::__none; 490*0fca6ea1SDimitry Andric if (__next_property == __EGC_property::__Regional_Indicator) 491*0fca6ea1SDimitry Andric return false; 492*0fca6ea1SDimitry Andric return __evaluate_none(__next_code_point, __next_property); 493*0fca6ea1SDimitry Andric } 494*0fca6ea1SDimitry Andric 495*0fca6ea1SDimitry Andric char32_t __prev_code_point_; 496*0fca6ea1SDimitry Andric __EGC_property __prev_property_; 497*0fca6ea1SDimitry Andric 498*0fca6ea1SDimitry Andric enum class __rule { 499*0fca6ea1SDimitry Andric __none, 500*0fca6ea1SDimitry Andric __GB9c_indic_conjunct_break, 501*0fca6ea1SDimitry Andric __GB11_emoji, 502*0fca6ea1SDimitry Andric __GB12_GB13_regional_indicator, 503*0fca6ea1SDimitry Andric }; 504*0fca6ea1SDimitry Andric __rule __active_rule_ = __rule::__none; 505*0fca6ea1SDimitry Andric 506*0fca6ea1SDimitry Andric enum class __GB11_emoji_state { 507*0fca6ea1SDimitry Andric __Extended_Pictographic, 508*0fca6ea1SDimitry Andric __Extend, 509*0fca6ea1SDimitry Andric __ZWJ, 510*0fca6ea1SDimitry Andric }; 511*0fca6ea1SDimitry Andric __GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic; 512*0fca6ea1SDimitry Andric 513*0fca6ea1SDimitry Andric enum class __GB9c_indic_conjunct_break_state { 514*0fca6ea1SDimitry Andric __Consonant, 515*0fca6ea1SDimitry Andric __Linker, 516*0fca6ea1SDimitry Andric }; 517*0fca6ea1SDimitry Andric 518*0fca6ea1SDimitry Andric __GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant; 519*0fca6ea1SDimitry Andric 520*0fca6ea1SDimitry Andric // NOLINTEND(readability-identifier-naming) 521*0fca6ea1SDimitry Andric }; 522*0fca6ea1SDimitry Andric 523fcaf7f86SDimitry Andric /// Helper class to extract an extended grapheme cluster from a Unicode character range. 524fcaf7f86SDimitry Andric /// 525fcaf7f86SDimitry Andric /// This function is used to determine the column width of an extended grapheme 526fcaf7f86SDimitry Andric /// cluster. In order to do that only the first code point is evaluated. 527fcaf7f86SDimitry Andric /// Therefore only this code point is extracted. 528fcaf7f86SDimitry Andric template <class _CharT> 529fcaf7f86SDimitry Andric class __extended_grapheme_cluster_view { 53006c3fb27SDimitry Andric using _Iterator = typename basic_string_view<_CharT>::const_iterator; 53106c3fb27SDimitry Andric 532fcaf7f86SDimitry Andric public: 53306c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last) 534*0fca6ea1SDimitry Andric : __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {} 535fcaf7f86SDimitry Andric 536fcaf7f86SDimitry Andric struct __cluster { 537fcaf7f86SDimitry Andric /// The first code point of the extended grapheme cluster. 538fcaf7f86SDimitry Andric /// 539fcaf7f86SDimitry Andric /// The first code point is used to estimate the width of the extended 540fcaf7f86SDimitry Andric /// grapheme cluster. 541fcaf7f86SDimitry Andric char32_t __code_point_; 542fcaf7f86SDimitry Andric 543fcaf7f86SDimitry Andric /// Points one beyond the last code unit in the extended grapheme cluster. 544fcaf7f86SDimitry Andric /// 545fcaf7f86SDimitry Andric /// It's expected the caller has the start position and thus can determine 546fcaf7f86SDimitry Andric /// the code unit range of the extended grapheme cluster. 54706c3fb27SDimitry Andric _Iterator __last_; 548fcaf7f86SDimitry Andric }; 549fcaf7f86SDimitry Andric 550*0fca6ea1SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() { 551*0fca6ea1SDimitry Andric char32_t __code_point = __at_break_.__current_code_point(); 552*0fca6ea1SDimitry Andric _Iterator __position = __code_point_view_.__position(); 553*0fca6ea1SDimitry Andric while (!__code_point_view_.__at_end()) { 554*0fca6ea1SDimitry Andric if (__at_break_(__code_point_view_.__consume().__code_point)) 555*0fca6ea1SDimitry Andric break; 556*0fca6ea1SDimitry Andric __position = __code_point_view_.__position(); 557*0fca6ea1SDimitry Andric } 558*0fca6ea1SDimitry Andric return {__code_point, __position}; 559fcaf7f86SDimitry Andric } 560fcaf7f86SDimitry Andric 561fcaf7f86SDimitry Andric private: 562fcaf7f86SDimitry Andric __code_point_view<_CharT> __code_point_view_; 563*0fca6ea1SDimitry Andric __extended_grapheme_cluster_break __at_break_; 564fcaf7f86SDimitry Andric }; 565fcaf7f86SDimitry Andric 56606c3fb27SDimitry Andric template <contiguous_iterator _Iterator> 56706c3fb27SDimitry Andric __extended_grapheme_cluster_view(_Iterator, _Iterator) -> __extended_grapheme_cluster_view<iter_value_t<_Iterator>>; 568bdd1243dSDimitry Andric 569bdd1243dSDimitry Andric # else // _LIBCPP_HAS_NO_UNICODE 570bdd1243dSDimitry Andric 571bdd1243dSDimitry Andric // For ASCII every character is a "code point". 572bdd1243dSDimitry Andric // This makes it easier to write code agnostic of the _LIBCPP_HAS_NO_UNICODE define. 573bdd1243dSDimitry Andric template <class _CharT> 574bdd1243dSDimitry Andric class __code_point_view { 57506c3fb27SDimitry Andric using _Iterator = typename basic_string_view<_CharT>::const_iterator; 57606c3fb27SDimitry Andric 577bdd1243dSDimitry Andric public: 57806c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last) 579bdd1243dSDimitry Andric : __first_(__first), __last_(__last) {} 580bdd1243dSDimitry Andric 581bdd1243dSDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; } 58206c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; } 583bdd1243dSDimitry Andric 58406c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept { 5851db9f3b2SDimitry Andric _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input"); 58606c3fb27SDimitry Andric return {static_cast<char32_t>(*__first_++)}; 587bdd1243dSDimitry Andric } 588bdd1243dSDimitry Andric 589bdd1243dSDimitry Andric private: 59006c3fb27SDimitry Andric _Iterator __first_; 59106c3fb27SDimitry Andric _Iterator __last_; 592bdd1243dSDimitry Andric }; 593fcaf7f86SDimitry Andric 594fcaf7f86SDimitry Andric # endif // _LIBCPP_HAS_NO_UNICODE 595fcaf7f86SDimitry Andric 596bdd1243dSDimitry Andric } // namespace __unicode 597bdd1243dSDimitry Andric 59806c3fb27SDimitry Andric #endif //_LIBCPP_STD_VER >= 20 599fcaf7f86SDimitry Andric 600fcaf7f86SDimitry Andric _LIBCPP_END_NAMESPACE_STD 601fcaf7f86SDimitry Andric 602fcaf7f86SDimitry Andric #endif // _LIBCPP___FORMAT_UNICODE_H 603