xref: /freebsd-src/contrib/llvm-project/libcxx/include/__format/unicode.h (revision 0fca6ea1d4eea4c934cfff25ac9ee8ad6fe95583)
1fcaf7f86SDimitry Andric // -*- C++ -*-
2fcaf7f86SDimitry Andric //===----------------------------------------------------------------------===//
3fcaf7f86SDimitry Andric //
4fcaf7f86SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5fcaf7f86SDimitry Andric // See https://llvm.org/LICENSE.txt for license information.
6fcaf7f86SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7fcaf7f86SDimitry Andric //
8fcaf7f86SDimitry Andric //===----------------------------------------------------------------------===//
9fcaf7f86SDimitry Andric 
10fcaf7f86SDimitry Andric #ifndef _LIBCPP___FORMAT_UNICODE_H
11fcaf7f86SDimitry Andric #define _LIBCPP___FORMAT_UNICODE_H
12fcaf7f86SDimitry Andric 
13fcaf7f86SDimitry Andric #include <__assert>
1406c3fb27SDimitry Andric #include <__bit/countl.h>
1506c3fb27SDimitry Andric #include <__concepts/same_as.h>
16fcaf7f86SDimitry Andric #include <__config>
17fcaf7f86SDimitry Andric #include <__format/extended_grapheme_cluster_table.h>
18*0fca6ea1SDimitry Andric #include <__format/indic_conjunct_break_table.h>
1906c3fb27SDimitry Andric #include <__iterator/concepts.h>
2006c3fb27SDimitry Andric #include <__iterator/readable_traits.h> // iter_value_t
21*0fca6ea1SDimitry Andric #include <__utility/unreachable.h>
2206c3fb27SDimitry Andric #include <string_view>
23fcaf7f86SDimitry Andric 
24fcaf7f86SDimitry Andric #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
25fcaf7f86SDimitry Andric #  pragma GCC system_header
26fcaf7f86SDimitry Andric #endif
27fcaf7f86SDimitry Andric 
28fcaf7f86SDimitry Andric _LIBCPP_BEGIN_NAMESPACE_STD
29fcaf7f86SDimitry Andric 
3006c3fb27SDimitry Andric #if _LIBCPP_STD_VER >= 20
31fcaf7f86SDimitry Andric 
32bdd1243dSDimitry Andric namespace __unicode {
33bdd1243dSDimitry Andric 
3406c3fb27SDimitry Andric // Helper struct for the result of a consume operation.
3506c3fb27SDimitry Andric //
3606c3fb27SDimitry Andric // The status value for a correct code point is 0. This allows a valid value to
3706c3fb27SDimitry Andric // be used without masking.
3806c3fb27SDimitry Andric // When the decoding fails it know the number of code units affected. For the
3906c3fb27SDimitry Andric // current use-cases that value is not needed, therefore it is not stored.
4006c3fb27SDimitry Andric // The escape routine needs the number of code units for both a valid and
4106c3fb27SDimitry Andric // invalid character and keeps track of it itself. Doing it in this result
4206c3fb27SDimitry Andric // unconditionally would give some overhead when the value is unneeded.
4306c3fb27SDimitry Andric struct __consume_result {
4406c3fb27SDimitry Andric   // When __status == __ok it contains the decoded code point.
4506c3fb27SDimitry Andric   // Else it contains the replacement character U+FFFD
4606c3fb27SDimitry Andric   char32_t __code_point : 31;
47bdd1243dSDimitry Andric 
4806c3fb27SDimitry Andric   enum : char32_t {
4906c3fb27SDimitry Andric     // Consumed a well-formed code point.
5006c3fb27SDimitry Andric     __ok = 0,
5106c3fb27SDimitry Andric     // Encountered invalid UTF-8
5206c3fb27SDimitry Andric     __error = 1
5306c3fb27SDimitry Andric   } __status : 1 {__ok};
54bdd1243dSDimitry Andric };
5506c3fb27SDimitry Andric static_assert(sizeof(__consume_result) == sizeof(char32_t));
56bdd1243dSDimitry Andric 
57fcaf7f86SDimitry Andric #  ifndef _LIBCPP_HAS_NO_UNICODE
58fcaf7f86SDimitry Andric 
59fcaf7f86SDimitry Andric /// Implements the grapheme cluster boundary rules
60fcaf7f86SDimitry Andric ///
61fcaf7f86SDimitry Andric /// These rules are used to implement format's width estimation as stated in
62fcaf7f86SDimitry Andric /// [format.string.std]/11
63fcaf7f86SDimitry Andric ///
64fcaf7f86SDimitry Andric /// The Standard refers to UAX \#29 for Unicode 12.0.0
65fcaf7f86SDimitry Andric /// https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundary_Rules
66fcaf7f86SDimitry Andric ///
67fcaf7f86SDimitry Andric /// The data tables used are
68fcaf7f86SDimitry Andric /// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakProperty.txt
69fcaf7f86SDimitry Andric /// https://www.unicode.org/Public/UCD/latest/ucd/emoji/emoji-data.txt
70fcaf7f86SDimitry Andric /// https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/GraphemeBreakTest.txt (for testing only)
71fcaf7f86SDimitry Andric 
72fcaf7f86SDimitry Andric inline constexpr char32_t __replacement_character = U'\ufffd';
73fcaf7f86SDimitry Andric 
7406c3fb27SDimitry Andric // The error of a consume operation.
7506c3fb27SDimitry Andric //
7606c3fb27SDimitry Andric // This sets the code point to the replacement character. This code point does
7706c3fb27SDimitry Andric // not participate in the grapheme clustering, so grapheme clustering code can
7806c3fb27SDimitry Andric // ignore the error status and always use the code point.
7906c3fb27SDimitry Andric inline constexpr __consume_result __consume_result_error{__replacement_character, __consume_result::__error};
8006c3fb27SDimitry Andric 
8106c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_high_surrogate(char32_t __value) {
8206c3fb27SDimitry Andric   return __value >= 0xd800 && __value <= 0xdbff;
8306c3fb27SDimitry Andric }
8406c3fb27SDimitry Andric 
8506c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool __is_low_surrogate(char32_t __value) {
8606c3fb27SDimitry Andric   return __value >= 0xdc00 && __value <= 0xdfff;
8706c3fb27SDimitry Andric }
8806c3fb27SDimitry Andric 
8906c3fb27SDimitry Andric // https://www.unicode.org/glossary/#surrogate_code_point
9006c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_surrogate(char32_t __value) {
9106c3fb27SDimitry Andric   return __value >= 0xd800 && __value <= 0xdfff;
9206c3fb27SDimitry Andric }
9306c3fb27SDimitry Andric 
9406c3fb27SDimitry Andric // https://www.unicode.org/glossary/#code_point
9506c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_code_point(char32_t __value) {
9606c3fb27SDimitry Andric   return __value <= 0x10ffff;
9706c3fb27SDimitry Andric }
9806c3fb27SDimitry Andric 
9906c3fb27SDimitry Andric // https://www.unicode.org/glossary/#unicode_scalar_value
10006c3fb27SDimitry Andric [[nodiscard]] _LIBCPP_HIDE_FROM_ABI inline constexpr bool __is_scalar_value(char32_t __value) {
10106c3fb27SDimitry Andric   return __unicode::__is_code_point(__value) && !__unicode::__is_surrogate(__value);
10206c3fb27SDimitry Andric }
10306c3fb27SDimitry Andric 
10406c3fb27SDimitry Andric template <contiguous_iterator _Iterator>
10506c3fb27SDimitry Andric   requires same_as<iter_value_t<_Iterator>, char>
10606c3fb27SDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __is_continuation(_Iterator __char, int __count) {
107fcaf7f86SDimitry Andric   do {
10806c3fb27SDimitry Andric     if ((*__char & 0b1100'0000) != 0b1000'0000)
109fcaf7f86SDimitry Andric       return false;
110fcaf7f86SDimitry Andric     --__count;
111fcaf7f86SDimitry Andric     ++__char;
112fcaf7f86SDimitry Andric   } while (__count);
113fcaf7f86SDimitry Andric   return true;
114fcaf7f86SDimitry Andric }
115fcaf7f86SDimitry Andric 
116fcaf7f86SDimitry Andric /// Helper class to extract a code unit from a Unicode character range.
117fcaf7f86SDimitry Andric ///
118fcaf7f86SDimitry Andric /// The stored range is a view. There are multiple specialization for different
119fcaf7f86SDimitry Andric /// character types.
120fcaf7f86SDimitry Andric template <class _CharT>
121fcaf7f86SDimitry Andric class __code_point_view;
122fcaf7f86SDimitry Andric 
123fcaf7f86SDimitry Andric /// UTF-8 specialization.
124fcaf7f86SDimitry Andric template <>
125fcaf7f86SDimitry Andric class __code_point_view<char> {
12606c3fb27SDimitry Andric   using _Iterator = basic_string_view<char>::const_iterator;
12706c3fb27SDimitry Andric 
128fcaf7f86SDimitry Andric public:
12906c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
130fcaf7f86SDimitry Andric       : __first_(__first), __last_(__last) {}
131fcaf7f86SDimitry Andric 
132fcaf7f86SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
13306c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
134fcaf7f86SDimitry Andric 
13506c3fb27SDimitry Andric   // https://www.unicode.org/versions/latest/ch03.pdf#G7404
13606c3fb27SDimitry Andric   // Based on Table 3-7, Well-Formed UTF-8 Byte Sequences
13706c3fb27SDimitry Andric   //
13806c3fb27SDimitry Andric   // Code Points        First Byte Second Byte Third Byte Fourth Byte  Remarks
13906c3fb27SDimitry Andric   // U+0000..U+007F     00..7F                                         U+0000..U+007F 1 code unit range
14006c3fb27SDimitry Andric   //                    C0..C1     80..BF                              invalid overlong encoding
14106c3fb27SDimitry Andric   // U+0080..U+07FF     C2..DF     80..BF                              U+0080..U+07FF 2 code unit range
14206c3fb27SDimitry Andric   //                    E0         80..9F      80..BF                  invalid overlong encoding
14306c3fb27SDimitry Andric   // U+0800..U+0FFF     E0         A0..BF      80..BF                  U+0800..U+FFFF 3 code unit range
14406c3fb27SDimitry Andric   // U+1000..U+CFFF     E1..EC     80..BF      80..BF
14506c3fb27SDimitry Andric   // U+D000..U+D7FF     ED         80..9F      80..BF
14606c3fb27SDimitry Andric   // U+D800..U+DFFF     ED         A0..BF      80..BF                  invalid encoding of surrogate code point
14706c3fb27SDimitry Andric   // U+E000..U+FFFF     EE..EF     80..BF      80..BF
14806c3fb27SDimitry Andric   //                    F0         80..8F      80..BF     80..BF       invalid overlong encoding
14906c3fb27SDimitry Andric   // U+10000..U+3FFFF   F0         90..BF      80..BF     80..BF       U+10000..U+10FFFF 4 code unit range
15006c3fb27SDimitry Andric   // U+40000..U+FFFFF   F1..F3     80..BF      80..BF     80..BF
15106c3fb27SDimitry Andric   // U+100000..U+10FFFF F4         80..8F      80..BF     80..BF
15206c3fb27SDimitry Andric   //                    F4         90..BF      80..BF     80..BF       U+110000.. invalid code point range
15306c3fb27SDimitry Andric   //
15406c3fb27SDimitry Andric   // Unlike other parsers, these invalid entries are tested after decoding.
15506c3fb27SDimitry Andric   // - The parser always needs to consume these code units
15606c3fb27SDimitry Andric   // - The code is optimized for well-formed UTF-8
15706c3fb27SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
1581db9f3b2SDimitry Andric     _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
159bdd1243dSDimitry Andric 
160bdd1243dSDimitry Andric     // Based on the number of leading 1 bits the number of code units in the
161bdd1243dSDimitry Andric     // code point can be determined. See
162bdd1243dSDimitry Andric     // https://en.wikipedia.org/wiki/UTF-8#Encoding
163bdd1243dSDimitry Andric     switch (std::countl_one(static_cast<unsigned char>(*__first_))) {
164bdd1243dSDimitry Andric     case 0:
16506c3fb27SDimitry Andric       return {static_cast<unsigned char>(*__first_++)};
166bdd1243dSDimitry Andric 
16706c3fb27SDimitry Andric     case 2: {
16806c3fb27SDimitry Andric       if (__last_ - __first_ < 2 || !__unicode::__is_continuation(__first_ + 1, 1)) [[unlikely]]
169bdd1243dSDimitry Andric         break;
170bdd1243dSDimitry Andric 
171bdd1243dSDimitry Andric       char32_t __value = static_cast<unsigned char>(*__first_++) & 0x1f;
172bdd1243dSDimitry Andric       __value <<= 6;
173bdd1243dSDimitry Andric       __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
17406c3fb27SDimitry Andric 
17506c3fb27SDimitry Andric       // These values should be encoded in 1 UTF-8 code unit.
17606c3fb27SDimitry Andric       if (__value < 0x0080) [[unlikely]]
17706c3fb27SDimitry Andric         return __consume_result_error;
17806c3fb27SDimitry Andric 
17906c3fb27SDimitry Andric       return {__value};
180bdd1243dSDimitry Andric     }
18106c3fb27SDimitry Andric 
18206c3fb27SDimitry Andric     case 3: {
18306c3fb27SDimitry Andric       if (__last_ - __first_ < 3 || !__unicode::__is_continuation(__first_ + 1, 2)) [[unlikely]]
184bdd1243dSDimitry Andric         break;
185bdd1243dSDimitry Andric 
186bdd1243dSDimitry Andric       char32_t __value = static_cast<unsigned char>(*__first_++) & 0x0f;
187bdd1243dSDimitry Andric       __value <<= 6;
188bdd1243dSDimitry Andric       __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
189bdd1243dSDimitry Andric       __value <<= 6;
190bdd1243dSDimitry Andric       __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
19106c3fb27SDimitry Andric 
19206c3fb27SDimitry Andric       // These values should be encoded in 1 or 2 UTF-8 code units.
19306c3fb27SDimitry Andric       if (__value < 0x0800) [[unlikely]]
19406c3fb27SDimitry Andric         return __consume_result_error;
19506c3fb27SDimitry Andric 
19606c3fb27SDimitry Andric       // A surrogate value is always encoded in 3 UTF-8 code units.
19706c3fb27SDimitry Andric       if (__unicode::__is_surrogate(__value)) [[unlikely]]
19806c3fb27SDimitry Andric         return __consume_result_error;
19906c3fb27SDimitry Andric 
20006c3fb27SDimitry Andric       return {__value};
201bdd1243dSDimitry Andric     }
20206c3fb27SDimitry Andric 
20306c3fb27SDimitry Andric     case 4: {
20406c3fb27SDimitry Andric       if (__last_ - __first_ < 4 || !__unicode::__is_continuation(__first_ + 1, 3)) [[unlikely]]
205bdd1243dSDimitry Andric         break;
206bdd1243dSDimitry Andric 
207bdd1243dSDimitry Andric       char32_t __value = static_cast<unsigned char>(*__first_++) & 0x07;
208bdd1243dSDimitry Andric       __value <<= 6;
209bdd1243dSDimitry Andric       __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
210bdd1243dSDimitry Andric       __value <<= 6;
211bdd1243dSDimitry Andric       __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
212bdd1243dSDimitry Andric       __value <<= 6;
213bdd1243dSDimitry Andric       __value |= static_cast<unsigned char>(*__first_++) & 0x3f;
214bdd1243dSDimitry Andric 
21506c3fb27SDimitry Andric       // These values should be encoded in 1, 2, or 3 UTF-8 code units.
21606c3fb27SDimitry Andric       if (__value < 0x10000) [[unlikely]]
21706c3fb27SDimitry Andric         return __consume_result_error;
218bdd1243dSDimitry Andric 
21906c3fb27SDimitry Andric       // A value too large is always encoded in 4 UTF-8 code units.
22006c3fb27SDimitry Andric       if (!__unicode::__is_code_point(__value)) [[unlikely]]
22106c3fb27SDimitry Andric         return __consume_result_error;
22206c3fb27SDimitry Andric 
22306c3fb27SDimitry Andric       return {__value};
224bdd1243dSDimitry Andric     }
225bdd1243dSDimitry Andric     }
226bdd1243dSDimitry Andric     // An invalid number of leading ones can be garbage or a code unit in the
227bdd1243dSDimitry Andric     // middle of a code point. By consuming one code unit the parser may get
228bdd1243dSDimitry Andric     // "in sync" after a few code units.
22906c3fb27SDimitry Andric     ++__first_;
23006c3fb27SDimitry Andric     return __consume_result_error;
231bdd1243dSDimitry Andric   }
232bdd1243dSDimitry Andric 
233fcaf7f86SDimitry Andric private:
23406c3fb27SDimitry Andric   _Iterator __first_;
23506c3fb27SDimitry Andric   _Iterator __last_;
236fcaf7f86SDimitry Andric };
237fcaf7f86SDimitry Andric 
238bdd1243dSDimitry Andric #    ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS
239bdd1243dSDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_high(wchar_t __value) {
240bdd1243dSDimitry Andric   return __value >= 0xd800 && __value <= 0xdbff;
241bdd1243dSDimitry Andric }
242bdd1243dSDimitry Andric 
243bdd1243dSDimitry Andric _LIBCPP_HIDE_FROM_ABI constexpr bool __is_surrogate_pair_low(wchar_t __value) {
244bdd1243dSDimitry Andric   return __value >= 0xdc00 && __value <= 0xdfff;
245bdd1243dSDimitry Andric }
246bdd1243dSDimitry Andric 
247fcaf7f86SDimitry Andric /// This specialization depends on the size of wchar_t
248fcaf7f86SDimitry Andric /// - 2 UTF-16 (for example Windows and AIX)
249fcaf7f86SDimitry Andric /// - 4 UTF-32 (for example Linux)
250fcaf7f86SDimitry Andric template <>
251fcaf7f86SDimitry Andric class __code_point_view<wchar_t> {
25206c3fb27SDimitry Andric   using _Iterator = typename basic_string_view<wchar_t>::const_iterator;
25306c3fb27SDimitry Andric 
254fcaf7f86SDimitry Andric public:
255bdd1243dSDimitry Andric   static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sizeof(wchar_t) has a not implemented value");
256bdd1243dSDimitry Andric 
25706c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
258fcaf7f86SDimitry Andric       : __first_(__first), __last_(__last) {}
259fcaf7f86SDimitry Andric 
26006c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
261fcaf7f86SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
262fcaf7f86SDimitry Andric 
26306c3fb27SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
2641db9f3b2SDimitry Andric     _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
265fcaf7f86SDimitry Andric 
26606c3fb27SDimitry Andric     char32_t __value = static_cast<char32_t>(*__first_++);
267fcaf7f86SDimitry Andric     if constexpr (sizeof(wchar_t) == 2) {
26806c3fb27SDimitry Andric       if (__unicode::__is_low_surrogate(__value)) [[unlikely]]
26906c3fb27SDimitry Andric         return __consume_result_error;
270fcaf7f86SDimitry Andric 
27106c3fb27SDimitry Andric       if (__unicode::__is_high_surrogate(__value)) {
27206c3fb27SDimitry Andric         if (__first_ == __last_ || !__unicode::__is_low_surrogate(static_cast<char32_t>(*__first_))) [[unlikely]]
27306c3fb27SDimitry Andric           return __consume_result_error;
27406c3fb27SDimitry Andric 
27506c3fb27SDimitry Andric         __value -= 0xd800;
27606c3fb27SDimitry Andric         __value <<= 10;
27706c3fb27SDimitry Andric         __value += static_cast<char32_t>(*__first_++) - 0xdc00;
27806c3fb27SDimitry Andric         __value += 0x10000;
27906c3fb27SDimitry Andric 
28006c3fb27SDimitry Andric         if (!__unicode::__is_code_point(__value)) [[unlikely]]
28106c3fb27SDimitry Andric           return __consume_result_error;
282fcaf7f86SDimitry Andric       }
283fcaf7f86SDimitry Andric     } else {
28406c3fb27SDimitry Andric       if (!__unicode::__is_scalar_value(__value)) [[unlikely]]
28506c3fb27SDimitry Andric         return __consume_result_error;
286fcaf7f86SDimitry Andric     }
287fcaf7f86SDimitry Andric 
28806c3fb27SDimitry Andric     return {__value};
289bdd1243dSDimitry Andric   }
290bdd1243dSDimitry Andric 
291fcaf7f86SDimitry Andric private:
29206c3fb27SDimitry Andric   _Iterator __first_;
29306c3fb27SDimitry Andric   _Iterator __last_;
294fcaf7f86SDimitry Andric };
295bdd1243dSDimitry Andric #    endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS
296fcaf7f86SDimitry Andric 
297*0fca6ea1SDimitry Andric // State machine to implement the Extended Grapheme Cluster Boundary
298*0fca6ea1SDimitry Andric //
299*0fca6ea1SDimitry Andric // The exact rules may change between Unicode versions.
300*0fca6ea1SDimitry Andric // This implements the extended rules see
301*0fca6ea1SDimitry Andric // https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
302*0fca6ea1SDimitry Andric class __extended_grapheme_cluster_break {
303*0fca6ea1SDimitry Andric   using __EGC_property  = __extended_grapheme_custer_property_boundary::__property;
304*0fca6ea1SDimitry Andric   using __inCB_property = __indic_conjunct_break::__property;
305fcaf7f86SDimitry Andric 
306*0fca6ea1SDimitry Andric public:
307*0fca6ea1SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_break(char32_t __first_code_point)
308*0fca6ea1SDimitry Andric       : __prev_code_point_(__first_code_point),
309*0fca6ea1SDimitry Andric         __prev_property_(__extended_grapheme_custer_property_boundary::__get_property(__first_code_point)) {
310*0fca6ea1SDimitry Andric     // Initializes the active rule.
311*0fca6ea1SDimitry Andric     if (__prev_property_ == __EGC_property::__Extended_Pictographic)
312*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__GB11_emoji;
313*0fca6ea1SDimitry Andric     else if (__prev_property_ == __EGC_property::__Regional_Indicator)
314*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__GB12_GB13_regional_indicator;
315*0fca6ea1SDimitry Andric     else if (__indic_conjunct_break::__get_property(__first_code_point) == __inCB_property::__Consonant)
316*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__GB9c_indic_conjunct_break;
317*0fca6ea1SDimitry Andric   }
318fcaf7f86SDimitry Andric 
319*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(char32_t __next_code_point) {
320*0fca6ea1SDimitry Andric     __EGC_property __next_property = __extended_grapheme_custer_property_boundary::__get_property(__next_code_point);
321*0fca6ea1SDimitry Andric     bool __result                  = __evaluate(__next_code_point, __next_property);
322*0fca6ea1SDimitry Andric     __prev_code_point_             = __next_code_point;
323*0fca6ea1SDimitry Andric     __prev_property_               = __next_property;
324*0fca6ea1SDimitry Andric     return __result;
325*0fca6ea1SDimitry Andric   }
326fcaf7f86SDimitry Andric 
327*0fca6ea1SDimitry Andric   // The code point whose break propery are considered during the next
328*0fca6ea1SDimitry Andric   // evaluation cyle.
329*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr char32_t __current_code_point() const { return __prev_code_point_; }
330*0fca6ea1SDimitry Andric 
331*0fca6ea1SDimitry Andric private:
332*0fca6ea1SDimitry Andric   // The naming of the identifiers matches the Unicode standard.
333*0fca6ea1SDimitry Andric   // NOLINTBEGIN(readability-identifier-naming)
334*0fca6ea1SDimitry Andric 
335*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
336*0fca6ea1SDimitry Andric   __evaluate(char32_t __next_code_point, __EGC_property __next_property) {
337*0fca6ea1SDimitry Andric     switch (__active_rule_) {
338*0fca6ea1SDimitry Andric     case __rule::__none:
339*0fca6ea1SDimitry Andric       return __evaluate_none(__next_code_point, __next_property);
340*0fca6ea1SDimitry Andric     case __rule::__GB9c_indic_conjunct_break:
341*0fca6ea1SDimitry Andric       return __evaluate_GB9c_indic_conjunct_break(__next_code_point, __next_property);
342*0fca6ea1SDimitry Andric     case __rule::__GB11_emoji:
343*0fca6ea1SDimitry Andric       return __evaluate_GB11_emoji(__next_code_point, __next_property);
344*0fca6ea1SDimitry Andric     case __rule::__GB12_GB13_regional_indicator:
345*0fca6ea1SDimitry Andric       return __evaluate_GB12_GB13_regional_indicator(__next_code_point, __next_property);
346*0fca6ea1SDimitry Andric     }
347*0fca6ea1SDimitry Andric     __libcpp_unreachable();
348*0fca6ea1SDimitry Andric   }
349*0fca6ea1SDimitry Andric 
350*0fca6ea1SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr bool __evaluate_none(char32_t __next_code_point, __EGC_property __next_property) {
351fcaf7f86SDimitry Andric     // *** Break at the start and end of text, unless the text is empty. ***
352fcaf7f86SDimitry Andric 
353*0fca6ea1SDimitry Andric     _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__sot, "should be handled in the constructor"); // GB1
354*0fca6ea1SDimitry Andric     _LIBCPP_ASSERT_INTERNAL(__prev_property_ != __EGC_property::__eot, "should be handled by our caller");      // GB2
355fcaf7f86SDimitry Andric 
356fcaf7f86SDimitry Andric     // *** Do not break between a CR and LF. Otherwise, break before and after controls. ***
357*0fca6ea1SDimitry Andric     if (__prev_property_ == __EGC_property::__CR && __next_property == __EGC_property::__LF) // GB3
358fcaf7f86SDimitry Andric       return false;
359fcaf7f86SDimitry Andric 
360*0fca6ea1SDimitry Andric     if (__prev_property_ == __EGC_property::__Control || __prev_property_ == __EGC_property::__CR ||
361*0fca6ea1SDimitry Andric         __prev_property_ == __EGC_property::__LF) // GB4
362fcaf7f86SDimitry Andric       return true;
363fcaf7f86SDimitry Andric 
364*0fca6ea1SDimitry Andric     if (__next_property == __EGC_property::__Control || __next_property == __EGC_property::__CR ||
365*0fca6ea1SDimitry Andric         __next_property == __EGC_property::__LF) // GB5
366fcaf7f86SDimitry Andric       return true;
367fcaf7f86SDimitry Andric 
368fcaf7f86SDimitry Andric     // *** Do not break Hangul syllable sequences. ***
369*0fca6ea1SDimitry Andric     if (__prev_property_ == __EGC_property::__L &&
370*0fca6ea1SDimitry Andric         (__next_property == __EGC_property::__L || __next_property == __EGC_property::__V ||
371*0fca6ea1SDimitry Andric          __next_property == __EGC_property::__LV || __next_property == __EGC_property::__LVT)) // GB6
372fcaf7f86SDimitry Andric       return false;
373fcaf7f86SDimitry Andric 
374*0fca6ea1SDimitry Andric     if ((__prev_property_ == __EGC_property::__LV || __prev_property_ == __EGC_property::__V) &&
375*0fca6ea1SDimitry Andric         (__next_property == __EGC_property::__V || __next_property == __EGC_property::__T)) // GB7
376fcaf7f86SDimitry Andric       return false;
377fcaf7f86SDimitry Andric 
378*0fca6ea1SDimitry Andric     if ((__prev_property_ == __EGC_property::__LVT || __prev_property_ == __EGC_property::__T) &&
379*0fca6ea1SDimitry Andric         __next_property == __EGC_property::__T) // GB8
380fcaf7f86SDimitry Andric       return false;
381fcaf7f86SDimitry Andric 
382fcaf7f86SDimitry Andric     // *** Do not break before extending characters or ZWJ. ***
383*0fca6ea1SDimitry Andric     if (__next_property == __EGC_property::__Extend || __next_property == __EGC_property::__ZWJ)
384fcaf7f86SDimitry Andric       return false; // GB9
385fcaf7f86SDimitry Andric 
386fcaf7f86SDimitry Andric     // *** Do not break before SpacingMarks, or after Prepend characters. ***
387*0fca6ea1SDimitry Andric     if (__next_property == __EGC_property::__SpacingMark) // GB9a
388fcaf7f86SDimitry Andric       return false;
389fcaf7f86SDimitry Andric 
390*0fca6ea1SDimitry Andric     if (__prev_property_ == __EGC_property::__Prepend) // GB9b
391fcaf7f86SDimitry Andric       return false;
392fcaf7f86SDimitry Andric 
393*0fca6ea1SDimitry Andric     // *** Do not break within certain combinations with Indic_Conjunct_Break (InCB)=Linker. ***
394*0fca6ea1SDimitry Andric     if (__indic_conjunct_break::__get_property(__next_code_point) == __inCB_property::__Consonant) {
395*0fca6ea1SDimitry Andric       __active_rule_                     = __rule::__GB9c_indic_conjunct_break;
396*0fca6ea1SDimitry Andric       __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
397*0fca6ea1SDimitry Andric       return true;
398*0fca6ea1SDimitry Andric     }
399*0fca6ea1SDimitry Andric 
400fcaf7f86SDimitry Andric     // *** Do not break within emoji modifier sequences or emoji zwj sequences. ***
401*0fca6ea1SDimitry Andric     if (__next_property == __EGC_property::__Extended_Pictographic) {
402*0fca6ea1SDimitry Andric       __active_rule_      = __rule::__GB11_emoji;
403*0fca6ea1SDimitry Andric       __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
404*0fca6ea1SDimitry Andric       return true;
405*0fca6ea1SDimitry Andric     }
406fcaf7f86SDimitry Andric 
407fcaf7f86SDimitry Andric     // *** Do not break within emoji flag sequences ***
408fcaf7f86SDimitry Andric 
409fcaf7f86SDimitry Andric     // That is, do not break between regional indicator (RI) symbols if there
410fcaf7f86SDimitry Andric     // is an odd number of RI characters before the break point.
411*0fca6ea1SDimitry Andric     if (__next_property == __EGC_property::__Regional_Indicator) { // GB12 + GB13
412*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__GB12_GB13_regional_indicator;
413*0fca6ea1SDimitry Andric       return true;
414fcaf7f86SDimitry Andric     }
415fcaf7f86SDimitry Andric 
416fcaf7f86SDimitry Andric     // *** Otherwise, break everywhere. ***
417fcaf7f86SDimitry Andric     return true; // GB999
418fcaf7f86SDimitry Andric   }
419fcaf7f86SDimitry Andric 
420*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
421*0fca6ea1SDimitry Andric   __evaluate_GB9c_indic_conjunct_break(char32_t __next_code_point, __EGC_property __next_property) {
422*0fca6ea1SDimitry Andric     __inCB_property __break = __indic_conjunct_break::__get_property(__next_code_point);
423*0fca6ea1SDimitry Andric     if (__break == __inCB_property::__none) {
424*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__none;
425*0fca6ea1SDimitry Andric       return __evaluate_none(__next_code_point, __next_property);
426*0fca6ea1SDimitry Andric     }
427*0fca6ea1SDimitry Andric 
428*0fca6ea1SDimitry Andric     switch (__GB9c_indic_conjunct_break_state_) {
429*0fca6ea1SDimitry Andric     case __GB9c_indic_conjunct_break_state::__Consonant:
430*0fca6ea1SDimitry Andric       if (__break == __inCB_property::__Extend) {
431*0fca6ea1SDimitry Andric         return false;
432*0fca6ea1SDimitry Andric       }
433*0fca6ea1SDimitry Andric       if (__break == __inCB_property::__Linker) {
434*0fca6ea1SDimitry Andric         __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Linker;
435*0fca6ea1SDimitry Andric         return false;
436*0fca6ea1SDimitry Andric       }
437*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__none;
438*0fca6ea1SDimitry Andric       return __evaluate_none(__next_code_point, __next_property);
439*0fca6ea1SDimitry Andric 
440*0fca6ea1SDimitry Andric     case __GB9c_indic_conjunct_break_state::__Linker:
441*0fca6ea1SDimitry Andric       if (__break == __inCB_property::__Extend) {
442*0fca6ea1SDimitry Andric         return false;
443*0fca6ea1SDimitry Andric       }
444*0fca6ea1SDimitry Andric       if (__break == __inCB_property::__Linker) {
445*0fca6ea1SDimitry Andric         return false;
446*0fca6ea1SDimitry Andric       }
447*0fca6ea1SDimitry Andric       if (__break == __inCB_property::__Consonant) {
448*0fca6ea1SDimitry Andric         __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
449*0fca6ea1SDimitry Andric         return false;
450*0fca6ea1SDimitry Andric       }
451*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__none;
452*0fca6ea1SDimitry Andric       return __evaluate_none(__next_code_point, __next_property);
453*0fca6ea1SDimitry Andric     }
454*0fca6ea1SDimitry Andric     __libcpp_unreachable();
455*0fca6ea1SDimitry Andric   }
456*0fca6ea1SDimitry Andric 
457*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
458*0fca6ea1SDimitry Andric   __evaluate_GB11_emoji(char32_t __next_code_point, __EGC_property __next_property) {
459*0fca6ea1SDimitry Andric     switch (__GB11_emoji_state_) {
460*0fca6ea1SDimitry Andric     case __GB11_emoji_state::__Extended_Pictographic:
461*0fca6ea1SDimitry Andric       if (__next_property == __EGC_property::__Extend) {
462*0fca6ea1SDimitry Andric         __GB11_emoji_state_ = __GB11_emoji_state::__Extend;
463*0fca6ea1SDimitry Andric         return false;
464*0fca6ea1SDimitry Andric       }
465*0fca6ea1SDimitry Andric       [[fallthrough]];
466*0fca6ea1SDimitry Andric     case __GB11_emoji_state::__Extend:
467*0fca6ea1SDimitry Andric       if (__next_property == __EGC_property::__ZWJ) {
468*0fca6ea1SDimitry Andric         __GB11_emoji_state_ = __GB11_emoji_state::__ZWJ;
469*0fca6ea1SDimitry Andric         return false;
470*0fca6ea1SDimitry Andric       }
471*0fca6ea1SDimitry Andric       if (__next_property == __EGC_property::__Extend)
472*0fca6ea1SDimitry Andric         return false;
473*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__none;
474*0fca6ea1SDimitry Andric       return __evaluate_none(__next_code_point, __next_property);
475*0fca6ea1SDimitry Andric 
476*0fca6ea1SDimitry Andric     case __GB11_emoji_state::__ZWJ:
477*0fca6ea1SDimitry Andric       if (__next_property == __EGC_property::__Extended_Pictographic) {
478*0fca6ea1SDimitry Andric         __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
479*0fca6ea1SDimitry Andric         return false;
480*0fca6ea1SDimitry Andric       }
481*0fca6ea1SDimitry Andric       __active_rule_ = __rule::__none;
482*0fca6ea1SDimitry Andric       return __evaluate_none(__next_code_point, __next_property);
483*0fca6ea1SDimitry Andric     }
484*0fca6ea1SDimitry Andric     __libcpp_unreachable();
485*0fca6ea1SDimitry Andric   }
486*0fca6ea1SDimitry Andric 
487*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr bool
488*0fca6ea1SDimitry Andric   __evaluate_GB12_GB13_regional_indicator(char32_t __next_code_point, __EGC_property __next_property) {
489*0fca6ea1SDimitry Andric     __active_rule_ = __rule::__none;
490*0fca6ea1SDimitry Andric     if (__next_property == __EGC_property::__Regional_Indicator)
491*0fca6ea1SDimitry Andric       return false;
492*0fca6ea1SDimitry Andric     return __evaluate_none(__next_code_point, __next_property);
493*0fca6ea1SDimitry Andric   }
494*0fca6ea1SDimitry Andric 
495*0fca6ea1SDimitry Andric   char32_t __prev_code_point_;
496*0fca6ea1SDimitry Andric   __EGC_property __prev_property_;
497*0fca6ea1SDimitry Andric 
498*0fca6ea1SDimitry Andric   enum class __rule {
499*0fca6ea1SDimitry Andric     __none,
500*0fca6ea1SDimitry Andric     __GB9c_indic_conjunct_break,
501*0fca6ea1SDimitry Andric     __GB11_emoji,
502*0fca6ea1SDimitry Andric     __GB12_GB13_regional_indicator,
503*0fca6ea1SDimitry Andric   };
504*0fca6ea1SDimitry Andric   __rule __active_rule_ = __rule::__none;
505*0fca6ea1SDimitry Andric 
506*0fca6ea1SDimitry Andric   enum class __GB11_emoji_state {
507*0fca6ea1SDimitry Andric     __Extended_Pictographic,
508*0fca6ea1SDimitry Andric     __Extend,
509*0fca6ea1SDimitry Andric     __ZWJ,
510*0fca6ea1SDimitry Andric   };
511*0fca6ea1SDimitry Andric   __GB11_emoji_state __GB11_emoji_state_ = __GB11_emoji_state::__Extended_Pictographic;
512*0fca6ea1SDimitry Andric 
513*0fca6ea1SDimitry Andric   enum class __GB9c_indic_conjunct_break_state {
514*0fca6ea1SDimitry Andric     __Consonant,
515*0fca6ea1SDimitry Andric     __Linker,
516*0fca6ea1SDimitry Andric   };
517*0fca6ea1SDimitry Andric 
518*0fca6ea1SDimitry Andric   __GB9c_indic_conjunct_break_state __GB9c_indic_conjunct_break_state_ = __GB9c_indic_conjunct_break_state::__Consonant;
519*0fca6ea1SDimitry Andric 
520*0fca6ea1SDimitry Andric   // NOLINTEND(readability-identifier-naming)
521*0fca6ea1SDimitry Andric };
522*0fca6ea1SDimitry Andric 
523fcaf7f86SDimitry Andric /// Helper class to extract an extended grapheme cluster from a Unicode character range.
524fcaf7f86SDimitry Andric ///
525fcaf7f86SDimitry Andric /// This function is used to determine the column width of an extended grapheme
526fcaf7f86SDimitry Andric /// cluster. In order to do that only the first code point is evaluated.
527fcaf7f86SDimitry Andric /// Therefore only this code point is extracted.
528fcaf7f86SDimitry Andric template <class _CharT>
529fcaf7f86SDimitry Andric class __extended_grapheme_cluster_view {
53006c3fb27SDimitry Andric   using _Iterator = typename basic_string_view<_CharT>::const_iterator;
53106c3fb27SDimitry Andric 
532fcaf7f86SDimitry Andric public:
53306c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr explicit __extended_grapheme_cluster_view(_Iterator __first, _Iterator __last)
534*0fca6ea1SDimitry Andric       : __code_point_view_(__first, __last), __at_break_(__code_point_view_.__consume().__code_point) {}
535fcaf7f86SDimitry Andric 
536fcaf7f86SDimitry Andric   struct __cluster {
537fcaf7f86SDimitry Andric     /// The first code point of the extended grapheme cluster.
538fcaf7f86SDimitry Andric     ///
539fcaf7f86SDimitry Andric     /// The first code point is used to estimate the width of the extended
540fcaf7f86SDimitry Andric     /// grapheme cluster.
541fcaf7f86SDimitry Andric     char32_t __code_point_;
542fcaf7f86SDimitry Andric 
543fcaf7f86SDimitry Andric     /// Points one beyond the last code unit in the extended grapheme cluster.
544fcaf7f86SDimitry Andric     ///
545fcaf7f86SDimitry Andric     /// It's expected the caller has the start position and thus can determine
546fcaf7f86SDimitry Andric     /// the code unit range of the extended grapheme cluster.
54706c3fb27SDimitry Andric     _Iterator __last_;
548fcaf7f86SDimitry Andric   };
549fcaf7f86SDimitry Andric 
550*0fca6ea1SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __cluster __consume() {
551*0fca6ea1SDimitry Andric     char32_t __code_point = __at_break_.__current_code_point();
552*0fca6ea1SDimitry Andric     _Iterator __position  = __code_point_view_.__position();
553*0fca6ea1SDimitry Andric     while (!__code_point_view_.__at_end()) {
554*0fca6ea1SDimitry Andric       if (__at_break_(__code_point_view_.__consume().__code_point))
555*0fca6ea1SDimitry Andric         break;
556*0fca6ea1SDimitry Andric       __position = __code_point_view_.__position();
557*0fca6ea1SDimitry Andric     }
558*0fca6ea1SDimitry Andric     return {__code_point, __position};
559fcaf7f86SDimitry Andric   }
560fcaf7f86SDimitry Andric 
561fcaf7f86SDimitry Andric private:
562fcaf7f86SDimitry Andric   __code_point_view<_CharT> __code_point_view_;
563*0fca6ea1SDimitry Andric   __extended_grapheme_cluster_break __at_break_;
564fcaf7f86SDimitry Andric };
565fcaf7f86SDimitry Andric 
56606c3fb27SDimitry Andric template <contiguous_iterator _Iterator>
56706c3fb27SDimitry Andric __extended_grapheme_cluster_view(_Iterator, _Iterator) -> __extended_grapheme_cluster_view<iter_value_t<_Iterator>>;
568bdd1243dSDimitry Andric 
569bdd1243dSDimitry Andric #  else //  _LIBCPP_HAS_NO_UNICODE
570bdd1243dSDimitry Andric 
571bdd1243dSDimitry Andric // For ASCII every character is a "code point".
572bdd1243dSDimitry Andric // This makes it easier to write code agnostic of the _LIBCPP_HAS_NO_UNICODE define.
573bdd1243dSDimitry Andric template <class _CharT>
574bdd1243dSDimitry Andric class __code_point_view {
57506c3fb27SDimitry Andric   using _Iterator = typename basic_string_view<_CharT>::const_iterator;
57606c3fb27SDimitry Andric 
577bdd1243dSDimitry Andric public:
57806c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr explicit __code_point_view(_Iterator __first, _Iterator __last)
579bdd1243dSDimitry Andric       : __first_(__first), __last_(__last) {}
580bdd1243dSDimitry Andric 
581bdd1243dSDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr bool __at_end() const noexcept { return __first_ == __last_; }
58206c3fb27SDimitry Andric   _LIBCPP_HIDE_FROM_ABI constexpr _Iterator __position() const noexcept { return __first_; }
583bdd1243dSDimitry Andric 
58406c3fb27SDimitry Andric   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI constexpr __consume_result __consume() noexcept {
5851db9f3b2SDimitry Andric     _LIBCPP_ASSERT_INTERNAL(__first_ != __last_, "can't move beyond the end of input");
58606c3fb27SDimitry Andric     return {static_cast<char32_t>(*__first_++)};
587bdd1243dSDimitry Andric   }
588bdd1243dSDimitry Andric 
589bdd1243dSDimitry Andric private:
59006c3fb27SDimitry Andric   _Iterator __first_;
59106c3fb27SDimitry Andric   _Iterator __last_;
592bdd1243dSDimitry Andric };
593fcaf7f86SDimitry Andric 
594fcaf7f86SDimitry Andric #  endif //  _LIBCPP_HAS_NO_UNICODE
595fcaf7f86SDimitry Andric 
596bdd1243dSDimitry Andric } // namespace __unicode
597bdd1243dSDimitry Andric 
59806c3fb27SDimitry Andric #endif //_LIBCPP_STD_VER >= 20
599fcaf7f86SDimitry Andric 
600fcaf7f86SDimitry Andric _LIBCPP_END_NAMESPACE_STD
601fcaf7f86SDimitry Andric 
602fcaf7f86SDimitry Andric #endif // _LIBCPP___FORMAT_UNICODE_H
603