#pragma once #include #include #include #include #ifdef __SSE2__ #include #endif namespace DB { namespace UTF8 { static const UInt8 CONTINUATION_OCTET_MASK = 0b11000000u; static const UInt8 CONTINUATION_OCTET = 0b10000000u; /// return true if `octet` binary repr starts with 10 (octet is a UTF-8 sequence continuation) inline bool isContinuationOctet(const UInt8 octet) { return (octet & CONTINUATION_OCTET_MASK) == CONTINUATION_OCTET; } /// moves `s` backward until either first non-continuation octet or begin inline void syncBackward(const UInt8 * & s, const UInt8 * const begin) { while (isContinuationOctet(*s) && s > begin) --s; } /// moves `s` forward until either first non-continuation octet or string end is met inline void syncForward(const UInt8 * & s, const UInt8 * const end) { while (s < end && isContinuationOctet(*s)) ++s; } /// returns UTF-8 code point sequence length judging by it's first octet inline size_t seqLength(const UInt8 first_octet) { if (first_octet < 0x80 || first_octet >= 0xF8) /// The specs of UTF-8. return 1; const size_t bits = 8; const auto first_zero = bitScanReverse(static_cast(~first_octet)); return bits - 1 - first_zero; } inline size_t countCodePoints(const UInt8 * data, size_t size) { size_t res = 0; const auto end = data + size; #ifdef __SSE2__ constexpr auto bytes_sse = sizeof(__m128i); const auto src_end_sse = data + size / bytes_sse * bytes_sse; const auto threshold = _mm_set1_epi8(0xBF); for (; data < src_end_sse; data += bytes_sse) res += __builtin_popcount(_mm_movemask_epi8( _mm_cmpgt_epi8(_mm_loadu_si128(reinterpret_cast(data)), threshold))); #endif for (; data < end; ++data) /// Skip UTF-8 continuation bytes. res += static_cast(*data) > static_cast(0xBF); return res; } template > size_t convertCodePointToUTF8(uint32_t code_point, CharT * out_bytes, size_t out_length) { static const Poco::UTF8Encoding utf8; int res = utf8.convert(code_point, reinterpret_cast(out_bytes), out_length); assert(res >= 0); return res; } template > std::optional convertUTF8ToCodePoint(const CharT * in_bytes, size_t in_length) { static const Poco::UTF8Encoding utf8; int res = utf8.queryConvert(reinterpret_cast(in_bytes), in_length); if (res >= 0) return res; return {}; } /// returns UTF-8 wcswidth. Invalid sequence is treated as zero width character. /// `prefix` is used to compute the `\t` width which extends the string before /// and include `\t` to the nearest longer length with multiple of eight. size_t computeWidth(const UInt8 * data, size_t size, size_t prefix = 0) noexcept; /** Calculate the maximum number of bytes, so that substring of this size fits in 'limit' width. * * For example, we have string "x你好", it has 3 code points and visible width of 5 and byte size of 7. * Suppose we have limit = 3. * Then we have to return 4 as maximum number of bytes * and the truncated string will be "x你": two code points, visible width 3, byte size 4. * * The same result will be for limit 4, because the last character would not fit. */ size_t computeBytesBeforeWidth(const UInt8 * data, size_t size, size_t prefix, size_t limit) noexcept; } }