ClickHouse/dbms/include/DB/Common/UTF8Helpers.h

55 lines
1.1 KiB
C++
Raw Normal View History

#pragma once
#include <DB/Core/Types.h>
2016-03-07 06:18:06 +00:00
#include <DB/Common/BitHelpers.h>
namespace DB
{
namespace UTF8
{
static const UInt8 CONTINUATION_OCTET_MASK = 0b11000000u;
static const UInt8 CONTINUATION_OCTET = 0b10000000u;
/// return true if `octet` binary repr starts with 10 (octet is a UTF-8 sequence continuation)
inline bool isContinuationOctet(const UInt8 octet)
{
return (octet & CONTINUATION_OCTET_MASK) == CONTINUATION_OCTET;
}
/// moves `s` backward until either first non-continuation octet or begin
inline void syncBackward(const UInt8 * & s, const UInt8 * const begin)
{
while (isContinuationOctet(*s) && s > begin)
--s;
}
/// moves `s` forward until either first non-continuation octet or string end is met
inline void syncForward(const UInt8 * & s, const UInt8 * const end)
{
while (s < end && isContinuationOctet(*s))
++s;
}
/// returns UTF-8 code point sequence length judging by it's first octet
inline std::size_t seqLength(const UInt8 first_octet)
{
if (first_octet < 0x80u)
return 1;
const std::size_t bits = 8;
const auto first_zero = bitScanReverse(static_cast<UInt8>(~first_octet));
return bits - 1 - first_zero;
}
}
}