mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
isValidUTF8 function added
This commit is contained in:
Danila Kutenin
parent
1c030e4ed1
commit
b603e51c6e
306
dbms/src/Functions/isValidUTF8.cpp
Normal file
306
dbms/src/Functions/isValidUTF8.cpp
Normal file
@ -0,0 +1,306 @@
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionStringOrArrayToT.h>
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#ifdef __SSE4_1__
|
||||
# include <emmintrin.h>
|
||||
# include <smmintrin.h>
|
||||
# include <tmmintrin.h>
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
struct ValidUTF8Impl
|
||||
{
|
||||
/*
|
||||
* inspired by https://github.com/cyb70289/utf8/
|
||||
* http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94
|
||||
*
|
||||
* Table 3-7. Well-Formed UTF-8 Byte Sequences
|
||||
*
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+0000..U+007F | 00..7F | | | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+0080..U+07FF | C2..DF | 80..BF | | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+0800..U+0FFF | E0 | A0..BF | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+D000..U+D7FF | ED | 80..9F | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
* | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF |
|
||||
* +--------------------+------------+-------------+------------+-------------+
|
||||
*/
|
||||
|
||||
static inline UInt8 isValidUTF8Naive(const UInt8 * data, UInt64 len)
|
||||
{
|
||||
while (len)
|
||||
{
|
||||
int bytes;
|
||||
const UInt8 byte1 = data[0];
|
||||
/* 00..7F */
|
||||
if (byte1 <= 0x7F)
|
||||
{
|
||||
bytes = 1;
|
||||
}
|
||||
/* C2..DF, 80..BF */
|
||||
else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF && static_cast<Int8>(data[1]) <= static_cast<Int8>(0xBF))
|
||||
{
|
||||
bytes = 2;
|
||||
}
|
||||
else if (len >= 3)
|
||||
{
|
||||
const UInt8 byte2 = data[1];
|
||||
bool byte2_ok = static_cast<Int8>(byte2) <= static_cast<Int8>(0xBF);
|
||||
bool byte3_ok = static_cast<Int8>(data[2]) <= static_cast<Int8>(0xBF);
|
||||
|
||||
if (byte2_ok && byte3_ok &&
|
||||
/* E0, A0..BF, 80..BF */
|
||||
((byte1 == 0xE0 && byte2 >= 0xA0) ||
|
||||
/* E1..EC, 80..BF, 80..BF */
|
||||
(byte1 >= 0xE1 && byte1 <= 0xEC) ||
|
||||
/* ED, 80..9F, 80..BF */
|
||||
(byte1 == 0xED && byte2 <= 0x9F) ||
|
||||
/* EE..EF, 80..BF, 80..BF */
|
||||
(byte1 >= 0xEE && byte1 <= 0xEF)))
|
||||
{
|
||||
bytes = 3;
|
||||
}
|
||||
else if (len >= 4)
|
||||
{
|
||||
bool byte4_ok = static_cast<Int8>(data[3]) <= static_cast<Int8>(0xBF);
|
||||
if (byte2_ok && byte3_ok && byte4_ok &&
|
||||
/* F0, 90..BF, 80..BF, 80..BF */
|
||||
((byte1 == 0xF0 && byte2 >= 0x90) ||
|
||||
/* F1..F3, 80..BF, 80..BF, 80..BF */
|
||||
(byte1 >= 0xF1 && byte1 <= 0xF3) ||
|
||||
/* F4, 80..8F, 80..BF, 80..BF */
|
||||
(byte1 == 0xF4 && byte2 <= 0x8F)))
|
||||
{
|
||||
bytes = 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return false;
|
||||
}
|
||||
len -= bytes;
|
||||
data += bytes;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#ifndef __SSE4_1__
|
||||
static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len) { return isValidUTF8Naive(data, len); }
|
||||
#else
|
||||
static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len)
|
||||
{
|
||||
/*
|
||||
* Map high nibble of "First Byte" to legal character length minus 1
|
||||
* 0x00 ~ 0xBF --> 0
|
||||
* 0xC0 ~ 0xDF --> 1
|
||||
* 0xE0 ~ 0xEF --> 2
|
||||
* 0xF0 ~ 0xFF --> 3
|
||||
*/
|
||||
const __m128i first_len_tbl = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3);
|
||||
|
||||
/* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */
|
||||
const __m128i first_range_tbl = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8);
|
||||
|
||||
/*
|
||||
* Range table, map range index to min and max values
|
||||
*/
|
||||
const __m128i range_min_tbl
|
||||
= _mm_setr_epi8(0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F);
|
||||
|
||||
const __m128i range_max_tbl
|
||||
= _mm_setr_epi8(0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80);
|
||||
|
||||
/*
|
||||
* Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after
|
||||
* which the Second Byte are not 80~BF. It contains "range index adjustment".
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | First Byte | original range| range adjustment | adjusted range |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | E0 | 2 | 2 | 4 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | ED | 2 | 3 | 5 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | F0 | 3 | 3 | 6 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
* | F4 | 4 | 4 | 8 |
|
||||
* +------------+---------------+------------------+----------------+
|
||||
*/
|
||||
|
||||
/* index1 -> E0, index14 -> ED */
|
||||
const __m128i df_ee_tbl = _mm_setr_epi8(0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0);
|
||||
|
||||
/* index1 -> F0, index5 -> F4 */
|
||||
const __m128i ef_fe_tbl = _mm_setr_epi8(0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
|
||||
|
||||
__m128i prev_input = _mm_set1_epi8(0);
|
||||
__m128i prev_first_len = _mm_set1_epi8(0);
|
||||
__m128i error = _mm_set1_epi8(0);
|
||||
|
||||
auto checkPacked = [&](__m128i input) noexcept
|
||||
{
|
||||
/* high_nibbles = input >> 4 */
|
||||
const __m128i high_nibbles = _mm_and_si128(_mm_srli_epi16(input, 4), _mm_set1_epi8(0x0F));
|
||||
|
||||
/* first_len = legal character length minus 1 */
|
||||
/* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
|
||||
/* first_len = first_len_tbl[high_nibbles] */
|
||||
__m128i first_len = _mm_shuffle_epi8(first_len_tbl, high_nibbles);
|
||||
|
||||
/* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */
|
||||
/* range = first_range_tbl[high_nibbles] */
|
||||
__m128i range = _mm_shuffle_epi8(first_range_tbl, high_nibbles);
|
||||
|
||||
/* Second Byte: set range index to first_len */
|
||||
/* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */
|
||||
/* range |= (first_len, prev_first_len) << 1 byte */
|
||||
range = _mm_or_si128(range, _mm_alignr_epi8(first_len, prev_first_len, 15));
|
||||
|
||||
/* Third Byte: set range index to saturate_sub(first_len, 1) */
|
||||
/* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */
|
||||
__m128i tmp1;
|
||||
__m128i tmp2;
|
||||
/* tmp1 = saturate_sub(first_len, 1) */
|
||||
tmp1 = _mm_subs_epu8(first_len, _mm_set1_epi8(1));
|
||||
/* tmp2 = saturate_sub(prev_first_len, 1) */
|
||||
tmp2 = _mm_subs_epu8(prev_first_len, _mm_set1_epi8(1));
|
||||
/* range |= (tmp1, tmp2) << 2 bytes */
|
||||
range = _mm_or_si128(range, _mm_alignr_epi8(tmp1, tmp2, 14));
|
||||
|
||||
/* Fourth Byte: set range index to saturate_sub(first_len, 2) */
|
||||
/* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */
|
||||
/* tmp1 = saturate_sub(first_len, 2) */
|
||||
tmp1 = _mm_subs_epu8(first_len, _mm_set1_epi8(2));
|
||||
/* tmp2 = saturate_sub(prev_first_len, 2) */
|
||||
tmp2 = _mm_subs_epu8(prev_first_len, _mm_set1_epi8(2));
|
||||
/* range |= (tmp1, tmp2) << 3 bytes */
|
||||
range = _mm_or_si128(range, _mm_alignr_epi8(tmp1, tmp2, 13));
|
||||
|
||||
/*
|
||||
* Now we have below range indices caluclated
|
||||
* Correct cases:
|
||||
* - 8 for C0~FF
|
||||
* - 3 for 1st byte after F0~FF
|
||||
* - 2 for 1st byte after E0~EF or 2nd byte after F0~FF
|
||||
* - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or
|
||||
* 3rd byte after F0~FF
|
||||
* - 0 for others
|
||||
* Error cases:
|
||||
* 9,10,11 if non ascii First Byte overlaps
|
||||
* E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error
|
||||
*/
|
||||
|
||||
/* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */
|
||||
/* Overlaps lead to index 9~15, which are illegal in range table */
|
||||
__m128i shift1, pos, range2;
|
||||
/* shift1 = (input, prev_input) << 1 byte */
|
||||
shift1 = _mm_alignr_epi8(input, prev_input, 15);
|
||||
pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF));
|
||||
/*
|
||||
* shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE |
|
||||
* pos: | 0 1 15 | 16 17 239| 240 241 255|
|
||||
* pos-240: | 0 0 0 | 0 0 0 | 0 1 15 |
|
||||
* pos+112: | 112 113 127| >= 128 | >= 128 |
|
||||
*/
|
||||
tmp1 = _mm_subs_epu8(pos, _mm_set1_epi8(240));
|
||||
range2 = _mm_shuffle_epi8(df_ee_tbl, tmp1);
|
||||
tmp2 = _mm_adds_epu8(pos, _mm_set1_epi8(112));
|
||||
range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp2));
|
||||
|
||||
range = _mm_add_epi8(range, range2);
|
||||
|
||||
/* Load min and max values per calculated range index */
|
||||
__m128i minv = _mm_shuffle_epi8(range_min_tbl, range);
|
||||
__m128i maxv = _mm_shuffle_epi8(range_max_tbl, range);
|
||||
|
||||
/* Check value range */
|
||||
error = _mm_or_si128(error, _mm_cmplt_epi8(input, minv));
|
||||
error = _mm_or_si128(error, _mm_cmpgt_epi8(input, maxv));
|
||||
|
||||
prev_input = input;
|
||||
prev_first_len = first_len;
|
||||
|
||||
data += 16;
|
||||
len -= 16;
|
||||
};
|
||||
|
||||
while (len >= 16)
|
||||
checkPacked(_mm_loadu_si128(reinterpret_cast<const __m128i *>(data)));
|
||||
|
||||
if (len)
|
||||
{
|
||||
alignas(16) char buf[32];
|
||||
_mm_store_si128(reinterpret_cast<__m128i *>(buf), _mm_loadu_si128(reinterpret_cast<const __m128i *>(data)));
|
||||
memset(reinterpret_cast<char *>(&buf) + len, 0, 16);
|
||||
checkPacked(_mm_load_si128(reinterpret_cast<__m128i *>(buf)));
|
||||
}
|
||||
/* Reduce error vector, error_reduced = 0xFFFF if error == 0 */
|
||||
return _mm_movemask_epi8(_mm_cmpeq_epi8(error, _mm_set1_epi8(0))) == 0xFFFF;
|
||||
}
|
||||
#endif
|
||||
|
||||
static constexpr bool is_fixed_to_constant = false;
|
||||
|
||||
static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res)
|
||||
{
|
||||
size_t size = offsets.size();
|
||||
size_t prev_offset = 0;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
res[i] = isValidUTF8(data.data() + prev_offset, offsets[i] - 1 - prev_offset);
|
||||
prev_offset = offsets[i];
|
||||
}
|
||||
}
|
||||
|
||||
static void vector_fixed_to_constant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt8 & /*res*/) {}
|
||||
|
||||
static void vector_fixed_to_vector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res)
|
||||
{
|
||||
size_t size = data.size() / n;
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
res[i] = isValidUTF8(data.data() + i * n, n);
|
||||
}
|
||||
|
||||
static void array(const ColumnString::Offsets &, PaddedPODArray<UInt8> &)
|
||||
{
|
||||
throw Exception("Cannot apply function isValidUTF8 to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
};
|
||||
|
||||
struct NameValidUTF8
|
||||
{
|
||||
static constexpr auto name = "isValidUTF8";
|
||||
};
|
||||
using FunctionValidUTF8 = FunctionStringOrArrayToT<ValidUTF8Impl, NameValidUTF8, UInt8>;
|
||||
|
||||
void registerFunctionValidUTF8(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionValidUTF8>();
|
||||
}
|
||||
|
||||
}
|
||||
@ -9,6 +9,7 @@ void registerFunctionEmpty(FunctionFactory &);
|
||||
void registerFunctionNotEmpty(FunctionFactory &);
|
||||
void registerFunctionLength(FunctionFactory &);
|
||||
void registerFunctionLengthUTF8(FunctionFactory &);
|
||||
void registerFunctionValidUTF8(FunctionFactory &);
|
||||
void registerFunctionLower(FunctionFactory &);
|
||||
void registerFunctionUpper(FunctionFactory &);
|
||||
void registerFunctionLowerUTF8(FunctionFactory &);
|
||||
@ -36,6 +37,7 @@ void registerFunctionsString(FunctionFactory & factory)
|
||||
registerFunctionNotEmpty(factory);
|
||||
registerFunctionLength(factory);
|
||||
registerFunctionLengthUTF8(factory);
|
||||
registerFunctionValidUTF8(factory);
|
||||
registerFunctionLower(factory);
|
||||
registerFunctionUpper(factory);
|
||||
registerFunctionLowerUTF8(factory);
|
||||
|
||||
1180
dbms/tests/queries/0_stateless/00934_is_valid_utf8.reference
Normal file
1180
dbms/tests/queries/0_stateless/00934_is_valid_utf8.reference
Normal file
File diff suppressed because it is too large
Load Diff
125
dbms/tests/queries/0_stateless/00934_is_valid_utf8.sql
Normal file
125
dbms/tests/queries/0_stateless/00934_is_valid_utf8.sql
Normal file
@ -0,0 +1,125 @@
|
||||
select 1 = isValidUTF8('') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('some text') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('какой-то текст') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\x00') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\x66') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\x7F') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\x00\x7F') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\x7F\x00') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xC2\x80') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xDF\xBF') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xE0\xA0\x80') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xE0\xA0\xBF') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xED\x9F\x80') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xEF\x80\xBF') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xF0\x90\xBF\x80') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xF2\x81\xBE\x99') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xF4\x8F\x88\xAA') from system.numbers limit 10;
|
||||
|
||||
select 1 = isValidUTF8('a') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xc3\xb1') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xe2\x82\xa1') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('\xf0\x90\x8c\xbc') from system.numbers limit 10;
|
||||
select 1 = isValidUTF8('안녕하세요, 세상') from system.numbers limit 10;
|
||||
|
||||
select 0 = isValidUTF8('\xc3\x28') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xa0\xa1') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xe2\x28\xa1') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xe2\x82\x28') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xf0\x28\x8c\xbc') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xf0\x90\x28\xbc') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xf0\x28\x8c\x28') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xc0\x9f') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xf5\xff\xff\xff') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xed\xa0\x81') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xf8\x90\x80\x80\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('123456789012345\xed') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('123456789012345\xf1') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('123456789012345\xc2') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xC2\x7F') from system.numbers limit 10;
|
||||
|
||||
select 0 = isValidUTF8('\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xBF') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xC0\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xC1\x00') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xC2\x7F') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xDF\xC0') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xE0\x9F\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xE0\xC2\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xED\xA0\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xED\x7F\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xEF\x80\x00') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xF0\x8F\x80\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xF0\xEE\x80\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xF2\x90\x91\x7F') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xF4\x90\x88\xAA') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\xF4\x00\xBF\xBF') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\xC2\x80\x00\x00\x00\xE1\x80\x80\x00\x00\xC2\xC2\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\xC2\xC2\x80\x00\x00\xE1\x80\x80\x00\x00\x00') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\xC2\x80') from system.numbers limit 10;
|
||||
select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0\x80\x80\x80') from system.numbers limit 10;
|
||||
|
||||
select 1 = isValidUTF8(toFixedString('some text', 9)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('какой-то текст', 27)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\x00', 1)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\x66', 1)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\x7F', 1)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\x00\x7F', 2)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\x7F\x00', 2)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xC2\x80', 2)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xDF\xBF', 2)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xE0\xA0\x80', 3)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xE0\xA0\xBF', 3)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xED\x9F\x80', 3)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xEF\x80\xBF', 3)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xF0\x90\xBF\x80', 4)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xF2\x81\xBE\x99', 4)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xF4\x8F\x88\xAA', 4)) from system.numbers limit 10;
|
||||
|
||||
select 0 = isValidUTF8(toFixedString('\x80', 1)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xBF', 1)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xC0\x80', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xC1\x00', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xC2\x7F', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xDF\xC0', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xE0\x9F\x80', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xE0\xC2\x80', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xED\xA0\x80', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xED\x7F\x80', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xEF\x80\x00', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xF0\x8F\x80\x80', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xF0\xEE\x80\x80', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xF2\x90\x91\x7F', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xF4\x90\x88\xAA', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xF4\x00\xBF\xBF', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\xC2\x80\x00\x00\x00\xE1\x80\x80\x00\x00\xC2\xC2\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 32)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\xC2\xC2\x80\x00\x00\xE1\x80\x80\x00\x00\x00', 16)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80', 32)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1', 32)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\x80', 33)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\xC2\x80', 34)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0\x80\x80\x80', 35)) from system.numbers limit 10;
|
||||
|
||||
select 1 = isValidUTF8(toFixedString('a', 1)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xc3\xb1', 2)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xe2\x82\xa1', 3)) from system.numbers limit 10;
|
||||
select 1 = isValidUTF8(toFixedString('\xf0\x90\x8c\xbc', 4)) from system.numbers limit 10;
|
||||
|
||||
select 0 = isValidUTF8(toFixedString('\xc3\x28', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xa0\xa1', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xe2\x28\xa1', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xe2\x82\x28', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xf0\x28\x8c\xbc', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xf0\x90\x28\xbc', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xf0\x28\x8c\x28', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xc0\x9f', 2)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xf5\xff\xff\xff', 4)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xed\xa0\x81', 3)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xf8\x90\x80\x80\x80', 5)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('123456789012345\xed', 16)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('123456789012345\xf1', 16)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('123456789012345\xc2', 16)) from system.numbers limit 10;
|
||||
select 0 = isValidUTF8(toFixedString('\xC2\x7F', 2)) from system.numbers limit 10;
|
||||
@ -56,6 +56,10 @@ It doesn't detect the language. So for Turkish the result might not be exactly c
|
||||
If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point.
|
||||
If the string contains a set of bytes that is not UTF-8, then the behavior is undefined.
|
||||
|
||||
## isValidUTF8
|
||||
|
||||
Returns 1, if set of bytes is valid UTF-8 encoded, otherwise 0.
|
||||
|
||||
## reverse
|
||||
|
||||
Reverses the string (as a sequence of bytes).
|
||||
|
||||
@ -38,6 +38,9 @@
|
||||
Если длина UTF-8 последовательности байт различна для верхнего и нижнего регистра кодовой точки, то для этой кодовой точки, результат работы может быть некорректным.
|
||||
Если строка содержит набор байт, не являющийся UTF-8, то поведение не определено.
|
||||
|
||||
## isValidUTF8
|
||||
Возвращает 1, если набор байт является корректным в кодировке UTF-8, 0 иначе.
|
||||
|
||||
## reverse
|
||||
Разворачивает строку (как последовательность байт).
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user