From b603e51c6efee5673a46f592fb26665ae2e7d79e Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Sun, 7 Apr 2019 21:58:13 +0300 Subject: [PATCH] isValidUTF8 function added --- dbms/src/Functions/isValidUTF8.cpp | 306 +++++ .../src/Functions/registerFunctionsString.cpp | 2 + .../0_stateless/00934_is_valid_utf8.reference | 1180 +++++++++++++++++ .../0_stateless/00934_is_valid_utf8.sql | 125 ++ .../functions/string_functions.md | 4 + .../functions/string_functions.md | 3 + 6 files changed, 1620 insertions(+) create mode 100644 dbms/src/Functions/isValidUTF8.cpp create mode 100644 dbms/tests/queries/0_stateless/00934_is_valid_utf8.reference create mode 100644 dbms/tests/queries/0_stateless/00934_is_valid_utf8.sql diff --git a/dbms/src/Functions/isValidUTF8.cpp b/dbms/src/Functions/isValidUTF8.cpp new file mode 100644 index 00000000000..0b573a0323a --- /dev/null +++ b/dbms/src/Functions/isValidUTF8.cpp @@ -0,0 +1,306 @@ +#include +#include +#include + +#include + +#ifdef __SSE4_1__ +# include +# include +# include +#endif + +namespace DB +{ +struct ValidUTF8Impl +{ + /* + * inspired by https://github.com/cyb70289/utf8/ + * http://www.unicode.org/versions/Unicode6.0.0/ch03.pdf - page 94 + * + * Table 3-7. Well-Formed UTF-8 Byte Sequences + * + * +--------------------+------------+-------------+------------+-------------+ + * | Code Points | First Byte | Second Byte | Third Byte | Fourth Byte | + * +--------------------+------------+-------------+------------+-------------+ + * | U+0000..U+007F | 00..7F | | | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+0080..U+07FF | C2..DF | 80..BF | | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+0800..U+0FFF | E0 | A0..BF | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+1000..U+CFFF | E1..EC | 80..BF | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+D000..U+D7FF | ED | 80..9F | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+E000..U+FFFF | EE..EF | 80..BF | 80..BF | | + * +--------------------+------------+-------------+------------+-------------+ + * | U+10000..U+3FFFF | F0 | 90..BF | 80..BF | 80..BF | + * +--------------------+------------+-------------+------------+-------------+ + * | U+40000..U+FFFFF | F1..F3 | 80..BF | 80..BF | 80..BF | + * +--------------------+------------+-------------+------------+-------------+ + * | U+100000..U+10FFFF | F4 | 80..8F | 80..BF | 80..BF | + * +--------------------+------------+-------------+------------+-------------+ + */ + + static inline UInt8 isValidUTF8Naive(const UInt8 * data, UInt64 len) + { + while (len) + { + int bytes; + const UInt8 byte1 = data[0]; + /* 00..7F */ + if (byte1 <= 0x7F) + { + bytes = 1; + } + /* C2..DF, 80..BF */ + else if (len >= 2 && byte1 >= 0xC2 && byte1 <= 0xDF && static_cast(data[1]) <= static_cast(0xBF)) + { + bytes = 2; + } + else if (len >= 3) + { + const UInt8 byte2 = data[1]; + bool byte2_ok = static_cast(byte2) <= static_cast(0xBF); + bool byte3_ok = static_cast(data[2]) <= static_cast(0xBF); + + if (byte2_ok && byte3_ok && + /* E0, A0..BF, 80..BF */ + ((byte1 == 0xE0 && byte2 >= 0xA0) || + /* E1..EC, 80..BF, 80..BF */ + (byte1 >= 0xE1 && byte1 <= 0xEC) || + /* ED, 80..9F, 80..BF */ + (byte1 == 0xED && byte2 <= 0x9F) || + /* EE..EF, 80..BF, 80..BF */ + (byte1 >= 0xEE && byte1 <= 0xEF))) + { + bytes = 3; + } + else if (len >= 4) + { + bool byte4_ok = static_cast(data[3]) <= static_cast(0xBF); + if (byte2_ok && byte3_ok && byte4_ok && + /* F0, 90..BF, 80..BF, 80..BF */ + ((byte1 == 0xF0 && byte2 >= 0x90) || + /* F1..F3, 80..BF, 80..BF, 80..BF */ + (byte1 >= 0xF1 && byte1 <= 0xF3) || + /* F4, 80..8F, 80..BF, 80..BF */ + (byte1 == 0xF4 && byte2 <= 0x8F))) + { + bytes = 4; + } + else + { + return false; + } + } + else + { + return false; + } + } + else + { + return false; + } + len -= bytes; + data += bytes; + } + return true; + } + +#ifndef __SSE4_1__ + static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len) { return isValidUTF8Naive(data, len); } +#else + static inline UInt8 isValidUTF8(const UInt8 * data, UInt64 len) + { + /* + * Map high nibble of "First Byte" to legal character length minus 1 + * 0x00 ~ 0xBF --> 0 + * 0xC0 ~ 0xDF --> 1 + * 0xE0 ~ 0xEF --> 2 + * 0xF0 ~ 0xFF --> 3 + */ + const __m128i first_len_tbl = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 2, 3); + + /* Map "First Byte" to 8-th item of range table (0xC2 ~ 0xF4) */ + const __m128i first_range_tbl = _mm_setr_epi8(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8); + + /* + * Range table, map range index to min and max values + */ + const __m128i range_min_tbl + = _mm_setr_epi8(0x00, 0x80, 0x80, 0x80, 0xA0, 0x80, 0x90, 0x80, 0xC2, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F, 0x7F); + + const __m128i range_max_tbl + = _mm_setr_epi8(0x7F, 0xBF, 0xBF, 0xBF, 0xBF, 0x9F, 0xBF, 0x8F, 0xF4, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80); + + /* + * Tables for fast handling of four special First Bytes(E0,ED,F0,F4), after + * which the Second Byte are not 80~BF. It contains "range index adjustment". + * +------------+---------------+------------------+----------------+ + * | First Byte | original range| range adjustment | adjusted range | + * +------------+---------------+------------------+----------------+ + * | E0 | 2 | 2 | 4 | + * +------------+---------------+------------------+----------------+ + * | ED | 2 | 3 | 5 | + * +------------+---------------+------------------+----------------+ + * | F0 | 3 | 3 | 6 | + * +------------+---------------+------------------+----------------+ + * | F4 | 4 | 4 | 8 | + * +------------+---------------+------------------+----------------+ + */ + + /* index1 -> E0, index14 -> ED */ + const __m128i df_ee_tbl = _mm_setr_epi8(0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0); + + /* index1 -> F0, index5 -> F4 */ + const __m128i ef_fe_tbl = _mm_setr_epi8(0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + __m128i prev_input = _mm_set1_epi8(0); + __m128i prev_first_len = _mm_set1_epi8(0); + __m128i error = _mm_set1_epi8(0); + + auto checkPacked = [&](__m128i input) noexcept + { + /* high_nibbles = input >> 4 */ + const __m128i high_nibbles = _mm_and_si128(_mm_srli_epi16(input, 4), _mm_set1_epi8(0x0F)); + + /* first_len = legal character length minus 1 */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* first_len = first_len_tbl[high_nibbles] */ + __m128i first_len = _mm_shuffle_epi8(first_len_tbl, high_nibbles); + + /* First Byte: set range index to 8 for bytes within 0xC0 ~ 0xFF */ + /* range = first_range_tbl[high_nibbles] */ + __m128i range = _mm_shuffle_epi8(first_range_tbl, high_nibbles); + + /* Second Byte: set range index to first_len */ + /* 0 for 00~7F, 1 for C0~DF, 2 for E0~EF, 3 for F0~FF */ + /* range |= (first_len, prev_first_len) << 1 byte */ + range = _mm_or_si128(range, _mm_alignr_epi8(first_len, prev_first_len, 15)); + + /* Third Byte: set range index to saturate_sub(first_len, 1) */ + /* 0 for 00~7F, 0 for C0~DF, 1 for E0~EF, 2 for F0~FF */ + __m128i tmp1; + __m128i tmp2; + /* tmp1 = saturate_sub(first_len, 1) */ + tmp1 = _mm_subs_epu8(first_len, _mm_set1_epi8(1)); + /* tmp2 = saturate_sub(prev_first_len, 1) */ + tmp2 = _mm_subs_epu8(prev_first_len, _mm_set1_epi8(1)); + /* range |= (tmp1, tmp2) << 2 bytes */ + range = _mm_or_si128(range, _mm_alignr_epi8(tmp1, tmp2, 14)); + + /* Fourth Byte: set range index to saturate_sub(first_len, 2) */ + /* 0 for 00~7F, 0 for C0~DF, 0 for E0~EF, 1 for F0~FF */ + /* tmp1 = saturate_sub(first_len, 2) */ + tmp1 = _mm_subs_epu8(first_len, _mm_set1_epi8(2)); + /* tmp2 = saturate_sub(prev_first_len, 2) */ + tmp2 = _mm_subs_epu8(prev_first_len, _mm_set1_epi8(2)); + /* range |= (tmp1, tmp2) << 3 bytes */ + range = _mm_or_si128(range, _mm_alignr_epi8(tmp1, tmp2, 13)); + + /* + * Now we have below range indices caluclated + * Correct cases: + * - 8 for C0~FF + * - 3 for 1st byte after F0~FF + * - 2 for 1st byte after E0~EF or 2nd byte after F0~FF + * - 1 for 1st byte after C0~DF or 2nd byte after E0~EF or + * 3rd byte after F0~FF + * - 0 for others + * Error cases: + * 9,10,11 if non ascii First Byte overlaps + * E.g., F1 80 C2 90 --> 8 3 10 2, where 10 indicates error + */ + + /* Adjust Second Byte range for special First Bytes(E0,ED,F0,F4) */ + /* Overlaps lead to index 9~15, which are illegal in range table */ + __m128i shift1, pos, range2; + /* shift1 = (input, prev_input) << 1 byte */ + shift1 = _mm_alignr_epi8(input, prev_input, 15); + pos = _mm_sub_epi8(shift1, _mm_set1_epi8(0xEF)); + /* + * shift1: | EF F0 ... FE | FF 00 ... ... DE | DF E0 ... EE | + * pos: | 0 1 15 | 16 17 239| 240 241 255| + * pos-240: | 0 0 0 | 0 0 0 | 0 1 15 | + * pos+112: | 112 113 127| >= 128 | >= 128 | + */ + tmp1 = _mm_subs_epu8(pos, _mm_set1_epi8(240)); + range2 = _mm_shuffle_epi8(df_ee_tbl, tmp1); + tmp2 = _mm_adds_epu8(pos, _mm_set1_epi8(112)); + range2 = _mm_add_epi8(range2, _mm_shuffle_epi8(ef_fe_tbl, tmp2)); + + range = _mm_add_epi8(range, range2); + + /* Load min and max values per calculated range index */ + __m128i minv = _mm_shuffle_epi8(range_min_tbl, range); + __m128i maxv = _mm_shuffle_epi8(range_max_tbl, range); + + /* Check value range */ + error = _mm_or_si128(error, _mm_cmplt_epi8(input, minv)); + error = _mm_or_si128(error, _mm_cmpgt_epi8(input, maxv)); + + prev_input = input; + prev_first_len = first_len; + + data += 16; + len -= 16; + }; + + while (len >= 16) + checkPacked(_mm_loadu_si128(reinterpret_cast(data))); + + if (len) + { + alignas(16) char buf[32]; + _mm_store_si128(reinterpret_cast<__m128i *>(buf), _mm_loadu_si128(reinterpret_cast(data))); + memset(reinterpret_cast(&buf) + len, 0, 16); + checkPacked(_mm_load_si128(reinterpret_cast<__m128i *>(buf))); + } + /* Reduce error vector, error_reduced = 0xFFFF if error == 0 */ + return _mm_movemask_epi8(_mm_cmpeq_epi8(error, _mm_set1_epi8(0))) == 0xFFFF; + } +#endif + + static constexpr bool is_fixed_to_constant = false; + + static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray & res) + { + size_t size = offsets.size(); + size_t prev_offset = 0; + for (size_t i = 0; i < size; ++i) + { + res[i] = isValidUTF8(data.data() + prev_offset, offsets[i] - 1 - prev_offset); + prev_offset = offsets[i]; + } + } + + static void vector_fixed_to_constant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt8 & /*res*/) {} + + static void vector_fixed_to_vector(const ColumnString::Chars & data, size_t n, PaddedPODArray & res) + { + size_t size = data.size() / n; + for (size_t i = 0; i < size; ++i) + res[i] = isValidUTF8(data.data() + i * n, n); + } + + static void array(const ColumnString::Offsets &, PaddedPODArray &) + { + throw Exception("Cannot apply function isValidUTF8 to Array argument", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } +}; + +struct NameValidUTF8 +{ + static constexpr auto name = "isValidUTF8"; +}; +using FunctionValidUTF8 = FunctionStringOrArrayToT; + +void registerFunctionValidUTF8(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/dbms/src/Functions/registerFunctionsString.cpp b/dbms/src/Functions/registerFunctionsString.cpp index 15d37d939b0..f04bbf84c4d 100644 --- a/dbms/src/Functions/registerFunctionsString.cpp +++ b/dbms/src/Functions/registerFunctionsString.cpp @@ -9,6 +9,7 @@ void registerFunctionEmpty(FunctionFactory &); void registerFunctionNotEmpty(FunctionFactory &); void registerFunctionLength(FunctionFactory &); void registerFunctionLengthUTF8(FunctionFactory &); +void registerFunctionValidUTF8(FunctionFactory &); void registerFunctionLower(FunctionFactory &); void registerFunctionUpper(FunctionFactory &); void registerFunctionLowerUTF8(FunctionFactory &); @@ -36,6 +37,7 @@ void registerFunctionsString(FunctionFactory & factory) registerFunctionNotEmpty(factory); registerFunctionLength(factory); registerFunctionLengthUTF8(factory); + registerFunctionValidUTF8(factory); registerFunctionLower(factory); registerFunctionUpper(factory); registerFunctionLowerUTF8(factory); diff --git a/dbms/tests/queries/0_stateless/00934_is_valid_utf8.reference b/dbms/tests/queries/0_stateless/00934_is_valid_utf8.reference new file mode 100644 index 00000000000..065b0a72373 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00934_is_valid_utf8.reference @@ -0,0 +1,1180 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/00934_is_valid_utf8.sql b/dbms/tests/queries/0_stateless/00934_is_valid_utf8.sql new file mode 100644 index 00000000000..6a0bdb3d345 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00934_is_valid_utf8.sql @@ -0,0 +1,125 @@ +select 1 = isValidUTF8('') from system.numbers limit 10; +select 1 = isValidUTF8('some text') from system.numbers limit 10; +select 1 = isValidUTF8('какой-то текст') from system.numbers limit 10; +select 1 = isValidUTF8('\x00') from system.numbers limit 10; +select 1 = isValidUTF8('\x66') from system.numbers limit 10; +select 1 = isValidUTF8('\x7F') from system.numbers limit 10; +select 1 = isValidUTF8('\x00\x7F') from system.numbers limit 10; +select 1 = isValidUTF8('\x7F\x00') from system.numbers limit 10; +select 1 = isValidUTF8('\xC2\x80') from system.numbers limit 10; +select 1 = isValidUTF8('\xDF\xBF') from system.numbers limit 10; +select 1 = isValidUTF8('\xE0\xA0\x80') from system.numbers limit 10; +select 1 = isValidUTF8('\xE0\xA0\xBF') from system.numbers limit 10; +select 1 = isValidUTF8('\xED\x9F\x80') from system.numbers limit 10; +select 1 = isValidUTF8('\xEF\x80\xBF') from system.numbers limit 10; +select 1 = isValidUTF8('\xF0\x90\xBF\x80') from system.numbers limit 10; +select 1 = isValidUTF8('\xF2\x81\xBE\x99') from system.numbers limit 10; +select 1 = isValidUTF8('\xF4\x8F\x88\xAA') from system.numbers limit 10; + +select 1 = isValidUTF8('a') from system.numbers limit 10; +select 1 = isValidUTF8('\xc3\xb1') from system.numbers limit 10; +select 1 = isValidUTF8('\xe2\x82\xa1') from system.numbers limit 10; +select 1 = isValidUTF8('\xf0\x90\x8c\xbc') from system.numbers limit 10; +select 1 = isValidUTF8('안녕하세요, 세상') from system.numbers limit 10; + +select 0 = isValidUTF8('\xc3\x28') from system.numbers limit 10; +select 0 = isValidUTF8('\xa0\xa1') from system.numbers limit 10; +select 0 = isValidUTF8('\xe2\x28\xa1') from system.numbers limit 10; +select 0 = isValidUTF8('\xe2\x82\x28') from system.numbers limit 10; +select 0 = isValidUTF8('\xf0\x28\x8c\xbc') from system.numbers limit 10; +select 0 = isValidUTF8('\xf0\x90\x28\xbc') from system.numbers limit 10; +select 0 = isValidUTF8('\xf0\x28\x8c\x28') from system.numbers limit 10; +select 0 = isValidUTF8('\xc0\x9f') from system.numbers limit 10; +select 0 = isValidUTF8('\xf5\xff\xff\xff') from system.numbers limit 10; +select 0 = isValidUTF8('\xed\xa0\x81') from system.numbers limit 10; +select 0 = isValidUTF8('\xf8\x90\x80\x80\x80') from system.numbers limit 10; +select 0 = isValidUTF8('123456789012345\xed') from system.numbers limit 10; +select 0 = isValidUTF8('123456789012345\xf1') from system.numbers limit 10; +select 0 = isValidUTF8('123456789012345\xc2') from system.numbers limit 10; +select 0 = isValidUTF8('\xC2\x7F') from system.numbers limit 10; + +select 0 = isValidUTF8('\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xBF') from system.numbers limit 10; +select 0 = isValidUTF8('\xC0\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xC1\x00') from system.numbers limit 10; +select 0 = isValidUTF8('\xC2\x7F') from system.numbers limit 10; +select 0 = isValidUTF8('\xDF\xC0') from system.numbers limit 10; +select 0 = isValidUTF8('\xE0\x9F\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xE0\xC2\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xED\xA0\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xED\x7F\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xEF\x80\x00') from system.numbers limit 10; +select 0 = isValidUTF8('\xF0\x8F\x80\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xF0\xEE\x80\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\xF2\x90\x91\x7F') from system.numbers limit 10; +select 0 = isValidUTF8('\xF4\x90\x88\xAA') from system.numbers limit 10; +select 0 = isValidUTF8('\xF4\x00\xBF\xBF') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\xC2\x80\x00\x00\x00\xE1\x80\x80\x00\x00\xC2\xC2\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\xC2\xC2\x80\x00\x00\xE1\x80\x80\x00\x00\x00') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\xC2\x80') from system.numbers limit 10; +select 0 = isValidUTF8('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0\x80\x80\x80') from system.numbers limit 10; + +select 1 = isValidUTF8(toFixedString('some text', 9)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('какой-то текст', 27)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\x00', 1)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\x66', 1)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\x7F', 1)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\x00\x7F', 2)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\x7F\x00', 2)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xC2\x80', 2)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xDF\xBF', 2)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xE0\xA0\x80', 3)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xE0\xA0\xBF', 3)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xED\x9F\x80', 3)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xEF\x80\xBF', 3)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xF0\x90\xBF\x80', 4)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xF2\x81\xBE\x99', 4)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xF4\x8F\x88\xAA', 4)) from system.numbers limit 10; + +select 0 = isValidUTF8(toFixedString('\x80', 1)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xBF', 1)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xC0\x80', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xC1\x00', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xC2\x7F', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xDF\xC0', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xE0\x9F\x80', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xE0\xC2\x80', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xED\xA0\x80', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xED\x7F\x80', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xEF\x80\x00', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xF0\x8F\x80\x80', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xF0\xEE\x80\x80', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xF2\x90\x91\x7F', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xF4\x90\x88\xAA', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xF4\x00\xBF\xBF', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\xC2\x80\x00\x00\x00\xE1\x80\x80\x00\x00\xC2\xC2\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', 32)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\xC2\xC2\x80\x00\x00\xE1\x80\x80\x00\x00\x00', 16)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80', 32)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1', 32)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\x80', 33)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF1\x80\xC2\x80', 34)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xF0\x80\x80\x80', 35)) from system.numbers limit 10; + +select 1 = isValidUTF8(toFixedString('a', 1)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xc3\xb1', 2)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xe2\x82\xa1', 3)) from system.numbers limit 10; +select 1 = isValidUTF8(toFixedString('\xf0\x90\x8c\xbc', 4)) from system.numbers limit 10; + +select 0 = isValidUTF8(toFixedString('\xc3\x28', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xa0\xa1', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xe2\x28\xa1', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xe2\x82\x28', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xf0\x28\x8c\xbc', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xf0\x90\x28\xbc', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xf0\x28\x8c\x28', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xc0\x9f', 2)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xf5\xff\xff\xff', 4)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xed\xa0\x81', 3)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xf8\x90\x80\x80\x80', 5)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('123456789012345\xed', 16)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('123456789012345\xf1', 16)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('123456789012345\xc2', 16)) from system.numbers limit 10; +select 0 = isValidUTF8(toFixedString('\xC2\x7F', 2)) from system.numbers limit 10; diff --git a/docs/en/query_language/functions/string_functions.md b/docs/en/query_language/functions/string_functions.md index 8d4dd838282..5b1fcc0cc80 100644 --- a/docs/en/query_language/functions/string_functions.md +++ b/docs/en/query_language/functions/string_functions.md @@ -56,6 +56,10 @@ It doesn't detect the language. So for Turkish the result might not be exactly c If the length of the UTF-8 byte sequence is different for upper and lower case of a code point, the result may be incorrect for this code point. If the string contains a set of bytes that is not UTF-8, then the behavior is undefined. +## isValidUTF8 + +Returns 1, if set of bytes is valid UTF-8 encoded, otherwise 0. + ## reverse Reverses the string (as a sequence of bytes). diff --git a/docs/ru/query_language/functions/string_functions.md b/docs/ru/query_language/functions/string_functions.md index 653342ebbd4..c180909a968 100644 --- a/docs/ru/query_language/functions/string_functions.md +++ b/docs/ru/query_language/functions/string_functions.md @@ -38,6 +38,9 @@ Если длина UTF-8 последовательности байт различна для верхнего и нижнего регистра кодовой точки, то для этой кодовой точки, результат работы может быть некорректным. Если строка содержит набор байт, не являющийся UTF-8, то поведение не определено. +## isValidUTF8 +Возвращает 1, если набор байт является корректным в кодировке UTF-8, 0 иначе. + ## reverse Разворачивает строку (как последовательность байт).