thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-04 13:32:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

ISSUES-1885 fix failed test

Browse Source
This commit is contained in:
zhang2014 2018-02-11 22:38:51 +08:00
parent 0f20952f2c
commit 73077e0245
1 changed files with 11 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dbms/src/Common/UTF8Helpers.h
View File

@ -2,6 +2,7 @@
#include <Core/Types.h>
#include <Common/BitHelpers.h>
#include <iostream>
#if __SSE2__
#include <emmintrin.h>
@ -58,23 +59,27 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
const auto bytes_sse = sizeof(__m128i);
const auto src_end_sse = (data + size) - (size % bytes_sse);
const auto fill_sse = _mm_set1_epi8(0xFF);
const auto upper_bound = _mm_set1_epi8(0x7F + 1);
const auto lower_bound = _mm_set1_epi8(0xC0 - 1);
for (; data < src_end_sse;)
{
UInt8 mem_res[16] = {0};
UInt16 mem_res[8] = {0};
auto sse_res = _mm_set1_epi8(0);
for (int i = 0; i < 0XFF && data < src_end_sse; ++i, data += bytes_sse)
for (int i = 0, limit = 0xFF; i < limit && data < src_end_sse; ++i, data += bytes_sse)
{
const auto chars = _mm_loadu_si128(reinterpret_cast<const __m128i *>(data));
sse_res = _mm_add_epi8(sse_res,
_mm_or_si128(_mm_cmplt_epi8(chars, upper_bound),
_mm_cmpgt_epi8(chars, lower_bound)));
const auto chars_cmpgt_res = _mm_or_si128(_mm_cmplt_epi8(chars, upper_bound),
_mm_cmpgt_epi8(chars, lower_bound));
sse_res = _mm_add_epi8(sse_res, _mm_add_epi8(_mm_xor_si128(chars_cmpgt_res, fill_sse), _mm_set1_epi8(1)));
}
_mm_store_si128(reinterpret_cast<__m128i *>(mem_res), sse_res);
auto horizontal_add = _mm_sad_epu8(sse_res, _mm_set1_epi8(0));
_mm_store_si128(reinterpret_cast<__m128i *>(mem_res), horizontal_add);
for (auto count : mem_res)
res += count;