thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-12 10:34:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removed useless code, added comments

Browse Source
This commit is contained in:
Alexey Milovidov 2020-01-08 13:20:55 +03:00
parent 98b0f1624d
commit 2084d26238
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp
View File

@ -636,15 +636,16 @@ bool SplitTokenExtractor::next(const char * data, size_t len, size_t * pos, size
{
if (isASCII(data[*pos]) && !isAlphaNumericASCII(data[*pos]))
{
/// Finish current token if any
if (*token_len > 0)
return true;
*token_start = ++*pos;
}
else
{
const size_t sz = UTF8::seqLength(static_cast<UInt8>(data[*pos]));
*pos += sz;
*token_len += sz;
/// Note that UTF-8 sequence is completely consisted of non-ASCII bytes.
++*pos;
++*token_len;
}
}
return *token_len > 0;