thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #31706 from yandd/master

Browse Source
This commit is contained in:
Vladimir C 2021-11-26 12:29:09 +03:00 committed by GitHub
commit 41c9067f48
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 52 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/en/sql-reference/functions/splitting-merging-functions.md
View File

@ -290,7 +290,7 @@ ngrams(string, ngramsize)
- Array with n-grams.
Type: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
Type: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
**Example**

2
docs/ru/sql-reference/functions/splitting-merging-functions.md
View File

@ -252,7 +252,7 @@ ngrams(string, ngramsize)
- Массив с n-граммами.
Тип: [Array](../../sql-reference/data-types/array.md)([FixedString](../../sql-reference/data-types/fixedstring.md)).
Тип: [Array](../../sql-reference/data-types/array.md)([String](../../sql-reference/data-types/string.md)).
**Пример**

20
src/Functions/tokenExtractors.cpp
View File

@ -68,17 +68,9 @@ public:
"Function {} second argument type should be constant UInt. Actual {}",
getName(),
arguments[1].type->getName());
Field ngram_argument_value;
ngram_argument_column->get(0, ngram_argument_value);
auto ngram_value = ngram_argument_value.safeGet<UInt64>();
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeFixedString>(ngram_value));
}
else
{
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
}
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
@ -93,16 +85,16 @@ public:
NgramTokenExtractor extractor(ngram_value);
auto result_column_fixed_string = ColumnFixedString::create(ngram_value);
auto result_column_string = ColumnString::create();
auto input_column = arguments[0].column;
if (const auto * column_string = checkAndGetColumn<ColumnString>(input_column.get()))
executeImpl(extractor, *column_string, *result_column_fixed_string, *column_offsets);
executeImpl(extractor, *column_string, *result_column_string, *column_offsets);
else if (const auto * column_fixed_string = checkAndGetColumn<ColumnFixedString>(input_column.get()))
executeImpl(extractor, *column_fixed_string, *result_column_fixed_string, *column_offsets);
executeImpl(extractor, *column_fixed_string, *result_column_string, *column_offsets);
return ColumnArray::create(std::move(result_column_fixed_string), std::move(column_offsets));
return ColumnArray::create(std::move(result_column_string), std::move(column_offsets));
}
else
{

28
tests/queries/0_stateless/02027_ngrams.reference
View File

@ -3,18 +3,38 @@
['Tes','est']
['Test']
[]
['😁','😈','😁','😈']
['😁😈','😈😁','😁😈']
['😁😈😁','😈😁😈']
['😁😈😁😈']
[]
['T','e','s','t']
['Te','es','st']
['Tes','est']
['Test']
[]
['T','e','s','t']
['Te','es','st']
['Tes','est']
['Test']
['😁','😈','😁','😈']
['😁😈','😈😁','😁😈']
['😁😈😁','😈😁😈']
['😁😈😁😈']
[]
['T','e','s','t']
['Te','es','st']
['Tes','est']
['Test']
[]
['😁','😈','😁','😈']
['😁😈','😈😁','😁😈']
['😁😈😁','😈😁😈']
['😁😈😁😈']
[]
['T','e','s','t']
['Te','es','st']
['Tes','est']
['Test']
[]
['😁','😈','😁','😈']
['😁😈','😈😁','😁😈']
['😁😈😁','😈😁😈']
['😁😈😁😈']
[]

20
tests/queries/0_stateless/02027_ngrams.sql
View File

@ -3,21 +3,41 @@ SELECT ngrams('Test', 2);
SELECT ngrams('Test', 3);
SELECT ngrams('Test', 4);
SELECT ngrams('Test', 5);
SELECT ngrams('😁😈😁😈', 1);
SELECT ngrams('😁😈😁😈', 2);
SELECT ngrams('😁😈😁😈', 3);
SELECT ngrams('😁😈😁😈', 4);
SELECT ngrams('😁😈😁😈', 5);
SELECT ngrams(materialize('Test'), 1);
SELECT ngrams(materialize('Test'), 2);
SELECT ngrams(materialize('Test'), 3);
SELECT ngrams(materialize('Test'), 4);
SELECT ngrams(materialize('Test'), 5);
SELECT ngrams(materialize('😁😈😁😈'), 1);
SELECT ngrams(materialize('😁😈😁😈'), 2);
SELECT ngrams(materialize('😁😈😁😈'), 3);
SELECT ngrams(materialize('😁😈😁😈'), 4);
SELECT ngrams(materialize('😁😈😁😈'), 5);
SELECT ngrams(toFixedString('Test', 4), 1);
SELECT ngrams(toFixedString('Test', 4), 2);
SELECT ngrams(toFixedString('Test', 4), 3);
SELECT ngrams(toFixedString('Test', 4), 4);
SELECT ngrams(toFixedString('Test', 4), 5);
SELECT ngrams(toFixedString('😁😈😁😈', 16), 1);
SELECT ngrams(toFixedString('😁😈😁😈', 16), 2);
SELECT ngrams(toFixedString('😁😈😁😈', 16), 3);
SELECT ngrams(toFixedString('😁😈😁😈', 16), 4);
SELECT ngrams(toFixedString('😁😈😁😈', 16), 5);
SELECT ngrams(materialize(toFixedString('Test', 4)), 1);
SELECT ngrams(materialize(toFixedString('Test', 4)), 2);
SELECT ngrams(materialize(toFixedString('Test', 4)), 3);
SELECT ngrams(materialize(toFixedString('Test', 4)), 4);
SELECT ngrams(materialize(toFixedString('Test', 4)), 5);
SELECT ngrams(materialize(toFixedString('😁😈😁😈', 16)), 1);
SELECT ngrams(materialize(toFixedString('😁😈😁😈', 16)), 2);
SELECT ngrams(materialize(toFixedString('😁😈😁😈', 16)), 3);
SELECT ngrams(materialize(toFixedString('😁😈😁😈', 16)), 4);
SELECT ngrams(materialize(toFixedString('😁😈😁😈', 16)), 5);