thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 02:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ability to work with different types besides Date in RangeHashed external dictionary created from DDL query.

Browse Source
This commit is contained in:
alesapin 2019-12-18 19:26:46 +03:00
parent 663b68cd08
commit 742caced3f
3 changed files with 76 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dbms/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
View File

@ -28,6 +28,7 @@ namespace ErrorCodes
namespace
{
using NamesToTypeNames = std::unordered_map<std::string, std::string>;
/// Get value from field and convert it to string.
/// Also remove quotes from strings.
String getUnescapedFieldString(const Field & field)
@ -112,21 +113,27 @@ void buildLayoutConfiguration(
* <range_min><name>StartDate</name></range_min>
* <range_max><name>EndDate</name></range_max>
*/
void buildRangeConfiguration(AutoPtr<Document> doc, AutoPtr<Element> root, const ASTDictionaryRange * range)
void buildRangeConfiguration(AutoPtr<Document> doc, AutoPtr<Element> root, const ASTDictionaryRange * range, const NamesToTypeNames & all_attrs)
{
// appends <key><name>value</name></key> to root
auto appendElem = [&doc, &root](const std::string & key, const std::string & value)
auto appendElem = [&doc, &root](const std::string & key, const std::string & name, const std::string & type)
{
AutoPtr<Element> element(doc->createElement(key));
AutoPtr<Element> name(doc->createElement("name"));
AutoPtr<Text> text(doc->createTextNode(value));
name->appendChild(text);
element->appendChild(name);
AutoPtr<Element> name_node(doc->createElement("name"));
AutoPtr<Text> name_text(doc->createTextNode(name));
name_node->appendChild(name_text);
element->appendChild(name_node);
AutoPtr<Element> type_node(doc->createElement("type"));
AutoPtr<Text> type_text(doc->createTextNode(type));
type_node->appendChild(type_text);
element->appendChild(type_node);
root->appendChild(element);
};
appendElem("range_min", range->min_attr_name);
appendElem("range_max", range->max_attr_name);
appendElem("range_min", range->min_attr_name, all_attrs.at(range->min_attr_name));
appendElem("range_max", range->max_attr_name, all_attrs.at(range->max_attr_name));
}
@ -296,25 +303,25 @@ void buildPrimaryKeyConfiguration(
/**
* Transforms list of ASTDictionaryAttributeDeclarations to list of dictionary attributes
*/
std::unordered_set<std::string> buildDictionaryAttributesConfiguration(
NamesToTypeNames buildDictionaryAttributesConfiguration(
AutoPtr<Document> doc,
AutoPtr<Element> root,
const ASTExpressionList * dictionary_attributes,
const Names & key_columns)
{
const auto & children = dictionary_attributes->children;
std::unordered_set<std::string> dictionary_attributes_names;
NamesToTypeNames attributes_names_and_types;
for (size_t i = 0; i < children.size(); ++i)
{
const ASTDictionaryAttributeDeclaration * dict_attr = children[i]->as<const ASTDictionaryAttributeDeclaration>();
if (!dict_attr->type)
throw Exception("Dictionary attribute must has type", ErrorCodes::INCORRECT_DICTIONARY_DEFINITION);
dictionary_attributes_names.insert(dict_attr->name);
attributes_names_and_types.emplace(dict_attr->name, queryToString(dict_attr->type));
if (std::find(key_columns.begin(), key_columns.end(), dict_attr->name) == key_columns.end())
buildSingleAttribute(doc, root, dict_attr);
}
return dictionary_attributes_names;
return attributes_names_and_types;
}
/** Transform function with key-value arguments to configuration
@ -404,7 +411,7 @@ void checkAST(const ASTCreateQuery & query)
/// Range can be empty
}
void checkPrimaryKey(const std::unordered_set<std::string> & all_attrs, const Names & key_attrs)
void checkPrimaryKey(const NamesToTypeNames & all_attrs, const Names & key_attrs)
{
for (const auto & key_attr : key_attrs)
if (all_attrs.count(key_attr) == 0)
@ -438,8 +445,8 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer
bool complex = DictionaryFactory::instance().isComplex(dictionary_layout->layout_type);
auto all_attr_names = buildDictionaryAttributesConfiguration(xml_document, structure_element, query.dictionary_attributes_list, pk_attrs);
checkPrimaryKey(all_attr_names, pk_attrs);
auto all_attr_names_and_types = buildDictionaryAttributesConfiguration(xml_document, structure_element, query.dictionary_attributes_list, pk_attrs);
checkPrimaryKey(all_attr_names_and_types, pk_attrs);
buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list);
@ -448,7 +455,7 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer
buildLifetimeConfiguration(xml_document, current_dictionary, query.dictionary->lifetime);
if (query.dictionary->range)
buildRangeConfiguration(xml_document, structure_element, query.dictionary->range);
buildRangeConfiguration(xml_document, structure_element, query.dictionary->range, all_attr_names_and_types);
conf->load(xml_document);
return conf;

8
dbms/tests/queries/0_stateless/01018_ddl_dictionaries_special.reference
View File

@ -1,9 +1,17 @@
***date dict***
0.33
0.42
0.46
0
***datetime dict***
0.33
0.42
0.46
0
***ip trie dict***
17501
NP
***hierarchy dict***
Moscow
[3,2,1,10000]
1

51
dbms/tests/queries/0_stateless/01018_ddl_dictionaries_special.sql
View File

@ -4,7 +4,9 @@ DROP DATABASE IF EXISTS database_for_dict;
CREATE DATABASE database_for_dict Engine = Ordinary;
CREATE TABLE database_for_dict.table_for_dict
SELECT '***date dict***';
CREATE TABLE database_for_dict.date_table
(
CountryID UInt64,
StartDate Date,
@ -14,9 +16,9 @@ CREATE TABLE database_for_dict.table_for_dict
ENGINE = MergeTree()
ORDER BY CountryID;
INSERT INTO database_for_dict.table_for_dict VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33);
INSERT INTO database_for_dict.table_for_dict VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42);
INSERT INTO database_for_dict.table_for_dict VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), 0.46);
INSERT INTO database_for_dict.date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33);
INSERT INTO database_for_dict.date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42);
INSERT INTO database_for_dict.date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), 0.46);
CREATE DICTIONARY database_for_dict.dict1
(
@ -26,7 +28,7 @@ CREATE DICTIONARY database_for_dict.dict1
Tax Float64
)
PRIMARY KEY CountryID
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' DB 'database_for_dict'))
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'date_table' DB 'database_for_dict'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(RANGE_HASHED())
RANGE(MIN StartDate MAX EndDate);
@ -36,6 +38,42 @@ SELECT dictGetFloat64('database_for_dict.dict1', 'Tax', toUInt64(1), toDate('201
SELECT dictGetFloat64('database_for_dict.dict1', 'Tax', toUInt64(2), toDate('2019-05-29'));
SELECT dictGetFloat64('database_for_dict.dict1', 'Tax', toUInt64(2), toDate('2019-05-31'));
SELECT '***datetime dict***';
CREATE TABLE database_for_dict.datetime_table
(
CountryID UInt64,
StartDate DateTime,
EndDate DateTime,
Tax Float64
)
ENGINE = MergeTree()
ORDER BY CountryID;
INSERT INTO database_for_dict.datetime_table VALUES(1, toDateTime('2019-05-05 00:00:00'), toDateTime('2019-05-20 00:00:00'), 0.33);
INSERT INTO database_for_dict.datetime_table VALUES(1, toDateTime('2019-05-21 00:00:00'), toDateTime('2019-05-30 00:00:00'), 0.42);
INSERT INTO database_for_dict.datetime_table VALUES(2, toDateTime('2019-05-21 00:00:00'), toDateTime('2019-05-30 00:00:00'), 0.46);
CREATE DICTIONARY database_for_dict.dict2
(
CountryID UInt64,
StartDate DateTime,
EndDate DateTime,
Tax Float64
)
PRIMARY KEY CountryID
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'datetime_table' DB 'database_for_dict'))
LIFETIME(MIN 1 MAX 1000)
LAYOUT(RANGE_HASHED())
RANGE(MIN StartDate MAX EndDate);
SELECT dictGetFloat64('database_for_dict.dict2', 'Tax', toUInt64(1), toDateTime('2019-05-15 00:00:00'));
SELECT dictGetFloat64('database_for_dict.dict2', 'Tax', toUInt64(1), toDateTime('2019-05-29 00:00:00'));
SELECT dictGetFloat64('database_for_dict.dict2', 'Tax', toUInt64(2), toDateTime('2019-05-29 00:00:00'));
SELECT dictGetFloat64('database_for_dict.dict2', 'Tax', toUInt64(2), toDateTime('2019-05-31 00:00:00'));
SELECT '***ip trie dict***';
CREATE TABLE database_for_dict.table_ip_trie
(
prefix String,
@ -61,6 +99,8 @@ LIFETIME(MIN 10 MAX 100);
SELECT dictGetUInt32('database_for_dict.dict_ip_trie', 'asn', tuple(IPv4StringToNum('202.79.32.0')));
SELECT dictGetString('database_for_dict.dict_ip_trie', 'cca2', tuple(IPv4StringToNum('202.79.32.0')));
SELECT '***hierarchy dict***';
CREATE TABLE database_for_dict.table_with_hierarchy
(
RegionID UInt64,
@ -91,4 +131,3 @@ SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(7), toUI
SELECT dictIsIn('database_for_dict.dictionary_with_hierarchy', toUInt64(1), toUInt64(5));
DROP DATABASE IF EXISTS database_for_dict;