diff --git a/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md b/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md index ca8aef24ea1..a5c40226f85 100644 --- a/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md +++ b/docs/en/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md @@ -35,11 +35,7 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration The source is configured in the `source` section. -For source types -[Local file](#dicts-external_dicts_dict_sources-local_file), -[Executable file](#dicts-external_dicts_dict_sources-executable), -[HTTP(s)](#dicts-external_dicts_dict_sources-http), -[ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) +For source types [Local file](#dicts-external_dicts_dict_sources-local_file), [Executable file](#dicts-external_dicts_dict_sources-executable), [HTTP(s)](#dicts-external_dicts_dict_sources-http), [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) optional settings are available: ``` xml @@ -53,6 +49,12 @@ optional settings are available: ``` +or +``` sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +SETTINGS(format_csv_allow_single_quotes = 0) +``` + Types of sources (`source_type`): diff --git a/docs/fr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md b/docs/fr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md index 55a982862ed..f5027311e76 100644 --- a/docs/fr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md +++ b/docs/fr/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md @@ -37,6 +37,20 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration La source est configurée dans le `source` section. +Pour les sources de types [Fichier Local](#dicts-external_dicts_dict_sources-local_file), [Fichier exécutable](#dicts-external_dicts_dict_sources-executable), [HTTP(s)](#dicts-external_dicts_dict_sources-http), [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) +les paramètres optionnels sont possibles: + +``` xml +@@ -53,6 +49,12 @@ optional settings are available: + + +``` +ou +``` sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +SETTINGS(format_csv_allow_single_quotes = 1) +``` + Les Types de sources (`source_type`): - [Fichier Local](#dicts-external_dicts_dict_sources-local_file) diff --git a/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md index 102a7cc0e7f..ae3bb57fae7 100644 --- a/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md +++ b/docs/ru/sql_reference/dictionaries/external_dictionaries/external_dicts_dict_sources.md @@ -30,11 +30,7 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration Источник настраивается в разделе `source`. -Для типов источников -[Локальный файл](#dicts-external_dicts_dict_sources-local_file), -[Исполняемый файл](#dicts-external_dicts_dict_sources-executable), -[HTTP(s)](#dicts-external_dicts_dict_sources-http), -[ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) +Для типов источников [Локальный файл](#dicts-external_dicts_dict_sources-local_file), [Исполняемый файл](#dicts-external_dicts_dict_sources-executable), [HTTP(s)](#dicts-external_dicts_dict_sources-http), [ClickHouse](#dicts-external_dicts_dict_sources-clickhouse) доступны дополнительные настройки: ``` xml @@ -48,6 +44,11 @@ SOURCE(SOURCE_TYPE(param1 val1 ... paramN valN)) -- Source configuration ``` +или +``` sql +SOURCE(FILE(path '/opt/dictionaries/os.tsv' format 'TabSeparated')) +SETTINGS(format_csv_allow_single_quotes = 0) +``` Типы источников (`source_type`): diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp index e60a0641066..dad16f11031 100644 --- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp +++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp @@ -375,13 +375,26 @@ void buildConfigurationFromFunctionWithKeyValueArguments( * * */ -void buildSourceConfiguration(AutoPtr doc, AutoPtr root, const ASTFunctionWithKeyValueArguments * source) +void buildSourceConfiguration(AutoPtr doc, AutoPtr root, const ASTFunctionWithKeyValueArguments * source, const ASTDictionarySettings * settings) { AutoPtr outer_element(doc->createElement("source")); root->appendChild(outer_element); AutoPtr source_element(doc->createElement(source->name)); outer_element->appendChild(source_element); buildConfigurationFromFunctionWithKeyValueArguments(doc, source_element, source->elements->as()); + + if (settings != nullptr) + { + AutoPtr settings_element(doc->createElement("settings")); + outer_element->appendChild(settings_element); + for (const auto & [name, value] : settings->changes) + { + AutoPtr setting_change_element(doc->createElement(name)); + settings_element->appendChild(setting_change_element); + AutoPtr setting_value(doc->createTextNode(getFieldAsString(value))); + setting_change_element->appendChild(setting_value); + } + } } /** Check all AST fields are filled, throws exception @@ -454,7 +467,7 @@ DictionaryConfigurationPtr getDictionaryConfigurationFromAST(const ASTCreateQuer buildPrimaryKeyConfiguration(xml_document, structure_element, complex, pk_attrs, query.dictionary_attributes_list); buildLayoutConfiguration(xml_document, current_dictionary, dictionary_layout); - buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source); + buildSourceConfiguration(xml_document, current_dictionary, query.dictionary->source, query.dictionary->dict_settings); buildLifetimeConfiguration(xml_document, current_dictionary, query.dictionary->lifetime); if (query.dictionary->range) diff --git a/src/Parsers/ASTDictionary.cpp b/src/Parsers/ASTDictionary.cpp index 9ff600333c5..c90c6defd27 100644 --- a/src/Parsers/ASTDictionary.cpp +++ b/src/Parsers/ASTDictionary.cpp @@ -107,6 +107,34 @@ void ASTDictionaryLayout::formatImpl(const FormatSettings & settings, settings.ostr << ")"; } +ASTPtr ASTDictionarySettings::clone() const +{ + auto res = std::make_shared(*this); + res->children.clear(); + res->changes = changes; + + return res; +} + +void ASTDictionarySettings::formatImpl(const FormatSettings & settings, + FormatState &, + FormatStateStacked) const +{ + + settings.ostr << (settings.hilite ? hilite_keyword : "") + << "SETTINGS" + << (settings.hilite ? hilite_none : "") + << "("; + for (auto it = changes.begin(); it != changes.end(); ++it) + { + if (it != changes.begin()) + settings.ostr << ", "; + + settings.ostr << it->name << " = " << applyVisitor(FieldVisitorToString(), it->value); + } + settings.ostr << (settings.hilite ? hilite_none : "") << ")"; +} + ASTPtr ASTDictionary::clone() const { @@ -128,6 +156,9 @@ ASTPtr ASTDictionary::clone() const if (range) res->set(res->range, range->clone()); + if (dict_settings) + res->set(res->dict_settings, dict_settings->clone()); + return res; } @@ -166,6 +197,12 @@ void ASTDictionary::formatImpl(const FormatSettings & settings, FormatState & st settings.ostr << settings.nl_or_ws; range->formatImpl(settings, state, frame); } + + if (dict_settings) + { + settings.ostr << settings.nl_or_ws; + dict_settings->formatImpl(settings, state, frame); + } } } diff --git a/src/Parsers/ASTDictionary.h b/src/Parsers/ASTDictionary.h index 6982381f14d..9685cc5a83a 100644 --- a/src/Parsers/ASTDictionary.h +++ b/src/Parsers/ASTDictionary.h @@ -5,6 +5,10 @@ #include #include +#include + +#include + namespace DB { @@ -60,6 +64,18 @@ public: void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; }; +class ASTDictionarySettings : public IAST +{ +public: + SettingsChanges changes; + + String getID(char) const override { return "Dictionary settings"; } + + ASTPtr clone() const override; + + void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; +}; + /// AST contains all parts of external dictionary definition except attributes class ASTDictionary : public IAST @@ -77,6 +93,8 @@ public: ASTDictionaryLayout * layout; /// Range for dictionary (only for range-hashed dictionaries) ASTDictionaryRange * range; + /// Settings for dictionary (optionally) + ASTDictionarySettings * dict_settings; String getID(char) const override { return "Dictionary definition"; } diff --git a/src/Parsers/ParserDictionary.cpp b/src/Parsers/ParserDictionary.cpp index 2680c700296..a18e5be8975 100644 --- a/src/Parsers/ParserDictionary.cpp +++ b/src/Parsers/ParserDictionary.cpp @@ -11,6 +11,8 @@ #include +#include + namespace DB { @@ -149,6 +151,31 @@ bool ParserDictionaryLayout::parseImpl(Pos & pos, ASTPtr & node, Expected & expe return true; } +bool ParserDictionarySettings::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserToken s_comma(TokenType::Comma); + + SettingsChanges changes; + + while (true) + { + if (!changes.empty() && !s_comma.ignore(pos)) + break; + + changes.push_back(SettingChange{}); + + if (!ParserSetQuery::parseNameValuePair(changes.back(), pos, expected)) + return false; + } + + auto query = std::make_shared(); + query->changes = std::move(changes); + + node = query; + + return true; +} + bool ParserDictionary::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { @@ -157,6 +184,7 @@ bool ParserDictionary::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword lifetime_keyword("LIFETIME"); ParserKeyword range_keyword("RANGE"); ParserKeyword layout_keyword("LAYOUT"); + ParserKeyword settings_keyword("SETTINGS"); ParserToken open(TokenType::OpeningRoundBracket); ParserToken close(TokenType::ClosingRoundBracket); ParserFunctionWithKeyValueArguments key_value_pairs_p; @@ -164,12 +192,14 @@ bool ParserDictionary::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserDictionaryLifetime lifetime_p; ParserDictionaryRange range_p; ParserDictionaryLayout layout_p; + ParserDictionarySettings settings_p; ASTPtr primary_key; ASTPtr ast_source; ASTPtr ast_lifetime; ASTPtr ast_layout; ASTPtr ast_range; + ASTPtr ast_settings; /// Primary is required to be the first in dictionary definition if (primary_key_keyword.ignore(pos) && !expression_list_p.parse(pos, primary_key, expected)) @@ -235,6 +265,20 @@ bool ParserDictionary::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) continue; } + if (!ast_settings && settings_keyword.ignore(pos, expected)) + { + if (!open.ignore(pos)) + return false; + + if (!settings_p.parse(pos, ast_settings, expected)) + return false; + + if (!close.ignore(pos)) + return false; + + continue; + } + break; } @@ -255,6 +299,9 @@ bool ParserDictionary::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (ast_range) query->set(query->range, ast_range); + if (ast_settings) + query->set(query->dict_settings, ast_settings); + return true; } diff --git a/src/Parsers/ParserDictionary.h b/src/Parsers/ParserDictionary.h index 3e91a72ee81..795a2c618db 100644 --- a/src/Parsers/ParserDictionary.h +++ b/src/Parsers/ParserDictionary.h @@ -3,6 +3,8 @@ #include #include +#include + namespace DB { @@ -37,6 +39,13 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserDictionarySettings: public IParserBase +{ +protected: + const char * getName() const override { return "settings definition"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + /// Combines together all parsers from above and also parses primary key and /// dictionary source, which consists of custom key-value pairs: diff --git a/src/Parsers/ParserSetQuery.cpp b/src/Parsers/ParserSetQuery.cpp index 99b08bff337..30d681cb126 100644 --- a/src/Parsers/ParserSetQuery.cpp +++ b/src/Parsers/ParserSetQuery.cpp @@ -13,7 +13,7 @@ namespace DB /// Parse `name = value`. -static bool parseNameValuePair(SettingChange & change, IParser::Pos & pos, Expected & expected) +bool ParserSetQuery::parseNameValuePair(SettingChange & change, IParser::Pos & pos, Expected & expected) { ParserIdentifier name_p; ParserLiteral value_p; diff --git a/src/Parsers/ParserSetQuery.h b/src/Parsers/ParserSetQuery.h index 13bc30acfce..59a6109ea48 100644 --- a/src/Parsers/ParserSetQuery.h +++ b/src/Parsers/ParserSetQuery.h @@ -14,11 +14,10 @@ class ParserSetQuery : public IParserBase { public: explicit ParserSetQuery(bool parse_only_internals_ = false) : parse_only_internals(parse_only_internals_) {} - + static bool parseNameValuePair(SettingChange & change, IParser::Pos & pos, Expected & expected); protected: const char * getName() const override { return "SET query"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; - /// Parse the list `name = value` pairs, without SET. bool parse_only_internals; }; diff --git a/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.reference b/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.reference new file mode 100644 index 00000000000..39998d57011 --- /dev/null +++ b/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.reference @@ -0,0 +1,2 @@ +INITIALIZING DICTIONARY +END diff --git a/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql b/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql new file mode 100644 index 00000000000..cbac234305d --- /dev/null +++ b/tests/queries/0_stateless/01259_dictionary_custom_settings_ddl.sql @@ -0,0 +1,47 @@ +DROP DATABASE IF EXISTS database_for_dict; + +CREATE DATABASE database_for_dict Engine = Ordinary; + +DROP TABLE IF EXISTS database_for_dict.table_for_dict; + +CREATE TABLE database_for_dict.table_for_dict +( + key_column UInt64, + second_column UInt64, + third_column String +) +ENGINE = MergeTree() +ORDER BY key_column; + +INSERT INTO database_for_dict.table_for_dict VALUES (100500, 10000000, 'Hello world'); + +DROP DATABASE IF EXISTS ordinary_db; + +CREATE DATABASE ordinary_db ENGINE = Ordinary; + +DROP DICTIONARY IF EXISTS ordinary_db.dict1; + +CREATE DICTIONARY ordinary_db.dict1 +( + key_column UInt64 DEFAULT 0, + second_column UInt64 DEFAULT 1, + third_column String DEFAULT 'qqq' +) +PRIMARY KEY key_column +SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'default' TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +LIFETIME(MIN 1 MAX 10) +LAYOUT(FLAT()) SETTINGS(max_result_bytes=1); + +SELECT 'INITIALIZING DICTIONARY'; + +SELECT dictGetUInt64('ordinary_db.dict1', 'second_column', toUInt64(100500)); -- { serverError 396 } + +SELECT 'END'; + +DROP DICTIONARY IF EXISTS ordinary_db.dict1; + +DROP DATABASE IF EXISTS ordinary_db; + +DROP TABLE IF EXISTS database_for_dict.table_for_dict; + +DROP DATABASE IF EXISTS database_for_dict;