From 1dce20e5daf60cbb0777b17b9e20a0c6b192345c Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 14 Nov 2020 16:44:45 +0300 Subject: [PATCH 1/4] Default host, port and user parameters for source(clickhouse(...)) --- .../ClickHouseDictionarySource.cpp | 18 +++++++++++----- src/Dictionaries/ClickHouseDictionarySource.h | 2 +- .../01018_ddl_dictionaries_select.reference | 2 ++ .../01018_ddl_dictionaries_select.sql | 21 +++++++++++++++++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 8199b16a94b..72cdb4a78e3 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -17,6 +17,14 @@ namespace DB { +namespace +{ + inline static UInt16 getPortFromContext(const Context & context, bool secure) + { + return secure ? context.getTCPPortSecure().value_or(0) : context.getTCPPort(); + } +} + namespace ErrorCodes { } @@ -59,10 +67,10 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( const std::string & default_database) : update_time{std::chrono::system_clock::from_time_t(0)} , dict_struct{dict_struct_} - , host{config.getString(config_prefix + ".host")} - , port(config.getInt(config_prefix + ".port")) , secure(config.getBool(config_prefix + ".secure", false)) - , user{config.getString(config_prefix + ".user", "")} + , host{config.getString(config_prefix + ".host", "localhost")} + , port(config.getInt(config_prefix + ".port", getPortFromContext(context_, secure))) + , user{config.getString(config_prefix + ".user", "default")} , password{config.getString(config_prefix + ".password", "")} , db{config.getString(config_prefix + ".db", default_database)} , table{config.getString(config_prefix + ".table")} @@ -72,7 +80,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( , query_builder{dict_struct, db, "", table, where, IdentifierQuotingStyle::Backticks} , sample_block{sample_block_} , context(context_) - , is_local{isLocalAddress({host, port}, secure ? context.getTCPPortSecure().value_or(0) : context.getTCPPort())} + , is_local{isLocalAddress({host, port}, getPortFromContext(context_, secure))} , pool{is_local ? nullptr : createPool(host, port, secure, db, user, password)} , load_all_query{query_builder.composeLoadAllQuery()} { @@ -92,9 +100,9 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionarySource & other) : update_time{other.update_time} , dict_struct{other.dict_struct} + , secure{other.secure} , host{other.host} , port{other.port} - , secure{other.secure} , user{other.user} , password{other.password} , db{other.db} diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 8017d458b7e..9ef77d061fd 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -61,9 +61,9 @@ private: std::chrono::time_point update_time; const DictionaryStructure dict_struct; + const bool secure; const std::string host; const UInt16 port; - const bool secure; const std::string user; const std::string password; const std::string db; diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference b/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference index 28ee2a8690b..f69302fb90f 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.reference @@ -7,6 +7,8 @@ 17 11 11 +17 +11 7 11 6 diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql index 4b548a913ea..78789388a89 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql @@ -51,6 +51,27 @@ DROP DICTIONARY database_for_dict.dict1; SELECT dictGetUInt8('database_for_dict.dict1', 'second_column', toUInt64(11)); -- {serverError 36} +-- SOURCE(CLICKHOUSE(...)) uses default params if not specified +DROP DICTIONARY IF EXISTS database_for_dict.dict1; + +CREATE DICTIONARY database_for_dict.dict1 +( + key_column UInt64 DEFAULT 0, + second_column UInt8 DEFAULT 1, + third_column String DEFAULT 'qqq', + fourth_column Float64 DEFAULT 42.0 +) +PRIMARY KEY key_column +SOURCE(CLICKHOUSE(TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +LIFETIME(MIN 1 MAX 10) +LAYOUT(FLAT()); + +SELECT dictGetUInt8('database_for_dict.dict1', 'second_column', toUInt64(11)); + +SELECT count(distinct(dictGetUInt8('database_for_dict.dict1', 'second_column', toUInt64(number)))) from numbers(100); + +DROP DICTIONARY database_for_dict.dict1; + CREATE DICTIONARY database_for_dict.dict1 ( key_column UInt64 DEFAULT 0, From 66fc54633134ae8d07a93ce265f159aeeda1d436 Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 14 Nov 2020 16:45:42 +0300 Subject: [PATCH 2/4] Fix doc for LIFETIME statement --- docs/en/sql-reference/statements/create/dictionary.md | 2 +- docs/es/sql-reference/statements/create.md | 2 +- docs/fa/sql-reference/statements/create.md | 2 +- docs/fr/sql-reference/statements/create.md | 2 +- docs/ja/sql-reference/statements/create.md | 2 +- docs/ru/sql-reference/statements/create/dictionary.md | 4 ++-- docs/tr/sql-reference/statements/create.md | 2 +- docs/zh/sql-reference/statements/create.md | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/statements/create/dictionary.md b/docs/en/sql-reference/statements/create/dictionary.md index b1098c54703..3fe94e267e4 100644 --- a/docs/en/sql-reference/statements/create/dictionary.md +++ b/docs/en/sql-reference/statements/create/dictionary.md @@ -20,7 +20,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` External dictionary structure consists of attributes. Dictionary attributes are specified similarly to table columns. The only required attribute property is its type, all other properties may have default values. diff --git a/docs/es/sql-reference/statements/create.md b/docs/es/sql-reference/statements/create.md index b851435286e..db3194ae114 100644 --- a/docs/es/sql-reference/statements/create.md +++ b/docs/es/sql-reference/statements/create.md @@ -291,7 +291,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` Crear [diccionario externo](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) con dado [estructura](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [fuente](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [diseño](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) y [vida](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). diff --git a/docs/fa/sql-reference/statements/create.md b/docs/fa/sql-reference/statements/create.md index c4b7ede05dd..970e8ee7535 100644 --- a/docs/fa/sql-reference/statements/create.md +++ b/docs/fa/sql-reference/statements/create.md @@ -291,7 +291,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` ایجاد [فرهنگ لغت خارجی](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) با توجه به [ساختار](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [متن](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [طرحبندی](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) و [طول عمر](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). diff --git a/docs/fr/sql-reference/statements/create.md b/docs/fr/sql-reference/statements/create.md index f7b3790baf2..e7c8040ee6e 100644 --- a/docs/fr/sql-reference/statements/create.md +++ b/docs/fr/sql-reference/statements/create.md @@ -291,7 +291,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` Crée [externe dictionnaire](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) avec le [structure](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [source](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [disposition](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) et [vie](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). diff --git a/docs/ja/sql-reference/statements/create.md b/docs/ja/sql-reference/statements/create.md index ae518dbfac8..1d1f2c57556 100644 --- a/docs/ja/sql-reference/statements/create.md +++ b/docs/ja/sql-reference/statements/create.md @@ -291,7 +291,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` 作成 [外部辞書](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) 与えられたと [構造](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [ソース](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [レイアウト](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) と [生涯](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). diff --git a/docs/ru/sql-reference/statements/create/dictionary.md b/docs/ru/sql-reference/statements/create/dictionary.md index a20dc812e02..3134a89483b 100644 --- a/docs/ru/sql-reference/statements/create/dictionary.md +++ b/docs/ru/sql-reference/statements/create/dictionary.md @@ -16,7 +16,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` Создаёт [внешний словарь](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) с заданной [структурой](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [источником](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [способом размещения в памяти](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) и [периодом обновления](../../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). @@ -27,5 +27,5 @@ LIFETIME([MIN val1] MAX val2) Смотрите [Внешние словари](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md). -[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/dictionary) +[Оригинальная статья](https://clickhouse.tech/docs/ru/sql-reference/statements/create/dictionary) \ No newline at end of file diff --git a/docs/tr/sql-reference/statements/create.md b/docs/tr/sql-reference/statements/create.md index 79bdb45f9e4..78390564880 100644 --- a/docs/tr/sql-reference/statements/create.md +++ b/docs/tr/sql-reference/statements/create.md @@ -291,7 +291,7 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` Oluşturuyor [dış sözlük](../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) verilen ile [yapılı](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-structure.md), [kaynaklı](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-sources.md), [düzen](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-layout.md) ve [ömür](../../sql-reference/dictionaries/external-dictionaries/external-dicts-dict-lifetime.md). diff --git a/docs/zh/sql-reference/statements/create.md b/docs/zh/sql-reference/statements/create.md index fa3cb8e5ea5..639af0841dc 100644 --- a/docs/zh/sql-reference/statements/create.md +++ b/docs/zh/sql-reference/statements/create.md @@ -259,5 +259,5 @@ CREATE DICTIONARY [IF NOT EXISTS] [db.]dictionary_name [ON CLUSTER cluster] PRIMARY KEY key1, key2 SOURCE(SOURCE_NAME([param1 value1 ... paramN valueN])) LAYOUT(LAYOUT_NAME([param_name param_value])) -LIFETIME([MIN val1] MAX val2) +LIFETIME({MIN min_val MAX max_val | max_val}) ``` From cd3cc329a12a4d3ec50eec6abf85b80a0fa9a827 Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 14 Nov 2020 18:23:48 +0300 Subject: [PATCH 3/4] Remove redundant anon namespace in ClickHouseDictionarySource.cpp --- src/Dictionaries/ClickHouseDictionarySource.cpp | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 72cdb4a78e3..8d733bcd90a 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -17,21 +17,14 @@ namespace DB { -namespace -{ - inline static UInt16 getPortFromContext(const Context & context, bool secure) - { - return secure ? context.getTCPPortSecure().value_or(0) : context.getTCPPort(); - } -} - -namespace ErrorCodes -{ -} - static const size_t MAX_CONNECTIONS = 16; +inline static UInt16 getPortFromContext(const Context & context, bool secure) +{ + return secure ? context.getTCPPortSecure().value_or(0) : context.getTCPPort(); +} + static ConnectionPoolWithFailoverPtr createPool( const std::string & host, UInt16 port, From 87baaa0d34d8a6923f6878b9209c5f06914b3e8b Mon Sep 17 00:00:00 2001 From: vdimir Date: Sat, 14 Nov 2020 23:59:29 +0300 Subject: [PATCH 4/4] Test default empty password for CLICKHOUSE dict source --- tests/queries/0_stateless/01018_ddl_dictionaries_select.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql index 78789388a89..4bb506579cc 100644 --- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql +++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql @@ -62,7 +62,7 @@ CREATE DICTIONARY database_for_dict.dict1 fourth_column Float64 DEFAULT 42.0 ) PRIMARY KEY key_column -SOURCE(CLICKHOUSE(TABLE 'table_for_dict' PASSWORD '' DB 'database_for_dict')) +SOURCE(CLICKHOUSE(TABLE 'table_for_dict' DB 'database_for_dict')) LIFETIME(MIN 1 MAX 10) LAYOUT(FLAT());