From 2cce811dda8fd356eacdfaf29861dc03548dfed9 Mon Sep 17 00:00:00 2001 From: feng lv Date: Tue, 7 Jul 2020 17:24:49 +0800 Subject: [PATCH 01/89] optimize trivial insert select style fix update fix fix fix --- src/Interpreters/InterpreterInsertQuery.cpp | 68 ++++++++++++++++++--- 1 file changed, 60 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 36b106d5d68..0f9133af34d 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -14,20 +15,20 @@ #include #include #include -#include #include #include #include #include #include +#include #include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include namespace DB @@ -200,9 +201,60 @@ BlockIO InterpreterInsertQuery::execute() size_t out_streams_size = 1; if (query.select) { - /// Passing 1 as subquery_depth will disable limiting size of intermediate result. - InterpreterSelectWithUnionQuery interpreter_select{ query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - res = interpreter_select.execute(); + auto is_trivial_select = [](const auto query_) + { + if (query_.tables()) + { + const auto & tables_in_select_query = query_.tables()->template as(); + for (const auto & child : tables_in_select_query.children) + { + const auto & table_element = child->template as(); + const auto & table_expr = table_element.table_expression->template as(); + if (table_expr.subquery) + { + return false; + } + } + } + if (!query_.distinct && !query_.limit_with_ties && !query_.prewhere() + && !query_.where() && !query_.groupBy() && !query_.having() && !query_.orderBy() && !query_.limitBy()) + return true; + return false; + }; + + const auto & ast = query.select->as(); + size_t num_selects = ast.list_of_selects->children.size(); + bool is_trivial_query = true; + for (size_t query_num = 0; query_num < num_selects; ++query_num) + { + const auto & query_select = ast.list_of_selects->children.at(query_num)->as(); + if (!is_trivial_select(query_select)) + { + is_trivial_query = false; + break; + } + } + + if (is_trivial_query) + { + auto new_context = context; + UInt64 max_threads = settings.max_insert_threads; + UInt64 min_insert_block_size_rows = settings.min_insert_block_size_rows; + new_context.setSetting("max_threads", std::max(1, max_threads)); + if (min_insert_block_size_rows) + new_context.setSetting("max_block_size", min_insert_block_size_rows); + + /// Passing 1 as subquery_depth will disable limiting size of intermediate result. + InterpreterSelectWithUnionQuery interpreter_select{ + query.select, new_context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + res = interpreter_select.execute(); + } + else + { + InterpreterSelectWithUnionQuery interpreter_select{ + query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + res = interpreter_select.execute(); + } if (table->supportsParallelInsert() && settings.max_insert_threads > 1) out_streams_size = std::min(size_t(settings.max_insert_threads), res.pipeline.getNumStreams()); From b5bc7c0d97dcf7695b1ebe0ad95157e40ccfa276 Mon Sep 17 00:00:00 2001 From: libenwang Date: Sat, 25 Jul 2020 12:01:16 +0800 Subject: [PATCH 02/89] Fix ClickHouseDictionarySource wrong access check --- src/Dictionaries/ClickHouseDictionarySource.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 180750d143a..edffe273ed0 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -74,7 +74,10 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( , load_all_query{query_builder.composeLoadAllQuery()} { /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). - context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0)); + if (is_local) + { + context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0)); + } context = copyContextAndApplySettings(path_to_settings, context, config); /// Query context is needed because some code in executeQuery function may assume it exists. From 76ef196c5dd5e577f6812c107f598626809e3624 Mon Sep 17 00:00:00 2001 From: libenwang Date: Mon, 27 Jul 2020 18:10:39 +0800 Subject: [PATCH 03/89] Remove settings apply when not local --- src/Dictionaries/ClickHouseDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index edffe273ed0..6cfb265afd6 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -77,8 +77,8 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( if (is_local) { context.setUser(user, password, Poco::Net::SocketAddress("127.0.0.1", 0)); + context = copyContextAndApplySettings(path_to_settings, context, config); } - context = copyContextAndApplySettings(path_to_settings, context, config); /// Query context is needed because some code in executeQuery function may assume it exists. /// Current example is Context::getSampleBlockCache from InterpreterSelectWithUnionQuery::getSampleBlock. From f6183559eb2918661c007ee3615d8b8a2d605864 Mon Sep 17 00:00:00 2001 From: libenwang Date: Wed, 29 Jul 2020 13:57:46 +0800 Subject: [PATCH 04/89] Add integration dictionaries tests --- .../configs/config_password.xml | 23 +++++++++++++ .../integration/test_dictionaries_ddl/test.py | 33 ++++++++++++++++++- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 tests/integration/test_dictionaries_ddl/configs/config_password.xml diff --git a/tests/integration/test_dictionaries_ddl/configs/config_password.xml b/tests/integration/test_dictionaries_ddl/configs/config_password.xml new file mode 100644 index 00000000000..3e9ef48f2db --- /dev/null +++ b/tests/integration/test_dictionaries_ddl/configs/config_password.xml @@ -0,0 +1,23 @@ + + + + + + + + + default + default + default + + + + + + ::/0 + + default + default + + + diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 220aeb6998a..91e92edd2b9 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -11,6 +11,7 @@ cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, node1 = cluster.add_instance('node1', with_mysql=True, main_configs=['configs/dictionaries/simple_dictionary.xml']) node2 = cluster.add_instance('node2', with_mysql=True, main_configs=['configs/dictionaries/simple_dictionary.xml', 'configs/dictionaries/lazy_load.xml']) node3 = cluster.add_instance('node3', main_configs=['configs/dictionaries/dictionary_with_conflict_name.xml']) +node4 = cluster.add_instance('node4', user_configs=['configs/config_password.xml']) # hardcoded value 33333 def create_mysql_conn(user, password, hostname, port): @@ -32,7 +33,7 @@ def execute_mysql_query(connection, query): def started_cluster(): try: cluster.start() - for clickhouse in [node1, node2, node3]: + for clickhouse in [node1, node2, node3, node4]: clickhouse.query("CREATE DATABASE test", user="admin") clickhouse.query("CREATE TABLE test.xml_dictionary_table (id UInt64, SomeValue1 UInt8, SomeValue2 String) ENGINE = MergeTree() ORDER BY id", user="admin") clickhouse.query("INSERT INTO test.xml_dictionary_table SELECT number, number % 23, hex(number) from numbers(1000)", user="admin") @@ -241,3 +242,33 @@ def test_dictionary_with_where(started_cluster): node1.query("SYSTEM RELOAD DICTIONARY default.special_dict") assert node1.query("SELECT dictGetString('default.special_dict', 'value1', toUInt64(2))") == 'qweqwe\n' + +def test_clickhouse_remote(started_cluster): + with pytest.raises(QueryRuntimeException): + node3.query(""" + CREATE DICTIONARY test.clickhouse_remote( + id UInt64, + SomeValue1 UInt8, + SomeValue2 String + ) + PRIMARY KEY id + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'node4' PORT 9000 USER 'default' TABLE 'xml_dictionary_table' DB 'test')) + LIFETIME(MIN 1 MAX 10) + """) + node3.query("system reload dictionaries") + + node3.query("detach dictionary if exists test.clickhouse_remote") + node3.query(""" + CREATE DICTIONARY test.clickhouse_remote( + id UInt64, + SomeValue1 UInt8, + SomeValue2 String + ) + PRIMARY KEY id + LAYOUT(FLAT()) + SOURCE(CLICKHOUSE(HOST 'node4' PORT 9000 USER 'default' PASSWORD 'default' TABLE 'xml_dictionary_table' DB 'test')) + LIFETIME(MIN 1 MAX 10) + """) + + node3.query("select dictGetUInt8('test.clickhouse_remote', 'SomeValue1', toUInt64(17))") == '17\n' \ No newline at end of file From 2e6edf5aa1e2a4fe1b83cb227508647c7ed55aa6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Fri, 31 Jul 2020 14:38:15 +0300 Subject: [PATCH 05/89] trigger --- tests/integration/test_dictionaries_ddl/test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 91e92edd2b9..0e613f74e9f 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -271,4 +271,5 @@ def test_clickhouse_remote(started_cluster): LIFETIME(MIN 1 MAX 10) """) - node3.query("select dictGetUInt8('test.clickhouse_remote', 'SomeValue1', toUInt64(17))") == '17\n' \ No newline at end of file + node3.query("select dictGetUInt8('test.clickhouse_remote', 'SomeValue1', toUInt64(17))") == '17\n' + From fdf646283dbf4aacf6d3d934694ce630a9ccd939 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 2 Aug 2020 09:57:24 +0800 Subject: [PATCH 06/89] fix fix --- src/Core/Settings.h | 1 + src/Interpreters/InterpreterInsertQuery.cpp | 85 ++++++++++--------- .../00340_squashing_insert_select.sql | 1 + ..._block_size_rows_for_materialized_views.sh | 2 +- 4 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ac325538923..66a1c512898 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -473,6 +473,7 @@ struct Settings : public SettingsCollection M(SettingBool, output_format_enable_streaming, false, "Enable streaming in output formats that support it.", 0) \ M(SettingBool, output_format_write_statistics, true, "Write statistics about read rows, bytes, time elapsed in suitable output formats.", 0) \ M(SettingBool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \ + M(SettingBool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ #define LIST_OF_SETTINGS(M) \ COMMON_SETTINGS(M) \ diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 0f9133af34d..4ec1e2bd50f 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -201,53 +201,62 @@ BlockIO InterpreterInsertQuery::execute() size_t out_streams_size = 1; if (query.select) { - auto is_trivial_select = [](const auto query_) - { - if (query_.tables()) - { - const auto & tables_in_select_query = query_.tables()->template as(); - for (const auto & child : tables_in_select_query.children) + auto optimize_trivial_insert_select = settings.optimize_trivial_insert_select; + if(optimize_trivial_insert_select) + { + auto is_trivial_select = [](const auto query_) { + if (query_.tables()) { - const auto & table_element = child->template as(); - const auto & table_expr = table_element.table_expression->template as(); - if (table_expr.subquery) + const auto & tables_in_select_query = query_.tables()->template as(); + for (const auto & child : tables_in_select_query.children) { - return false; + const auto & table_element = child->template as(); + const auto & table_expr = table_element.table_expression->template as(); + if (table_expr.subquery) + { + return false; + } } } - } - if (!query_.distinct && !query_.limit_with_ties && !query_.prewhere() - && !query_.where() && !query_.groupBy() && !query_.having() && !query_.orderBy() && !query_.limitBy()) - return true; - return false; - }; + if (!query_.distinct && !query_.limit_with_ties && !query_.prewhere() && !query_.where() && !query_.groupBy() + && !query_.having() && !query_.orderBy() && !query_.limitBy()) + return true; + return false; + }; - const auto & ast = query.select->as(); - size_t num_selects = ast.list_of_selects->children.size(); - bool is_trivial_query = true; - for (size_t query_num = 0; query_num < num_selects; ++query_num) - { - const auto & query_select = ast.list_of_selects->children.at(query_num)->as(); - if (!is_trivial_select(query_select)) + const auto & ast = query.select->as(); + size_t num_selects = ast.list_of_selects->children.size(); + bool is_trivial_query = true; + for (size_t query_num = 0; query_num < num_selects; ++query_num) { - is_trivial_query = false; - break; + const auto & query_select = ast.list_of_selects->children.at(query_num)->as(); + if (!is_trivial_select(query_select)) + { + is_trivial_query = false; + break; + } } - } - if (is_trivial_query) - { - auto new_context = context; - UInt64 max_threads = settings.max_insert_threads; - UInt64 min_insert_block_size_rows = settings.min_insert_block_size_rows; - new_context.setSetting("max_threads", std::max(1, max_threads)); - if (min_insert_block_size_rows) - new_context.setSetting("max_block_size", min_insert_block_size_rows); + if (is_trivial_query) + { + auto new_context = context; + UInt64 max_threads = settings.max_insert_threads; + UInt64 min_insert_block_size_rows = settings.min_insert_block_size_rows; + new_context.setSetting("max_threads", std::max(1, max_threads)); + if (min_insert_block_size_rows) + new_context.setSetting("max_block_size", min_insert_block_size_rows); - /// Passing 1 as subquery_depth will disable limiting size of intermediate result. - InterpreterSelectWithUnionQuery interpreter_select{ - query.select, new_context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - res = interpreter_select.execute(); + /// Passing 1 as subquery_depth will disable limiting size of intermediate result. + InterpreterSelectWithUnionQuery interpreter_select{ + query.select, new_context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + res = interpreter_select.execute(); + } + else + { + InterpreterSelectWithUnionQuery interpreter_select{ + query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + res = interpreter_select.execute(); + } } else { diff --git a/tests/queries/0_stateless/00340_squashing_insert_select.sql b/tests/queries/0_stateless/00340_squashing_insert_select.sql index 65b82f4b4f9..102eb061bad 100644 --- a/tests/queries/0_stateless/00340_squashing_insert_select.sql +++ b/tests/queries/0_stateless/00340_squashing_insert_select.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS numbers_squashed; CREATE TABLE numbers_squashed AS system.numbers ENGINE = StripeLog; +SET optimize_trivial_insert_select = 'false'; SET max_block_size = 10000; SET min_insert_block_size_rows = 1000000; diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh index c9a4de647b9..416bbf82620 100755 --- a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh +++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh @@ -78,7 +78,7 @@ insert into data_01278 select from numbers(100000); -- { serverError 241; } EOL } | { - execute --max_memory_usage=$TEST_01278_MEMORY "$@" + execute --max_memory_usage=$TEST_01278_MEMORY --optimize_trivial_insert_select='false' "$@" } echo 'select count() from out_01278' | execute } From 88e48168d91c79f6e3842345dcfa54ab7f44909d Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 2 Aug 2020 13:35:58 +0800 Subject: [PATCH 07/89] fix --- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterInsertQuery.cpp | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 0fb017122f5..57413e69cdb 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -395,7 +395,7 @@ class IColumn; \ M(Bool, force_optimize_skip_unused_shards_no_nested, false, "Obsolete setting, does nothing. Will be removed after 2020-12-01. Use force_optimize_skip_unused_shards_nesting instead.", 0) \ M(Bool, experimental_use_processors, true, "Obsolete setting, does nothing. Will be removed after 2020-11-29.", 0) \ - M(SettingBool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ + M(Bool, optimize_trivial_insert_select, true, "Optimize trivial 'INSERT INTO table SELECT ... FROM TABLES' query", 0) \ #define FORMAT_FACTORY_SETTINGS(M) \ M(Char, format_csv_delimiter, ',', "The character to be considered as a delimiter in CSV data. If setting with a string, a string has to have a length of 1.", 0) \ diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 4ec1e2bd50f..6cd706b4eb9 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -202,9 +202,10 @@ BlockIO InterpreterInsertQuery::execute() if (query.select) { auto optimize_trivial_insert_select = settings.optimize_trivial_insert_select; - if(optimize_trivial_insert_select) - { - auto is_trivial_select = [](const auto query_) { + if (optimize_trivial_insert_select) + { + auto is_trivial_select = [](const auto query_) + { if (query_.tables()) { const auto & tables_in_select_query = query_.tables()->template as(); From a827462f55f17e7eae0b069a9b40c9060cdc3d67 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 2 Aug 2020 20:30:41 +0800 Subject: [PATCH 08/89] fix --- src/Interpreters/InterpreterInsertQuery.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 6cd706b4eb9..40549fca4b1 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -204,7 +204,7 @@ BlockIO InterpreterInsertQuery::execute() auto optimize_trivial_insert_select = settings.optimize_trivial_insert_select; if (optimize_trivial_insert_select) { - auto is_trivial_select = [](const auto query_) + auto is_trivial_select = [](const auto && query_) { if (query_.tables()) { @@ -219,10 +219,8 @@ BlockIO InterpreterInsertQuery::execute() } } } - if (!query_.distinct && !query_.limit_with_ties && !query_.prewhere() && !query_.where() && !query_.groupBy() - && !query_.having() && !query_.orderBy() && !query_.limitBy()) - return true; - return false; + return (!query_.distinct && !query_.limit_with_ties && !query_.prewhere() && !query_.where() && !query_.groupBy() + && !query_.having() && !query_.orderBy() && !query_.limitBy()); }; const auto & ast = query.select->as(); @@ -231,7 +229,7 @@ BlockIO InterpreterInsertQuery::execute() for (size_t query_num = 0; query_num < num_selects; ++query_num) { const auto & query_select = ast.list_of_selects->children.at(query_num)->as(); - if (!is_trivial_select(query_select)) + if (!is_trivial_select(std::move(query_select))) { is_trivial_query = false; break; From c6780148ec37188f4fe6106b313de275160c4a37 Mon Sep 17 00:00:00 2001 From: Javi Santana Date: Fri, 14 Aug 2020 14:29:16 +0200 Subject: [PATCH 09/89] return 200 when replicas status is ok and verbose = 1 --- src/Server/ReplicasStatusHandler.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 5ead756ee1e..ef5be2e39e2 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -75,16 +75,17 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request const auto & config = context.getConfigRef(); setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - if (ok && !verbose) - { + if (!ok) { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_SERVICE_UNAVAILABLE); + verbose = true; + } + + if (verbose) + response.send() << message.rdbuf(); + else { const char * data = "Ok.\n"; response.sendBuffer(data, strlen(data)); } - else - { - response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_SERVICE_UNAVAILABLE); - response.send() << message.rdbuf(); - } } catch (...) { From cb80eba410002c8c58f5c2025bf882f4d8d63939 Mon Sep 17 00:00:00 2001 From: Javi Santana Date: Fri, 14 Aug 2020 15:16:07 +0200 Subject: [PATCH 10/89] added tests --- tests/integration/test_http_handlers_config/test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index b15cd1fdb89..9925b8c5267 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -121,8 +121,11 @@ def test_defaults_http_handlers(): assert 200 == cluster.instance.http_request('ping', method='GET').status_code assert 'Ok.\n' == cluster.instance.http_request('ping', method='GET').content - assert 200 == cluster.instance.http_request('replicas_status', method='GET').status_code - assert 'Ok.\n' == cluster.instance.http_request('replicas_status', method='GET').content + assert 200 == cluster.instance.http_request('replicas_status', method='get').status_code + assert 'ok.\n' == cluster.instance.http_request('replicas_status', method='get').content + + assert 200 == cluster.instance.http_request('replicas_status?verbose=1', method='get').status_code + assert 'ok.\n' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content From f04f984fb23cc67d141aaf0f8cf6729c6ed210d1 Mon Sep 17 00:00:00 2001 From: Javi Santana Date: Fri, 14 Aug 2020 15:31:54 +0200 Subject: [PATCH 11/89] change response content for test --- tests/integration/test_http_handlers_config/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index 9925b8c5267..df93ed3b877 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -125,7 +125,7 @@ def test_defaults_http_handlers(): assert 'ok.\n' == cluster.instance.http_request('replicas_status', method='get').content assert 200 == cluster.instance.http_request('replicas_status?verbose=1', method='get').status_code - assert 'ok.\n' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content + assert '\n' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content From ce64a73342ff90f2f2562d5723ae2af413011e60 Mon Sep 17 00:00:00 2001 From: Javi Santana Date: Mon, 17 Aug 2020 09:46:38 +0200 Subject: [PATCH 12/89] fixed tests and style --- src/Server/ReplicasStatusHandler.cpp | 6 ++++-- tests/integration/test_http_handlers_config/test.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index ef5be2e39e2..de68635d26e 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -75,14 +75,16 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request const auto & config = context.getConfigRef(); setResponseDefaultHeaders(response, config.getUInt("keep_alive_timeout", 10)); - if (!ok) { + if (!ok) + { response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_SERVICE_UNAVAILABLE); verbose = true; } if (verbose) response.send() << message.rdbuf(); - else { + else + { const char * data = "Ok.\n"; response.sendBuffer(data, strlen(data)); } diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index df93ed3b877..c138c725ffb 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -122,7 +122,7 @@ def test_defaults_http_handlers(): assert 'Ok.\n' == cluster.instance.http_request('ping', method='GET').content assert 200 == cluster.instance.http_request('replicas_status', method='get').status_code - assert 'ok.\n' == cluster.instance.http_request('replicas_status', method='get').content + assert 'Ok.\n' == cluster.instance.http_request('replicas_status', method='get').content assert 200 == cluster.instance.http_request('replicas_status?verbose=1', method='get').status_code assert '\n' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content From ca953418239c090b13dce076b220300e017ebe31 Mon Sep 17 00:00:00 2001 From: Olga Revyakina Date: Tue, 18 Aug 2020 09:32:40 +0300 Subject: [PATCH 13/89] English docs for the lock_acquire_timeout setting. --- docs/en/operations/settings/settings.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index c844a88613d..665d944eaed 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1864,4 +1864,17 @@ Default value: `0`. - [Synchronicity of ALTER Queries](../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) - [Mutations](../../sql-reference/statements/alter/index.md#mutations) +## lock_acquire_timeout {#lock_acquire_timeout} + +Defines how many seconds locking request waits before failing. + +Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When timeout expires and locking request fails, the Clickhouse server throws an exeption "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. + +Possible values: + +- Positive integer. +- 0 — No locking timeout. + +Default value: `120`. + [Original article](https://clickhouse.tech/docs/en/operations/settings/settings/) From cd22abbfe3e886c1ab35264c82760168af021741 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Tue, 18 Aug 2020 14:59:57 +0300 Subject: [PATCH 14/89] Expect empty content on verbose response --- tests/integration/test_http_handlers_config/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_http_handlers_config/test.py b/tests/integration/test_http_handlers_config/test.py index c138c725ffb..6b0ced11204 100644 --- a/tests/integration/test_http_handlers_config/test.py +++ b/tests/integration/test_http_handlers_config/test.py @@ -125,7 +125,7 @@ def test_defaults_http_handlers(): assert 'Ok.\n' == cluster.instance.http_request('replicas_status', method='get').content assert 200 == cluster.instance.http_request('replicas_status?verbose=1', method='get').status_code - assert '\n' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content + assert '' == cluster.instance.http_request('replicas_status?verbose=1', method='get').content assert 200 == cluster.instance.http_request('?query=SELECT+1', method='GET').status_code assert '1\n' == cluster.instance.http_request('?query=SELECT+1', method='GET').content From 4bbcec3d33cf2b5dcc9a54c7bac8e5c9294f5422 Mon Sep 17 00:00:00 2001 From: hcz Date: Wed, 19 Aug 2020 14:44:11 +0800 Subject: [PATCH 15/89] Add defaultValueOfTypeName --- src/Functions/defaultValueOfTypeName.cpp | 69 +++++++++++++++++++ .../registerFunctionsMiscellaneous.cpp | 2 + 2 files changed, 71 insertions(+) create mode 100644 src/Functions/defaultValueOfTypeName.cpp diff --git a/src/Functions/defaultValueOfTypeName.cpp b/src/Functions/defaultValueOfTypeName.cpp new file mode 100644 index 00000000000..5b57642aa0e --- /dev/null +++ b/src/Functions/defaultValueOfTypeName.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + +/// Returns global default value for type name (example: 0 for numeric types, '' for String). +class FunctionDefaultValueOfTypeName : public IFunction +{ +public: + static constexpr auto name = "defaultValueOfTypeName"; + static FunctionPtr create(const Context &) + { + return std::make_shared(); + } + + String getName() const override + { + return name; + } + + bool useDefaultImplementationForNulls() const override { return false; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } + + size_t getNumberOfArguments() const override + { + return 1; + } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + auto col_type_const = typeid_cast(arguments.front().column.get()); + if (!col_type_const || !isString(arguments.front().type)) + throw Exception("The argument of function " + getName() + " must be a constant string describing type. " + "Instead there is a column with the following structure: " + arguments.front().column->dumpStructure(), + ErrorCodes::ILLEGAL_COLUMN); + + return DataTypeFactory::instance().get(col_type_const->getValue()); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + { + const IDataType & type = *block.getByPosition(arguments[0]).type; + block.getByPosition(result).column = type.createColumnConst(input_rows_count, type.getDefault()); + } + + ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const Block & block, const ColumnNumbers & arguments) const override + { + const IDataType & type = *block.getByPosition(arguments[0]).type; + return type.createColumnConst(1, type.getDefault()); + } +}; + + +void registerFunctionDefaultValueOfTypeName(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 11d3e129c28..25e64eba999 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -18,6 +18,7 @@ void registerFunctionBlockSerializedSize(FunctionFactory &); void registerFunctionToColumnTypeName(FunctionFactory &); void registerFunctionDumpColumnStructure(FunctionFactory &); void registerFunctionDefaultValueOfArgumentType(FunctionFactory &); +void registerFunctionDefaultValueOfTypeName(FunctionFactory &); void registerFunctionBlockSize(FunctionFactory &); void registerFunctionBlockNumber(FunctionFactory &); void registerFunctionRowNumberInBlock(FunctionFactory &); @@ -79,6 +80,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionToColumnTypeName(factory); registerFunctionDumpColumnStructure(factory); registerFunctionDefaultValueOfArgumentType(factory); + registerFunctionDefaultValueOfTypeName(factory); registerFunctionBlockSize(factory); registerFunctionBlockNumber(factory); registerFunctionRowNumberInBlock(factory); From be2ffe07eef5f20f100ceee1f9afff0ad8d5db5f Mon Sep 17 00:00:00 2001 From: hcz Date: Wed, 19 Aug 2020 15:52:33 +0800 Subject: [PATCH 16/89] Add docs and tests --- .../functions/other-functions.md | 46 ++++++++++++++++++- ...01425_default_value_of_type_name.reference | 1 + .../01425_default_value_of_type_name.sql | 8 ++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/01425_default_value_of_type_name.reference create mode 100644 tests/queries/0_stateless/01425_default_value_of_type_name.sql diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md index 798f0eb3739..05247b6db7d 100644 --- a/docs/en/sql-reference/functions/other-functions.md +++ b/docs/en/sql-reference/functions/other-functions.md @@ -917,6 +917,48 @@ SELECT defaultValueOfArgumentType( CAST(1 AS Nullable(Int8) ) ) └───────────────────────────────────────────────────────┘ ``` +## defaultValueOfTypeName {#defaultvalueoftypename} + +Outputs the default value for given type name. + +Does not include default values for custom columns set by the user. + +``` sql +defaultValueOfTypeName(type) +``` + +**Parameters:** + +- `type` — A string representing a type name. + +**Returned values** + +- `0` for numbers. +- Empty string for strings. +- `ᴺᵁᴸᴸ` for [Nullable](../../sql-reference/data-types/nullable.md). + +**Example** + +``` sql +SELECT defaultValueOfTypeName('Int8') +``` + +``` text +┌─defaultValueOfTypeName('Int8')─┐ +│ 0 │ +└────────────────────────────────┘ +``` + +``` sql +SELECT defaultValueOfTypeName('Nullable(Int8)') +``` + +``` text +┌─defaultValueOfTypeName('Nullable(Int8)')─┐ +│ ᴺᵁᴸᴸ │ +└──────────────────────────────────────────┘ +``` + ## replicate {#other-functions-replicate} Creates an array with a single value. @@ -1413,13 +1455,13 @@ Type: [String](../../sql-reference/data-types/string.md). Query: -```sql +```sql SELECT randomStringUTF8(13) ``` Result: -```text +```text ┌─randomStringUTF8(13)─┐ │ 𘤗𙉝д兠庇󡅴󱱎󦐪􂕌𔊹𓰛 │ └──────────────────────┘ diff --git a/tests/queries/0_stateless/01425_default_value_of_type_name.reference b/tests/queries/0_stateless/01425_default_value_of_type_name.reference new file mode 100644 index 00000000000..659c7ddf960 --- /dev/null +++ b/tests/queries/0_stateless/01425_default_value_of_type_name.reference @@ -0,0 +1 @@ +0 00000000-0000-0000-0000-000000000000 0.0.0.0 :: 0.000 ('1970-01-01','1970-01-01 08:00:00',[],NULL) diff --git a/tests/queries/0_stateless/01425_default_value_of_type_name.sql b/tests/queries/0_stateless/01425_default_value_of_type_name.sql new file mode 100644 index 00000000000..7072d705347 --- /dev/null +++ b/tests/queries/0_stateless/01425_default_value_of_type_name.sql @@ -0,0 +1,8 @@ +SELECT + defaultValueOfTypeName('Int64'), + defaultValueOfTypeName('String'), + defaultValueOfTypeName('UUID'), + defaultValueOfTypeName('IPv4'), + defaultValueOfTypeName('IPv6'), + defaultValueOfTypeName('Decimal128(3)'), + defaultValueOfTypeName('Tuple(Date, DateTime, Array(Array(String)), Nullable(UInt8))'); From c0f8fb10bcf02532c723919a9fe563356e578e29 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Wed, 19 Aug 2020 17:20:59 +0300 Subject: [PATCH 17/89] Update docs/en/operations/settings/settings.md Co-authored-by: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> --- docs/en/operations/settings/settings.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 665d944eaed..2e2483725a3 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1868,7 +1868,7 @@ Default value: `0`. Defines how many seconds locking request waits before failing. -Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When timeout expires and locking request fails, the Clickhouse server throws an exeption "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. +Locking timeout is used to protect from deadlocks while executing read/write operations with tables. When timeout expires and locking request fails, the ClickHouse server throws an exeption "Locking attempt timed out! Possible deadlock avoided. Client should retry." with error code `DEADLOCK_AVOIDED`. Possible values: From 4d093e8e26cb7eb58c073986a69ef7eb1f6b884f Mon Sep 17 00:00:00 2001 From: hcz Date: Thu, 20 Aug 2020 10:50:34 +0800 Subject: [PATCH 18/89] Fix type error --- src/Functions/defaultValueOfTypeName.cpp | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Functions/defaultValueOfTypeName.cpp b/src/Functions/defaultValueOfTypeName.cpp index 5b57642aa0e..15ef11da64e 100644 --- a/src/Functions/defaultValueOfTypeName.cpp +++ b/src/Functions/defaultValueOfTypeName.cpp @@ -47,17 +47,11 @@ public: return DataTypeFactory::instance().get(col_type_const->getValue()); } - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) const override + void executeImpl(Block & block, const ColumnNumbers &, size_t result, size_t input_rows_count) const override { - const IDataType & type = *block.getByPosition(arguments[0]).type; + const IDataType & type = *block.getByPosition(result).type; block.getByPosition(result).column = type.createColumnConst(input_rows_count, type.getDefault()); } - - ColumnPtr getResultIfAlwaysReturnsConstantAndHasArguments(const Block & block, const ColumnNumbers & arguments) const override - { - const IDataType & type = *block.getByPosition(arguments[0]).type; - return type.createColumnConst(1, type.getDefault()); - } }; From cc328532af5c9bcfb8c180a4496573be57565146 Mon Sep 17 00:00:00 2001 From: hcz Date: Thu, 20 Aug 2020 12:07:01 +0800 Subject: [PATCH 19/89] Fix timezone --- .../0_stateless/01425_default_value_of_type_name.reference | 2 +- tests/queries/0_stateless/01425_default_value_of_type_name.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01425_default_value_of_type_name.reference b/tests/queries/0_stateless/01425_default_value_of_type_name.reference index 659c7ddf960..c7dfcb7607d 100644 --- a/tests/queries/0_stateless/01425_default_value_of_type_name.reference +++ b/tests/queries/0_stateless/01425_default_value_of_type_name.reference @@ -1 +1 @@ -0 00000000-0000-0000-0000-000000000000 0.0.0.0 :: 0.000 ('1970-01-01','1970-01-01 08:00:00',[],NULL) +0 00000000-0000-0000-0000-000000000000 0.0.0.0 :: 0.000 ('1970-01-01','1970-01-01 00:00:00',[],NULL) diff --git a/tests/queries/0_stateless/01425_default_value_of_type_name.sql b/tests/queries/0_stateless/01425_default_value_of_type_name.sql index 7072d705347..c783d005a0e 100644 --- a/tests/queries/0_stateless/01425_default_value_of_type_name.sql +++ b/tests/queries/0_stateless/01425_default_value_of_type_name.sql @@ -5,4 +5,4 @@ SELECT defaultValueOfTypeName('IPv4'), defaultValueOfTypeName('IPv6'), defaultValueOfTypeName('Decimal128(3)'), - defaultValueOfTypeName('Tuple(Date, DateTime, Array(Array(String)), Nullable(UInt8))'); + defaultValueOfTypeName('Tuple(Date, DateTime(\'UTC\'), Array(Array(String)), Nullable(UInt8))'); From 0a7fb4eb8054c442025fb4d3a78e278b133efec3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Aug 2020 16:27:47 +0300 Subject: [PATCH 20/89] Remove strange files --- src/Interpreters/Common/config.h | 15 ------- src/Interpreters/Common/config_version.h | 50 --------------------- src/Interpreters/Core/include/config_core.h | 11 ----- 3 files changed, 76 deletions(-) delete mode 100644 src/Interpreters/Common/config.h delete mode 100644 src/Interpreters/Common/config_version.h delete mode 100644 src/Interpreters/Core/include/config_core.h diff --git a/src/Interpreters/Common/config.h b/src/Interpreters/Common/config.h deleted file mode 100644 index ae6376b12c2..00000000000 --- a/src/Interpreters/Common/config.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -// .h autogenerated by cmake! - -#define USE_RE2_ST 0 -#define USE_SSL 0 -#define USE_HDFS 0 -#define USE_AWS_S3 0 -#define USE_BROTLI 0 -#define USE_UNWIND 0 -#define USE_OPENCL 0 -#define USE_CASSANDRA 0 -#define USE_SENTRY 0 -#define USE_GRPC 0 -#define CLICKHOUSE_SPLIT_BINARY 0 diff --git a/src/Interpreters/Common/config_version.h b/src/Interpreters/Common/config_version.h deleted file mode 100644 index 7948a6c853c..00000000000 --- a/src/Interpreters/Common/config_version.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -// .h autogenerated by cmake! - -#define USE_DBMS_TCP_PROTOCOL_VERSION 0 - -#if USE_DBMS_TCP_PROTOCOL_VERSION - #include "Core/Defines.h" - #ifndef VERSION_REVISION - #define VERSION_REVISION DBMS_TCP_PROTOCOL_VERSION - #endif -#else -/* #undef VERSION_REVISION */ -#endif - - -#define VERSION_NAME "Project" -#define DBMS_NAME VERSION_NAME -/* #undef VERSION_MAJOR */ -/* #undef VERSION_MINOR */ -/* #undef VERSION_PATCH */ -/* #undef VERSION_STRING */ -#define VERSION_STRING_SHORT "." -/* #undef VERSION_OFFICIAL */ -#define VERSION_FULL "Project " -/* #undef VERSION_DESCRIBE */ -/* #undef VERSION_GITHASH */ -#define VERSION_INTEGER ERROR - -#if defined(VERSION_MAJOR) -#define DBMS_VERSION_MAJOR VERSION_MAJOR -#else -#define DBMS_VERSION_MAJOR 0 -#endif - -#if defined(VERSION_MINOR) -#define DBMS_VERSION_MINOR VERSION_MINOR -#else -#define DBMS_VERSION_MINOR 0 -#endif - -#if defined(VERSION_PATCH) -#define DBMS_VERSION_PATCH VERSION_PATCH -#else -#define DBMS_VERSION_PATCH 0 -#endif - -#if !defined(VERSION_OFFICIAL) -# define VERSION_OFFICIAL "" -#endif diff --git a/src/Interpreters/Core/include/config_core.h b/src/Interpreters/Core/include/config_core.h deleted file mode 100644 index a2e610aa053..00000000000 --- a/src/Interpreters/Core/include/config_core.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -// .h autogenerated by cmake! - -#define USE_ICU 0 -#define USE_MYSQL 0 -#define USE_RDKAFKA 0 -#define USE_EMBEDDED_COMPILER 0 -#define USE_INTERNAL_LLVM_LIBRARY 0 -#define USE_SSL 0 -#define USE_OPENCL 0 From 1fa4978685ffd93f34f0c5ca74e9b0abeaf7f22c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Aug 2020 17:29:31 +0300 Subject: [PATCH 21/89] Make the code more clear; add comments --- src/Interpreters/InterpreterInsertQuery.cpp | 116 +++++++++++--------- 1 file changed, 65 insertions(+), 51 deletions(-) diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index 3f979abd4d8..1557f065b32 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -93,7 +93,8 @@ Block InterpreterInsertQuery::getSampleBlock( /// The table does not have a column with that name if (!table_sample.has(current_name)) - throw Exception("No such column " + current_name + " in table " + query.table_id.getNameForLogs(), ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); + throw Exception("No such column " + current_name + " in table " + query.table_id.getNameForLogs(), + ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); if (!allow_materialized && !table_sample_non_materialized.has(current_name)) throw Exception("Cannot insert column " + current_name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); @@ -106,6 +107,40 @@ Block InterpreterInsertQuery::getSampleBlock( } +/** A query that just reads all data without any complex computations or filetering. + * If we just pipe the result to INSERT, we don't have to use too many threads for read. + */ +static bool isTrivialSelect(const ASTSelectQuery & select_query) +{ + const auto & tables = select_query.tables(); + + if (!tables) + return false; + + const auto & tables_in_select_query = tables->as(); + + if (tables_in_select_query.children.size() != 1) + return false; + + const auto & child = tables_in_select_query.children.front(); + const auto & table_element = child->as(); + const auto & table_expr = table_element.table_expression->as(); + + if (table_expr.subquery) + return false; + + /// Note: how to write it in more generic way? + return (!select_query.distinct + && !select_query.limit_with_ties + && !select_query.prewhere() + && !select_query.where() + && !select_query.groupBy() + && !select_query.having() + && !select_query.orderBy() + && !select_query.limitBy()); +}; + + BlockIO InterpreterInsertQuery::execute() { const Settings & settings = context.getSettingsRef(); @@ -201,64 +236,43 @@ BlockIO InterpreterInsertQuery::execute() size_t out_streams_size = 1; if (query.select) { - auto optimize_trivial_insert_select = settings.optimize_trivial_insert_select; - if (optimize_trivial_insert_select) + bool is_trivial_insert_select = false; + + if (settings.optimize_trivial_insert_select) { - auto is_trivial_select = [](const auto && query_) - { - if (query_.tables()) - { - const auto & tables_in_select_query = query_.tables()->template as(); - for (const auto & child : tables_in_select_query.children) - { - const auto & table_element = child->template as(); - const auto & table_expr = table_element.table_expression->template as(); - if (table_expr.subquery) - { - return false; - } - } - } - return (!query_.distinct && !query_.limit_with_ties && !query_.prewhere() && !query_.where() && !query_.groupBy() - && !query_.having() && !query_.orderBy() && !query_.limitBy()); - }; + const auto & selects = query.select->as().list_of_selects->children; - const auto & ast = query.select->as(); - size_t num_selects = ast.list_of_selects->children.size(); - bool is_trivial_query = true; - for (size_t query_num = 0; query_num < num_selects; ++query_num) + is_trivial_insert_select = std::all_of(selects.begin(), selects.end(), [](const ASTPtr & select) { - const auto & query_select = ast.list_of_selects->children.at(query_num)->as(); - if (!is_trivial_select(std::move(query_select))) - { - is_trivial_query = false; - break; - } - } + return isTrivialSelect(select->as()); + }); + } - if (is_trivial_query) - { - auto new_context = context; - UInt64 max_threads = settings.max_insert_threads; - UInt64 min_insert_block_size_rows = settings.min_insert_block_size_rows; - new_context.setSetting("max_threads", std::max(1, max_threads)); - if (min_insert_block_size_rows) - new_context.setSetting("max_block_size", min_insert_block_size_rows); + if (is_trivial_insert_select) + { + /** When doing trivial INSERT INTO ... SELECT ... FROM table, + * don't need to process SELECT with more than max_insert_threads + * and it's reasonable to set block size for SELECT to the desired block size for INSERT + * to avoid unnecessary squashing. + */ - /// Passing 1 as subquery_depth will disable limiting size of intermediate result. - InterpreterSelectWithUnionQuery interpreter_select{ - query.select, new_context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - res = interpreter_select.execute(); - } - else - { - InterpreterSelectWithUnionQuery interpreter_select{ - query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; - res = interpreter_select.execute(); - } + Settings new_settings = context.getSettings(); + + new_settings.max_threads = std::max(1, settings.max_insert_threads); + + if (settings.min_insert_block_size_rows) + new_settings.max_block_size = settings.min_insert_block_size_rows; + + Context new_context = context; + new_context.setSettings(new_settings); + + InterpreterSelectWithUnionQuery interpreter_select{ + query.select, new_context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; + res = interpreter_select.execute(); } else { + /// Passing 1 as subquery_depth will disable limiting size of intermediate result. InterpreterSelectWithUnionQuery interpreter_select{ query.select, context, SelectQueryOptions(QueryProcessingStage::Complete, 1)}; res = interpreter_select.execute(); From e1ccebe8f36bbb4c16952fe80f07271b99618503 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Aug 2020 17:42:12 +0300 Subject: [PATCH 22/89] Minor modification --- src/DataStreams/PushingToViewsBlockOutputStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 6e29b4fede8..4f6950f7717 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -57,9 +57,9 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( insert_context->setSetting("insert_deduplicate", false); // Separate min_insert_block_size_rows/min_insert_block_size_bytes for children - if (insert_settings.min_insert_block_size_rows_for_materialized_views.changed) + if (insert_settings.min_insert_block_size_rows_for_materialized_views) insert_context->setSetting("min_insert_block_size_rows", insert_settings.min_insert_block_size_rows_for_materialized_views.value); - if (insert_settings.min_insert_block_size_bytes_for_materialized_views.changed) + if (insert_settings.min_insert_block_size_bytes_for_materialized_views) insert_context->setSetting("min_insert_block_size_bytes", insert_settings.min_insert_block_size_bytes_for_materialized_views.value); } From 612382b74e088880dd092da945117c1fd5ca8263 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Aug 2020 17:45:00 +0300 Subject: [PATCH 23/89] Minor modification --- src/DataStreams/PushingToViewsBlockOutputStream.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 4f6950f7717..12ee22b20bb 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -14,6 +14,7 @@ #include #include + namespace DB { @@ -33,7 +34,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( * but it's clear that here is not the best place for this functionality. */ addTableLock( - storage->lockForShare(context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout)); + storage->lockForShare(context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout)); /// If the "root" table deduplicates blocks, there are no need to make deduplication for children /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks @@ -74,7 +75,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( if (auto * materialized_view = dynamic_cast(dependent_table.get())) { addTableLock( - materialized_view->lockForShare(context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout)); + materialized_view->lockForShare(context.getInitialQueryId(), context.getSettingsRef().lock_acquire_timeout)); StoragePtr inner_table = materialized_view->getTargetTable(); auto inner_table_id = inner_table->getStorageID(); @@ -86,15 +87,17 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( /// Get list of columns we get from select query. auto header = InterpreterSelectQuery(query, *select_context, SelectQueryOptions().analyze()) - .getSampleBlock(); + .getSampleBlock(); /// Insert only columns returned by select. auto list = std::make_shared(); const auto & inner_table_columns = inner_metadata_snapshot->getColumns(); - for (auto & column : header) + for (const auto & column : header) + { /// But skip columns which storage doesn't have. if (inner_table_columns.hasPhysical(column.name)) list->children.emplace_back(std::make_shared(column.name)); + } insert->columns = std::move(list); From ec24bfe1286fd8bd84d24a824cce6227d1fc321c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Aug 2020 22:54:06 +0300 Subject: [PATCH 24/89] Added a test that proves feature exists --- .../01455_optimize_trivial_insert_select.reference | 2 ++ .../0_stateless/01455_optimize_trivial_insert_select.sql | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 tests/queries/0_stateless/01455_optimize_trivial_insert_select.reference create mode 100644 tests/queries/0_stateless/01455_optimize_trivial_insert_select.sql diff --git a/tests/queries/0_stateless/01455_optimize_trivial_insert_select.reference b/tests/queries/0_stateless/01455_optimize_trivial_insert_select.reference new file mode 100644 index 00000000000..ebe978eef95 --- /dev/null +++ b/tests/queries/0_stateless/01455_optimize_trivial_insert_select.reference @@ -0,0 +1,2 @@ +1000000 0 +1000000 1 diff --git a/tests/queries/0_stateless/01455_optimize_trivial_insert_select.sql b/tests/queries/0_stateless/01455_optimize_trivial_insert_select.sql new file mode 100644 index 00000000000..de470fe6a57 --- /dev/null +++ b/tests/queries/0_stateless/01455_optimize_trivial_insert_select.sql @@ -0,0 +1,7 @@ +SET max_insert_threads = 1, max_threads = 100, min_insert_block_size_rows = 1048576, max_block_size = 65536; +CREATE TEMPORARY TABLE t (x UInt64); +-- For trivial INSERT SELECT, max_threads is lowered to max_insert_threads and max_block_size is changed to min_insert_block_size_rows. +INSERT INTO t SELECT * FROM numbers_mt(1000000); +SET max_threads = 1; +-- If data was inserted by more threads, we will probably see data out of order. +SELECT DISTINCT blockSize(), runningDifference(x) FROM t; From 99009a3468169414e6ac166790a6777ae3a07707 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 24 Aug 2020 23:02:29 +0300 Subject: [PATCH 25/89] utils/generate-ya-make/generate-ya-make.sh --- src/Functions/ya.make | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 3a0c0f36bb8..d337a574d4f 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -137,6 +137,7 @@ SRCS( currentUser.cpp dateDiff.cpp defaultValueOfArgumentType.cpp + defaultValueOfTypeName.cpp demange.cpp divide.cpp dumpColumnStructure.cpp From b0908144b3a49650e5c488bf8dd2c5461c84d9df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 25 Aug 2020 01:20:43 +0300 Subject: [PATCH 26/89] Fix clang-tidy --- src/Functions/defaultValueOfTypeName.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/defaultValueOfTypeName.cpp b/src/Functions/defaultValueOfTypeName.cpp index 15ef11da64e..42db312dee1 100644 --- a/src/Functions/defaultValueOfTypeName.cpp +++ b/src/Functions/defaultValueOfTypeName.cpp @@ -38,7 +38,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - auto col_type_const = typeid_cast(arguments.front().column.get()); + const ColumnConst * col_type_const = typeid_cast(arguments.front().column.get()); if (!col_type_const || !isString(arguments.front().type)) throw Exception("The argument of function " + getName() + " must be a constant string describing type. " "Instead there is a column with the following structure: " + arguments.front().column->dumpStructure(), From 3560736a226d0c92ce97366a57e506f8b719af6e Mon Sep 17 00:00:00 2001 From: Y Lu <1838324+drag0nglass@users.noreply.github.com> Date: Tue, 25 Aug 2020 11:39:38 +0800 Subject: [PATCH 27/89] Update json-import.md The format spec shall be 'JSONEachRow' rather than '20JSONEachRow' --- docs/en/faq/integration/json-import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/faq/integration/json-import.md b/docs/en/faq/integration/json-import.md index ef939b2c0d6..067b407a079 100644 --- a/docs/en/faq/integration/json-import.md +++ b/docs/en/faq/integration/json-import.md @@ -19,7 +19,7 @@ $ echo '{"foo":"bar"}' | curl 'http://localhost:8123/?query=INSERT%20INTO%20test Using [CLI interface](../../interfaces/cli.md): ``` bash -$ echo '{"foo":"bar"}' | clickhouse-client ---query="INSERT INTO test FORMAT 20JSONEachRow" +$ echo '{"foo":"bar"}' | clickhouse-client ---query="INSERT INTO test FORMAT JSONEachRow" ``` Instead of inserting data manually, you might consider to use one of [client libraries](../../interfaces/index.md) instead. From 8d2122c62c4d00a99067202e366dd53d02f01571 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 25 Aug 2020 15:46:31 +0300 Subject: [PATCH 28/89] add test --- ...t_optimizations_over_distributed.reference | 12 ++++++++ ...456_ast_optimizations_over_distributed.sql | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/queries/0_stateless/01456_ast_optimizations_over_distributed.reference create mode 100644 tests/queries/0_stateless/01456_ast_optimizations_over_distributed.sql diff --git a/tests/queries/0_stateless/01456_ast_optimizations_over_distributed.reference b/tests/queries/0_stateless/01456_ast_optimizations_over_distributed.reference new file mode 100644 index 00000000000..8c76b239991 --- /dev/null +++ b/tests/queries/0_stateless/01456_ast_optimizations_over_distributed.reference @@ -0,0 +1,12 @@ +1 +1 +1 +other +google +1 +1 +2 +other +other +google +google diff --git a/tests/queries/0_stateless/01456_ast_optimizations_over_distributed.sql b/tests/queries/0_stateless/01456_ast_optimizations_over_distributed.sql new file mode 100644 index 00000000000..1470d8c8d4f --- /dev/null +++ b/tests/queries/0_stateless/01456_ast_optimizations_over_distributed.sql @@ -0,0 +1,28 @@ +SET optimize_move_functions_out_of_any = 1; +SET optimize_injective_functions_inside_uniq = 1; +SET optimize_arithmetic_operations_in_aggregate_functions = 1; +SET optimize_if_transform_strings_to_enum = 1; + +SELECT any(number + 1) FROM numbers(1); +SELECT uniq(bitNot(number)) FROM numbers(1); +SELECT sum(number + 1) FROM numbers(1); +SELECT transform(number, [1, 2], ['google', 'yandex'], 'other') FROM numbers(1); +SELECT number > 0 ? 'yandex' : 'google' FROM numbers(1); + + +DROP TABLE IF EXISTS local_table; +DROP TABLE IF EXISTS dist; + +CREATE TABLE local_table (number UInt64) ENGINE = Memory; +CREATE TABLE dist AS local_table ENGINE = Distributed(test_cluster_two_shards_localhost, currentDatabase(), local_table); + +INSERT INTO local_table SELECT number FROM numbers(1); + +SELECT any(number + 1) FROM dist; +SELECT uniq(bitNot(number)) FROM dist; +SELECT sum(number + 1) FROM dist; +SELECT transform(number, [1, 2], ['google', 'yandex'], 'other') FROM dist; +SELECT number > 0 ? 'yandex' : 'google' FROM dist; + +DROP TABLE local_table; +DROP TABLE dist; From 82c8f5671314c7d2a89eff59b135bf62f10d9262 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 25 Aug 2020 16:27:47 +0300 Subject: [PATCH 29/89] Add test for changing column's type by performing ADD COLUMN + UPDATE + DROP COLUMN. --- ..._column_type_via_add_drop_update.reference | 24 +++++++++ ...modify_column_type_via_add_drop_update.sql | 52 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.reference create mode 100644 tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql diff --git a/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.reference b/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.reference new file mode 100644 index 00000000000..0d0a0e003b7 --- /dev/null +++ b/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.reference @@ -0,0 +1,24 @@ +a b c d e +Int64 String UInt8 Float64 UInt32 +0 0 0 0 0 +1 2 3 4 5 +2 4 6 8 10 +3 6 9 12 15 +4 8 12 16 20 +5 10 15 20 25 +6 12 18 24 30 +7 14 21 28 35 +8 16 24 32 40 +9 18 27 36 45 +a b c d e +Int64 String UInt8 Float64 UInt32 +0 0 0 0 0 +1 2 3 4 5 +2 4 6 8 10 +3 6 9 12 15 +4 8 12 16 20 +5 10 15 20 25 +6 12 18 24 30 +7 14 21 28 35 +8 16 24 32 40 +9 18 27 36 45 diff --git a/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql b/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql new file mode 100644 index 00000000000..e512cd960c5 --- /dev/null +++ b/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql @@ -0,0 +1,52 @@ +DROP TABLE IF EXISTS tbl; +CREATE TABLE tbl(a String, b UInt32, c Float64, d Int64, e UInt8) ENGINE=MergeTree ORDER BY tuple(); +INSERT INTO tbl SELECT number, number * 2, number * 3, number * 4, number * 5 FROM system.numbers LIMIT 10; + +-- Change the types of columns by adding a temporary column and updating and dropping. +-- Alters should be executed in sequential order. +ALTER TABLE tbl ADD COLUMN xi Int64; +ALTER TABLE tbl UPDATE xi = a WHERE 1; +ALTER TABLE tbl DROP COLUMN a; +ALTER TABLE tbl ADD COLUMN a Int64; +ALTER TABLE tbl UPDATE a = xi WHERE 1; +ALTER TABLE tbl DROP COLUMN xi; + +ALTER TABLE tbl ADD COLUMN xi String; +ALTER TABLE tbl UPDATE xi = b WHERE 1; +ALTER TABLE tbl DROP COLUMN b; +ALTER TABLE tbl ADD COLUMN b String; +ALTER TABLE tbl UPDATE b = xi WHERE 1; +ALTER TABLE tbl DROP COLUMN xi; + +ALTER TABLE tbl ADD COLUMN xi UInt8; +ALTER TABLE tbl UPDATE xi = c WHERE 1; +ALTER TABLE tbl DROP COLUMN c; +ALTER TABLE tbl ADD COLUMN c UInt8; +ALTER TABLE tbl UPDATE c = xi WHERE 1; +ALTER TABLE tbl DROP COLUMN xi; + +ALTER TABLE tbl ADD COLUMN xi Float64; +ALTER TABLE tbl UPDATE xi = d WHERE 1; +ALTER TABLE tbl DROP COLUMN d; +ALTER TABLE tbl ADD COLUMN d Float64; +ALTER TABLE tbl UPDATE d = xi WHERE 1; +ALTER TABLE tbl DROP COLUMN xi; + +ALTER TABLE tbl ADD COLUMN xi UInt32; +ALTER TABLE tbl UPDATE xi = e WHERE 1; +ALTER TABLE tbl DROP COLUMN e; +ALTER TABLE tbl ADD COLUMN e UInt32; +ALTER TABLE tbl UPDATE e = xi WHERE 1; +ALTER TABLE tbl DROP COLUMN xi; + +SELECT * FROM tbl FORMAT TabSeparatedWithNamesAndTypes; + +DROP TABLE tbl; + +-- Do the same thing again but with MODIFY COLUMN. +CREATE TABLE tbl(a String, b UInt32, c Float64, d Int64, e UInt8) ENGINE=MergeTree ORDER BY tuple(); +INSERT INTO tbl SELECT number, number * 2, number * 3, number * 4, number * 5 FROM system.numbers LIMIT 10; +ALTER TABLE tbl MODIFY COLUMN a Int64, MODIFY COLUMN b String, MODIFY COLUMN c UInt8, MODIFY COLUMN d Float64, MODIFY COLUMN e UInt32; +SELECT * FROM tbl FORMAT TabSeparatedWithNamesAndTypes; + +DROP TABLE tbl; From 59d879b1fea9e1d598ccc15cd81d870c977aad9c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 25 Aug 2020 18:06:24 +0300 Subject: [PATCH 30/89] fix "no space left" extra info for Poco::Exception --- src/Common/Exception.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp index fa24bb6377e..4dbd685d562 100644 --- a/src/Common/Exception.cpp +++ b/src/Common/Exception.cpp @@ -205,7 +205,19 @@ static std::string getExtraExceptionInfo(const std::exception & e) if (const auto * file_exception = dynamic_cast(&e)) { if (file_exception->code() == ENOSPC) - getNoSpaceLeftInfoMessage(file_exception->message(), msg); + { + /// See Poco::FileImpl::handleLastErrorImpl(...) + constexpr const char * expected_error_message = "no space left on device: "; + if (startsWith(file_exception->message(), expected_error_message)) + { + String path = file_exception->message().substr(strlen(expected_error_message)); + getNoSpaceLeftInfoMessage(path, msg); + } + else + { + msg += "\nCannot print extra info for Poco::Exception"; + } + } } else if (const auto * errno_exception = dynamic_cast(&e)) { From 899f93356f6d410cc9c0ea162c3feeded697c0d8 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Tue, 25 Aug 2020 18:38:36 +0300 Subject: [PATCH 31/89] romanzhukov-DOCSUP-2192: Split ALTER to separate files (ru) (#13905) * DOCSUP-2192: Split alter section to separete files. * DOCSUP-2192: Test. * DOCSUP-2192: Fixed cross links. * DOCSUP-2192: Fix links. * DOCSUP-2192: Fix links. * DOCSUP-2192: Fix links. * DOCSUP-2192: Added alter delete and alter update topics. * DOCSUP-2192: Fix one layout mistake and update links. * DOCSUP-2192: Added delete and update sections. Updated alter index.md * DOCSUP-2192: Fix links. Co-authored-by: romanzhukov --- .../statements/alter/order-by.md | 3 +- .../engines/table-engines/log-family/index.md | 2 +- .../custom-partitioning-key.md | 2 +- .../mergetree-family/mergetree.md | 2 +- docs/ru/operations/access-rights.md | 10 +- docs/ru/operations/backup.md | 2 +- docs/ru/operations/settings/settings.md | 6 +- docs/ru/operations/system-tables/mutations.md | 4 +- docs/ru/operations/system-tables/part_log.md | 2 +- docs/ru/operations/system-tables/parts.md | 4 +- docs/ru/sql-reference/index.md | 2 +- docs/ru/sql-reference/statements/alter.md | 613 ------------------ .../sql-reference/statements/alter/column.md | 140 ++++ .../statements/alter/constraint.md | 23 + .../sql-reference/statements/alter/delete.md | 29 + .../sql-reference/statements/alter/index.md | 72 ++ .../statements/alter/index/index.md | 24 + .../statements/alter/order-by.md | 22 + .../statements/alter/partition.md | 259 ++++++++ .../sql-reference/statements/alter/quota.md | 22 + .../ru/sql-reference/statements/alter/role.md | 18 + .../statements/alter/row-policy.md | 21 + .../statements/alter/settings-profile.md | 18 + docs/ru/sql-reference/statements/alter/ttl.md | 14 + .../sql-reference/statements/alter/update.md | 29 + .../ru/sql-reference/statements/alter/user.md | 49 ++ .../sql-reference/statements/create/role.md | 2 +- docs/ru/sql-reference/statements/grant.md | 2 +- docs/ru/sql-reference/statements/kill.md | 2 +- docs/ru/sql-reference/statements/optimize.md | 2 +- 30 files changed, 763 insertions(+), 637 deletions(-) delete mode 100644 docs/ru/sql-reference/statements/alter.md create mode 100644 docs/ru/sql-reference/statements/alter/column.md create mode 100644 docs/ru/sql-reference/statements/alter/constraint.md create mode 100644 docs/ru/sql-reference/statements/alter/delete.md create mode 100644 docs/ru/sql-reference/statements/alter/index.md create mode 100644 docs/ru/sql-reference/statements/alter/index/index.md create mode 100644 docs/ru/sql-reference/statements/alter/order-by.md create mode 100644 docs/ru/sql-reference/statements/alter/partition.md create mode 100644 docs/ru/sql-reference/statements/alter/quota.md create mode 100644 docs/ru/sql-reference/statements/alter/role.md create mode 100644 docs/ru/sql-reference/statements/alter/row-policy.md create mode 100644 docs/ru/sql-reference/statements/alter/settings-profile.md create mode 100644 docs/ru/sql-reference/statements/alter/ttl.md create mode 100644 docs/ru/sql-reference/statements/alter/update.md create mode 100644 docs/ru/sql-reference/statements/alter/user.md diff --git a/docs/en/sql-reference/statements/alter/order-by.md b/docs/en/sql-reference/statements/alter/order-by.md index 1e2dadb2946..d41b9a91724 100644 --- a/docs/en/sql-reference/statements/alter/order-by.md +++ b/docs/en/sql-reference/statements/alter/order-by.md @@ -14,5 +14,4 @@ The command changes the [sorting key](../../../engines/table-engines/mergetree-f The command is lightweight in a sense that it only changes metadata. To keep the property that data part rows are ordered by the sorting key expression you cannot add expressions containing existing columns to the sorting key (only columns added by the `ADD COLUMN` command in the same `ALTER` query). !!! note "Note" - It only works for tables in the [`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) family (including -[replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). + It only works for tables in the [`MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) family (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) tables). diff --git a/docs/ru/engines/table-engines/log-family/index.md b/docs/ru/engines/table-engines/log-family/index.md index 4aa7540c0a8..d2cb0df5a8a 100644 --- a/docs/ru/engines/table-engines/log-family/index.md +++ b/docs/ru/engines/table-engines/log-family/index.md @@ -26,7 +26,7 @@ toc_priority: 29 Во время запросов `INSERT` таблица блокируется, а другие запросы на чтение и запись ожидают разблокировки таблицы. Если запросов на запись данных нет, то можно выполнять любое количество конкуретных запросов на чтение. -- Не поддерживают операции [мутации](../../../sql-reference/statements/alter.md#mutations). +- Не поддерживают операции [мутации](../../../sql-reference/statements/alter/index.md#mutations). - Не поддерживают индексы. diff --git a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md index 2e8b946ee8c..89980a3bd73 100644 --- a/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md +++ b/docs/ru/engines/table-engines/mergetree-family/custom-partitioning-key.md @@ -113,7 +113,7 @@ drwxr-xr-x 2 clickhouse clickhouse 4096 Feb 1 16:48 detached ‘201901\_1\_1\_0’, ‘201901\_1\_7\_1’ и т. д. – это директории кусков партиции. Каждый кусок содержит данные только для соответствующего месяца (таблица в данном примере содержит партиционирование по месяцам). -Директория `detached` содержит куски, отсоединенные от таблицы с помощью запроса [DETACH](../../../sql-reference/statements/alter.md#alter_detach-partition). Поврежденные куски также попадают в эту директорию – они не удаляются с сервера. +Директория `detached` содержит куски, отсоединенные от таблицы с помощью запроса [DETACH](../../../sql-reference/statements/alter/partition.md#alter_detach-partition). Поврежденные куски также попадают в эту директорию – они не удаляются с сервера. Сервер не использует куски из директории `detached`. Вы можете в любое время добавлять, удалять, модифицировать данные в директории detached - сервер не будет об этом знать, пока вы не сделаете запрос [ATTACH](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md#alter_attach-partition). diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 5eed6ddd3b4..f04fbae18ba 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -601,7 +601,7 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' В таблицах `MergeTree` данные попадают на диск несколькими способами: - В результате вставки (запрос `INSERT`). -- В фоновых операциях слияний и [мутаций](../../../sql-reference/statements/alter.md#mutations). +- В фоновых операциях слияний и [мутаций](../../../sql-reference/statements/alter/index.md#mutations). - При скачивании данных с другой реплики. - В результате заморозки партиций [ALTER TABLE … FREEZE PARTITION](../../../engines/table-engines/mergetree-family/mergetree.md#alter_freeze-partition). diff --git a/docs/ru/operations/access-rights.md b/docs/ru/operations/access-rights.md index edce9f3920e..27dbc2fbf62 100644 --- a/docs/ru/operations/access-rights.md +++ b/docs/ru/operations/access-rights.md @@ -56,7 +56,7 @@ ClickHouse поддерживает управление доступом на Запросы управления: - [CREATE USER](../sql-reference/statements/create/user.md#create-user-statement) -- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement) +- [ALTER USER](../sql-reference/statements/alter/user.md) - [DROP USER](../sql-reference/statements/misc.md#drop-user-statement) - [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement) @@ -83,7 +83,7 @@ ClickHouse поддерживает управление доступом на Запросы управления: - [CREATE ROLE](../sql-reference/statements/create/index.md#create-role-statement) -- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement) +- [ALTER ROLE](../sql-reference/statements/alter/role.md) - [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement) - [SET ROLE](../sql-reference/statements/misc.md#set-role-statement) - [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement) @@ -98,7 +98,7 @@ ClickHouse поддерживает управление доступом на Запросы управления: - [CREATE ROW POLICY](../sql-reference/statements/create/index.md#create-row-policy-statement) -- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement) +- [ALTER ROW POLICY](../sql-reference/statements/alter/row-policy.md) - [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement) - [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement) @@ -110,7 +110,7 @@ ClickHouse поддерживает управление доступом на Запросы управления: - [CREATE SETTINGS PROFILE](../sql-reference/statements/create/index.md#create-settings-profile-statement) -- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement) +- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter/settings-profile.md) - [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement) - [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement) @@ -124,7 +124,7 @@ ClickHouse поддерживает управление доступом на Запросы управления: - [CREATE QUOTA](../sql-reference/statements/create/index.md#create-quota-statement) -- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement) +- [ALTER QUOTA](../sql-reference/statements/alter/quota.md) - [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement) - [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement) diff --git a/docs/ru/operations/backup.md b/docs/ru/operations/backup.md index 6badaa7f724..89fb0403543 100644 --- a/docs/ru/operations/backup.md +++ b/docs/ru/operations/backup.md @@ -27,7 +27,7 @@ ClickHouse позволяет использовать запрос `ALTER TABLE ... FREEZE PARTITION ...` для создания локальной копии партиций таблицы. Это реализуется с помощью жестких ссылок (hardlinks) на каталог `/var/lib/clickhouse/shadow/`, поэтому такая копия обычно не занимает дополнительное место на диске для старых данных. Созданные копии файлов не обрабатываются сервером ClickHouse, поэтому вы можете просто оставить их там: у вас будет простая резервная копия, которая не требует дополнительной внешней системы, однако при аппаратных проблемах вы можете утратить и актуальные данные и сохраненную копию. По этой причине, лучше удаленно скопировать их в другое место, а затем удалить локальную копию. Распределенные файловые системы и хранилища объектов по-прежнему являются хорошими вариантами для этого, однако можно использовать и обычные присоединенные файловые серверы с достаточно большой ёмкостью (в этом случае передача будет происходить через сетевую файловую систему или, возможно, [rsync](https://en.wikipedia.org/wiki/Rsync)). -Дополнительные сведения о запросах, связанных с манипуляциями партициями, см. в разделе [ALTER](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). +Дополнительные сведения о запросах, связанных с манипуляциями партициями, см. в разделе [ALTER](../sql-reference/statements/alter/partition.md#alter_manipulations-with-partitions). Для автоматизации этого подхода доступен инструмент от сторонних разработчиков: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup). diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 4b817396eb7..e8d3f1057df 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1606,7 +1606,7 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; ## mutations_sync {#mutations_sync} -Позволяет выполнять запросы `ALTER TABLE ... UPDATE|DELETE` ([мутации](../../sql-reference/statements/alter.md#mutations)) синхронно. +Позволяет выполнять запросы `ALTER TABLE ... UPDATE|DELETE` ([мутации](../../sql-reference/statements/alter/index.md#mutations)) синхронно. Возможные значения: @@ -1618,8 +1618,8 @@ SELECT idx, i FROM null_in WHERE i IN (1, NULL) SETTINGS transform_null_in = 1; **См. также** -- [Синхронность запросов ALTER](../../sql-reference/statements/alter.md#synchronicity-of-alter-queries) -- [Мутации](../../sql-reference/statements/alter.md#mutations) +- [Синхронность запросов ALTER](../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) +- [Мутации](../../sql-reference/statements/alter/index.md#mutations) [Оригинальная статья](https://clickhouse.tech/docs/ru/operations/settings/settings/) diff --git a/docs/ru/operations/system-tables/mutations.md b/docs/ru/operations/system-tables/mutations.md index 4653e6ad418..044677030ba 100644 --- a/docs/ru/operations/system-tables/mutations.md +++ b/docs/ru/operations/system-tables/mutations.md @@ -1,6 +1,6 @@ # system.mutations {#system_tables-mutations} -Таблица содержит информацию о ходе выполнения [мутаций](../../sql-reference/statements/alter.md#mutations) таблиц семейства MergeTree. Каждой команде мутации соответствует одна строка таблицы. +Таблица содержит информацию о ходе выполнения [мутаций](../../sql-reference/statements/alter/index.md#mutations) таблиц семейства MergeTree. Каждой команде мутации соответствует одна строка таблицы. Столбцы: @@ -41,7 +41,7 @@ **См. также** -- [Мутации](../../sql-reference/statements/alter.md#mutations) +- [Мутации](../../sql-reference/statements/alter/index.md#mutations) - [Движок MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) - [Репликация данных](../../engines/table-engines/mergetree-family/replication.md) (семейство ReplicatedMergeTree) diff --git a/docs/ru/operations/system-tables/part_log.md b/docs/ru/operations/system-tables/part_log.md index 409947a1bda..255ece76ee2 100644 --- a/docs/ru/operations/system-tables/part_log.md +++ b/docs/ru/operations/system-tables/part_log.md @@ -10,7 +10,7 @@ - `NEW_PART` — вставка нового куска. - `MERGE_PARTS` — слияние кусков. - `DOWNLOAD_PART` — загрузка с реплики. - - `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter.md#alter_detach-partition). + - `REMOVE_PART` — удаление или отсоединение из таблицы с помощью [DETACH PARTITION](../../sql-reference/statements/alter/partition.md#alter_detach-partition). - `MUTATE_PART` — изменение куска. - `MOVE_PART` — перемещение куска между дисками. - `event_date` (Date) — дата события. diff --git a/docs/ru/operations/system-tables/parts.md b/docs/ru/operations/system-tables/parts.md index afedf20caf0..950e652332d 100644 --- a/docs/ru/operations/system-tables/parts.md +++ b/docs/ru/operations/system-tables/parts.md @@ -6,7 +6,7 @@ Столбцы: -- `partition` ([String](../../sql-reference/data-types/string.md)) – имя партиции. Что такое партиция можно узнать из описания запроса [ALTER](../../sql-reference/statements/alter.md#query_language_queries_alter). +- `partition` ([String](../../sql-reference/data-types/string.md)) – имя партиции. Что такое партиция можно узнать из описания запроса [ALTER](../../sql-reference/statements/alter/index.md#query_language_queries_alter). Форматы: @@ -66,7 +66,7 @@ - `primary_key_bytes_in_memory_allocated` ([UInt64](../../sql-reference/data-types/int-uint.md)) – объём памяти (в байтах) выделенный для размещения первичных ключей. -- `is_frozen` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Признак, показывающий существование бэкапа партиции. 1, бэкап есть. 0, бэкапа нет. Смотрите раздел [FREEZE PARTITION](../../sql-reference/statements/alter.md#alter_freeze-partition). +- `is_frozen` ([UInt8](../../sql-reference/data-types/int-uint.md)) – Признак, показывающий существование бэкапа партиции. 1, бэкап есть. 0, бэкапа нет. Смотрите раздел [FREEZE PARTITION](../../sql-reference/statements/alter/partition.md#alter_freeze-partition). - `database` ([String](../../sql-reference/data-types/string.md)) – имя базы данных. diff --git a/docs/ru/sql-reference/index.md b/docs/ru/sql-reference/index.md index 92983d6d18a..f59232ee047 100644 --- a/docs/ru/sql-reference/index.md +++ b/docs/ru/sql-reference/index.md @@ -10,7 +10,7 @@ toc_title: hidden - [SELECT](statements/select/index.md) - [INSERT INTO](statements/insert-into.md) - [CREATE](statements/create/index.md) -- [ALTER](statements/alter.md#query_language_queries_alter) +- [ALTER](statements/alter/index.md#query_language_queries_alter) - [Прочие виды запросов](statements/misc.md) [Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/) diff --git a/docs/ru/sql-reference/statements/alter.md b/docs/ru/sql-reference/statements/alter.md deleted file mode 100644 index 7cac767e842..00000000000 --- a/docs/ru/sql-reference/statements/alter.md +++ /dev/null @@ -1,613 +0,0 @@ ---- -toc_priority: 36 -toc_title: ALTER ---- - -## ALTER {#query_language_queries_alter} - -Запрос `ALTER` поддерживается только для таблиц типа `*MergeTree`, а также `Merge` и `Distributed`. Запрос имеет несколько вариантов. - -### Манипуляции со столбцами {#manipuliatsii-so-stolbtsami} - -Изменение структуры таблицы. - -``` sql -ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ... -``` - -В запросе указывается список из одного или более действий через запятую. -Каждое действие — операция над столбцом. - -Существуют следующие действия: - -- [ADD COLUMN](#alter_add-column) — добавляет столбец в таблицу; -- [DROP COLUMN](#alter_drop-column) — удаляет столбец; -- [CLEAR COLUMN](#alter_clear-column) — сбрасывает все значения в столбце для заданной партиции; -- [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу; -- [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL. - -Подробное описание для каждого действия приведено ниже. - -#### ADD COLUMN {#alter_add-column} - -``` sql -ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] -``` - -Добавляет в таблицу новый столбец с именем `name`, типом `type`, [кодеком](create/table.md#codecs) `codec` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](create/index.md#create-default-values)). - -Если указано `IF NOT EXISTS`, запрос не будет возвращать ошибку, если столбец уже существует. Если указано `AFTER name_after` (имя другого столбца), то столбец добавляется (в список столбцов таблицы) после указанного. Иначе, столбец добавляется в конец таблицы. Обратите внимание, ClickHouse не позволяет добавлять столбцы в начало таблицы. Для цепочки действий, `name_after` может быть именем столбца, который добавляется в одном из предыдущих действий. - -Добавление столбца всего лишь меняет структуру таблицы, и не производит никаких действий с данными - соответствующие данные не появляются на диске после ALTER-а. При чтении из таблицы, если для какого-либо столбца отсутствуют данные, то он заполняется значениями по умолчанию (выполняя выражение по умолчанию, если такое есть, или нулями, пустыми строками). Также, столбец появляется на диске при слиянии кусков данных (см. [MergeTree](../../sql-reference/statements/alter.md)). - -Такая схема позволяет добиться мгновенной работы запроса `ALTER` и отсутствия необходимости увеличивать объём старых данных. - -Пример: - -``` sql -ALTER TABLE visits ADD COLUMN browser String AFTER user_id -``` - -#### DROP COLUMN {#alter_drop-column} - -``` sql -DROP COLUMN [IF EXISTS] name -``` - -Удаляет столбец с именем `name`. Если указано `IF EXISTS`, запрос не будет возвращать ошибку, если столбца не существует. - -Запрос удаляет данные из файловой системы. Так как это представляет собой удаление целых файлов, запрос выполняется почти мгновенно. - -Пример: - -``` sql -ALTER TABLE visits DROP COLUMN browser -``` - -#### CLEAR COLUMN {#alter_clear-column} - -``` sql -CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name -``` - -Сбрасывает все значения в столбце для заданной партиции. Если указано `IF EXISTS`, запрос не будет возвращать ошибку, если столбца не существует. - -Как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -Пример: - -``` sql -ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple() -``` - -#### COMMENT COLUMN {#alter_comment-column} - -``` sql -COMMENT COLUMN [IF EXISTS] name 'Text comment' -``` - -Добавляет комментарий к таблице. Если указано `IF EXISTS`, запрос не будет возвращать ошибку, если столбца не существует. - -Каждый столбец может содержать только один комментарий. При выполнении запроса существующий комментарий заменяется на новый. - -Посмотреть комментарии можно в столбце `comment_expression` из запроса [DESCRIBE TABLE](misc.md#misc-describe-table). - -Пример: - -``` sql -ALTER TABLE visits COMMENT COLUMN browser 'Столбец показывает, из каких браузеров пользователи заходили на сайт.' -``` - -#### MODIFY COLUMN {#alter_modify-column} - -``` sql -MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] -``` - -Запрос изменяет следующие свойства столбца `name`: - -- Тип - -- Значение по умолчанию - -- TTL - - Примеры изменения TTL столбца смотрите в разделе [TTL столбца](../../sql_reference/statements/alter.md#mergetree-column-ttl). - -Если указано `IF EXISTS`, запрос не возвращает ошибку, если столбца не существует. - -При изменении типа, значения преобразуются так, как если бы к ним была применена функция [toType](../../sql-reference/statements/alter.md). Если изменяется только выражение для умолчания, запрос не делает никакой сложной работы и выполняется мгновенно. - -Пример запроса: - -``` sql -ALTER TABLE visits MODIFY COLUMN browser Array(String) -``` - -Изменение типа столбца - это единственное действие, которое выполняет сложную работу - меняет содержимое файлов с данными. Для больших таблиц, выполнение может занять длительное время. - -Выполнение производится в несколько стадий: - -- подготовка временных (новых) файлов с изменёнными данными; -- переименование старых файлов; -- переименование временных (новых) файлов в старые; -- удаление старых файлов. - -Из них, длительной является только первая стадия. Если на этой стадии возникнет сбой, то данные не поменяются. -Если на одной из следующих стадий возникнет сбой, то данные будет можно восстановить вручную. За исключением случаев, когда старые файлы удалены из файловой системы, а данные для новых файлов не доехали на диск и потеряны. - -Запрос `ALTER` на изменение столбцов реплицируется. Соответствующие инструкции сохраняются в ZooKeeper, и затем каждая реплика их применяет. Все запросы `ALTER` выполняются в одном и том же порядке. Запрос ждёт выполнения соответствующих действий на всех репликах. Но при этом, запрос на изменение столбцов в реплицируемой таблице можно прервать, и все действия будут осуществлены асинхронно. - -#### Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} - -Запрос `ALTER` позволяет создавать и удалять отдельные элементы (столбцы) вложенных структур данных, но не вложенные структуры данных целиком. Для добавления вложенной структуры данных, вы можете добавить столбцы с именем вида `name.nested_name` и типом `Array(T)` - вложенная структура данных полностью эквивалентна нескольким столбцам-массивам с именем, имеющим одинаковый префикс до точки. - -Отсутствует возможность удалять столбцы, входящие в первичный ключ или ключ для сэмплирования (в общем, входящие в выражение `ENGINE`). Изменение типа у столбцов, входящих в первичный ключ возможно только в том случае, если это изменение не приводит к изменению данных (например, разрешено добавление значения в Enum или изменение типа с `DateTime` на `UInt32`). - -Если возможностей запроса `ALTER` не хватает для нужного изменения таблицы, вы можете создать новую таблицу, скопировать туда данные с помощью запроса [INSERT SELECT](insert-into.md#insert_query_insert-select), затем поменять таблицы местами с помощью запроса [RENAME](misc.md#misc_operations-rename), и удалить старую таблицу. В качестве альтернативы для запроса `INSERT SELECT`, можно использовать инструмент [clickhouse-copier](../../sql-reference/statements/alter.md). - -Запрос `ALTER` блокирует все чтения и записи для таблицы. То есть, если на момент запроса `ALTER`, выполнялся долгий `SELECT`, то запрос `ALTER` сначала дождётся его выполнения. И в это время, все новые запросы к той же таблице, будут ждать, пока завершится этот `ALTER`. - -Для таблиц, которые не хранят данные самостоятельно (типа [Merge](../../sql-reference/statements/alter.md) и [Distributed](../../sql-reference/statements/alter.md)), `ALTER` всего лишь меняет структуру таблицы, но не меняет структуру подчинённых таблиц. Для примера, при ALTER-е таблицы типа `Distributed`, вам также потребуется выполнить запрос `ALTER` для таблиц на всех удалённых серверах. - -### Манипуляции с ключевыми выражениями таблиц {#manipuliatsii-s-kliuchevymi-vyrazheniiami-tablits} - -Поддерживается операция: - -``` sql -MODIFY ORDER BY new_expression -``` - -Работает только для таблиц семейства [`MergeTree`](../../sql-reference/statements/alter.md) (в том числе [реплицированных](../../sql-reference/statements/alter.md)). После выполнения запроса -[ключ сортировки](../../sql-reference/statements/alter.md) таблицы -заменяется на `new_expression` (выражение или кортеж выражений). Первичный ключ при этом остаётся прежним. - -Операция затрагивает только метаданные. Чтобы сохранить свойство упорядоченности кусков данных по ключу -сортировки, разрешено добавлять в ключ только новые столбцы (т.е. столбцы, добавляемые командой `ADD COLUMN` -в том же запросе `ALTER`), у которых нет выражения по умолчанию. - -### Манипуляции с индексами {#manipuliatsii-s-indeksami} - -Добавить или удалить индекс можно с помощью операций - -``` sql -ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value [AFTER name] -ALTER TABLE [db].name DROP INDEX name -``` - -Поддерживается только таблицами семейства `*MergeTree`. - -Команда `ADD INDEX` добавляет описание индексов в метаданные, а `DROP INDEX` удаляет индекс из метаданных и стирает файлы индекса с диска, поэтому они легковесные и работают мгновенно. - -Если индекс появился в метаданных, то он начнет считаться в последующих слияниях и записях в таблицу, а не сразу после выполнения операции `ALTER`. - -Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. - -### Манипуляции с ограничениями (constraints) {#manipuliatsii-s-ogranicheniiami-constraints} - -Про ограничения подробнее написано [тут](create/index.md#constraints). - -Добавить или удалить ограничение можно с помощью запросов - -``` sql -ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; -ALTER TABLE [db].name DROP CONSTRAINT constraint_name; -``` - -Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мнгновенно. - -Если ограничение появилось для непустой таблицы, то *проверка ограничения для имеющихся данных не производится*. - -Запрос на изменение ограничений для Replicated таблиц реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. - -### Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} - -Для работы с [партициями](../../sql-reference/statements/alter.md) доступны следующие операции: - -- [DETACH PARTITION](#alter_detach-partition) — перенести партицию в директорию `detached`; -- [DROP PARTITION](#alter_drop-partition) — удалить партицию; -- [ATTACH PARTITION\|PART](#alter_attach-partition) — добавить партицию/кусок в таблицу из директории `detached`; -- [ATTACH PARTITION FROM](#alter_attach-partition-from) — скопировать партицию из другой таблицы; -- [REPLACE PARTITION](#alter_replace-partition) — скопировать партицию из другой таблицы с заменой; -- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — переместить партицию в другую таблицу; -- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — удалить все значения в столбце для заданной партиции; -- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — очистить построенные вторичные индексы для заданной партиции; -- [FREEZE PARTITION](#alter_freeze-partition) — создать резервную копию партиции; -- [FETCH PARTITION](#alter_fetch-partition) — скачать партицию с другого сервера; -- [MOVE PARTITION\|PART](#alter_move-partition) — переместить партицию/кускок на другой диск или том. - -#### DETACH PARTITION {#alter_detach-partition} - -``` sql -ALTER TABLE table_name DETACH PARTITION partition_expr -``` - -Перемещает заданную партицию в директорию `detached`. Сервер не будет знать об этой партиции до тех пор, пока вы не выполните запрос [ATTACH](#alter_attach-partition). - -Пример: - -``` sql -ALTER TABLE visits DETACH PARTITION 201901 -``` - -Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -После того как запрос будет выполнен, вы сможете производить любые операции с данными в директории `detached`. Например, можно удалить их из файловой системы. - -Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../operations/system-tables/replicas.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплики-лидера, запрос вернет ошибку. - -#### DROP PARTITION {#alter_drop-partition} - -``` sql -ALTER TABLE table_name DROP PARTITION partition_expr -``` - -Удаляет партицию. Партиция помечается как неактивная и будет полностью удалена примерно через 10 минут. - -Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -Запрос реплицируется — данные будут удалены на всех репликах. - -#### DROP DETACHED PARTITION\|PART {#alter_drop-detached} - -``` sql -ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr -``` - -Удаляет из `detached` кусок или все куски, принадлежащие партиции. -Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -#### ATTACH PARTITION\|PART {#alter_attach-partition} - -``` sql -ALTER TABLE table_name ATTACH PARTITION|PART partition_expr -``` - -Добавляет данные в таблицу из директории `detached`. Можно добавить данные как для целой партиции, так и для отдельного куска. Примеры: - -``` sql -ALTER TABLE visits ATTACH PARTITION 201901; -ALTER TABLE visits ATTACH PART 201901_2_2_0; -``` - -Как корректно задать имя партиции или куска, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`. Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу. Все остальные реплики загружают данные с реплики-инициатора запроса. - -Это означает, что вы можете разместить данные в директории `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах. - -#### ATTACH PARTITION FROM {#alter_attach-partition-from} - -``` sql -ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 -``` - -Копирует партицию из таблицы `table1` в таблицу `table2` и добавляет к существующим данным `table2`. Данные из `table1` не удаляются. - -Следует иметь в виду: - -- Таблицы должны иметь одинаковую структуру. -- Для таблиц должен быть задан одинаковый ключ партиционирования. - -Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -#### REPLACE PARTITION {#alter_replace-partition} - -``` sql -ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 -``` - -Копирует партицию из таблицы `table1` в таблицу `table2` с заменой существующих данных в `table2`. Данные из `table1` не удаляются. - -Следует иметь в виду: - -- Таблицы должны иметь одинаковую структуру. -- Для таблиц должен быть задан одинаковый ключ партиционирования. - -Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -#### MOVE PARTITION TO TABLE {#alter_move_to_table-partition} - -``` sql -ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest -``` - -Перемещает партицию из таблицы `table_source` в таблицу `table_dest` (добавляет к существующим данным в `table_dest`) с удалением данных из таблицы `table_source`. - -Следует иметь в виду: - -- Таблицы должны иметь одинаковую структуру. -- Для таблиц должен быть задан одинаковый ключ партиционирования. -- Движки таблиц должны быть одинакового семейства (реплицированные или нереплицированные). -- Для таблиц должна быть задана одинаковая политика хранения. - -#### CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} - -``` sql -ALTER TABLE table_name CLEAR COLUMN column_name IN PARTITION partition_expr -``` - -Сбрасывает все значения в столбце для заданной партиции. Если для столбца определено значение по умолчанию (в секции `DEFAULT`), то будет выставлено это значение. - -Пример: - -``` sql -ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 -``` - -#### CLEAR INDEX IN PARTITION {#alter_clear-index-partition} - -``` sql -ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr -``` - -Работает как `CLEAR COLUMN`, но сбрасывает индексы вместо данных в столбцах. - -#### FREEZE PARTITION {#alter_freeze-partition} - -``` sql -ALTER TABLE table_name FREEZE [PARTITION partition_expr] -``` - -Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. - -!!! note "Примечание" - Создание резервной копии не требует остановки сервера. - -Для таблиц старого стиля имя партиций можно задавать в виде префикса (например, ‘2019’). В этом случае резервные копии будут созданы для всех соответствующих партиций. Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). - -Запрос делает следующее — для текущего состояния таблицы он формирует жесткие ссылки на данные в этой таблице. Ссылки размещаются в директории `/var/lib/clickhouse/shadow/N/...`, где: - -- `/var/lib/clickhouse/` — рабочая директория ClickHouse, заданная в конфигурационном файле; -- `N` — инкрементальный номер резервной копии. - -!!! note "Примечание" - При использовании [нескольких дисков для хранения данных таблицы](../../sql-reference/statements/alter.md#table_engine-mergetree-multiple-volumes) директория `shadow/N` появляется на каждом из дисков, на которых были куски, попавшие под выражение `PARTITION`. - -Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит ‘chmod’ для всех файлов, запрещая запись в них. - -Обратите внимание, запрос `ALTER TABLE t FREEZE PARTITION` не реплицируется. Он создает резервную копию только на локальном сервере. После создания резервной копии данные из `/var/lib/clickhouse/shadow/` можно скопировать на удалённый сервер, а локальную копию удалить. - -Резервная копия создается почти мгновенно (однако сначала запрос дожидается завершения всех запросов, которые выполняются для соответствующей таблицы). - -`ALTER TABLE t FREEZE PARTITION` копирует только данные, но не метаданные таблицы. Чтобы сделать резервную копию метаданных таблицы, скопируйте файл `/var/lib/clickhouse/metadata/database/table.sql` - -Чтобы восстановить данные из резервной копии, выполните следующее: - -1. Создайте таблицу, если она ещё не существует. Запрос на создание можно взять из .sql файла (замените в нём `ATTACH` на `CREATE`). -2. Скопируйте данные из директории `data/database/table/` внутри резервной копии в директорию `/var/lib/clickhouse/data/database/table/detached/`. -3. С помощью запросов `ALTER TABLE t ATTACH PARTITION` добавьте данные в таблицу. - -Восстановление данных из резервной копии не требует остановки сервера. - -Подробнее о резервном копировании и восстановлении данных читайте в разделе [Резервное копирование данных](../../operations/backup.md). - -#### FETCH PARTITION {#alter_fetch-partition} - -``` sql -ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' -``` - -Загружает партицию с другого сервера. Этот запрос работает только для реплицированных таблиц. - -Запрос выполняет следующее: - -1. Загружает партицию с указанного шарда. Путь к шарду задается в секции `FROM` (‘path-in-zookeeper’). Обратите внимание, нужно задавать путь к шарду в ZooKeeper. -2. Помещает загруженные данные в директорию `detached` таблицы `table_name`. Чтобы прикрепить эти данные к таблице, используйте запрос [ATTACH PARTITION\|PART](#alter_attach-partition). - -Например: - -``` sql -ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; -ALTER TABLE users ATTACH PARTITION 201902; -``` - -Следует иметь в виду: - -- Запрос `ALTER TABLE t FETCH PARTITION` не реплицируется. Он загружает партицию в директорию `detached` только на локальном сервере. -- Запрос `ALTER TABLE t ATTACH` реплицируется — он добавляет данные в таблицу сразу на всех репликах. На одной из реплик данные будут добавлены из директории `detached`, а на других — из соседних реплик. - -Перед загрузкой данных система проверяет, существует ли партиция и совпадает ли её структура со структурой таблицы. При этом автоматически выбирается наиболее актуальная реплика среди всех живых реплик. - -Несмотря на то что запрос называется `ALTER TABLE`, он не изменяет структуру таблицы и не изменяет сразу доступные данные в таблице. - -#### MOVE PARTITION\|PART {#alter_move-partition} - -Перемещает партицию или кусок данных на другой том или диск для таблиц с движком `MergeTree`. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](../../sql-reference/statements/alter.md#table_engine-mergetree-multiple-volumes). - -``` sql -ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' -``` - -Запрос `ALTER TABLE t MOVE`: - -- Не реплицируется, т.к. на разных репликах могут быть различные конфигурации политик хранения. -- Возвращает ошибку, если указан несконфигурированный том или диск. Ошибка также возвращается в случае невыполнения условий перемещения данных, которые указаны в конфигурации политики хранения. -- Может возвращать ошибку в случае, когда перемещаемые данные уже оказались перемещены в результате фонового процесса, конкурентного запроса `ALTER TABLE t MOVE` или как часть результата фоновой операции слияния. В данном случае никаких дополнительных действий от пользователя не требуется. - -Примеры: - -``` sql -ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow' -ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd' -``` - -#### Как задавать имя партиции в запросах ALTER {#alter-how-to-specify-part-expr} - -Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: - -- Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../../operations/system-tables/parts.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. -- Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. -- Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. -- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached_parts](../../operations/system-tables/detached_parts.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. - -Использование кавычек в имени партиций зависит от типа данных столбца, по которому задано партиционирование. Например, для столбца с типом `String` имя партиции необходимо указывать в кавычках (одинарных). Для типов `Date` и `Int*` кавычки указывать не нужно. - -Замечание: для таблиц старого стиля партицию можно указывать и как число `201901`, и как строку `'201901'`. Синтаксис для таблиц нового типа более строг к типам (аналогично парсеру входного формата VALUES). - -Правила, сформулированные выше, актуальны также для запросов [OPTIMIZE](misc.md#misc_operations-optimize). Чтобы указать единственную партицию непартиционированной таблицы, укажите `PARTITION tuple()`. Например: - -``` sql -OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; -``` - -Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). - -### Манипуляции с TTL таблицы {#manipuliatsii-s-ttl-tablitsy} - -Вы можете изменить [TTL для таблицы](../../sql-reference/statements/alter.md#mergetree-table-ttl) запросом следующего вида: - -``` sql -ALTER TABLE table-name MODIFY TTL ttl-expression -``` - -### Мутации {#mutations} - -Мутации - разновидность запроса ALTER, позволяющая изменять или удалять данные в таблице. В отличие от стандартных запросов `DELETE` и `UPDATE`, рассчитанных на точечное изменение данных, область применения мутаций - достаточно тяжёлые изменения, затрагивающие много строк в таблице. Поддержана для движков таблиц семейства `MergeTree`, в том числе для движков с репликацией. - -Конвертировать существующие таблицы для работы с мутациями не нужно. Но после применения первой мутации формат данных таблицы становится несовместимым с предыдущими версиями и откатиться на предыдущую версию уже не получится. - -На данный момент доступны команды: - -``` sql -ALTER TABLE [db.]table DELETE WHERE filter_expr -``` - -Выражение `filter_expr` должно иметь тип `UInt8`. Запрос удаляет строки таблицы, для которых это выражение принимает ненулевое значение. - -``` sql -ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr -``` - -Выражение `filter_expr` должно иметь тип `UInt8`. Запрос изменяет значение указанных столбцов на вычисленное значение соответствующих выражений в каждой строке, для которой `filter_expr` принимает ненулевое значение. Вычисленные значения преобразуются к типу столбца с помощью оператора `CAST`. Изменение столбцов, которые используются при вычислении первичного ключа или ключа партиционирования, не поддерживается. - -``` sql -ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name -``` - -Команда перестроит вторичный индекс `name` для партиции `partition_name`. - -В одном запросе можно указать несколько команд через запятую. - -Для \*MergeTree-таблиц мутации выполняются, перезаписывая данные по кускам (parts). При этом атомарности нет — куски заменяются на помутированные по мере выполнения и запрос `SELECT`, заданный во время выполнения мутации, увидит данные как из измененных кусков, так и из кусков, которые еще не были изменены. - -Мутации линейно упорядочены между собой и накладываются на каждый кусок в порядке добавления. Мутации также упорядочены со вставками - гарантируется, что данные, вставленные в таблицу до начала выполнения запроса мутации, будут изменены, а данные, вставленные после окончания запроса мутации, изменены не будут. При этом мутации никак не блокируют вставки. - -Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../operations/system-tables/mutations.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](../../sql-reference/statements/kill.md#kill-mutation). - -Записи о последних выполненных мутациях удаляются не сразу (количество сохраняемых мутаций определяется параметром движка таблиц `finished_mutations_to_keep`). Более старые записи удаляются. - -### Синхронность запросов ALTER {#synchronicity-of-alter-queries} - -Для нереплицируемых таблиц, все запросы `ALTER` выполняются синхронно. Для реплицируемых таблиц, запрос всего лишь добавляет инструкцию по соответствующим действиям в `ZooKeeper`, а сами действия осуществляются при первой возможности. Но при этом, запрос может ждать завершения выполнения этих действий на всех репликах. - -Для запросов `ALTER ... ATTACH|DETACH|DROP` можно настроить ожидание, с помощью настройки `replication_alter_partitions_sync`. -Возможные значения: `0` - не ждать, `1` - ждать выполнения только у себя (по умолчанию), `2` - ждать всех. - -Для запросов `ALTER TABLE ... UPDATE|DELETE` синхронность выполнения определяется настройкой [mutations_sync](../../operations/settings/settings.md#mutations_sync). - -## ALTER USER {#alter-user-statement} - -Изменяет аккаунт пользователя ClickHouse. - -### Синтаксис {#alter-user-syntax} - -``` sql -ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] - [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] - [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] -``` - -### Описание {#alter-user-dscr} - -Для выполнения `ALTER USER` необходима привилегия [ALTER USER](grant.md#grant-access-management). - -### Примеры {#alter-user-examples} - -Установить ролями по умолчанию роли, назначенные пользователю: - -``` sql -ALTER USER user DEFAULT ROLE role1, role2 -``` - -Если роли не были назначены пользователю, ClickHouse выбрасывает исключение. - -Установить ролями по умолчанию все роли, назначенные пользователю: - -``` sql -ALTER USER user DEFAULT ROLE ALL -``` - -Если роль будет впоследствии назначена пользователю, она автоматически станет ролью по умолчанию. - -Установить ролями по умолчанию все назначенные пользователю роли кроме `role1` и `role2`: - -``` sql -ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 -``` - - -## ALTER ROLE {#alter-role-statement} - -Изменяет роль. - -### Синтаксис {#alter-role-syntax} - -``` sql -ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] -``` - - -## ALTER ROW POLICY {#alter-row-policy-statement} - -Изменяет политику доступа к строкам. - -### Синтаксис {#alter-row-policy-syntax} - -``` sql -ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table - [RENAME TO new_name] - [AS {PERMISSIVE | RESTRICTIVE}] - [FOR SELECT] - [USING {condition | NONE}][,...] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - - -## ALTER QUOTA {#alter-quota-statement} - -Изменяет квоту. - -### Синтаксис {#alter-quota-syntax} - -``` sql -ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] - [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR} - {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | - NO LIMITS | TRACKING ONLY} [,...]] - [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] -``` - - -## ALTER SETTINGS PROFILE {#alter-settings-profile-statement} - -Изменяет профили настроек. - -### Синтаксис {#alter-settings-profile-syntax} - -``` sql -ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] - [RENAME TO new_name] - [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] -``` - - - -[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/) diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md new file mode 100644 index 00000000000..811539d60d3 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -0,0 +1,140 @@ +--- +toc_priority: 37 +toc_title: COLUMN +--- + +# Манипуляции со столбцами {#manipuliatsii-so-stolbtsami} + +Существуют следующие действия: + +- [ADD COLUMN](#alter_add-column) — добавляет столбец в таблицу; +- [DROP COLUMN](#alter_drop-column) — удаляет столбец; +- [CLEAR COLUMN](#alter_clear-column) — сбрасывает все значения в столбце для заданной партиции; +- [COMMENT COLUMN](#alter_comment-column) — добавляет комментарий к столбцу; +- [MODIFY COLUMN](#alter_modify-column) — изменяет тип столбца, выражение для значения по умолчанию и TTL. + +Подробное описание для каждого действия приведено ниже. + +## ADD COLUMN {#alter_add-column} + +``` sql +ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after] +``` + +Добавляет в таблицу новый столбец с именем `name`, типом `type`, [кодеком](../create/table.md#codecs) `codec` и выражением для умолчания `default_expr` (смотрите раздел [Значения по умолчанию](../create/index.md#create-default-values)). + +Если указано `IF NOT EXISTS`, запрос не будет возвращать ошибку, если столбец уже существует. Если указано `AFTER name_after` (имя другого столбца), то столбец добавляется (в список столбцов таблицы) после указанного. Иначе, столбец добавляется в конец таблицы. Обратите внимание, ClickHouse не позволяет добавлять столбцы в начало таблицы. Для цепочки действий, `name_after` может быть именем столбца, который добавляется в одном из предыдущих действий. + +Добавление столбца всего лишь меняет структуру таблицы, и не производит никаких действий с данными - соответствующие данные не появляются на диске после ALTER-а. При чтении из таблицы, если для какого-либо столбца отсутствуют данные, то он заполняется значениями по умолчанию (выполняя выражение по умолчанию, если такое есть, или нулями, пустыми строками). Также, столбец появляется на диске при слиянии кусков данных (см. [MergeTree](../../../sql-reference/statements/alter/index.md)). + +Такая схема позволяет добиться мгновенной работы запроса `ALTER` и отсутствия необходимости увеличивать объём старых данных. + +Пример: + +``` sql +ALTER TABLE visits ADD COLUMN browser String AFTER user_id +``` + +## DROP COLUMN {#alter_drop-column} + +``` sql +DROP COLUMN [IF EXISTS] name +``` + +Удаляет столбец с именем `name`. Если указано `IF EXISTS`, запрос не будет возвращать ошибку, если столбца не существует. + +Запрос удаляет данные из файловой системы. Так как это представляет собой удаление целых файлов, запрос выполняется почти мгновенно. + +Пример: + +``` sql +ALTER TABLE visits DROP COLUMN browser +``` + +## CLEAR COLUMN {#alter_clear-column} + +``` sql +CLEAR COLUMN [IF EXISTS] name IN PARTITION partition_name +``` + +Сбрасывает все значения в столбце для заданной партиции. Если указано `IF EXISTS`, запрос не будет возвращать ошибку, если столбца не существует. + +Как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +Пример: + +``` sql +ALTER TABLE visits CLEAR COLUMN browser IN PARTITION tuple() +``` + +## COMMENT COLUMN {#alter_comment-column} + +``` sql +COMMENT COLUMN [IF EXISTS] name 'Text comment' +``` + +Добавляет комментарий к таблице. Если указано `IF EXISTS`, запрос не будет возвращать ошибку, если столбца не существует. + +Каждый столбец может содержать только один комментарий. При выполнении запроса существующий комментарий заменяется на новый. + +Посмотреть комментарии можно в столбце `comment_expression` из запроса [DESCRIBE TABLE](../misc.md#misc-describe-table). + +Пример: + +``` sql +ALTER TABLE visits COMMENT COLUMN browser 'Столбец показывает, из каких браузеров пользователи заходили на сайт.' +``` + +## MODIFY COLUMN {#alter_modify-column} + +``` sql +MODIFY COLUMN [IF EXISTS] name [type] [default_expr] [TTL] +``` + +Запрос изменяет следующие свойства столбца `name`: + +- Тип + +- Значение по умолчанию + +- TTL + + Примеры изменения TTL столбца смотрите в разделе [TTL столбца](ttl.md#mergetree-column-ttl). + +Если указано `IF EXISTS`, запрос не возвращает ошибку, если столбца не существует. + +При изменении типа, значения преобразуются так, как если бы к ним была применена функция [toType](../../../sql-reference/statements/alter/index.md). Если изменяется только выражение для умолчания, запрос не делает никакой сложной работы и выполняется мгновенно. + +Пример запроса: + +``` sql +ALTER TABLE visits MODIFY COLUMN browser Array(String) +``` + +Изменение типа столбца - это единственное действие, которое выполняет сложную работу - меняет содержимое файлов с данными. Для больших таблиц, выполнение может занять длительное время. + +Выполнение производится в несколько стадий: + +- подготовка временных (новых) файлов с изменёнными данными; +- переименование старых файлов; +- переименование временных (новых) файлов в старые; +- удаление старых файлов. + +Из них, длительной является только первая стадия. Если на этой стадии возникнет сбой, то данные не поменяются. +Если на одной из следующих стадий возникнет сбой, то данные будет можно восстановить вручную. За исключением случаев, когда старые файлы удалены из файловой системы, а данные для новых файлов не доехали на диск и потеряны. + +Запрос `ALTER` на изменение столбцов реплицируется. Соответствующие инструкции сохраняются в ZooKeeper, и затем каждая реплика их применяет. Все запросы `ALTER` выполняются в одном и том же порядке. Запрос ждёт выполнения соответствующих действий на всех репликах. Но при этом, запрос на изменение столбцов в реплицируемой таблице можно прервать, и все действия будут осуществлены асинхронно. + +## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} + +Запрос `ALTER` позволяет создавать и удалять отдельные элементы (столбцы) вложенных структур данных, но не вложенные структуры данных целиком. Для добавления вложенной структуры данных, вы можете добавить столбцы с именем вида `name.nested_name` и типом `Array(T)` - вложенная структура данных полностью эквивалентна нескольким столбцам-массивам с именем, имеющим одинаковый префикс до точки. + +Отсутствует возможность удалять столбцы, входящие в первичный ключ или ключ для сэмплирования (в общем, входящие в выражение `ENGINE`). Изменение типа у столбцов, входящих в первичный ключ возможно только в том случае, если это изменение не приводит к изменению данных (например, разрешено добавление значения в Enum или изменение типа с `DateTime` на `UInt32`). + +Если возможностей запроса `ALTER` не хватает для нужного изменения таблицы, вы можете создать новую таблицу, скопировать туда данные с помощью запроса [INSERT SELECT](../insert-into.md#insert_query_insert-select), затем поменять таблицы местами с помощью запроса [RENAME](../misc.md#misc_operations-rename), и удалить старую таблицу. В качестве альтернативы для запроса `INSERT SELECT`, можно использовать инструмент [clickhouse-copier](../../../sql-reference/statements/alter/index.md). + +Запрос `ALTER` блокирует все чтения и записи для таблицы. То есть, если на момент запроса `ALTER`, выполнялся долгий `SELECT`, то запрос `ALTER` сначала дождётся его выполнения. И в это время, все новые запросы к той же таблице, будут ждать, пока завершится этот `ALTER`. + +Для таблиц, которые не хранят данные самостоятельно (типа [Merge](../../../sql-reference/statements/alter/index.md) и [Distributed](../../../sql-reference/statements/alter/index.md)), `ALTER` всего лишь меняет структуру таблицы, но не меняет структуру подчинённых таблиц. Для примера, при ALTER-е таблицы типа `Distributed`, вам также потребуется выполнить запрос `ALTER` для таблиц на всех удалённых серверах. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/column/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/constraint.md b/docs/ru/sql-reference/statements/alter/constraint.md new file mode 100644 index 00000000000..bacdff9ff57 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/constraint.md @@ -0,0 +1,23 @@ +--- +toc_priority: 43 +toc_title: CONSTRAINT +--- + +# Манипуляции с ограничениями (constraints) {#manipuliatsii-s-ogranicheniiami-constraints} + +Про ограничения подробнее написано [тут](../create/table.md#constraints). + +Добавить или удалить ограничение можно с помощью запросов + +``` sql +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +Запросы выполняют добавление или удаление метаданных об ограничениях таблицы `[db].name`, поэтому выполняются мгновенно. + +Если ограничение появилось для непустой таблицы, то *проверка ограничения для имеющихся данных не производится*. + +Запрос на изменение ограничений для Replicated таблиц реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/constraint/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/delete.md b/docs/ru/sql-reference/statements/alter/delete.md new file mode 100644 index 00000000000..29e1ae564d2 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/delete.md @@ -0,0 +1,29 @@ +--- +toc_priority: 39 +toc_title: DELETE +--- + +# ALTER TABLE … DELETE {#alter-mutations} + +``` sql +ALTER TABLE [db.]table [ON CLUSTER cluster] DELETE WHERE filter_expr +``` + +Позволяет удалить данные, соответствующие указанному выражению фильтрации. Реализовано как [мутация](../../../sql-reference/statements/alter/index.md#mutations). + +!!! note "Note" + Префикс `ALTER TABLE` делает этот синтаксис отличным от большинства других систем, поддерживающих SQL. Он предназначен для обозначения того, что в отличие от аналогичных запросов в базах данных OLTP это тяжелая операция, не предназначенная для частого использования. + +Выражение `filter_expr` должно иметь тип `UInt8`. Запрос удаляет строки в таблице, для которых это выражение принимает ненулевое значение. + +Один запрос может содержать несколько команд, разделенных запятыми. + +Синхронность обработки запроса определяется параметром [mutations_sync](../../../operations/settings/settings.md#mutations_sync). По умолчанию он является асинхронным. + +**Смотрите также** + +- [Мутации](../../../sql-reference/statements/alter/index.md#mutations) +- [Синхронность запросов ALTER](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) +- [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/delete/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/index.md b/docs/ru/sql-reference/statements/alter/index.md new file mode 100644 index 00000000000..035be934eb4 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/index.md @@ -0,0 +1,72 @@ +--- +toc_priority: 36 +toc_title: ALTER +--- + +## ALTER {#query_language_queries_alter} + +Изменение структуры таблицы. + +``` sql +ALTER TABLE [db].name [ON CLUSTER cluster] ADD|DROP|CLEAR|COMMENT|MODIFY COLUMN ... +``` + +В запросе указывается список из одного или более действий через запятую. +Каждое действие — операция над столбцом. + +Большинство запросов `ALTER` изменяют настройки таблицы или данные: + +- [COLUMN](../../../sql-reference/statements/alter/column.md) +- [PARTITION](../../../sql-reference/statements/alter/partition.md) +- [DELETE](../../../sql-reference/statements/alter/delete.md) +- [UPDATE](../../../sql-reference/statements/alter/update.md) +- [ORDER BY](../../../sql-reference/statements/alter/order-by.md) +- [INDEX](../../../sql-reference/statements/alter/index/index.md) +- [CONSTRAINT](../../../sql-reference/statements/alter/constraint.md) +- [TTL](../../../sql-reference/statements/alter/ttl.md) + +!!! note "Note" + Запрос `ALTER` поддерживается только для таблиц типа `*MergeTree`, а также `Merge` и `Distributed`. Запрос имеет несколько вариантов. + +Следующие запросы `ALTER` изменяют сущности, связанные с управлением доступом на основе ролей: + +- [USER](../../../sql-reference/statements/alter/user.md) +- [ROLE](../../../sql-reference/statements/alter/role.md) +- [QUOTA](../../../sql-reference/statements/alter/quota.md) +- [ROW POLICY](../../../sql-reference/statements/alter/row-policy.md) +- [SETTINGS PROFILE](../../../sql-reference/statements/alter/settings-profile.md) + +### Мутации {#mutations} + +Мутации - разновидность запроса ALTER, позволяющая изменять или удалять данные в таблице. В отличие от стандартных запросов [ALTER TABLE … DELETE](../../../sql-reference/statements/alter/delete.md) и [ALTER TABLE … UPDATE](../../../sql-reference/statements/alter/update.md), рассчитанных на точечное изменение данных, область применения мутаций - достаточно тяжёлые изменения, затрагивающие много строк в таблице. Поддержана для движков таблиц семейства [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md), в том числе для движков с репликацией. + +Конвертировать существующие таблицы для работы с мутациями не нужно. Но после применения первой мутации формат данных таблицы становится несовместимым с предыдущими версиями и откатиться на предыдущую версию уже не получится. + +На данный момент доступны команды: + +``` sql +ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name +``` + +Команда перестроит вторичный индекс `name` для партиции `partition_name`. + +В одном запросе можно указать несколько команд через запятую. + +Для `*MergeTree`-таблиц мутации выполняются, перезаписывая данные по кускам (parts). При этом атомарности нет — куски заменяются на помутированные по мере выполнения и запрос `SELECT`, заданный во время выполнения мутации, увидит данные как из измененных кусков, так и из кусков, которые еще не были изменены. + +Мутации линейно упорядочены между собой и накладываются на каждый кусок в порядке добавления. Мутации также упорядочены со вставками - гарантируется, что данные, вставленные в таблицу до начала выполнения запроса мутации, будут изменены, а данные, вставленные после окончания запроса мутации, изменены не будут. При этом мутации никак не блокируют вставки. + +Запрос завершается немедленно после добавления информации о мутации (для реплицированных таблиц - в ZooKeeper, для нереплицированных - на файловую систему). Сама мутация выполняется асинхронно, используя настройки системного профиля. Следить за ходом её выполнения можно по таблице [`system.mutations`](../../../operations/system-tables/mutations.md#system_tables-mutations). Добавленные мутации будут выполняться до конца даже в случае перезапуска серверов ClickHouse. Откатить мутацию после её добавления нельзя, но если мутация по какой-то причине не может выполниться до конца, её можно остановить с помощью запроса [`KILL MUTATION`](../../../sql-reference/statements/kill.md#kill-mutation). + +Записи о последних выполненных мутациях удаляются не сразу (количество сохраняемых мутаций определяется параметром движка таблиц `finished_mutations_to_keep`). Более старые записи удаляются. + +### Синхронность запросов ALTER {#synchronicity-of-alter-queries} + +Для нереплицируемых таблиц, все запросы `ALTER` выполняются синхронно. Для реплицируемых таблиц, запрос всего лишь добавляет инструкцию по соответствующим действиям в `ZooKeeper`, а сами действия осуществляются при первой возможности. Но при этом, запрос может ждать завершения выполнения этих действий на всех репликах. + +Для запросов `ALTER ... ATTACH|DETACH|DROP` можно настроить ожидание, с помощью настройки `replication_alter_partitions_sync`. +Возможные значения: `0` - не ждать, `1` - ждать выполнения только у себя (по умолчанию), `2` - ждать всех. + +Для запросов `ALTER TABLE ... UPDATE|DELETE` синхронность выполнения определяется настройкой [mutations_sync](../../../operations/settings/settings.md#mutations_sync). + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/index/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/index/index.md b/docs/ru/sql-reference/statements/alter/index/index.md new file mode 100644 index 00000000000..863f8b875dd --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/index/index.md @@ -0,0 +1,24 @@ +--- +toc_hidden_folder: true +toc_priority: 42 +toc_title: INDEX +--- + +# Манипуляции с индексами {#manipuliatsii-s-indeksami} + +Добавить или удалить индекс можно с помощью операций + +``` sql +ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value [AFTER name] +ALTER TABLE [db].name DROP INDEX name +``` + +Поддерживается только таблицами семейства `*MergeTree`. + +Команда `ADD INDEX` добавляет описание индексов в метаданные, а `DROP INDEX` удаляет индекс из метаданных и стирает файлы индекса с диска, поэтому они легковесные и работают мгновенно. + +Если индекс появился в метаданных, то он начнет считаться в последующих слияниях и записях в таблицу, а не сразу после выполнения операции `ALTER`. + +Запрос на изменение индексов реплицируется, сохраняя новые метаданные в ZooKeeper и применяя изменения на всех репликах. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/index/index/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/order-by.md b/docs/ru/sql-reference/statements/alter/order-by.md new file mode 100644 index 00000000000..32c0e382445 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/order-by.md @@ -0,0 +1,22 @@ +--- +toc_priority: 41 +toc_title: ORDER BY +--- + +# Манипуляции с ключевыми выражениями таблиц {#manipuliatsii-s-kliuchevymi-vyrazheniiami-tablits} + +Поддерживается операция: + +``` sql +MODIFY ORDER BY new_expression +``` + +Работает только для таблиц семейства [`MergeTree`](../../../sql-reference/statements/alter/index.md) (в том числе [реплицированных](../../../sql-reference/statements/alter/index.md)). После выполнения запроса +[ключ сортировки](../../../sql-reference/statements/alter/index.md) таблицы +заменяется на `new_expression` (выражение или кортеж выражений). Первичный ключ при этом остаётся прежним. + +Операция затрагивает только метаданные. Чтобы сохранить свойство упорядоченности кусков данных по ключу +сортировки, разрешено добавлять в ключ только новые столбцы (т.е. столбцы, добавляемые командой `ADD COLUMN` +в том же запросе `ALTER`), у которых нет выражения по умолчанию. + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/order-by/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md new file mode 100644 index 00000000000..5c4a23428ad --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/partition.md @@ -0,0 +1,259 @@ +--- +toc_priority: 38 +toc_title: PARTITION +--- + +# Манипуляции с партициями и кусками {#alter_manipulations-with-partitions} + +Для работы с [партициями](../../../engines/table-engines/mergetree-family/custom-partitioning-key.md) доступны следующие операции: + +- [DETACH PARTITION](#alter_detach-partition) — перенести партицию в директорию `detached`; +- [DROP PARTITION](#alter_drop-partition) — удалить партицию; +- [ATTACH PARTITION\|PART](#alter_attach-partition) — добавить партицию/кусок в таблицу из директории `detached`; +- [ATTACH PARTITION FROM](#alter_attach-partition-from) — скопировать партицию из другой таблицы; +- [REPLACE PARTITION](#alter_replace-partition) — скопировать партицию из другой таблицы с заменой; +- [MOVE PARTITION TO TABLE](#alter_move_to_table-partition) — переместить партицию в другую таблицу; +- [CLEAR COLUMN IN PARTITION](#alter_clear-column-partition) — удалить все значения в столбце для заданной партиции; +- [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — очистить построенные вторичные индексы для заданной партиции; +- [FREEZE PARTITION](#alter_freeze-partition) — создать резервную копию партиции; +- [FETCH PARTITION](#alter_fetch-partition) — скачать партицию с другого сервера; +- [MOVE PARTITION\|PART](#alter_move-partition) — переместить партицию/кускок на другой диск или том. + +## DETACH PARTITION {#alter_detach-partition} + +``` sql +ALTER TABLE table_name DETACH PARTITION partition_expr +``` + +Перемещает заданную партицию в директорию `detached`. Сервер не будет знать об этой партиции до тех пор, пока вы не выполните запрос [ATTACH](#alter_attach-partition). + +Пример: + +``` sql +ALTER TABLE visits DETACH PARTITION 201901 +``` + +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +После того как запрос будет выполнен, вы сможете производить любые операции с данными в директории `detached`. Например, можно удалить их из файловой системы. + +Запрос реплицируется — данные будут перенесены в директорию `detached` и забыты на всех репликах. Обратите внимание, запрос может быть отправлен только на реплику-лидер. Чтобы узнать, является ли реплика лидером, выполните запрос `SELECT` к системной таблице [system.replicas](../../../operations/system-tables/replicas.md#system_tables-replicas). Либо можно выполнить запрос `DETACH` на всех репликах — тогда на всех репликах, кроме реплики-лидера, запрос вернет ошибку. + +## DROP PARTITION {#alter_drop-partition} + +``` sql +ALTER TABLE table_name DROP PARTITION partition_expr +``` + +Удаляет партицию. Партиция помечается как неактивная и будет полностью удалена примерно через 10 минут. + +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +Запрос реплицируется — данные будут удалены на всех репликах. + +## DROP DETACHED PARTITION\|PART {#alter_drop-detached} + +``` sql +ALTER TABLE table_name DROP DETACHED PARTITION|PART partition_expr +``` + +Удаляет из `detached` кусок или все куски, принадлежащие партиции. +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +## ATTACH PARTITION\|PART {#alter_attach-partition} + +``` sql +ALTER TABLE table_name ATTACH PARTITION|PART partition_expr +``` + +Добавляет данные в таблицу из директории `detached`. Можно добавить данные как для целой партиции, так и для отдельного куска. Примеры: + +``` sql +ALTER TABLE visits ATTACH PARTITION 201901; +ALTER TABLE visits ATTACH PART 201901_2_2_0; +``` + +Как корректно задать имя партиции или куска, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +Этот запрос реплицируется. Реплика-иницатор проверяет, есть ли данные в директории `detached`. Если данные есть, то запрос проверяет их целостность. В случае успеха данные добавляются в таблицу. Все остальные реплики загружают данные с реплики-инициатора запроса. + +Это означает, что вы можете разместить данные в директории `detached` на одной реплике и с помощью запроса `ALTER ... ATTACH` добавить их в таблицу на всех репликах. + +## ATTACH PARTITION FROM {#alter_attach-partition-from} + +``` sql +ALTER TABLE table2 ATTACH PARTITION partition_expr FROM table1 +``` + +Копирует партицию из таблицы `table1` в таблицу `table2` и добавляет к существующим данным `table2`. Данные из `table1` не удаляются. + +Следует иметь в виду: + +- Таблицы должны иметь одинаковую структуру. +- Для таблиц должен быть задан одинаковый ключ партиционирования. + +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +## REPLACE PARTITION {#alter_replace-partition} + +``` sql +ALTER TABLE table2 REPLACE PARTITION partition_expr FROM table1 +``` + +Копирует партицию из таблицы `table1` в таблицу `table2` с заменой существующих данных в `table2`. Данные из `table1` не удаляются. + +Следует иметь в виду: + +- Таблицы должны иметь одинаковую структуру. +- Для таблиц должен быть задан одинаковый ключ партиционирования. + +Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +## MOVE PARTITION TO TABLE {#alter_move_to_table-partition} + +``` sql +ALTER TABLE table_source MOVE PARTITION partition_expr TO TABLE table_dest +``` + +Перемещает партицию из таблицы `table_source` в таблицу `table_dest` (добавляет к существующим данным в `table_dest`) с удалением данных из таблицы `table_source`. + +Следует иметь в виду: + +- Таблицы должны иметь одинаковую структуру. +- Для таблиц должен быть задан одинаковый ключ партиционирования. +- Движки таблиц должны быть одинакового семейства (реплицированные или нереплицированные). +- Для таблиц должна быть задана одинаковая политика хранения. + +## CLEAR COLUMN IN PARTITION {#alter_clear-column-partition} + +``` sql +ALTER TABLE table_name CLEAR COLUMN column_name IN PARTITION partition_expr +``` + +Сбрасывает все значения в столбце для заданной партиции. Если для столбца определено значение по умолчанию (в секции `DEFAULT`), то будет выставлено это значение. + +Пример: + +``` sql +ALTER TABLE visits CLEAR COLUMN hour in PARTITION 201902 +``` + +## CLEAR INDEX IN PARTITION {#alter_clear-index-partition} + +``` sql +ALTER TABLE table_name CLEAR INDEX index_name IN PARTITION partition_expr +``` + +Работает как `CLEAR COLUMN`, но сбрасывает индексы вместо данных в столбцах. + +## FREEZE PARTITION {#alter_freeze-partition} + +``` sql +ALTER TABLE table_name FREEZE [PARTITION partition_expr] +``` + +Создаёт резервную копию для заданной партиции. Если выражение `PARTITION` опущено, резервные копии будут созданы для всех партиций. + +!!! note "Примечание" + Создание резервной копии не требует остановки сервера. + +Для таблиц старого стиля имя партиций можно задавать в виде префикса (например, ‘2019’). В этом случае резервные копии будут созданы для всех соответствующих партиций. Подробнее о том, как корректно задать имя партиции, см. в разделе [Как задавать имя партиции в запросах ALTER](#alter-how-to-specify-part-expr). + +Запрос делает следующее — для текущего состояния таблицы он формирует жесткие ссылки на данные в этой таблице. Ссылки размещаются в директории `/var/lib/clickhouse/shadow/N/...`, где: + +- `/var/lib/clickhouse/` — рабочая директория ClickHouse, заданная в конфигурационном файле; +- `N` — инкрементальный номер резервной копии. + +!!! note "Примечание" + При использовании [нескольких дисков для хранения данных таблицы](../../statements/alter/index.md#table_engine-mergetree-multiple-volumes) директория `shadow/N` появляется на каждом из дисков, на которых были куски, попавшие под выражение `PARTITION`. + +Структура директорий внутри резервной копии такая же, как внутри `/var/lib/clickhouse/`. Запрос выполнит ‘chmod’ для всех файлов, запрещая запись в них. + +Обратите внимание, запрос `ALTER TABLE t FREEZE PARTITION` не реплицируется. Он создает резервную копию только на локальном сервере. После создания резервной копии данные из `/var/lib/clickhouse/shadow/` можно скопировать на удалённый сервер, а локальную копию удалить. + +Резервная копия создается почти мгновенно (однако сначала запрос дожидается завершения всех запросов, которые выполняются для соответствующей таблицы). + +`ALTER TABLE t FREEZE PARTITION` копирует только данные, но не метаданные таблицы. Чтобы сделать резервную копию метаданных таблицы, скопируйте файл `/var/lib/clickhouse/metadata/database/table.sql` + +Чтобы восстановить данные из резервной копии, выполните следующее: + +1. Создайте таблицу, если она ещё не существует. Запрос на создание можно взять из .sql файла (замените в нём `ATTACH` на `CREATE`). +2. Скопируйте данные из директории `data/database/table/` внутри резервной копии в директорию `/var/lib/clickhouse/data/database/table/detached/`. +3. С помощью запросов `ALTER TABLE t ATTACH PARTITION` добавьте данные в таблицу. + +Восстановление данных из резервной копии не требует остановки сервера. + +Подробнее о резервном копировании и восстановлении данных читайте в разделе [Резервное копирование данных](../../../operations/backup.md). + +## FETCH PARTITION {#alter_fetch-partition} + +``` sql +ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' +``` + +Загружает партицию с другого сервера. Этот запрос работает только для реплицированных таблиц. + +Запрос выполняет следующее: + +1. Загружает партицию с указанного шарда. Путь к шарду задается в секции `FROM` (‘path-in-zookeeper’). Обратите внимание, нужно задавать путь к шарду в ZooKeeper. +2. Помещает загруженные данные в директорию `detached` таблицы `table_name`. Чтобы прикрепить эти данные к таблице, используйте запрос [ATTACH PARTITION\|PART](#alter_attach-partition). + +Например: + +``` sql +ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; +ALTER TABLE users ATTACH PARTITION 201902; +``` + +Следует иметь в виду: + +- Запрос `ALTER TABLE t FETCH PARTITION` не реплицируется. Он загружает партицию в директорию `detached` только на локальном сервере. +- Запрос `ALTER TABLE t ATTACH` реплицируется — он добавляет данные в таблицу сразу на всех репликах. На одной из реплик данные будут добавлены из директории `detached`, а на других — из соседних реплик. + +Перед загрузкой данных система проверяет, существует ли партиция и совпадает ли её структура со структурой таблицы. При этом автоматически выбирается наиболее актуальная реплика среди всех живых реплик. + +Несмотря на то что запрос называется `ALTER TABLE`, он не изменяет структуру таблицы и не изменяет сразу доступные данные в таблице. + +## MOVE PARTITION\|PART {#alter_move-partition} + +Перемещает партицию или кусок данных на другой том или диск для таблиц с движком `MergeTree`. Смотрите [Хранение данных таблицы на нескольких блочных устройствах](../../statements/alter/index.md#table_engine-mergetree-multiple-volumes). + +``` sql +ALTER TABLE table_name MOVE PARTITION|PART partition_expr TO DISK|VOLUME 'disk_name' +``` + +Запрос `ALTER TABLE t MOVE`: + +- Не реплицируется, т.к. на разных репликах могут быть различные конфигурации политик хранения. +- Возвращает ошибку, если указан несконфигурированный том или диск. Ошибка также возвращается в случае невыполнения условий перемещения данных, которые указаны в конфигурации политики хранения. +- Может возвращать ошибку в случае, когда перемещаемые данные уже оказались перемещены в результате фонового процесса, конкурентного запроса `ALTER TABLE t MOVE` или как часть результата фоновой операции слияния. В данном случае никаких дополнительных действий от пользователя не требуется. + +Примеры: + +``` sql +ALTER TABLE hits MOVE PART '20190301_14343_16206_438' TO VOLUME 'slow' +ALTER TABLE hits MOVE PARTITION '2019-09-01' TO DISK 'fast_ssd' +``` + +## Как задавать имя партиции в запросах ALTER {#alter-how-to-specify-part-expr} + +Чтобы задать нужную партицию в запросах `ALTER ... PARTITION`, можно использовать: + +- Имя партиции. Посмотреть имя партиции можно в столбце `partition` системной таблицы [system.parts](../../../operations/system-tables/parts.md#system_tables-parts). Например, `ALTER TABLE visits DETACH PARTITION 201901`. +- Произвольное выражение из столбцов исходной таблицы. Также поддерживаются константы и константные выражения. Например, `ALTER TABLE visits DETACH PARTITION toYYYYMM(toDate('2019-01-25'))`. +- Строковый идентификатор партиции. Идентификатор партиции используется для именования кусков партиции на файловой системе и в ZooKeeper. В запросах `ALTER` идентификатор партиции нужно указывать в секции `PARTITION ID`, в одинарных кавычках. Например, `ALTER TABLE visits DETACH PARTITION ID '201901'`. +- Для запросов [ATTACH PART](#alter_attach-partition) и [DROP DETACHED PART](#alter_drop-detached): чтобы задать имя куска партиции, используйте строковой литерал со значением из столбца `name` системной таблицы [system.detached_parts](../../../operations/system-tables/detached_parts.md#system_tables-detached_parts). Например, `ALTER TABLE visits ATTACH PART '201901_1_1_0'`. + +Использование кавычек в имени партиций зависит от типа данных столбца, по которому задано партиционирование. Например, для столбца с типом `String` имя партиции необходимо указывать в кавычках (одинарных). Для типов `Date` и `Int*` кавычки указывать не нужно. + +Замечание: для таблиц старого стиля партицию можно указывать и как число `201901`, и как строку `'201901'`. Синтаксис для таблиц нового типа более строг к типам (аналогично парсеру входного формата VALUES). + +Правила, сформулированные выше, актуальны также для запросов [OPTIMIZE](../../../sql-reference/statements/optimize.md). Чтобы указать единственную партицию непартиционированной таблицы, укажите `PARTITION tuple()`. Например: + +``` sql +OPTIMIZE TABLE table_not_partitioned PARTITION tuple() FINAL; +``` + +Примеры запросов `ALTER ... PARTITION` можно посмотреть в тестах: [`00502_custom_partitioning_local`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_local.sql) и [`00502_custom_partitioning_replicated_zookeeper`](https://github.com/ClickHouse/ClickHouse/blob/master/tests/queries/0_stateless/00502_custom_partitioning_replicated_zookeeper.sql). + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/partition/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/quota.md b/docs/ru/sql-reference/statements/alter/quota.md new file mode 100644 index 00000000000..707f56e7cd4 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/quota.md @@ -0,0 +1,22 @@ +--- +toc_priority: 46 +toc_title: QUOTA +--- + +# ALTER QUOTA {#alter-quota-statement} + +Изменяет квоту. + +## Синтаксис {#alter-quota-syntax} + +``` sql +ALTER QUOTA [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [KEYED BY {'none' | 'user name' | 'ip address' | 'client key' | 'client key or user name' | 'client key or ip address'}] + [FOR [RANDOMIZED] INTERVAL number {SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR} + {MAX { {QUERIES | ERRORS | RESULT ROWS | RESULT BYTES | READ ROWS | READ BYTES | EXECUTION TIME} = number } [,...] | + NO LIMITS | TRACKING ONLY} [,...]] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/quota/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/role.md b/docs/ru/sql-reference/statements/alter/role.md new file mode 100644 index 00000000000..c56f34e016f --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/role.md @@ -0,0 +1,18 @@ +--- +toc_priority: 46 +toc_title: ROLE +--- + +# ALTER ROLE {#alter-role-statement} + +Изменяет роль. + +## Синтаксис {#alter-role-syntax} + +``` sql +ALTER ROLE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/role/) diff --git a/docs/ru/sql-reference/statements/alter/row-policy.md b/docs/ru/sql-reference/statements/alter/row-policy.md new file mode 100644 index 00000000000..0a7931d23de --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/row-policy.md @@ -0,0 +1,21 @@ +--- +toc_priority: 47 +toc_title: ROW POLICY +--- + +# ALTER ROW POLICY {#alter-row-policy-statement} + +Изменяет политику доступа к строкам. + +## Синтаксис {#alter-row-policy-syntax} + +``` sql +ALTER [ROW] POLICY [IF EXISTS] name [ON CLUSTER cluster_name] ON [database.]table + [RENAME TO new_name] + [AS {PERMISSIVE | RESTRICTIVE}] + [FOR SELECT] + [USING {condition | NONE}][,...] + [TO {role [,...] | ALL | ALL EXCEPT role [,...]}] +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/row-policy/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/settings-profile.md b/docs/ru/sql-reference/statements/alter/settings-profile.md new file mode 100644 index 00000000000..7a39a0efae2 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/settings-profile.md @@ -0,0 +1,18 @@ +--- +toc_priority: 48 +toc_title: SETTINGS PROFILE +--- + +# ALTER SETTINGS PROFILE {#alter-settings-profile-statement} + +Изменяет профили настроек. + +## Синтаксис {#alter-settings-profile-syntax} + +``` sql +ALTER SETTINGS PROFILE [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | INHERIT 'profile_name'] [,...] +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/settings-profile) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/ttl.md b/docs/ru/sql-reference/statements/alter/ttl.md new file mode 100644 index 00000000000..5e5f47c22e3 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/ttl.md @@ -0,0 +1,14 @@ +--- +toc_priority: 44 +toc_title: TTL +--- + +# Манипуляции с TTL таблицы {#manipuliatsii-s-ttl-tablitsy} + +Вы можете изменить [TTL для таблицы](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-column-ttl) запросом следующего вида: + +``` sql +ALTER TABLE table-name MODIFY TTL ttl-expression +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/ttl/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/update.md b/docs/ru/sql-reference/statements/alter/update.md new file mode 100644 index 00000000000..f497b2c4511 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/update.md @@ -0,0 +1,29 @@ +--- +toc_priority: 40 +toc_title: UPDATE +--- + +# ALTER TABLE … UPDATE {#alter-table-update-statements} + +``` sql +ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr +``` + +Позволяет манипулировать данными, соответствующими заданному выражению фильтрации. Реализовано как [мутация](../../../sql-reference/statements/alter/index.md#mutations). + +!!! note "Note" + Префикс `ALTER TABLE` делает этот синтаксис отличным от большинства других систем, поддерживающих SQL. Он предназначен для обозначения того, что в отличие от аналогичных запросов в базах данных OLTP это тяжелая операция, не предназначенная для частого использования. + +Выражение `filter_expr` должно иметь тип `UInt8`. Запрос изменяет значение указанных столбцов на вычисленное значение соответствующих выражений в каждой строке, для которой `filter_expr` принимает ненулевое значение. Вычисленные значения преобразуются к типу столбца с помощью оператора `CAST`. Изменение столбцов, которые используются при вычислении первичного ключа или ключа партиционирования, не поддерживается. + +Один запрос может содержать несколько команд, разделенных запятыми. + +Синхронность обработки запроса определяется параметром [mutations_sync](../../../operations/settings/settings.md#mutations_sync). По умолчанию он является асинхронным. + +**Смотрите также** + +- [Мутации](../../../sql-reference/statements/alter/index.md#mutations) +- [Синхронность запросов ALTER](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) +- [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/update/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/alter/user.md b/docs/ru/sql-reference/statements/alter/user.md new file mode 100644 index 00000000000..0e8fda23380 --- /dev/null +++ b/docs/ru/sql-reference/statements/alter/user.md @@ -0,0 +1,49 @@ +--- +toc_priority: 45 +toc_title: USER +--- + +# ALTER USER {#alter-user-statement} + +Изменяет аккаунт пользователя ClickHouse. + +## Синтаксис {#alter-user-syntax} + +``` sql +ALTER USER [IF EXISTS] name [ON CLUSTER cluster_name] + [RENAME TO new_name] + [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] + [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] + [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] +``` + +## Описание {#alter-user-dscr} + +Для выполнения `ALTER USER` необходима привилегия [ALTER USER](../grant.md#grant-access-management). + +## Примеры {#alter-user-examples} + +Установить ролями по умолчанию роли, назначенные пользователю: + +``` sql +ALTER USER user DEFAULT ROLE role1, role2 +``` + +Если роли не были назначены пользователю, ClickHouse выбрасывает исключение. + +Установить ролями по умолчанию все роли, назначенные пользователю: + +``` sql +ALTER USER user DEFAULT ROLE ALL +``` + +Если роль будет впоследствии назначена пользователю, она автоматически станет ролью по умолчанию. + +Установить ролями по умолчанию все назначенные пользователю роли кроме `role1` и `role2`: + +``` sql +ALTER USER user DEFAULT ROLE ALL EXCEPT role1, role2 +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/alter/user/) \ No newline at end of file diff --git a/docs/ru/sql-reference/statements/create/role.md b/docs/ru/sql-reference/statements/create/role.md index e8374320546..b8c0fc2b453 100644 --- a/docs/ru/sql-reference/statements/create/role.md +++ b/docs/ru/sql-reference/statements/create/role.md @@ -20,7 +20,7 @@ CREATE ROLE [IF NOT EXISTS | OR REPLACE] name Одному пользователю можно назначить несколько ролей. Пользователи могут применять назначенные роли в произвольных комбинациях с помощью выражения [SET ROLE](../misc.md#set-role-statement). Конечный объем привилегий — это комбинация всех привилегий всех примененных ролей. Если у пользователя имеются привилегии, присвоенные его аккаунту напрямую, они также прибавляются к привилегиям, присвоенным через роли. -Роли по умолчанию применяются при входе пользователя в систему. Установить роли по умолчанию можно с помощью выражений [SET DEFAULT ROLE](../misc.md#set-default-role-statement) или [ALTER USER](../alter.md#alter-user-statement). +Роли по умолчанию применяются при входе пользователя в систему. Установить роли по умолчанию можно с помощью выражений [SET DEFAULT ROLE](../misc.md#set-default-role-statement) или [ALTER USER](../alter/index.md#alter-user-statement). Для отзыва роли используется выражение [REVOKE](../../../sql-reference/statements/revoke.md). diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 043df0f7397..8eea84ac594 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -249,7 +249,7 @@ GRANT INSERT(x,y) ON db.table TO john ### ALTER {#grant-alter} -Разрешает выполнять запросы [ALTER](alter.md) в соответствии со следующей иерархией привилегий: +Разрешает выполнять запросы [ALTER](alter/index.md) в соответствии со следующей иерархией привилегий: - `ALTER`. Уровень: `COLUMN`. - `ALTER TABLE`. Уровень: `GROUP` diff --git a/docs/ru/sql-reference/statements/kill.md b/docs/ru/sql-reference/statements/kill.md index 4fcfa9a61ce..dd2f24ffa27 100644 --- a/docs/ru/sql-reference/statements/kill.md +++ b/docs/ru/sql-reference/statements/kill.md @@ -51,7 +51,7 @@ KILL MUTATION [ON CLUSTER cluster] [FORMAT format] ``` -Пытается остановить выполняющиеся в данные момент [мутации](alter.md#mutations). Мутации для остановки выбираются из таблицы [`system.mutations`](../../operations/system-tables/mutations.md#system_tables-mutations) с помощью условия, указанного в секции `WHERE` запроса `KILL`. +Пытается остановить выполняющиеся в данные момент [мутации](alter/index.md#mutations). Мутации для остановки выбираются из таблицы [`system.mutations`](../../operations/system-tables/mutations.md#system_tables-mutations) с помощью условия, указанного в секции `WHERE` запроса `KILL`. Тестовый вариант запроса (`TEST`) только проверяет права пользователя и выводит список запросов для остановки. diff --git a/docs/ru/sql-reference/statements/optimize.md b/docs/ru/sql-reference/statements/optimize.md index c150f02c3d6..d94ba2aa5da 100644 --- a/docs/ru/sql-reference/statements/optimize.md +++ b/docs/ru/sql-reference/statements/optimize.md @@ -14,7 +14,7 @@ OPTIMIZE TABLE [db.]name [ON CLUSTER cluster] [PARTITION partition | PARTITION I Если `OPTIMIZE` применяется к таблицам семейства [ReplicatedMergeTree](../../engines/table-engines/mergetree-family/replication.md), ClickHouse создаёт задачу на мёрж и ожидает её исполнения на всех узлах (если активирована настройка `replication_alter_partitions_sync`). - Если `OPTIMIZE` не выполняет мёрж по любой причине, ClickHouse не оповещает об этом клиента. Чтобы включить оповещения, используйте настройку [optimize\_throw\_if\_noop](../../operations/settings/settings.md#setting-optimize_throw_if_noop). -- Если указать `PARTITION`, то оптимизация выполняется только для указанной партиции. [Как задавать имя партиции в запросах](alter.md#alter-how-to-specify-part-expr). +- Если указать `PARTITION`, то оптимизация выполняется только для указанной партиции. [Как задавать имя партиции в запросах](alter/index.md#alter-how-to-specify-part-expr). - Если указать `FINAL`, то оптимизация выполняется даже в том случае, если все данные уже лежат в одном куске. - Если указать `DEDUPLICATE`, то произойдет схлопывание полностью одинаковых строк (сравниваются значения во всех колонках), имеет смысл только для движка MergeTree. From f95acff93cb17afda047f6fe2d40ebb59e3fb2f8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 25 Aug 2020 19:00:00 +0300 Subject: [PATCH 32/89] Update 01456_modify_column_type_via_add_drop_update.sql --- .../01456_modify_column_type_via_add_drop_update.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql b/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql index e512cd960c5..a2e4804f12e 100644 --- a/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql +++ b/tests/queries/0_stateless/01456_modify_column_type_via_add_drop_update.sql @@ -2,6 +2,8 @@ DROP TABLE IF EXISTS tbl; CREATE TABLE tbl(a String, b UInt32, c Float64, d Int64, e UInt8) ENGINE=MergeTree ORDER BY tuple(); INSERT INTO tbl SELECT number, number * 2, number * 3, number * 4, number * 5 FROM system.numbers LIMIT 10; +SET mutations_sync = 1; + -- Change the types of columns by adding a temporary column and updating and dropping. -- Alters should be executed in sequential order. ALTER TABLE tbl ADD COLUMN xi Int64; From 4942d85090864f7f3f22ad04d7c290023dad7b64 Mon Sep 17 00:00:00 2001 From: Javi Santana Date: Tue, 25 Aug 2020 18:09:22 +0200 Subject: [PATCH 33/89] adds some info to the exception about zookeeper path --- src/Storages/StorageReplicatedMergeTree.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 59e3ca4cdc8..49ceebf226c 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3484,7 +3484,10 @@ std::optional StorageReplicatedMergeTree::totalBytes() const void StorageReplicatedMergeTree::assertNotReadonly() const { if (is_readonly) - throw Exception("Table is in readonly mode", ErrorCodes::TABLE_IS_READ_ONLY); + { + auto err = "Table is in readonly mode (zookeeper path: " + zookeeper_path + ")"; + throw Exception(err, ErrorCodes::TABLE_IS_READ_ONLY); + } } From 737b255aaeeb97e859d45e5bfb66bd699ec98345 Mon Sep 17 00:00:00 2001 From: Artem Zuikov Date: Tue, 25 Aug 2020 19:32:58 +0300 Subject: [PATCH 34/89] ignore test in ya check --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 0c3ce88b439..aa1ecf03fde 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -135,3 +135,4 @@ 01391_join_on_dict_crash 01401_FORMAT_SETTINGS 01411_bayesian_ab_testing +01456_ast_optimizations_over_distributed From ad589c59e679c17a88379befad718fab2b966499 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 25 Aug 2020 19:58:02 +0300 Subject: [PATCH 35/89] Update PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3dffbd7ecfc..db923369296 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -24,3 +24,6 @@ Detailed description / Documentation draft: By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder. If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first. + + +Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/ From ea4aa3469788c5611ef54c09fff039280aae0f3a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 25 Aug 2020 20:09:35 +0300 Subject: [PATCH 36/89] Little bit more pretty --- src/Storages/StorageReplicatedMergeTree.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 49ceebf226c..7bffa267750 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -3484,10 +3484,7 @@ std::optional StorageReplicatedMergeTree::totalBytes() const void StorageReplicatedMergeTree::assertNotReadonly() const { if (is_readonly) - { - auto err = "Table is in readonly mode (zookeeper path: " + zookeeper_path + ")"; - throw Exception(err, ErrorCodes::TABLE_IS_READ_ONLY); - } + throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is in readonly mode (zookeeper path: {})", zookeeper_path); } From 906d94f72746ee9a7ad2200b6589b3e8b3d617a3 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 22:15:54 +0500 Subject: [PATCH 37/89] Yet another .net core client library Tested and used in production. Windows\Linux supported. Supported new advanced memory management dotnet core features like Span<> and Memory<> Almost all CH types supported Full support for Compression Nullable types fully tested and supported --- docs/en/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index 44186ec9135..aa13ebd436b 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -46,6 +46,7 @@ toc_title: Client Libraries - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient] (https://github.com/Octonica/ClickHouseClient) - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From d456aa86ef3864456df3af2df210cafdd56f1cf5 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 14 Aug 2020 18:44:04 +0300 Subject: [PATCH 38/89] Revert "Revert "cmake: Add option to fail configuration instead of auto-reconfiguration"" This reverts commit 89419ceb9bd60121a029430b26aa4506c517f01d (curl part moved to separate commit) Squashed: - termcap removed - fix for cassandra on apple merged - cmake: fix "Can't find system zlib library" in unbundled build - cmake: fix condition when testing for rdkafka platform - cmake: PROTOBUF_OLD_ABI_COMPAT shouldn't be enabled for internal protobuf Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 14 +- cmake/analysis.cmake | 4 +- cmake/contrib_finder.cmake | 4 +- cmake/find/amqpcpp.cmake | 25 +- cmake/find/avro.cmake | 18 +- cmake/find/base64.cmake | 26 +- cmake/find/brotli.cmake | 14 +- cmake/find/capnp.cmake | 26 +- cmake/find/cassandra.cmake | 46 +- cmake/find/ccache.cmake | 10 +- cmake/find/cxx.cmake | 71 ++- cmake/find/fastops.cmake | 27 +- cmake/find/gperf.cmake | 11 +- cmake/find/grpc.cmake | 50 +- cmake/find/gtest.cmake | 17 +- cmake/find/h3.cmake | 38 +- cmake/find/hdfs3.cmake | 22 +- cmake/find/icu.cmake | 13 +- cmake/find/ldap.cmake | 151 +++--- cmake/find/libgsasl.cmake | 17 +- cmake/find/libxml2.cmake | 5 + cmake/find/llvm.cmake | 139 +++--- cmake/find/ltdl.cmake | 2 +- cmake/find/msgpack.cmake | 32 +- cmake/find/mysqlclient.cmake | 93 ++-- cmake/find/opencl.cmake | 10 +- cmake/find/orc.cmake | 19 +- cmake/find/parquet.cmake | 40 +- cmake/find/protobuf.cmake | 13 +- cmake/find/rapidjson.cmake | 7 + cmake/find/rdkafka.cmake | 51 +- cmake/find/re2.cmake | 7 +- cmake/find/sentry.cmake | 5 + cmake/find/simdjson.cmake | 7 +- cmake/find/snappy.cmake | 26 +- cmake/find/sparsehash.cmake | 3 + cmake/find/ssl.cmake | 15 +- cmake/find/stats.cmake | 4 + cmake/find/zlib.cmake | 5 + cmake/find/zstd.cmake | 6 +- cmake/tools.cmake | 22 +- contrib/arrow-cmake/CMakeLists.txt | 2 +- contrib/boost-cmake/CMakeLists.txt | 82 ++-- contrib/cctz-cmake/CMakeLists.txt | 64 +-- contrib/hyperscan-cmake/CMakeLists.txt | 494 +++++++++---------- contrib/jemalloc-cmake/CMakeLists.txt | 293 ++++++------ contrib/libcpuid-cmake/CMakeLists.txt | 55 ++- contrib/libhdfs3-cmake/CMakeLists.txt | 14 +- contrib/lz4-cmake/CMakeLists.txt | 37 +- contrib/poco-cmake/CMakeLists.txt | 3 + contrib/replxx-cmake/CMakeLists.txt | 88 ++-- contrib/unixodbc-cmake/CMakeLists.txt | 628 +++++++++++++------------ 52 files changed, 1654 insertions(+), 1221 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ec168723b83..32edc1b7f9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,14 @@ endforeach() project(ClickHouse) +option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION + "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" OFF) +if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) + set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR) +else() + set(RECONFIGURE_MESSAGE_LEVEL STATUS) +endif() + include (cmake/arch.cmake) include (cmake/target.cmake) include (cmake/tools.cmake) @@ -57,7 +65,7 @@ if(ENABLE_IPO) message(STATUS "IPO/LTO is supported, enabling") set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) else() - message(STATUS "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>") + message (${RECONFIGURE_MESSAGE_LEVEL} "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>") endif() else() message(STATUS "IPO/LTO not enabled.") @@ -133,6 +141,8 @@ option (ENABLE_TESTS "Enables tests" ON) if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0") option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY." ON) +elseif(GLIBC_COMPATIBILITY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration") endif () if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0") @@ -263,6 +273,8 @@ if (COMPILER_CLANG) set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin") set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin") + elseif (ENABLE_THINLTO) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO") endif () # Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled diff --git a/cmake/analysis.cmake b/cmake/analysis.cmake index 287c36a8de7..daaa730ac4b 100644 --- a/cmake/analysis.cmake +++ b/cmake/analysis.cmake @@ -12,7 +12,9 @@ if (ENABLE_CLANG_TIDY) set (USE_CLANG_TIDY 1) # The variable CMAKE_CXX_CLANG_TIDY will be set inside src and base directories with non third-party code. # set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") + elseif (FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) + message(FATAL_ERROR "clang-tidy is not found") else () - message(STATUS "clang-tidy is not found. This is normal - the tool is used only for static code analysis and not essential for build.") + message(STATUS "clang-tidy is not found. This is normal - the tool is only used for static code analysis and isn't essential for the build.") endif () endif () diff --git a/cmake/contrib_finder.cmake b/cmake/contrib_finder.cmake index f9bf3ce2837..64c6d5f5c0a 100644 --- a/cmake/contrib_finder.cmake +++ b/cmake/contrib_finder.cmake @@ -8,6 +8,9 @@ macro(find_contrib_lib LIB_NAME) if (NOT USE_INTERNAL_${LIB_NAME_UC}_LIBRARY) find_package ("${LIB_NAME}") + if (NOT ${LIB_NAME_UC}_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use system ${LIB_NAME}") + endif() endif () if (NOT ${LIB_NAME_UC}_FOUND) @@ -17,5 +20,4 @@ macro(find_contrib_lib LIB_NAME) endif () message (STATUS "Using ${LIB_NAME}: ${${LIB_NAME_UC}_INCLUDE_DIR} : ${${LIB_NAME_UC}_LIBRARIES}") - endmacro() diff --git a/cmake/find/amqpcpp.cmake b/cmake/find/amqpcpp.cmake index b3e193c72ff..4191dce26bb 100644 --- a/cmake/find/amqpcpp.cmake +++ b/cmake/find/amqpcpp.cmake @@ -1,21 +1,22 @@ option(ENABLE_AMQPCPP "Enalbe AMQP-CPP" ${ENABLE_LIBRARIES}) +if (NOT ENABLE_AMQPCPP) + return() +endif() + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt") message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init --recursive") - set (ENABLE_AMQPCPP 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal AMQP-CPP library") + set (USE_AMQPCPP 0) + return() endif () -if (ENABLE_AMQPCPP) +set (USE_AMQPCPP 1) +set (AMQPCPP_LIBRARY AMQP-CPP) - set (USE_AMQPCPP 1) - set (AMQPCPP_LIBRARY AMQP-CPP) - - set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include") - - list (APPEND AMQPCPP_INCLUDE_DIR - "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include" - "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP") - -endif() +set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include") +list (APPEND AMQPCPP_INCLUDE_DIR + "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include" + "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP") message (STATUS "Using AMQP-CPP=${USE_AMQPCPP}: ${AMQPCPP_INCLUDE_DIR} : ${AMQPCPP_LIBRARY}") diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake index cdb3fc84d3d..a07e6353212 100644 --- a/cmake/find/avro.cmake +++ b/cmake/find/avro.cmake @@ -1,28 +1,34 @@ option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) -if (ENABLE_AVRO) +if (NOT ENABLE_AVRO) + if (USE_INTERNAL_AVRO_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal avro library with ENABLE_AVRO=OFF") + endif() + return() +endif() option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED}) -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") - if(USE_INTERNAL_AVRO_LIBRARY) +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") + if (USE_INTERNAL_AVRO_LIBRARY) message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal avro") + set(USE_INTERNAL_AVRO_LIBRARY 0) endif() set(MISSING_INTERNAL_AVRO_LIBRARY 1) - set(USE_INTERNAL_AVRO_LIBRARY 0) endif() if (NOT USE_INTERNAL_AVRO_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Using system avro library is not supported yet") elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY) include(cmake/find/snappy.cmake) set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") set(AVROCPP_LIBRARY avrocpp) + set(USE_INTERNAL_AVRO_LIBRARY 1) endif () if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) set(USE_AVRO 1) endif() -endif() - message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_INCLUDE_DIR} : ${AVROCPP_LIBRARY}") diff --git a/cmake/find/base64.cmake b/cmake/find/base64.cmake index f72397597d7..7427baf9cad 100644 --- a/cmake/find/base64.cmake +++ b/cmake/find/base64.cmake @@ -1,17 +1,21 @@ +option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_BASE64) + return() +endif() + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE") set (MISSING_INTERNAL_BASE64_LIBRARY 1) message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") endif () -if (NOT MISSING_INTERNAL_BASE64_LIBRARY) - option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES}) -endif () +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") + message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") +else() + set (BASE64_LIBRARY base64) + set (USE_BASE64 1) +endif() -if (ENABLE_BASE64) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") - message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive") - else() - set (BASE64_LIBRARY base64) - set (USE_BASE64 1) - endif() -endif () +if (NOT USE_BASE64) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable base64") +endif() diff --git a/cmake/find/brotli.cmake b/cmake/find/brotli.cmake index 5453cf15236..eca3df0d2c8 100644 --- a/cmake/find/brotli.cmake +++ b/cmake/find/brotli.cmake @@ -1,12 +1,18 @@ option (ENABLE_BROTLI "Enable brotli" ${ENABLE_LIBRARIES}) -if (ENABLE_BROTLI) +if (NOT ENABLE_BROTLI) + if (USE_INTERNAL_BROTLI_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal brotly library with ENABLE_BROTLI=OFF") + endif() + return() +endif() option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h") if (USE_INTERNAL_BROTLI_LIBRARY) message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal brotli") set (USE_INTERNAL_BROTLI_LIBRARY 0) endif () set (MISSING_INTERNAL_BROTLI_LIBRARY 1) @@ -19,18 +25,18 @@ if(NOT USE_INTERNAL_BROTLI_LIBRARY) find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h brotli/encode.h brotli/port.h brotli/types.h PATHS ${BROTLI_INCLUDE_PATHS}) if(BROTLI_LIBRARY_DEC AND BROTLI_LIBRARY_ENC AND BROTLI_LIBRARY_COMMON) set(BROTLI_LIBRARY ${BROTLI_LIBRARY_DEC} ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_COMMON}) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use system brotli") endif() endif() if (BROTLI_LIBRARY AND BROTLI_INCLUDE_DIR) set (USE_BROTLI 1) elseif (NOT MISSING_INTERNAL_BROTLI_LIBRARY) - set (BROTLI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include) + set (BROTLI_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include") set (USE_INTERNAL_BROTLI_LIBRARY 1) set (BROTLI_LIBRARY brotli) set (USE_BROTLI 1) endif () -endif() - message (STATUS "Using brotli=${USE_BROTLI}: ${BROTLI_INCLUDE_DIR} : ${BROTLI_LIBRARY}") diff --git a/cmake/find/capnp.cmake b/cmake/find/capnp.cmake index 0620a66808b..ee4735bd175 100644 --- a/cmake/find/capnp.cmake +++ b/cmake/find/capnp.cmake @@ -1,15 +1,21 @@ option (ENABLE_CAPNP "Enable Cap'n Proto" ${ENABLE_LIBRARIES}) -if (ENABLE_CAPNP) +if (NOT ENABLE_CAPNP) + if (USE_INTERNAL_CAPNP_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal capnproto library with ENABLE_CAPNP=OFF") + endif() + return() +endif() option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt") if(USE_INTERNAL_CAPNP_LIBRARY) message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive") + message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal capnproto") + set(USE_INTERNAL_CAPNP_LIBRARY 0) endif() set(MISSING_INTERNAL_CAPNP_LIBRARY 1) - set(USE_INTERNAL_CAPNP_LIBRARY 0) endif() # FIXME: refactor to use `add_library(… IMPORTED)` if possible. @@ -18,17 +24,21 @@ if (NOT USE_INTERNAL_CAPNP_LIBRARY) find_library (CAPNP capnp) find_library (CAPNPC capnpc) - set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) + if(KJ AND CAPNP AND CAPNPC) + set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system capnproto") + endif() +endif() + +if (CAPNP_LIBRARIES) + set (USE_CAPNP 1) elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY) add_subdirectory(contrib/capnproto-cmake) set (CAPNP_LIBRARIES capnpc) -endif () - -if (CAPNP_LIBRARIES) set (USE_CAPNP 1) -endif () - + set (USE_INTERNAL_CAPNP_LIBRARY 1) endif () message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_LIBRARIES}") diff --git a/cmake/find/cassandra.cmake b/cmake/find/cassandra.cmake index 6ff7697ea57..037d6c3f131 100644 --- a/cmake/find/cassandra.cmake +++ b/cmake/find/cassandra.cmake @@ -1,29 +1,33 @@ option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES}) -if (ENABLE_CASSANDRA) - if (APPLE) - SET(CMAKE_MACOSX_RPATH ON) - endif() +if (NOT ENABLE_CASSANDRA) + return() +endif() - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") - message (ERROR "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") - elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") - message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") + message (ERROR "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libuv needed for Cassandra") +elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") + message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal Cassandra") +else() + set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") + set (CASSANDRA_INCLUDE_DIR + "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") + if (MAKE_STATIC_LIBRARIES) + set (LIBUV_LIBRARY uv_a) + set (CASSANDRA_LIBRARY cassandra_static) else() - set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") - set (CASSANDRA_INCLUDE_DIR - "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") - if (USE_STATIC_LIBRARIES) - set (LIBUV_LIBRARY uv_a) - set (CASSANDRA_LIBRARY cassandra_static) - else() - set (LIBUV_LIBRARY uv) - set (CASSANDRA_LIBRARY cassandra) - endif() - set (USE_CASSANDRA 1) - set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") - + set (LIBUV_LIBRARY uv) + set (CASSANDRA_LIBRARY cassandra) endif() + + set (USE_CASSANDRA 1) + set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") endif() message (STATUS "Using cassandra=${USE_CASSANDRA}: ${CASSANDRA_INCLUDE_DIR} : ${CASSANDRA_LIBRARY}") diff --git a/cmake/find/ccache.cmake b/cmake/find/ccache.cmake index 95d6b208cfa..e20e6416e97 100644 --- a/cmake/find/ccache.cmake +++ b/cmake/find/ccache.cmake @@ -1,3 +1,9 @@ +option(ENABLE_CCACHE "Speedup re-compilations using ccache" ON) + +if (NOT ENABLE_CCACHE) + return() +endif() + find_program (CCACHE_FOUND ccache) if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache") @@ -9,6 +15,8 @@ if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CM set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND}) set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND}) else () - message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") + message(${RECONFIGURE_MESSAGE_LEVEL} "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118") endif () +elseif (NOT CCACHE_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use ccache") endif () diff --git a/cmake/find/cxx.cmake b/cmake/find/cxx.cmake index f6c999351e8..02f7113e6fb 100644 --- a/cmake/find/cxx.cmake +++ b/cmake/find/cxx.cmake @@ -1,41 +1,68 @@ -set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED}) +option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt") - message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") - set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0) +if (NOT USE_LIBCXX) + if (USE_INTERNAL_LIBCXX_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with USE_LIBCXX=OFF") + endif() + target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) + return() endif() -option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED}) +set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED}) option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT}) -if (USE_LIBCXX) - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt") + if (USE_INTERNAL_LIBCXX_LIBRARY) + message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libcxx") + set(USE_INTERNAL_LIBCXX_LIBRARY 0) + endif() + set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0) + set(MISSING_INTERNAL_LIBCXX_LIBRARY 1) +endif() - if (NOT USE_INTERNAL_LIBCXX_LIBRARY) - find_library (LIBCXX_LIBRARY c++) - find_library (LIBCXXFS_LIBRARY c++fs) - find_library (LIBCXXABI_LIBRARY c++abi) +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") +if (NOT USE_INTERNAL_LIBCXX_LIBRARY) + find_library (LIBCXX_LIBRARY c++) + find_library (LIBCXXFS_LIBRARY c++fs) + find_library (LIBCXXABI_LIBRARY c++abi) - target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) + if(LIBCXX_LIBRARY AND LIBCXXABI_LIBRARY) # c++fs is now a part of the libc++ + set (HAVE_LIBCXX 1) else () - set (LIBCXX_LIBRARY cxx) - set (LIBCXXABI_LIBRARY cxxabi) - add_subdirectory(contrib/libcxxabi-cmake) - add_subdirectory(contrib/libcxx-cmake) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libcxx") + endif() - # Exception handling library is embedded into libcxxabi. - endif () + if(NOT LIBCXXFS_LIBRARY) + set(LIBCXXFS_LIBRARY ${LIBCXX_LIBRARY}) + endif() - target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + + target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) +endif () + +if (NOT HAVE_LIBCXX AND NOT MISSING_INTERNAL_LIBCXX_LIBRARY) + set (LIBCXX_LIBRARY cxx) + set (LIBCXXABI_LIBRARY cxxabi) + add_subdirectory(contrib/libcxxabi-cmake) + add_subdirectory(contrib/libcxx-cmake) + + # Exception handling library is embedded into libcxxabi. set (HAVE_LIBCXX 1) + set(USE_INTERNAL_LIBCXX_LIBRARY 1) +endif () + +if (HAVE_LIBCXX) + target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY}) message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}") message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}") message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}") -else () +else() target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) -endif () +endif() diff --git a/cmake/find/fastops.cmake b/cmake/find/fastops.cmake index 1296ba586a0..5ab320bdb7a 100644 --- a/cmake/find/fastops.cmake +++ b/cmake/find/fastops.cmake @@ -1,19 +1,24 @@ if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT OS_DARWIN) option(ENABLE_FASTOPS "Enable fast vectorized mathematical functions library by Mikhail Parakhin" ${ENABLE_LIBRARIES}) +elseif(ENABLE_FASTOPS) + message (${RECONFIGURE_MESSAGE_LEVEL} "Fastops library is not supported on ARM, FreeBSD and Darwin") endif() -if(ENABLE_FASTOPS) - if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fastops/fastops/fastops.h") - message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init --recursive") - set(MISSING_INTERNAL_FASTOPS_LIBRARY 1) - endif() - if(NOT MISSING_INTERNAL_FASTOPS_LIBRARY) - set(USE_FASTOPS 1) - set(FASTOPS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/fastops/) - set(FASTOPS_LIBRARY fastops) - endif() -else() +if(NOT ENABLE_FASTOPS) set(USE_FASTOPS 0) + return() +endif() + +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fastops/fastops/fastops.h") + message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal fastops library") + set(MISSING_INTERNAL_FASTOPS_LIBRARY 1) +endif() + +if(NOT MISSING_INTERNAL_FASTOPS_LIBRARY) + set(USE_FASTOPS 1) + set(FASTOPS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/fastops/) + set(FASTOPS_LIBRARY fastops) endif() message(STATUS "Using fastops=${USE_FASTOPS}: ${FASTOPS_INCLUDE_DIR} : ${FASTOPS_LIBRARY}") diff --git a/cmake/find/gperf.cmake b/cmake/find/gperf.cmake index 451f5c21571..9b806598c57 100644 --- a/cmake/find/gperf.cmake +++ b/cmake/find/gperf.cmake @@ -1,8 +1,11 @@ -# Check if gperf was installed -find_program(GPERF gperf) -if(GPERF) - option(ENABLE_GPERF "Use gperf function hash generator tool" ${ENABLE_LIBRARIES}) +if(NOT DEFINED ENABLE_GPERF OR ENABLE_GPERF) + # Check if gperf was installed + find_program(GPERF gperf) + if(GPERF) + option(ENABLE_GPERF "Use gperf function hash generator tool" ${ENABLE_LIBRARIES}) + endif() endif() + if (ENABLE_GPERF) if(NOT GPERF) message(FATAL_ERROR "Could not find the program gperf") diff --git a/cmake/find/grpc.cmake b/cmake/find/grpc.cmake index 0019dbd5eed..216f8ae1878 100644 --- a/cmake/find/grpc.cmake +++ b/cmake/find/grpc.cmake @@ -1,26 +1,38 @@ option (ENABLE_GRPC "Use gRPC" ${ENABLE_LIBRARIES}) -if (ENABLE_GRPC) - option (USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled" ${NOT_UNBUNDLED}) - +if (NOT ENABLE_GRPC) if (USE_INTERNAL_GRPC_LIBRARY) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include/grpc++/grpc++.h") - message(WARNING "submodule contrib/grpc is missing. To fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_GRPC_LIBRARY OFF) - elif (NOT USE_PROTOBUF) - message(WARNING "gRPC requires protobuf which is disabled") - set (USE_INTERNAL_GRPC_LIBRARY OFF) - else() - set (GRPC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc/include") - set (GRPC_LIBRARY "libgrpc++") - set (USE_GRPC ON) - endif() + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal gRPC library with ENABLE_GRPC=OFF") + endif() + return() +endif() + +option (USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled" ${NOT_UNBUNDLED}) + +if (NOT USE_INTERNAL_GRPC_LIBRARY) + find_package(grpc) + if (GRPC_INCLUDE_DIR AND GRPC_LIBRARY) + set (USE_GRPC ON) else() - find_package(grpc) - if (GRPC_INCLUDE_DIR AND GRPC_LIBRARY) - set (USE_GRPC ON) - endif() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system gRPC") endif() endif() -message(STATUS "Using gRPC=${USE_GRPC}: ${GRPC_INCLUDE_DIR} : ${GRPC_LIBRARY}") +if (NOT USE_GRPC) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include/grpc++/grpc++.h") + message (WARNING "submodule contrib/grpc is missing. To fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal gRPC") + set (USE_INTERNAL_GRPC_LIBRARY OFF) + elseif (NOT USE_PROTOBUF) + message (WARNING "gRPC requires protobuf which is disabled") + message (${RECONFIGURE_MESSAGE_LEVEL} "Will not use internal gRPC without protobuf") + set (USE_INTERNAL_GRPC_LIBRARY OFF) + else() + set (GRPC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc/include") + set (GRPC_LIBRARY "libgrpc++") + set (USE_GRPC ON) + set (USE_INTERNAL_GRPC_LIBRARY ON) + endif() +endif() + +message (STATUS "Using gRPC=${USE_GRPC}: ${GRPC_INCLUDE_DIR} : ${GRPC_LIBRARY}") diff --git a/cmake/find/gtest.cmake b/cmake/find/gtest.cmake index b41c4cc0af8..36e45a1381e 100644 --- a/cmake/find/gtest.cmake +++ b/cmake/find/gtest.cmake @@ -1,22 +1,30 @@ option (ENABLE_GTEST_LIBRARY "Enable gtest library" ${ENABLE_LIBRARIES}) -if (ENABLE_GTEST_LIBRARY) +if (NOT ENABLE_GTEST_LIBRARY) + if(USE_INTERNAL_GTEST_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal Google Test when ENABLE_GTEST_LIBRARY=OFF") + endif() + return() +endif() option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeLists.txt") if (USE_INTERNAL_GTEST_LIBRARY) message (WARNING "submodule contrib/googletest is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal gtest") set (USE_INTERNAL_GTEST_LIBRARY 0) endif () set (MISSING_INTERNAL_GTEST_LIBRARY 1) endif () - if(NOT USE_INTERNAL_GTEST_LIBRARY) # TODO: autodetect of GTEST_SRC_DIR by EXISTS /usr/src/googletest/CMakeLists.txt if(NOT GTEST_SRC_DIR) find_package(GTest) + if (NOT GTEST_INCLUDE_DIRS) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system Google Test") + endif() endif() endif() @@ -26,12 +34,13 @@ if (NOT GTEST_SRC_DIR AND NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_ set (GTEST_LIBRARIES gtest) set (GTEST_BOTH_LIBRARIES ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) set (GTEST_INCLUDE_DIRS ${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest) +elseif(USE_INTERNAL_GTEST_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Wouldn't use internal Google Test library") + set (USE_INTERNAL_GTEST_LIBRARY 0) endif () if((GTEST_INCLUDE_DIRS AND GTEST_BOTH_LIBRARIES) OR GTEST_SRC_DIR) set(USE_GTEST 1) endif() -endif() - message (STATUS "Using gtest=${USE_GTEST}: ${GTEST_INCLUDE_DIRS} : ${GTEST_BOTH_LIBRARIES} : ${GTEST_SRC_DIR}") diff --git a/cmake/find/h3.cmake b/cmake/find/h3.cmake index e01f0269507..7ca070b45fe 100644 --- a/cmake/find/h3.cmake +++ b/cmake/find/h3.cmake @@ -1,28 +1,38 @@ option (ENABLE_H3 "Enable H3" ${ENABLE_LIBRARIES}) -if (ENABLE_H3) +if(NOT ENABLE_H3) + if(USE_INTERNAL_H3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal h3 library with ENABLE_H3=OFF") + endif () + return() +endif() -option (USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of bundled" ${NOT_UNBUNDLED}) +option(USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include/h3Index.h") if(USE_INTERNAL_H3_LIBRARY) - message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init --recursive") + message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init --recursive") + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal h3 library") + set(USE_INTERNAL_H3_LIBRARY 0) endif() set(MISSING_INTERNAL_H3_LIBRARY 1) - set(USE_INTERNAL_H3_LIBRARY 0) endif() -if (USE_INTERNAL_H3_LIBRARY) - set (H3_LIBRARY h3) - set (H3_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include) -elseif (NOT MISSING_INTERNAL_H3_LIBRARY) - find_library (H3_LIBRARY h3) - find_path (H3_INCLUDE_DIR NAMES h3/h3api.h PATHS ${H3_INCLUDE_PATHS}) -endif () +if(NOT USE_INTERNAL_H3_LIBRARY) + find_library(H3_LIBRARY h3) + find_path(H3_INCLUDE_DIR NAMES h3/h3api.h PATHS ${H3_INCLUDE_PATHS}) + + if(NOT H3_LIBRARY OR NOT H3_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system h3 library") + endif() +endif() if (H3_LIBRARY AND H3_INCLUDE_DIR) set (USE_H3 1) -endif () - -endif () +elseif(NOT MISSING_INTERNAL_H3_LIBRARY) + set (H3_LIBRARY h3) + set (H3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include") + set (USE_H3 1) + set (USE_INTERNAL_H3_LIBRARY 1) +endif() message (STATUS "Using h3=${USE_H3}: ${H3_INCLUDE_DIR} : ${H3_LIBRARY}") diff --git a/cmake/find/hdfs3.cmake b/cmake/find/hdfs3.cmake index 6a0d01d2245..e8e9fb359c9 100644 --- a/cmake/find/hdfs3.cmake +++ b/cmake/find/hdfs3.cmake @@ -1,21 +1,33 @@ if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND USE_PROTOBUF) option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) +elseif(ENABLE_HDFS OR USE_INTERNAL_HDFS3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") +endif() + +if(NOT ENABLE_HDFS) + if(USE_INTERNAL_HDFS3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library with ENABLE_HDFS3=OFF") + endif() + return() endif() -if(ENABLE_HDFS) option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") if(USE_INTERNAL_HDFS3_LIBRARY) - message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive") + message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library") + set(USE_INTERNAL_HDFS3_LIBRARY 0) endif() set(MISSING_INTERNAL_HDFS3_LIBRARY 1) - set(USE_INTERNAL_HDFS3_LIBRARY 0) endif() if(NOT USE_INTERNAL_HDFS3_LIBRARY) find_library(HDFS3_LIBRARY hdfs3) find_path(HDFS3_INCLUDE_DIR NAMES hdfs/hdfs.h PATHS ${HDFS3_INCLUDE_PATHS}) + if(NOT HDFS3_LIBRARY OR NOT HDFS3_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find system HDFS3 library") + endif() endif() if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) @@ -26,9 +38,7 @@ elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRA set(USE_INTERNAL_HDFS3_LIBRARY 1) set(USE_HDFS 1) else() - set(USE_INTERNAL_HDFS3_LIBRARY 0) -endif() - + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannout enable HDFS3") endif() message(STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/cmake/find/icu.cmake b/cmake/find/icu.cmake index 7beb25626b9..40fb391656d 100644 --- a/cmake/find/icu.cmake +++ b/cmake/find/icu.cmake @@ -4,13 +4,20 @@ else () option(ENABLE_ICU "Enable ICU" 0) endif () -if (ENABLE_ICU) +if (NOT ENABLE_ICU) + if(USE_INTERNAL_ICU_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal icu library with ENABLE_ICU=OFF") + endif() + message(STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)") + return() +endif() option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE") if (USE_INTERNAL_ICU_LIBRARY) message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ICU") set (USE_INTERNAL_ICU_LIBRARY 0) endif () set (MISSING_INTERNAL_ICU_LIBRARY 1) @@ -24,6 +31,8 @@ if(NOT USE_INTERNAL_ICU_LIBRARY) #set (ICU_LIBRARIES ${ICU_I18N_LIBRARY} ${ICU_UC_LIBRARY} ${ICU_DATA_LIBRARY} CACHE STRING "") if(ICU_FOUND) set(USE_ICU 1) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ICU") endif() endif() @@ -35,8 +44,6 @@ elseif (NOT MISSING_INTERNAL_ICU_LIBRARY) set (USE_ICU 1) endif () -endif() - if(USE_ICU) message(STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBRARIES}") else() diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index 99c9007d6b5..3f7871617d4 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -1,84 +1,97 @@ option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES}) -if (ENABLE_LDAP) - option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ${NOT_UNBUNDLED}) +if (NOT ENABLE_LDAP) + if(USE_INTERNAL_LDAP_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal LDAP library with ENABLE_LDAP=OFF") + endif () + return() +endif() - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README") - if (USE_INTERNAL_LDAP_LIBRARY) - message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init --recursive") - endif () +option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ${NOT_UNBUNDLED}) - set (USE_INTERNAL_LDAP_LIBRARY 0) - set (MISSING_INTERNAL_LDAP_LIBRARY 1) +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README") + if (USE_INTERNAL_LDAP_LIBRARY) + message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal LDAP library") endif () - set (OPENLDAP_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) - set (OPENLDAP_USE_REENTRANT_LIBS 1) + set (USE_INTERNAL_LDAP_LIBRARY 0) + set (MISSING_INTERNAL_LDAP_LIBRARY 1) +endif () - if (NOT USE_INTERNAL_LDAP_LIBRARY) - if (OPENLDAP_USE_STATIC_LIBS) - message (WARNING "Unable to use external static OpenLDAP libraries, falling back to the bundled version.") - set (USE_INTERNAL_LDAP_LIBRARY 1) - else () - if (APPLE AND NOT OPENLDAP_ROOT_DIR) - set (OPENLDAP_ROOT_DIR "/usr/local/opt/openldap") - endif () +set (OPENLDAP_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) +set (OPENLDAP_USE_REENTRANT_LIBS 1) - find_package (OpenLDAP) - endif () - endif () - - if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) - string (TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) - string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) - - if ( - "${_system_processor}" STREQUAL "amd64" OR - "${_system_processor}" STREQUAL "x64" - ) - set (_system_processor "x86_64") - elseif ( - "${_system_processor}" STREQUAL "arm64" - ) - set (_system_processor "aarch64") +if (NOT USE_INTERNAL_LDAP_LIBRARY) + if (OPENLDAP_USE_STATIC_LIBS) + message (WARNING "Unable to use external static OpenLDAP libraries, falling back to the bundled version.") + message (${RECONFIGURE_MESSAGE_LEVEL} "Unable to use external OpenLDAP") + set (USE_INTERNAL_LDAP_LIBRARY 1) + else () + if (APPLE AND NOT OPENLDAP_ROOT_DIR) + set (OPENLDAP_ROOT_DIR "/usr/local/opt/openldap") endif () - if ( - ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR - ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) - ) - set (_ldap_supported_platform TRUE) - endif () + find_package (OpenLDAP) - if (NOT _ldap_supported_platform) - message (WARNING "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") - elseif (NOT USE_SSL) - message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") - else () - set (USE_INTERNAL_LDAP_LIBRARY 1) - set (OPENLDAP_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") - set (OPENLDAP_INCLUDE_DIRS - "${ClickHouse_SOURCE_DIR}/contrib/openldap-cmake/${_system_name}_${_system_processor}/include" - "${ClickHouse_SOURCE_DIR}/contrib/openldap/include" - ) - # Below, 'ldap'/'ldap_r' and 'lber' will be resolved to - # the targets defined in contrib/openldap-cmake/CMakeLists.txt - if (OPENLDAP_USE_REENTRANT_LIBS) - set (OPENLDAP_LDAP_LIBRARY "ldap_r") - else () - set (OPENLDAP_LDAP_LIBRARY "ldap") - endif() - set (OPENLDAP_LBER_LIBRARY "lber") - set (OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) - set (OPENLDAP_FOUND 1) - endif () - endif () - - if (OPENLDAP_FOUND) - set (USE_LDAP 1) + if (NOT OPENLDAP_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system OpenLDAP") + endif() endif () endif () +if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) + string (TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) + string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) + + if ( + "${_system_processor}" STREQUAL "amd64" OR + "${_system_processor}" STREQUAL "x64" + ) + set (_system_processor "x86_64") + elseif ( + "${_system_processor}" STREQUAL "arm64" + ) + set (_system_processor "aarch64") + endif () + + if ( + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR + ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) + ) + set (_ldap_supported_platform TRUE) + endif () + + if (NOT _ldap_supported_platform) + message (WARNING "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") + elseif (NOT USE_SSL) + message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") + else () + set (USE_INTERNAL_LDAP_LIBRARY 1) + set (OPENLDAP_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") + set (OPENLDAP_INCLUDE_DIRS + "${ClickHouse_SOURCE_DIR}/contrib/openldap-cmake/${_system_name}_${_system_processor}/include" + "${ClickHouse_SOURCE_DIR}/contrib/openldap/include" + ) + # Below, 'ldap'/'ldap_r' and 'lber' will be resolved to + # the targets defined in contrib/openldap-cmake/CMakeLists.txt + if (OPENLDAP_USE_REENTRANT_LIBS) + set (OPENLDAP_LDAP_LIBRARY "ldap_r") + else () + set (OPENLDAP_LDAP_LIBRARY "ldap") + endif() + set (OPENLDAP_LBER_LIBRARY "lber") + set (OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) + set (OPENLDAP_FOUND 1) + endif () +endif () + +if (OPENLDAP_FOUND) + set (USE_LDAP 1) +endif () + message (STATUS "Using ldap=${USE_LDAP}: ${OPENLDAP_INCLUDE_DIRS} : ${OPENLDAP_LIBRARIES}") diff --git a/cmake/find/libgsasl.cmake b/cmake/find/libgsasl.cmake index e9c45a09010..3b580a4ddb2 100644 --- a/cmake/find/libgsasl.cmake +++ b/cmake/find/libgsasl.cmake @@ -1,12 +1,18 @@ option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES}) -if (ENABLE_GSASL_LIBRARY) +if (NOT ENABLE_GSASL_LIBRARY) + if(USE_INTERNAL_LIBGSASL_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libgsasl library with ENABLE_GSASL_LIBRARY=OFF") + endif() + return() +endif() option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${NOT_UNBUNDLED}) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h") if (USE_INTERNAL_LIBGSASL_LIBRARY) message (WARNING "submodule contrib/libgsasl is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libgsasl") set (USE_INTERNAL_LIBGSASL_LIBRARY 0) endif () set (MISSING_INTERNAL_LIBGSASL_LIBRARY 1) @@ -14,12 +20,15 @@ endif () if (NOT USE_INTERNAL_LIBGSASL_LIBRARY) find_library (LIBGSASL_LIBRARY gsasl) - find_path (LIBGSASL_INCLUDE_DIR NAMES gsasl.h PATHS ${LIBGSASL_INCLUDE_PATHS}) + find_path (LIBGSASL_INCLUDE_DIR NAMES gsasl.h PATHS ${LIBGSASL_INCLUDE_PATHS}) + if (NOT LIBGSASL_LIBRARY OR NOT LIBGSASL_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libgsasl") + endif () endif () if (LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR) elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY) - set (LIBGSASL_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src ${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include) + set (LIBGSASL_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src" "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include") set (USE_INTERNAL_LIBGSASL_LIBRARY 1) set (LIBGSASL_LIBRARY libgsasl) endif () @@ -28,6 +37,4 @@ if(LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR) set (USE_LIBGSASL 1) endif() -endif() - message (STATUS "Using libgsasl=${USE_LIBGSASL}: ${LIBGSASL_INCLUDE_DIR} : ${LIBGSASL_LIBRARY}") diff --git a/cmake/find/libxml2.cmake b/cmake/find/libxml2.cmake index 73732e139c5..04609e497a8 100644 --- a/cmake/find/libxml2.cmake +++ b/cmake/find/libxml2.cmake @@ -3,6 +3,7 @@ option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libxml2/libxml.h") if (USE_INTERNAL_LIBXML2_LIBRARY) message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libxml") set (USE_INTERNAL_LIBXML2_LIBRARY 0) endif () set (MISSING_INTERNAL_LIBXML2_LIBRARY 1) @@ -12,6 +13,10 @@ if (NOT USE_INTERNAL_LIBXML2_LIBRARY) find_package (LibXml2) #find_library (LIBXML2_LIBRARY libxml2) #find_path (LIBXML2_INCLUDE_DIR NAMES libxml.h PATHS ${LIBXML2_INCLUDE_PATHS}) + + if (NOT LIBXML2_LIBRARY OR NOT LIBXML2_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libxml2") + endif () endif () if (LIBXML2_LIBRARY AND LIBXML2_INCLUDE_DIR) diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index 7cb67d1a990..e46b8c9335e 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -2,78 +2,89 @@ if (NOT APPLE) option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile_expressions' option for query execution" ${ENABLE_LIBRARIES}) option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library." ${NOT_UNBUNDLED}) +elseif(ENABLE_EMBEDDED_COMPILER OR USE_INTERNAL_LLVM_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Embedded compiler is not currently supported on Apple") endif () -if (ENABLE_EMBEDDED_COMPILER) - if (USE_INTERNAL_LLVM_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt") - message (WARNING "submodule contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_LLVM_LIBRARY 0) - endif () - - if (NOT USE_INTERNAL_LLVM_LIBRARY) - set (LLVM_PATHS "/usr/local/lib/llvm") - - foreach(llvm_v 9 8) - if (NOT LLVM_FOUND) - find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS}) - endif () - endforeach () - - if (LLVM_FOUND) - # Remove dynamically-linked zlib and libedit from LLVM's dependencies: - set_target_properties(LLVMSupport PROPERTIES INTERFACE_LINK_LIBRARIES "-lpthread;LLVMDemangle;${ZLIB_LIBRARIES}") - set_target_properties(LLVMLineEditor PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMSupport") - - option(LLVM_HAS_RTTI "Enable if LLVM was build with RTTI enabled" ON) - set (USE_EMBEDDED_COMPILER 1) - else() - set (USE_EMBEDDED_COMPILER 0) - endif() - - if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX) - message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages in Linux is incompatible with libc++ ABI. LLVM Will be disabled.") - set (LLVM_FOUND 0) - set (USE_EMBEDDED_COMPILER 0) - endif () - else() - if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) - message(WARNING "Option ENABLE_EMBEDDED_COMPILER is set but LLVM library cannot build if build directory is the same as source directory.") - set (LLVM_FOUND 0) - set (USE_EMBEDDED_COMPILER 0) - elseif (SPLIT_SHARED_LIBRARIES) - # llvm-tablegen cannot find shared libraries that we build. Probably can be easily fixed. - message(WARNING "Option ENABLE_EMBEDDED_COMPILER is not compatible with SPLIT_SHARED_LIBRARIES. Build of LLVM will be disabled.") - set (LLVM_FOUND 0) - set (USE_EMBEDDED_COMPILER 0) - elseif (NOT ARCH_AMD64) - # It's not supported yet, but you can help. - message(WARNING "Option ENABLE_EMBEDDED_COMPILER is only available for x86_64. Build of LLVM will be disabled.") - set (LLVM_FOUND 0) - set (USE_EMBEDDED_COMPILER 0) - elseif (SANITIZE STREQUAL "undefined") - # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. - message(WARNING "Option ENABLE_EMBEDDED_COMPILER does not work with UBSan, because 'llvm-tblgen' tool from LLVM has undefined behaviour. Build of LLVM will be disabled.") - set (LLVM_FOUND 0) - set (USE_EMBEDDED_COMPILER 0) - else () - set (LLVM_FOUND 1) - set (USE_EMBEDDED_COMPILER 1) - set (LLVM_VERSION "9.0.0bundled") - set (LLVM_INCLUDE_DIRS - ${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include - ) - set (LLVM_LIBRARY_DIRS ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm) - endif() +if (NOT ENABLE_EMBEDDED_COMPILER) + if(USE_INTERNAL_LLVM_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal LLVM library with ENABLE_EMBEDDED_COMPILER=OFF") endif() + return() +endif() + +if (USE_INTERNAL_LLVM_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt") + message (WARNING "submodule contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't fidd internal LLVM library") + set (USE_INTERNAL_LLVM_LIBRARY 0) +endif () + +if (NOT USE_INTERNAL_LLVM_LIBRARY) + set (LLVM_PATHS "/usr/local/lib/llvm") + + foreach(llvm_v 9 8) + if (NOT LLVM_FOUND) + find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS}) + endif () + endforeach () if (LLVM_FOUND) - message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}") - message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}") - message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") + # Remove dynamically-linked zlib and libedit from LLVM's dependencies: + set_target_properties(LLVMSupport PROPERTIES INTERFACE_LINK_LIBRARIES "-lpthread;LLVMDemangle;${ZLIB_LIBRARIES}") + set_target_properties(LLVMLineEditor PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMSupport") + + option(LLVM_HAS_RTTI "Enable if LLVM was build with RTTI enabled" ON) + set (USE_EMBEDDED_COMPILER 1) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system LLVM") + set (USE_EMBEDDED_COMPILER 0) + endif() + + if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX) + message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages in Linux is incompatible with libc++ ABI. LLVM Will be disabled.") + message (${RECONFIGURE_MESSAGE_LEVEL} "Unsupported LLVM configuration, cannot enable LLVM") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + endif () +else() + if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is set but LLVM library cannot build if build directory is the same as source directory.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + elseif (SPLIT_SHARED_LIBRARIES) + # llvm-tablegen cannot find shared libraries that we build. Probably can be easily fixed. + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is not compatible with SPLIT_SHARED_LIBRARIES. Build of LLVM will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + elseif (NOT ARCH_AMD64) + # It's not supported yet, but you can help. + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is only available for x86_64. Build of LLVM will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + elseif (SANITIZE STREQUAL "undefined") + # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. + message(WARNING "Option ENABLE_EMBEDDED_COMPILER does not work with UBSan, because 'llvm-tblgen' tool from LLVM has undefined behaviour. Build of LLVM will be disabled.") + set (LLVM_FOUND 0) + set (USE_EMBEDDED_COMPILER 0) + else () + set (LLVM_FOUND 1) + set (USE_EMBEDDED_COMPILER 1) + set (LLVM_VERSION "9.0.0bundled") + set (LLVM_INCLUDE_DIRS + ${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include + ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include + ) + set (LLVM_LIBRARY_DIRS ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm) endif() endif() +if (LLVM_FOUND) + message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}") + message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}") + message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") +else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable LLVM") +endif() # This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles. set (REQUIRED_LLVM_LIBRARIES diff --git a/cmake/find/ltdl.cmake b/cmake/find/ltdl.cmake index 18003618dbd..b48a3630222 100644 --- a/cmake/find/ltdl.cmake +++ b/cmake/find/ltdl.cmake @@ -1,5 +1,5 @@ if (ENABLE_ODBC AND NOT USE_INTERNAL_ODBC_LIBRARY) set (LTDL_PATHS "/usr/local/opt/libtool/lib") - find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS}) + find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS} REQUIRED) message (STATUS "Using ltdl: ${LTDL_LIBRARY}") endif () diff --git a/cmake/find/msgpack.cmake b/cmake/find/msgpack.cmake index 102ea619f6a..130aa007ad5 100644 --- a/cmake/find/msgpack.cmake +++ b/cmake/find/msgpack.cmake @@ -1,27 +1,37 @@ option (ENABLE_MSGPACK "Enable msgpack library" ${ENABLE_LIBRARIES}) -if (ENABLE_MSGPACK) +if(NOT ENABLE_MSGPACK) + if(USE_INTERNAL_MSGPACK_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal msgpack with ENABLE_MSGPACK=OFF") + endif() + return() +endif() option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library instead of bundled" ${NOT_UNBUNDLED}) -if (USE_INTERNAL_MSGPACK_LIBRARY) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp") +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp") + if(USE_INTERNAL_MSGPACK_LIBRARY) message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init --recursive") - set(USE_INTERNAL_MSGPACK_LIBRARY 0) - set(MISSING_INTERNAL_MSGPACK_LIBRARY 1) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal msgpack") + set(USE_INTERNAL_MSGPACK_LIBRARY 0) + endif() + set(MISSING_INTERNAL_MSGPACK_LIBRARY 1) +endif() + +if(NOT USE_INTERNAL_MSGPACK_LIBRARY) + find_path(MSGPACK_INCLUDE_DIR NAMES msgpack.hpp PATHS ${MSGPACK_INCLUDE_PATHS}) + if(NOT MSGPACK_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system msgpack") endif() endif() -if (USE_INTERNAL_MSGPACK_LIBRARY) - set(MSGPACK_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include) -else() - find_path(MSGPACK_INCLUDE_DIR NAMES msgpack.hpp PATHS ${MSGPACK_INCLUDE_PATHS}) +if(NOT MSGPACK_INCLUDE_DIR AND NOT MISSING_INTERNAL_MSGPACK_LIBRARY) + set(MSGPACK_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include") + set(USE_INTERNAL_MSGPACK_LIBRARY 1) endif() if (MSGPACK_INCLUDE_DIR) set(USE_MSGPACK 1) endif() -endif() - message(STATUS "Using msgpack=${USE_MSGPACK}: ${MSGPACK_INCLUDE_DIR}") diff --git a/cmake/find/mysqlclient.cmake b/cmake/find/mysqlclient.cmake index 886c5b9bb59..b0bfd02fabc 100644 --- a/cmake/find/mysqlclient.cmake +++ b/cmake/find/mysqlclient.cmake @@ -4,56 +4,71 @@ else () option(ENABLE_MYSQL "Enable MySQL" FALSE) endif () -if(ENABLE_MYSQL) - option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ${NOT_UNBUNDLED}) +if(NOT ENABLE_MYSQL) + if (USE_INTERNAL_MYSQL_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal mysql library with ENABLE_MYSQL=OFF") + endif () + message (STATUS "Build without mysqlclient (support for MYSQL dictionary source will be disabled)") + return() +endif() - if(USE_INTERNAL_MYSQL_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README") +option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ${NOT_UNBUNDLED}) + +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README") + if(USE_INTERNAL_MYSQL_LIBRARY) message(WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal mysql library") set(USE_INTERNAL_MYSQL_LIBRARY 0) endif() + set(MISSING_INTERNAL_MYSQL_LIBRARY 1) +endif() - if (USE_INTERNAL_MYSQL_LIBRARY) - set (MYSQLCLIENT_LIBRARIES mariadbclient) +if (NOT USE_INTERNAL_MYSQL_LIBRARY) + set (MYSQL_LIB_PATHS + "/usr/local/opt/mysql/lib" + "/usr/local/lib" + "/usr/local/lib64" + "/usr/local/lib/mariadb" # macos brew mariadb-connector-c + "/usr/mysql/lib" + "/usr/mysql/lib64" + "/usr/lib" + "/usr/lib64" + "/lib" + "/lib64") + + set (MYSQL_INCLUDE_PATHS + "/usr/local/opt/mysql/include" + "/usr/mysql/include" + "/usr/local/include" + "/usr/include") + + find_path (MYSQL_INCLUDE_DIR NAMES mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) + + if (USE_STATIC_LIBRARIES) + find_library (STATIC_MYSQLCLIENT_LIB NAMES mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) + else () + find_library (MYSQLCLIENT_LIBRARIES NAMES mariadb mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) + endif () + + if (MYSQL_INCLUDE_DIR AND (STATIC_MYSQLCLIENT_LIB OR MYSQLCLIENT_LIBRARIES)) set (USE_MYSQL 1) set (MYSQLXX_LIBRARY mysqlxx) + if (APPLE) + # /usr/local/include/mysql/mysql_com.h:1011:10: fatal error: mysql/udf_registration_types.h: No such file or directory + set(MYSQL_INCLUDE_DIR ${MYSQL_INCLUDE_DIR} ${MYSQL_INCLUDE_DIR}/mysql) + endif () else () - set (MYSQL_LIB_PATHS - "/usr/local/opt/mysql/lib" - "/usr/local/lib" - "/usr/local/lib64" - "/usr/local/lib/mariadb" # macos brew mariadb-connector-c - "/usr/mysql/lib" - "/usr/mysql/lib64" - "/usr/lib" - "/usr/lib64" - "/lib" - "/lib64") - - set (MYSQL_INCLUDE_PATHS - "/usr/local/opt/mysql/include" - "/usr/mysql/include" - "/usr/local/include" - "/usr/include") - - find_path (MYSQL_INCLUDE_DIR NAMES mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) - - if (USE_STATIC_LIBRARIES) - find_library (STATIC_MYSQLCLIENT_LIB NAMES mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) - else () - find_library (MYSQLCLIENT_LIBRARIES NAMES mariadb mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) - endif () - - if (MYSQL_INCLUDE_DIR AND (STATIC_MYSQLCLIENT_LIB OR MYSQLCLIENT_LIBRARIES)) - set (USE_MYSQL 1) - set (MYSQLXX_LIBRARY mysqlxx) - if (APPLE) - # /usr/local/include/mysql/mysql_com.h:1011:10: fatal error: mysql/udf_registration_types.h: No such file or directory - set(MYSQL_INCLUDE_DIR ${MYSQL_INCLUDE_DIR} ${MYSQL_INCLUDE_DIR}/mysql) - endif () - endif () + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system mysql library") endif () endif () +if (NOT USE_MYSQL AND NOT MISSING_INTERNAL_MYSQL_LIBRARY) + set (MYSQLCLIENT_LIBRARIES mariadbclient) + set (MYSQLXX_LIBRARY mysqlxx) + set (USE_MYSQL 1) + set (USE_INTERNAL_MYSQL_LIBRARY 1) +endif() + if (USE_MYSQL) message (STATUS "Using mysqlclient=${USE_MYSQL}: ${MYSQL_INCLUDE_DIR} : ${MYSQLCLIENT_LIBRARIES}; staticlib=${STATIC_MYSQLCLIENT_LIB}") else () diff --git a/cmake/find/opencl.cmake b/cmake/find/opencl.cmake index 0f307350cb8..2b0cc7c5dd4 100644 --- a/cmake/find/opencl.cmake +++ b/cmake/find/opencl.cmake @@ -3,12 +3,14 @@ if(0) option(ENABLE_OPENCL "Enable OpenCL support" ${ENABLE_LIBRARIES}) endif() -if(ENABLE_OPENCL) +if(NOT ENABLE_OPENCL) + return() +endif() # Intel OpenCl driver: sudo apt install intel-opencl-icd # @sa https://github.com/intel/compute-runtime/releases -# OpenCL applications should link wiht ICD loader +# OpenCL applications should link with ICD loader # sudo apt install opencl-headers ocl-icd-libopencl1 # sudo ln -s /usr/lib/x86_64-linux-gnu/libOpenCL.so.1.0.0 /usr/lib/libOpenCL.so # TODO: add https://github.com/OCL-dev/ocl-icd as submodule instead @@ -16,8 +18,8 @@ if(ENABLE_OPENCL) find_package(OpenCL) if(OpenCL_FOUND) set(USE_OPENCL 1) -endif() - +else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable OpenCL support") endif() message(STATUS "Using opencl=${USE_OPENCL}: ${OpenCL_INCLUDE_DIRS} : ${OpenCL_LIBRARIES}") diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index 26253687c80..c9474a095c4 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -1,12 +1,20 @@ option (ENABLE_ORC "Enable ORC" ${ENABLE_LIBRARIES}) -if(ENABLE_ORC) -include(cmake/find/snappy.cmake) +if(NOT ENABLE_ORC) + if(USE_INTERNAL_ORC_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal ORC library with ENABLE_ORD=OFF") + endif() + return() +endif() + option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) +include(cmake/find/snappy.cmake) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh") if(USE_INTERNAL_ORC_LIBRARY) message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ORC") set(USE_INTERNAL_ORC_LIBRARY 0) endif() set(MISSING_INTERNAL_ORC_LIBRARY 1) @@ -14,6 +22,9 @@ endif () if (NOT USE_INTERNAL_ORC_LIBRARY) find_package(orc) + if (NOT ORC_LIBRARY OR NOT ORC_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ORC") + endif () endif () #if (USE_INTERNAL_ORC_LIBRARY) @@ -30,10 +41,10 @@ elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY AND SNAPPY_LIBRARY) # set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") set(ORC_LIBRARY orc) set(USE_ORC 1) + set(USE_INTERNAL_ORC_LIBRARY 1) else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable ORC support - missing dependencies") set(USE_INTERNAL_ORC_LIBRARY 0) endif() -endif() - message (STATUS "Using internal=${USE_INTERNAL_ORC_LIBRARY} orc=${USE_ORC}: ${ORC_INCLUDE_DIR} : ${ORC_LIBRARY}") diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index d4f62b87d29..3cb6be38c3f 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -1,24 +1,39 @@ if (Protobuf_PROTOC_EXECUTABLE) option (ENABLE_PARQUET "Enable parquet" ${ENABLE_LIBRARIES}) +elseif(ENABLE_PARQUET OR USE_INTERNAL_PARQUET_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use parquet without protoc executable") endif() -if (ENABLE_PARQUET) +if (NOT ENABLE_PARQUET) + if(USE_INTERNAL_PARQUET_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal parquet with ENABLE_PARQUET=OFF") + endif() + message(STATUS "Building without Parquet support") + return() +endif() if (NOT OS_FREEBSD) # Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory option(USE_INTERNAL_PARQUET_LIBRARY "Set to FALSE to use system parquet library instead of bundled" ${NOT_UNBUNDLED}) +elseif(USE_INTERNAL_PARQUET_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal parquet is not supported on freebsd") endif() if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/CMakeLists.txt") if(USE_INTERNAL_PARQUET_LIBRARY) message(WARNING "submodule contrib/arrow (required for Parquet) is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet library") + set(USE_INTERNAL_PARQUET_LIBRARY 0) endif() - set(USE_INTERNAL_PARQUET_LIBRARY 0) set(MISSING_INTERNAL_PARQUET_LIBRARY 1) endif() if(NOT USE_INTERNAL_PARQUET_LIBRARY) find_package(Arrow) find_package(Parquet) + + if(NOT ARROW_INCLUDE_DIR OR PARQUET_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system parquet: arrow=${ARROW_INCLUDE_DIR} parquet=${PARQUET_INCLUDE_DIR}" ) + endif() endif() if(ARROW_INCLUDE_DIR AND PARQUET_INCLUDE_DIR) @@ -26,6 +41,8 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) include(cmake/find/snappy.cmake) if(SNAPPY_LIBRARY) set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet library without snappy") endif() include(CheckCXXSourceCompiles) @@ -33,19 +50,20 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) set(CMAKE_REQUIRED_LIBRARIES ${DOUBLE_CONVERSION_LIBRARIES}) set(CMAKE_REQUIRED_INCLUDES ${DOUBLE_CONVERSION_INCLUDE_DIR}) check_cxx_source_compiles(" - #include - int main() { static const int flags_ = double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY; return 0;} + #include + int main() { static const int flags_ = double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY; return 0;} " HAVE_DOUBLE_CONVERSION_ALLOW_CASE_INSENSIBILITY) if(NOT HAVE_DOUBLE_CONVERSION_ALLOW_CASE_INSENSIBILITY) # HAVE_STD_RANDOM_SHUFFLE - message(STATUS "Disabling internal parquet library because arrow is broken (can't use old double_conversion)") - set(CAN_USE_INTERNAL_PARQUET_LIBRARY 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Disabling internal parquet library because arrow is broken (can't use old double_conversion)") + set(CAN_USE_INTERNAL_PARQUET_LIBRARY 0) endif() endif() - if(NOT CAN_USE_INTERNAL_PARQUET_LIBRARY) + if(NOT CAN_USE_INTERNAL_PARQUET_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet") set(USE_INTERNAL_PARQUET_LIBRARY 0) - else() + else() set(USE_INTERNAL_PARQUET_LIBRARY 1) if(USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) @@ -53,7 +71,7 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) set(PARQUET_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src" ${ClickHouse_BINARY_DIR}/contrib/arrow/cpp/src) endif() - if(${USE_STATIC_LIBRARIES}) + if(MAKE_STATIC_LIBRARIES) set(FLATBUFFERS_LIBRARY flatbuffers) set(ARROW_LIBRARY arrow_static) set(PARQUET_LIBRARY parquet_static) @@ -72,8 +90,8 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) set(USE_ORC 1) set(USE_ARROW 1) endif() -endif() - +elseif(OS_FREEBSD) + message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal parquet library on FreeBSD is not supported") endif() if(USE_PARQUET) diff --git a/cmake/find/protobuf.cmake b/cmake/find/protobuf.cmake index 51110d3b6a3..49d09299f27 100644 --- a/cmake/find/protobuf.cmake +++ b/cmake/find/protobuf.cmake @@ -1,12 +1,18 @@ option(ENABLE_PROTOBUF "Enable protobuf" ${ENABLE_LIBRARIES}) -if(ENABLE_PROTOBUF) +if(NOT ENABLE_PROTOBUF) + if(USE_INTERNAL_PROTOBUF_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf with ENABLE_PROTOBUF=OFF") + endif() + return() +endif() option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt") if(USE_INTERNAL_PROTOBUF_LIBRARY) message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf") set(USE_INTERNAL_PROTOBUF_LIBRARY 0) endif() set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1) @@ -14,6 +20,9 @@ endif() if(NOT USE_INTERNAL_PROTOBUF_LIBRARY) find_package(Protobuf) + if (NOT Protobuf_LIBRARY OR NOT Protobuf_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf") + endif() endif() if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR) @@ -36,11 +45,11 @@ if(OS_FREEBSD AND SANITIZE STREQUAL "address") if(LLVM_INCLUDE_DIRS) set(Protobuf_INCLUDE_DIR ${Protobuf_INCLUDE_DIR} ${LLVM_INCLUDE_DIRS}) else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use protobuf on FreeBSD with address sanitizer without LLVM") set(USE_PROTOBUF 0) endif() endif() include (${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake) -endif() message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE}") diff --git a/cmake/find/rapidjson.cmake b/cmake/find/rapidjson.cmake index df896b644e5..f880d19143e 100644 --- a/cmake/find/rapidjson.cmake +++ b/cmake/find/rapidjson.cmake @@ -1,5 +1,8 @@ option(ENABLE_RAPIDJSON "Use rapidjson" ${ENABLE_LIBRARIES}) if(NOT ENABLE_RAPIDJSON) + if(USE_INTERNAL_RAPIDJSON_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal rapidjson library with ENABLE_RAPIDJSON=OFF") + endif() return() endif() @@ -8,6 +11,7 @@ option(USE_INTERNAL_RAPIDJSON_LIBRARY "Set to FALSE to use system rapidjson libr if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include/rapidjson/rapidjson.h") if(USE_INTERNAL_RAPIDJSON_LIBRARY) message(WARNING "submodule contrib/rapidjson is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rapidjson library") set(USE_INTERNAL_RAPIDJSON_LIBRARY 0) endif() set(MISSING_INTERNAL_RAPIDJSON_LIBRARY 1) @@ -15,6 +19,9 @@ endif() if(NOT USE_INTERNAL_RAPIDJSON_LIBRARY) find_path(RAPIDJSON_INCLUDE_DIR NAMES rapidjson/rapidjson.h PATHS ${RAPIDJSON_INCLUDE_PATHS}) + if(NOT RAPIDJSON_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system rapidjson") + endif() endif() if(RAPIDJSON_INCLUDE_DIR) diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake index f18674dd440..32a78320e12 100644 --- a/cmake/find/rdkafka.cmake +++ b/cmake/find/rdkafka.cmake @@ -1,30 +1,55 @@ # Freebsd: contrib/cppkafka/include/cppkafka/detail/endianness.h:53:23: error: 'betoh16' was not declared in this scope if (NOT ARCH_ARM AND NOT OS_FREEBSD AND OPENSSL_FOUND) option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES}) +elseif(ENABLE_RDKAFKA AND NOT OPENSSL_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use librdkafka without SSL") +elseif(ENABLE_RDKAFKA) + message (${RECONFIGURE_MESSAGE_LEVEL} "librdafka is not supported on ARM and on FreeBSD") endif () -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt") - message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init --recursive") - set (ENABLE_RDKAFKA 0) -endif () - -if (ENABLE_RDKAFKA) +if (NOT ENABLE_RDKAFKA) + if (USE_INTERNAL_RDKAFKA_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ENABLE_RDKAFKA=OFF") + endif() + return() +endif() if (NOT ARCH_ARM AND USE_LIBGSASL) option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED}) +elseif(USE_INTERNAL_RDKAFKA_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM} AND USE_LIBGSASL=${USE_LIBGSASL}") endif () -if (USE_INTERNAL_RDKAFKA_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt") - message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive") - set (USE_INTERNAL_RDKAFKA_LIBRARY 0) - set (MISSING_INTERNAL_RDKAFKA_LIBRARY 1) +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt") + if(USE_INTERNAL_RDKAFKA_LIBRARY) + message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka") + set (USE_INTERNAL_RDKAFKA_LIBRARY 0) + endif() + set (MISSING_INTERNAL_CPPKAFKA_LIBRARY 1) +endif () + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/CMakeLists.txt") + if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY) + message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka") + set (USE_INTERNAL_RDKAFKA_LIBRARY 0) + endif() + set (MISSING_INTERNAL_RDKAFKA_LIBRARY 1) endif () if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) find_library (RDKAFKA_LIB rdkafka) find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS}) + if (NOT RDKAFKA_LIB OR NOT RDKAFKA_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system librdkafka") + endif() + if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD) find_library (SASL2_LIBRARY sasl2) + if (NOT SASL2_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sasl2 library needed for static librdkafka") + endif() endif () set (CPPKAFKA_LIBRARY cppkafka) # TODO: try to use unbundled version. endif () @@ -39,14 +64,14 @@ if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR) if (LZ4_LIBRARY) list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY}) endif () -elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT ARCH_ARM) +elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT MISSING_INTERNAL_CPPKAFKA_LIBRARY AND NOT ARCH_ARM) set (USE_INTERNAL_RDKAFKA_LIBRARY 1) set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") set (RDKAFKA_LIBRARY rdkafka) set (CPPKAFKA_LIBRARY cppkafka) set (USE_RDKAFKA 1) -endif () - +elseif(ARCH_ARM) + message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal rdkafka on ARM is not supported") endif () message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY} ${CPPKAFKA_LIBRARY}") diff --git a/cmake/find/re2.cmake b/cmake/find/re2.cmake index 05ba80f143f..87bc974c788 100644 --- a/cmake/find/re2.cmake +++ b/cmake/find/re2.cmake @@ -3,6 +3,7 @@ option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/CMakeLists.txt") if(USE_INTERNAL_RE2_LIBRARY) message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal re2 library") endif() set(USE_INTERNAL_RE2_LIBRARY 0) set(MISSING_INTERNAL_RE2_LIBRARY 1) @@ -11,6 +12,9 @@ endif() if (NOT USE_INTERNAL_RE2_LIBRARY) find_library (RE2_LIBRARY re2) find_path (RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_INCLUDE_PATHS}) + if (NOT RE2_LIBRARY OR NOT RE2_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system re2 library") + endif () endif () string(FIND ${CMAKE_CURRENT_BINARY_DIR} " " _have_space) @@ -24,12 +28,13 @@ if (RE2_LIBRARY AND RE2_INCLUDE_DIR) elseif (NOT MISSING_INTERNAL_RE2_LIBRARY) set (USE_INTERNAL_RE2_LIBRARY 1) set (RE2_LIBRARY re2) - set (RE2_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/re2) + set (RE2_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/re2") if (NOT MISSING_INTERNAL_RE2_ST_LIBRARY) set (RE2_ST_LIBRARY re2_st) set (USE_RE2_ST 1) else () set (RE2_ST_LIBRARY ${RE2_LIBRARY}) + message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal re2 library instead of re2_st") endif () endif () diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake index 84425220f12..2936c045f99 100644 --- a/cmake/find/sentry.cmake +++ b/cmake/find/sentry.cmake @@ -2,6 +2,9 @@ set (SENTRY_LIBRARY "sentry") set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include") if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init --recursive") + if (USE_SENTRY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sentry library") + endif() return() endif () @@ -16,4 +19,6 @@ if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT_UNBUNDLED AND NOT (OS_ message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") include_directories("${SENTRY_INCLUDE_DIR}") +elseif (USE_SENTRY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Sentry is not supported in current configuration") endif () diff --git a/cmake/find/simdjson.cmake b/cmake/find/simdjson.cmake index 4b1f3224076..cffe20bdb2d 100644 --- a/cmake/find/simdjson.cmake +++ b/cmake/find/simdjson.cmake @@ -1,8 +1,11 @@ +option (USE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include/simdjson.h") message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init --recursive") + if (USE_SIMDJSON) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal simdjson library") + endif() return() endif () -option (USE_SIMDJSON "Use simdjson" ON) - message(STATUS "Using simdjson=${USE_SIMDJSON}") diff --git a/cmake/find/snappy.cmake b/cmake/find/snappy.cmake index 504db87d613..e719231c338 100644 --- a/cmake/find/snappy.cmake +++ b/cmake/find/snappy.cmake @@ -1,13 +1,21 @@ option(USE_SNAPPY "Enable support of snappy library" ${ENABLE_LIBRARIES}) -if(USE_SNAPPY) - option (USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ${NOT_UNBUNDLED}) - - if(NOT USE_INTERNAL_SNAPPY_LIBRARY) - find_library(SNAPPY_LIBRARY snappy) - else () - set(SNAPPY_LIBRARY snappy) +if(NOT USE_SNAPPY) + if (USE_INTERNAL_SNAPPY_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal snappy library with USE_SNAPPY=OFF") endif() + return() +endif() - message (STATUS "Using snappy: ${SNAPPY_LIBRARY}") -endif () +option (USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ${NOT_UNBUNDLED}) + +if(NOT USE_INTERNAL_SNAPPY_LIBRARY) + find_library(SNAPPY_LIBRARY snappy) + if (NOT SNAPPY_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system snappy library") + endif() +else () + set(SNAPPY_LIBRARY snappy) +endif() + +message (STATUS "Using snappy: ${SNAPPY_LIBRARY}") diff --git a/cmake/find/sparsehash.cmake b/cmake/find/sparsehash.cmake index d34ed8e048b..f8b6d293705 100644 --- a/cmake/find/sparsehash.cmake +++ b/cmake/find/sparsehash.cmake @@ -2,6 +2,9 @@ option (USE_INTERNAL_SPARSEHASH_LIBRARY "Set to FALSE to use system sparsehash l if (NOT USE_INTERNAL_SPARSEHASH_LIBRARY) find_path (SPARSEHASH_INCLUDE_DIR NAMES sparsehash/sparse_hash_map PATHS ${SPARSEHASH_INCLUDE_PATHS}) + if (NOT SPARSEHASH_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sparsehash library") + endif () endif () if (SPARSEHASH_INCLUDE_DIR) diff --git a/cmake/find/ssl.cmake b/cmake/find/ssl.cmake index efc9127309c..9058857c173 100644 --- a/cmake/find/ssl.cmake +++ b/cmake/find/ssl.cmake @@ -1,12 +1,18 @@ option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES}) -if(ENABLE_SSL) +if(NOT ENABLE_SSL) + if (USE_INTERNAL_SSL_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal ssl library with ENABLE_SSL=OFF") + endif() + return() +endif() option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openssl/README") if(USE_INTERNAL_SSL_LIBRARY) message(WARNING "submodule contrib/openssl is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ssl library") endif() set(USE_INTERNAL_SSL_LIBRARY 0) set(MISSING_INTERNAL_SSL_LIBRARY 1) @@ -36,6 +42,10 @@ if (NOT USE_INTERNAL_SSL_LIBRARY) set (OPENSSL_FOUND 1) endif () endif () + + if (NOT OPENSSL_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ssl") + endif() endif () if (NOT OPENSSL_FOUND AND NOT MISSING_INTERNAL_SSL_LIBRARY) @@ -123,7 +133,4 @@ if(OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY) endif() endif() - -endif () - message (STATUS "Using ssl=${USE_SSL}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}") diff --git a/cmake/find/stats.cmake b/cmake/find/stats.cmake index ef5b1a73659..339e8524598 100644 --- a/cmake/find/stats.cmake +++ b/cmake/find/stats.cmake @@ -14,6 +14,10 @@ if (ENABLE_STATS) set(GCEM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/gcem/include) set (USE_STATS 1) endif() + + if (NOT USE_STATS) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable stats library") + endif() endif() message (STATUS "Using stats=${USE_STATS} : ${STATS_INCLUDE_DIR}") diff --git a/cmake/find/zlib.cmake b/cmake/find/zlib.cmake index f65d379f577..33b317a9a5e 100644 --- a/cmake/find/zlib.cmake +++ b/cmake/find/zlib.cmake @@ -6,12 +6,14 @@ else () set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "") if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal zlib library") endif () endif () if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}/zlib.h") if(USE_INTERNAL_ZLIB_LIBRARY) message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zlib library") endif() set(USE_INTERNAL_ZLIB_LIBRARY 0) set(MISSING_INTERNAL_ZLIB_LIBRARY 1) @@ -19,6 +21,9 @@ endif() if (NOT USE_INTERNAL_ZLIB_LIBRARY) find_package (ZLIB) + if (NOT ZLIB_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system zlib library") + endif() endif () if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY) diff --git a/cmake/find/zstd.cmake b/cmake/find/zstd.cmake index e4f32d4b170..b12bb701e0e 100644 --- a/cmake/find/zstd.cmake +++ b/cmake/find/zstd.cmake @@ -3,14 +3,18 @@ option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library inste if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h") if(USE_INTERNAL_ZSTD_LIBRARY) message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zstd library") + set(USE_INTERNAL_ZSTD_LIBRARY 0) endif() - set(USE_INTERNAL_ZSTD_LIBRARY 0) set(MISSING_INTERNAL_ZSTD_LIBRARY 1) endif() if (NOT USE_INTERNAL_ZSTD_LIBRARY) find_library (ZSTD_LIBRARY zstd) find_path (ZSTD_INCLUDE_DIR NAMES zstd.h PATHS ${ZSTD_INCLUDE_PATHS}) + if (NOT ZSTD_LIBRARY OR NOT ZSTD_INCLUDE_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system zstd library") + endif () endif () if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 95e00ad9951..a6ea573a59d 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -41,25 +41,23 @@ STRING(REGEX MATCHALL "[0-9]+" COMPILER_VERSION_LIST ${CMAKE_CXX_COMPILER_VERSIO LIST(GET COMPILER_VERSION_LIST 0 COMPILER_VERSION_MAJOR) option (LINKER_NAME "Linker name or full path") -if (COMPILER_GCC) +if (COMPILER_GCC AND NOT LINKER_NAME) find_program (LLD_PATH NAMES "ld.lld") find_program (GOLD_PATH NAMES "ld.gold") -else () +elseif (NOT LINKER_NAME) find_program (LLD_PATH NAMES "ld.lld-${COMPILER_VERSION_MAJOR}" "lld-${COMPILER_VERSION_MAJOR}" "ld.lld" "lld") find_program (GOLD_PATH NAMES "ld.gold" "gold") endif () -if (OS_LINUX) +if (OS_LINUX AND NOT LINKER_NAME) # We prefer LLD linker over Gold or BFD on Linux. - if (NOT LINKER_NAME) - if (LLD_PATH) - if (COMPILER_GCC) - # GCC driver requires one of supported linker names like "lld". - set (LINKER_NAME "lld") - else () - # Clang driver simply allows full linker path. - set (LINKER_NAME ${LLD_PATH}) - endif () + if (LLD_PATH) + if (COMPILER_GCC) + # GCC driver requires one of supported linker names like "lld". + set (LINKER_NAME "lld") + else () + # Clang driver simply allows full linker path. + set (LINKER_NAME ${LLD_PATH}) endif () endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index be8cb9a9708..442f2534f6a 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -80,7 +80,7 @@ set(FLATBUFFERS_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/flatbuffers) set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_SRC_DIR}/include) # set flatbuffers CMake options -if (${USE_STATIC_LIBRARIES}) +if (MAKE_STATIC_LIBRARIES) set(FLATBUFFERS_BUILD_FLATLIB ON CACHE BOOL "Enable the build of the flatbuffers library") set(FLATBUFFERS_BUILD_SHAREDLIB OFF CACHE BOOL "Disable the build of the flatbuffers shared library") else () diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index e92fe4b7159..4d6a2ff0e62 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -1,6 +1,52 @@ option (USE_INTERNAL_BOOST_LIBRARY "Use internal Boost library" ${NOT_UNBUNDLED}) -if (USE_INTERNAL_BOOST_LIBRARY) +if (NOT USE_INTERNAL_BOOST_LIBRARY) + # 1.70 like in contrib/boost + # 1.67 on CI + set(BOOST_VERSION 1.67) + + find_package(Boost ${BOOST_VERSION} COMPONENTS + system + filesystem + iostreams + program_options + regex + ) + + if(Boost_INCLUDE_DIR AND Boost_FILESYSTEM_LIBRARY AND Boost_FILESYSTEM_LIBRARY AND + Boost_PROGRAM_OPTIONS_LIBRARY AND Boost_REGEX_LIBRARY AND Boost_SYSTEM_LIBRARY) + + set(EXTERNAL_BOOST_FOUND 1) + + add_library (_boost_headers_only INTERFACE) + add_library (boost::headers_only ALIAS _boost_headers_only) + target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${Boost_INCLUDE_DIR}) + + add_library (_boost_filesystem INTERFACE) + add_library (_boost_iostreams INTERFACE) + add_library (_boost_program_options INTERFACE) + add_library (_boost_regex INTERFACE) + add_library (_boost_system INTERFACE) + + target_link_libraries (_boost_filesystem INTERFACE ${Boost_FILESYSTEM_LIBRARY}) + target_link_libraries (_boost_iostreams INTERFACE ${Boost_IOSTREAMS_LIBRARY}) + target_link_libraries (_boost_program_options INTERFACE ${Boost_PROGRAM_OPTIONS_LIBRARY}) + target_link_libraries (_boost_regex INTERFACE ${Boost_REGEX_LIBRARY}) + target_link_libraries (_boost_system INTERFACE ${Boost_SYSTEM_LIBRARY}) + + add_library (boost::filesystem ALIAS _boost_filesystem) + add_library (boost::iostreams ALIAS _boost_iostreams) + add_library (boost::program_options ALIAS _boost_program_options) + add_library (boost::regex ALIAS _boost_regex) + add_library (boost::system ALIAS _boost_system) + else() + set(EXTERNAL_BOOST_FOUND 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system boost") + endif() +endif() + +if (NOT EXTERNAL_BOOST_FOUND) + set (USE_INTERNAL_BOOST_LIBRARY 1) set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/boost) # filesystem @@ -96,38 +142,4 @@ if (USE_INTERNAL_BOOST_LIBRARY) add_library (_boost_system ${SRCS_SYSTEM}) add_library (boost::system ALIAS _boost_system) target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR}) -else () - # 1.70 like in contrib/boost - # 1.67 on CI - set(BOOST_VERSION 1.67) - - find_package(Boost ${BOOST_VERSION} COMPONENTS - system - filesystem - iostreams - program_options - regex - REQUIRED) - - add_library (_boost_headers_only INTERFACE) - add_library (boost::headers_only ALIAS _boost_headers_only) - target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${Boost_INCLUDE_DIR}) - - add_library (_boost_filesystem INTERFACE) - add_library (_boost_iostreams INTERFACE) - add_library (_boost_program_options INTERFACE) - add_library (_boost_regex INTERFACE) - add_library (_boost_system INTERFACE) - - target_link_libraries (_boost_filesystem INTERFACE ${Boost_FILESYSTEM_LIBRARY}) - target_link_libraries (_boost_iostreams INTERFACE ${Boost_IOSTREAMS_LIBRARY}) - target_link_libraries (_boost_program_options INTERFACE ${Boost_PROGRAM_OPTIONS_LIBRARY}) - target_link_libraries (_boost_regex INTERFACE ${Boost_REGEX_LIBRARY}) - target_link_libraries (_boost_system INTERFACE ${Boost_SYSTEM_LIBRARY}) - - add_library (boost::filesystem ALIAS _boost_filesystem) - add_library (boost::iostreams ALIAS _boost_iostreams) - add_library (boost::program_options ALIAS _boost_program_options) - add_library (boost::regex ALIAS _boost_regex) - add_library (boost::system ALIAS _boost_system) endif () diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt index eab3210d556..e9075444b2c 100644 --- a/contrib/cctz-cmake/CMakeLists.txt +++ b/contrib/cctz-cmake/CMakeLists.txt @@ -1,9 +1,42 @@ -option (USE_INTERNAL_CCTZ "Use internal cctz library" ${NOT_UNBUNDLED}) +option (USE_INTERNAL_CCTZ_LIBRARY "Use internal cctz library" ${NOT_UNBUNDLED}) -if (USE_INTERNAL_CCTZ) - SET(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz) +if (NOT USE_INTERNAL_CCTZ_LIBRARY) + find_library (LIBRARY_CCTZ cctz) + find_path (INCLUDE_CCTZ NAMES cctz/civil_time.h) - SET (SRCS + if (LIBRARY_CCTZ AND INCLUDE_CCTZ) + set (EXTERNAL_CCTZ_LIBRARY_FOUND 1) + + set(CMAKE_REQUIRED_LIBRARIES ${LIBRARY_CCTZ}) + set(CMAKE_REQUIRED_INCLUDES ${INCLUDE_CCTZ}) + check_cxx_source_compiles( + " + #include + int main() { + cctz::civil_day date; + } + " + EXTERNAL_CCTZ_LIBRARY_WORKS + ) + + if (NOT EXTERNAL_CCTZ_LIBRARY_WORKS) + message (${RECONFIGURE_MESSAGE_LEVEL} "External cctz is not working: ${LIBRARY_CCTZ} ${INCLUDE_CCTZ}") + else() + add_library (cctz UNKNOWN IMPORTED) + set_property (TARGET cctz PROPERTY IMPORTED_LOCATION ${LIBRARY_CCTZ}) + set_property (TARGET cctz PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_CCTZ}) + endif() + else() + set (EXTERNAL_CCTZ_LIBRARY_FOUND 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system cctz") + endif() +endif() + +if (NOT EXTERNAL_CCTZ_LIBRARY_FOUND OR NOT EXTERNAL_CCTZ_LIBRARY_WORKS) + set(USE_INTERNAL_CCTZ_LIBRARY 1) + set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/cctz) + + set (SRCS ${LIBRARY_DIR}/src/civil_time_detail.cc ${LIBRARY_DIR}/src/time_zone_fixed.cc ${LIBRARY_DIR}/src/time_zone_format.cc @@ -63,29 +96,6 @@ if (USE_INTERNAL_CCTZ) add_dependencies(cctz tzdata) target_link_libraries(cctz INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") endif () - -else () - find_library (LIBRARY_CCTZ cctz) - find_path (INCLUDE_CCTZ NAMES cctz/civil_time.h) - - add_library (cctz UNKNOWN IMPORTED) - set_property (TARGET cctz PROPERTY IMPORTED_LOCATION ${LIBRARY_CCTZ}) - set_property (TARGET cctz PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_CCTZ}) - - set(CMAKE_REQUIRED_LIBRARIES cctz) - check_cxx_source_compiles( - " - #include - int main() { - cctz::civil_day date; - } - " - EXTERNAL_CCTZ_WORKS - ) - - if (NOT EXTERNAL_CCTZ_WORKS) - message (FATAL_ERROR "cctz is unusable: ${LIBRARY_CCTZ} ${INCLUDE_CCTZ}") - endif () endif () message (STATUS "Using cctz") diff --git a/contrib/hyperscan-cmake/CMakeLists.txt b/contrib/hyperscan-cmake/CMakeLists.txt index bc2f6c4e89d..e403eeb9d3e 100644 --- a/contrib/hyperscan-cmake/CMakeLists.txt +++ b/contrib/hyperscan-cmake/CMakeLists.txt @@ -1,252 +1,272 @@ -option (ENABLE_HYPERSCAN "Enable hyperscan library" ${ENABLE_LIBRARIES}) - -if (NOT HAVE_SSSE3) +if (HAVE_SSSE3) + option (ENABLE_HYPERSCAN "Enable hyperscan library" ${ENABLE_LIBRARIES}) +elseif(ENABLE_HYPERSCAN) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use hyperscan without SSSE3") set (ENABLE_HYPERSCAN OFF) endif () -if (ENABLE_HYPERSCAN) - option (USE_INTERNAL_HYPERSCAN_LIBRARY "Use internal hyperscan library" ${NOT_UNBUNDLED}) - +if (NOT ENABLE_HYPERSCAN) if (USE_INTERNAL_HYPERSCAN_LIBRARY) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/hyperscan) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal hyperscan with ENABLE_HYPERSCAN=OFF") + endif() - set (SRCS - ${LIBRARY_DIR}/src/alloc.c - ${LIBRARY_DIR}/src/compiler/asserts.cpp - ${LIBRARY_DIR}/src/compiler/compiler.cpp - ${LIBRARY_DIR}/src/compiler/error.cpp - ${LIBRARY_DIR}/src/crc32.c - ${LIBRARY_DIR}/src/database.c - ${LIBRARY_DIR}/src/fdr/engine_description.cpp - ${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp - ${LIBRARY_DIR}/src/fdr/fdr_compile.cpp - ${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp - ${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp - ${LIBRARY_DIR}/src/fdr/fdr.c - ${LIBRARY_DIR}/src/fdr/flood_compile.cpp - ${LIBRARY_DIR}/src/fdr/teddy_compile.cpp - ${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp - ${LIBRARY_DIR}/src/fdr/teddy.c - ${LIBRARY_DIR}/src/grey.cpp - ${LIBRARY_DIR}/src/hs_valid_platform.c - ${LIBRARY_DIR}/src/hs_version.c - ${LIBRARY_DIR}/src/hs.cpp - ${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp - ${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp - ${LIBRARY_DIR}/src/hwlm/hwlm.c - ${LIBRARY_DIR}/src/hwlm/noodle_build.cpp - ${LIBRARY_DIR}/src/hwlm/noodle_engine.c - ${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp - ${LIBRARY_DIR}/src/nfa/accel.c - ${LIBRARY_DIR}/src/nfa/accelcompile.cpp - ${LIBRARY_DIR}/src/nfa/castle.c - ${LIBRARY_DIR}/src/nfa/castlecompile.cpp - ${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp - ${LIBRARY_DIR}/src/nfa/dfa_min.cpp - ${LIBRARY_DIR}/src/nfa/gough.c - ${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp - ${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp - ${LIBRARY_DIR}/src/nfa/goughcompile.cpp - ${LIBRARY_DIR}/src/nfa/lbr.c - ${LIBRARY_DIR}/src/nfa/limex_64.c - ${LIBRARY_DIR}/src/nfa/limex_accel.c - ${LIBRARY_DIR}/src/nfa/limex_compile.cpp - ${LIBRARY_DIR}/src/nfa/limex_native.c - ${LIBRARY_DIR}/src/nfa/limex_simd128.c - ${LIBRARY_DIR}/src/nfa/limex_simd256.c - ${LIBRARY_DIR}/src/nfa/limex_simd384.c - ${LIBRARY_DIR}/src/nfa/limex_simd512.c - ${LIBRARY_DIR}/src/nfa/mcclellan.c - ${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp - ${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp - ${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp - ${LIBRARY_DIR}/src/nfa/mcsheng_data.c - ${LIBRARY_DIR}/src/nfa/mcsheng.c - ${LIBRARY_DIR}/src/nfa/mpv.c - ${LIBRARY_DIR}/src/nfa/mpvcompile.cpp - ${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c - ${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp - ${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp - ${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp - ${LIBRARY_DIR}/src/nfa/rdfa.cpp - ${LIBRARY_DIR}/src/nfa/repeat.c - ${LIBRARY_DIR}/src/nfa/repeatcompile.cpp - ${LIBRARY_DIR}/src/nfa/sheng.c - ${LIBRARY_DIR}/src/nfa/shengcompile.cpp - ${LIBRARY_DIR}/src/nfa/shufti.c - ${LIBRARY_DIR}/src/nfa/shufticompile.cpp - ${LIBRARY_DIR}/src/nfa/tamarama.c - ${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp - ${LIBRARY_DIR}/src/nfa/truffle.c - ${LIBRARY_DIR}/src/nfa/trufflecompile.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_builder.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_calc_components.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_cyclic_redundancy.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_depth.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_dominators.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_edge_redundancy.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_equivalence.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_execute.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_expr_info.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_extparam.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_fixed_width.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_fuzzy.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_haig.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_mcclellan.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_misc_opt.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_netflow.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_prefilter.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_region.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_som.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_split.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_uncalc_components.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_utf8.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_util.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp - ${LIBRARY_DIR}/src/nfagraph/ng_width.cpp - ${LIBRARY_DIR}/src/nfagraph/ng.cpp - ${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp - ${LIBRARY_DIR}/src/parser/buildstate.cpp - ${LIBRARY_DIR}/src/parser/check_refs.cpp - ${LIBRARY_DIR}/src/parser/Component.cpp - ${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp - ${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp - ${LIBRARY_DIR}/src/parser/ComponentAtomicGroup.cpp - ${LIBRARY_DIR}/src/parser/ComponentBackReference.cpp - ${LIBRARY_DIR}/src/parser/ComponentBoundary.cpp - ${LIBRARY_DIR}/src/parser/ComponentByte.cpp - ${LIBRARY_DIR}/src/parser/ComponentClass.cpp - ${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp - ${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp - ${LIBRARY_DIR}/src/parser/ComponentEUS.cpp - ${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp - ${LIBRARY_DIR}/src/parser/ComponentSequence.cpp - ${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp - ${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp - ${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp - ${LIBRARY_DIR}/src/parser/control_verbs.cpp - ${LIBRARY_DIR}/src/parser/logical_combination.cpp - ${LIBRARY_DIR}/src/parser/parse_error.cpp - ${LIBRARY_DIR}/src/parser/parser_util.cpp - ${LIBRARY_DIR}/src/parser/Parser.cpp - ${LIBRARY_DIR}/src/parser/prefilter.cpp - ${LIBRARY_DIR}/src/parser/shortcut_literal.cpp - ${LIBRARY_DIR}/src/parser/ucp_table.cpp - ${LIBRARY_DIR}/src/parser/unsupported.cpp - ${LIBRARY_DIR}/src/parser/utf8_validate.cpp - ${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp - ${LIBRARY_DIR}/src/rose/block.c - ${LIBRARY_DIR}/src/rose/catchup.c - ${LIBRARY_DIR}/src/rose/init.c - ${LIBRARY_DIR}/src/rose/match.c - ${LIBRARY_DIR}/src/rose/program_runtime.c - ${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp - ${LIBRARY_DIR}/src/rose/rose_build_add.cpp - ${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp - ${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp - ${LIBRARY_DIR}/src/rose/rose_build_castle.cpp - ${LIBRARY_DIR}/src/rose/rose_build_compile.cpp - ${LIBRARY_DIR}/src/rose/rose_build_convert.cpp - ${LIBRARY_DIR}/src/rose/rose_build_dedupe.cpp - ${LIBRARY_DIR}/src/rose/rose_build_engine_blob.cpp - ${LIBRARY_DIR}/src/rose/rose_build_exclusive.cpp - ${LIBRARY_DIR}/src/rose/rose_build_groups.cpp - ${LIBRARY_DIR}/src/rose/rose_build_infix.cpp - ${LIBRARY_DIR}/src/rose/rose_build_instructions.cpp - ${LIBRARY_DIR}/src/rose/rose_build_lit_accel.cpp - ${LIBRARY_DIR}/src/rose/rose_build_long_lit.cpp - ${LIBRARY_DIR}/src/rose/rose_build_lookaround.cpp - ${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp - ${LIBRARY_DIR}/src/rose/rose_build_merge.cpp - ${LIBRARY_DIR}/src/rose/rose_build_misc.cpp - ${LIBRARY_DIR}/src/rose/rose_build_program.cpp - ${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp - ${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp - ${LIBRARY_DIR}/src/rose/rose_build_width.cpp - ${LIBRARY_DIR}/src/rose/rose_in_util.cpp - ${LIBRARY_DIR}/src/rose/stream.c - ${LIBRARY_DIR}/src/runtime.c - ${LIBRARY_DIR}/src/scratch.c - ${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp - ${LIBRARY_DIR}/src/som/slot_manager.cpp - ${LIBRARY_DIR}/src/som/som_runtime.c - ${LIBRARY_DIR}/src/som/som_stream.c - ${LIBRARY_DIR}/src/stream_compress.c - ${LIBRARY_DIR}/src/util/alloc.cpp - ${LIBRARY_DIR}/src/util/charreach.cpp - ${LIBRARY_DIR}/src/util/clique.cpp - ${LIBRARY_DIR}/src/util/compile_context.cpp - ${LIBRARY_DIR}/src/util/compile_error.cpp - ${LIBRARY_DIR}/src/util/cpuid_flags.c - ${LIBRARY_DIR}/src/util/depth.cpp - ${LIBRARY_DIR}/src/util/fatbit_build.cpp - ${LIBRARY_DIR}/src/util/multibit_build.cpp - ${LIBRARY_DIR}/src/util/multibit.c - ${LIBRARY_DIR}/src/util/report_manager.cpp - ${LIBRARY_DIR}/src/util/simd_utils.c - ${LIBRARY_DIR}/src/util/state_compress.c - ${LIBRARY_DIR}/src/util/target_info.cpp - ${LIBRARY_DIR}/src/util/ue2string.cpp - ) + add_library (hyperscan INTERFACE) + target_compile_definitions (hyperscan INTERFACE USE_HYPERSCAN=0) - add_library (hyperscan ${SRCS}) + message (STATUS "Not using hyperscan") + return() +endif() - target_compile_definitions (hyperscan PUBLIC USE_HYPERSCAN=1) - target_compile_options (hyperscan - PRIVATE -g0 # Library has too much debug information - -march=corei7 -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # The options from original build system - -fno-sanitize=undefined # Assume the library takes care of itself - ) - target_include_directories (hyperscan - PRIVATE - common - ${LIBRARY_DIR}/include - ) - target_include_directories (hyperscan SYSTEM PUBLIC ${LIBRARY_DIR}/src) - if (ARCH_AMD64) - target_include_directories (hyperscan PRIVATE x86_64) - endif () - target_link_libraries (hyperscan PRIVATE boost::headers_only) - else () - find_library (LIBRARY_HYPERSCAN hs) - find_path (INCLUDE_HYPERSCAN NAMES hs.h HINTS /usr/include/hs) # Ubuntu puts headers in this folder +option (USE_INTERNAL_HYPERSCAN_LIBRARY "Use internal hyperscan library" ${NOT_UNBUNDLED}) + +if (NOT USE_INTERNAL_HYPERSCAN_LIBRARY) + find_library (LIBRARY_HYPERSCAN hs) + find_path (INCLUDE_HYPERSCAN NAMES hs.h HINTS /usr/include/hs) # Ubuntu puts headers in this folder + + if (LIBRARY_HYPERSCAN AND INCLUDE_HYPERSCAN) + set (EXTERNAL_HYPERSCAN_LIBRARY_FOUND 1) add_library (hyperscan UNKNOWN IMPORTED GLOBAL) set_target_properties (hyperscan PROPERTIES IMPORTED_LOCATION ${LIBRARY_HYPERSCAN}) set_target_properties (hyperscan PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_HYPERSCAN}) set_property(TARGET hyperscan APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_HYPERSCAN=1) + else () + set (EXTERNAL_HYPERSCAN_LIBRARY_FOUND 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system hyperscan library") endif () - message (STATUS "Using hyperscan") -else () - add_library (hyperscan INTERFACE) - target_compile_definitions (hyperscan INTERFACE USE_HYPERSCAN=0) - - message (STATUS "Not using hyperscan") endif () + +if (NOT EXTERNAL_HYPERSCAN_LIBRARY_FOUND) + set (USE_INTERNAL_HYPERSCAN_LIBRARY 1) + + set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/hyperscan) + + set (SRCS + ${LIBRARY_DIR}/src/alloc.c + ${LIBRARY_DIR}/src/compiler/asserts.cpp + ${LIBRARY_DIR}/src/compiler/compiler.cpp + ${LIBRARY_DIR}/src/compiler/error.cpp + ${LIBRARY_DIR}/src/crc32.c + ${LIBRARY_DIR}/src/database.c + ${LIBRARY_DIR}/src/fdr/engine_description.cpp + ${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp + ${LIBRARY_DIR}/src/fdr/fdr_compile.cpp + ${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp + ${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp + ${LIBRARY_DIR}/src/fdr/fdr.c + ${LIBRARY_DIR}/src/fdr/flood_compile.cpp + ${LIBRARY_DIR}/src/fdr/teddy_compile.cpp + ${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp + ${LIBRARY_DIR}/src/fdr/teddy.c + ${LIBRARY_DIR}/src/grey.cpp + ${LIBRARY_DIR}/src/hs_valid_platform.c + ${LIBRARY_DIR}/src/hs_version.c + ${LIBRARY_DIR}/src/hs.cpp + ${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp + ${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp + ${LIBRARY_DIR}/src/hwlm/hwlm.c + ${LIBRARY_DIR}/src/hwlm/noodle_build.cpp + ${LIBRARY_DIR}/src/hwlm/noodle_engine.c + ${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp + ${LIBRARY_DIR}/src/nfa/accel.c + ${LIBRARY_DIR}/src/nfa/accelcompile.cpp + ${LIBRARY_DIR}/src/nfa/castle.c + ${LIBRARY_DIR}/src/nfa/castlecompile.cpp + ${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp + ${LIBRARY_DIR}/src/nfa/dfa_min.cpp + ${LIBRARY_DIR}/src/nfa/gough.c + ${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp + ${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp + ${LIBRARY_DIR}/src/nfa/goughcompile.cpp + ${LIBRARY_DIR}/src/nfa/lbr.c + ${LIBRARY_DIR}/src/nfa/limex_64.c + ${LIBRARY_DIR}/src/nfa/limex_accel.c + ${LIBRARY_DIR}/src/nfa/limex_compile.cpp + ${LIBRARY_DIR}/src/nfa/limex_native.c + ${LIBRARY_DIR}/src/nfa/limex_simd128.c + ${LIBRARY_DIR}/src/nfa/limex_simd256.c + ${LIBRARY_DIR}/src/nfa/limex_simd384.c + ${LIBRARY_DIR}/src/nfa/limex_simd512.c + ${LIBRARY_DIR}/src/nfa/mcclellan.c + ${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp + ${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp + ${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp + ${LIBRARY_DIR}/src/nfa/mcsheng_data.c + ${LIBRARY_DIR}/src/nfa/mcsheng.c + ${LIBRARY_DIR}/src/nfa/mpv.c + ${LIBRARY_DIR}/src/nfa/mpvcompile.cpp + ${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c + ${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp + ${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp + ${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp + ${LIBRARY_DIR}/src/nfa/rdfa.cpp + ${LIBRARY_DIR}/src/nfa/repeat.c + ${LIBRARY_DIR}/src/nfa/repeatcompile.cpp + ${LIBRARY_DIR}/src/nfa/sheng.c + ${LIBRARY_DIR}/src/nfa/shengcompile.cpp + ${LIBRARY_DIR}/src/nfa/shufti.c + ${LIBRARY_DIR}/src/nfa/shufticompile.cpp + ${LIBRARY_DIR}/src/nfa/tamarama.c + ${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp + ${LIBRARY_DIR}/src/nfa/truffle.c + ${LIBRARY_DIR}/src/nfa/trufflecompile.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_builder.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_calc_components.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_cyclic_redundancy.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_depth.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_dominators.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_edge_redundancy.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_equivalence.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_execute.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_expr_info.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_extparam.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_fixed_width.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_fuzzy.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_haig.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_mcclellan.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_misc_opt.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_netflow.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_prefilter.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_region.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_som.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_split.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_uncalc_components.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_utf8.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_util.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp + ${LIBRARY_DIR}/src/nfagraph/ng_width.cpp + ${LIBRARY_DIR}/src/nfagraph/ng.cpp + ${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp + ${LIBRARY_DIR}/src/parser/buildstate.cpp + ${LIBRARY_DIR}/src/parser/check_refs.cpp + ${LIBRARY_DIR}/src/parser/Component.cpp + ${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp + ${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp + ${LIBRARY_DIR}/src/parser/ComponentAtomicGroup.cpp + ${LIBRARY_DIR}/src/parser/ComponentBackReference.cpp + ${LIBRARY_DIR}/src/parser/ComponentBoundary.cpp + ${LIBRARY_DIR}/src/parser/ComponentByte.cpp + ${LIBRARY_DIR}/src/parser/ComponentClass.cpp + ${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp + ${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp + ${LIBRARY_DIR}/src/parser/ComponentEUS.cpp + ${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp + ${LIBRARY_DIR}/src/parser/ComponentSequence.cpp + ${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp + ${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp + ${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp + ${LIBRARY_DIR}/src/parser/control_verbs.cpp + ${LIBRARY_DIR}/src/parser/logical_combination.cpp + ${LIBRARY_DIR}/src/parser/parse_error.cpp + ${LIBRARY_DIR}/src/parser/parser_util.cpp + ${LIBRARY_DIR}/src/parser/Parser.cpp + ${LIBRARY_DIR}/src/parser/prefilter.cpp + ${LIBRARY_DIR}/src/parser/shortcut_literal.cpp + ${LIBRARY_DIR}/src/parser/ucp_table.cpp + ${LIBRARY_DIR}/src/parser/unsupported.cpp + ${LIBRARY_DIR}/src/parser/utf8_validate.cpp + ${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp + ${LIBRARY_DIR}/src/rose/block.c + ${LIBRARY_DIR}/src/rose/catchup.c + ${LIBRARY_DIR}/src/rose/init.c + ${LIBRARY_DIR}/src/rose/match.c + ${LIBRARY_DIR}/src/rose/program_runtime.c + ${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp + ${LIBRARY_DIR}/src/rose/rose_build_add.cpp + ${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp + ${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp + ${LIBRARY_DIR}/src/rose/rose_build_castle.cpp + ${LIBRARY_DIR}/src/rose/rose_build_compile.cpp + ${LIBRARY_DIR}/src/rose/rose_build_convert.cpp + ${LIBRARY_DIR}/src/rose/rose_build_dedupe.cpp + ${LIBRARY_DIR}/src/rose/rose_build_engine_blob.cpp + ${LIBRARY_DIR}/src/rose/rose_build_exclusive.cpp + ${LIBRARY_DIR}/src/rose/rose_build_groups.cpp + ${LIBRARY_DIR}/src/rose/rose_build_infix.cpp + ${LIBRARY_DIR}/src/rose/rose_build_instructions.cpp + ${LIBRARY_DIR}/src/rose/rose_build_lit_accel.cpp + ${LIBRARY_DIR}/src/rose/rose_build_long_lit.cpp + ${LIBRARY_DIR}/src/rose/rose_build_lookaround.cpp + ${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp + ${LIBRARY_DIR}/src/rose/rose_build_merge.cpp + ${LIBRARY_DIR}/src/rose/rose_build_misc.cpp + ${LIBRARY_DIR}/src/rose/rose_build_program.cpp + ${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp + ${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp + ${LIBRARY_DIR}/src/rose/rose_build_width.cpp + ${LIBRARY_DIR}/src/rose/rose_in_util.cpp + ${LIBRARY_DIR}/src/rose/stream.c + ${LIBRARY_DIR}/src/runtime.c + ${LIBRARY_DIR}/src/scratch.c + ${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp + ${LIBRARY_DIR}/src/som/slot_manager.cpp + ${LIBRARY_DIR}/src/som/som_runtime.c + ${LIBRARY_DIR}/src/som/som_stream.c + ${LIBRARY_DIR}/src/stream_compress.c + ${LIBRARY_DIR}/src/util/alloc.cpp + ${LIBRARY_DIR}/src/util/charreach.cpp + ${LIBRARY_DIR}/src/util/clique.cpp + ${LIBRARY_DIR}/src/util/compile_context.cpp + ${LIBRARY_DIR}/src/util/compile_error.cpp + ${LIBRARY_DIR}/src/util/cpuid_flags.c + ${LIBRARY_DIR}/src/util/depth.cpp + ${LIBRARY_DIR}/src/util/fatbit_build.cpp + ${LIBRARY_DIR}/src/util/multibit_build.cpp + ${LIBRARY_DIR}/src/util/multibit.c + ${LIBRARY_DIR}/src/util/report_manager.cpp + ${LIBRARY_DIR}/src/util/simd_utils.c + ${LIBRARY_DIR}/src/util/state_compress.c + ${LIBRARY_DIR}/src/util/target_info.cpp + ${LIBRARY_DIR}/src/util/ue2string.cpp + ) + + add_library (hyperscan ${SRCS}) + + target_compile_definitions (hyperscan PUBLIC USE_HYPERSCAN=1) + target_compile_options (hyperscan + PRIVATE -g0 # Library has too much debug information + -march=corei7 -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # The options from original build system + -fno-sanitize=undefined # Assume the library takes care of itself + ) + target_include_directories (hyperscan + PRIVATE + common + ${LIBRARY_DIR}/include + ) + target_include_directories (hyperscan SYSTEM PUBLIC ${LIBRARY_DIR}/src) + if (ARCH_AMD64) + target_include_directories (hyperscan PRIVATE x86_64) + endif () + target_link_libraries (hyperscan PRIVATE boost::headers_only) + + set (USE_INTERNAL_HYPERSCAN_LIBRARY 1) +endif () + +message (STATUS "Using hyperscan") diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 13f7ea3326b..e8a690ff149 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -2,140 +2,40 @@ option (ENABLE_JEMALLOC "Enable jemalloc allocator" ${ENABLE_LIBRARIES}) if (SANITIZE OR NOT (ARCH_AMD64 OR ARCH_ARM) OR NOT (OS_LINUX OR OS_FREEBSD OR OS_DARWIN)) set (ENABLE_JEMALLOC OFF) - message (STATUS "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64 or aarch64 on linux or freebsd.") + message (${RECONFIGURE_MESSAGE_LEVEL} + "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64 or aarch64 on linux or freebsd.") endif () -if (ENABLE_JEMALLOC) - if (NOT OS_LINUX) - message (WARNING "jemalloc support on non-linux is EXPERIMENTAL") +if (NOT ENABLE_JEMALLOC) + if(USE_INTERNAL_JEMALLOC_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal jemalloc with ENABLE_JEMALLOC=OFF") endif() - option (USE_INTERNAL_JEMALLOC "Use internal jemalloc library" ${NOT_UNBUNDLED}) + add_library(jemalloc INTERFACE) + target_compile_definitions(jemalloc INTERFACE USE_JEMALLOC=0) - if (USE_INTERNAL_JEMALLOC) - if (OS_LINUX) - # ThreadPool select job randomly, and there can be some threads that had been - # performed some memory heavy task before and will be inactive for some time, - # but until it will became active again, the memory will not be freed since by - # default each thread has it's own arena, but there should be not more then - # 4*CPU arenas (see opt.nareans description). - # - # By enabling percpu_arena number of arenas limited to number of CPUs and hence - # this problem should go away. - # - # muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to - # avoid spurious latencies and additional work associated with - # MADV_DONTNEED. See - # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. - set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") - else() - set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") - endif() - # CACHE variable is empty, to allow changing defaults without necessity - # to purge cache - set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of JEMalloc" ) - if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE) - set (JEMALLOC_CONFIG_MALLOC_CONF "${JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE}") - endif() - message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}") + message (STATUS "Not using jemalloc") + return() +endif () - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc") +if (NOT OS_LINUX) + message (WARNING "jemalloc support on non-linux is EXPERIMENTAL") +endif() - set (SRCS - ${LIBRARY_DIR}/src/arena.c - ${LIBRARY_DIR}/src/background_thread.c - ${LIBRARY_DIR}/src/base.c - ${LIBRARY_DIR}/src/bin.c - ${LIBRARY_DIR}/src/bitmap.c - ${LIBRARY_DIR}/src/ckh.c - ${LIBRARY_DIR}/src/ctl.c - ${LIBRARY_DIR}/src/div.c - ${LIBRARY_DIR}/src/extent.c - ${LIBRARY_DIR}/src/extent_dss.c - ${LIBRARY_DIR}/src/extent_mmap.c - ${LIBRARY_DIR}/src/hash.c - ${LIBRARY_DIR}/src/hook.c - ${LIBRARY_DIR}/src/jemalloc.c - ${LIBRARY_DIR}/src/large.c - ${LIBRARY_DIR}/src/log.c - ${LIBRARY_DIR}/src/malloc_io.c - ${LIBRARY_DIR}/src/mutex.c - ${LIBRARY_DIR}/src/mutex_pool.c - ${LIBRARY_DIR}/src/nstime.c - ${LIBRARY_DIR}/src/pages.c - ${LIBRARY_DIR}/src/prng.c - ${LIBRARY_DIR}/src/prof.c - ${LIBRARY_DIR}/src/rtree.c - ${LIBRARY_DIR}/src/sc.c - ${LIBRARY_DIR}/src/stats.c - ${LIBRARY_DIR}/src/sz.c - ${LIBRARY_DIR}/src/tcache.c - ${LIBRARY_DIR}/src/test_hooks.c - ${LIBRARY_DIR}/src/ticker.c - ${LIBRARY_DIR}/src/tsd.c - ${LIBRARY_DIR}/src/witness.c - ${LIBRARY_DIR}/src/safety_check.c - ) - if (OS_DARWIN) - list(APPEND SRCS ${LIBRARY_DIR}/src/zone.c) - endif () +option (USE_INTERNAL_JEMALLOC_LIBRARY "Use internal jemalloc library" ${NOT_UNBUNDLED}) - add_library(jemalloc ${SRCS}) - target_include_directories(jemalloc PRIVATE ${LIBRARY_DIR}/include) - target_include_directories(jemalloc SYSTEM PUBLIC include) +if (NOT USE_INTERNAL_JEMALLOC_LIBRARY) + find_library(LIBRARY_JEMALLOC jemalloc) + find_path(INCLUDE_JEMALLOC jemalloc/jemalloc.h) - set (JEMALLOC_INCLUDE_PREFIX) - # OS_ - if (OS_LINUX) - set (JEMALLOC_INCLUDE_PREFIX "include_linux") - elseif (OS_FREEBSD) - set (JEMALLOC_INCLUDE_PREFIX "include_freebsd") - elseif (OS_DARWIN) - set (JEMALLOC_INCLUDE_PREFIX "include_darwin") - else () - message (FATAL_ERROR "This OS is not supported") - endif () - # ARCH_ - if (ARCH_AMD64) - set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_x86_64") - elseif (ARCH_ARM) - set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_aarch64") - else () - message (FATAL_ERROR "This arch is not supported") - endif () - - configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in - ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h) - target_include_directories(jemalloc SYSTEM PRIVATE - ${CMAKE_CURRENT_BINARY_DIR}/${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal) - - target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) - - if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") - target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1) - - if (USE_UNWIND) - target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1) - target_link_libraries (jemalloc PRIVATE unwind) - endif () - endif () - - target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) - # for RTLD_NEXT - target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) - else () - find_library(LIBRARY_JEMALLOC jemalloc) - find_path(INCLUDE_JEMALLOC jemalloc/jemalloc.h) + if (LIBRARY_JEMALLOC AND INCLUDE_JEMALLOC) + set(EXTERNAL_JEMALLOC_LIBRARY_FOUND 1) set(THREADS_PREFER_PTHREAD_FLAG ON) - find_package(Threads REQUIRED) + find_package(Threads) - add_library (jemalloc STATIC IMPORTED) - set_property (TARGET jemalloc PROPERTY IMPORTED_LOCATION ${LIBRARY_JEMALLOC}) - set_property (TARGET jemalloc PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_JEMALLOC}) - set_property (TARGET jemalloc PROPERTY INTERFACE_LINK_LIBRARIES Threads::Threads dl) - - set (CMAKE_REQUIRED_LIBRARIES jemalloc) + set (CMAKE_REQUIRED_LIBRARIES ${LIBRARY_JEMALLOC} Threads::Threads "dl") + set (CMAKE_REQUIRED_INCLUDES ${INCLUDE_JEMALLOC}) check_cxx_source_compiles ( " #include @@ -144,24 +44,145 @@ if (ENABLE_JEMALLOC) free(mallocx(1, 0)); } " - EXTERNAL_JEMALLOC_WORKS + EXTERNAL_JEMALLOC_LIBRARY_WORKS ) - if (NOT EXTERNAL_JEMALLOC_WORKS) - message (FATAL_ERROR "jemalloc is unusable: ${LIBRARY_JEMALLOC} ${INCLUDE_JEMALLOC}") + if (EXTERNAL_JEMALLOC_LIBRARY_WORKS) + add_library (jemalloc STATIC IMPORTED) + set_property (TARGET jemalloc PROPERTY IMPORTED_LOCATION ${LIBRARY_JEMALLOC}) + set_property (TARGET jemalloc PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_JEMALLOC}) + set_property (TARGET jemalloc PROPERTY INTERFACE_LINK_LIBRARIES Threads::Threads dl) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "External jemalloc is unusable: ${LIBRARY_JEMALLOC} ${INCLUDE_JEMALLOC}") + endif () + + else() + set(EXTERNAL_JEMALLOC_LIBRARY_FOUND 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system jemalloc") + endif() +endif () + +if (NOT EXTERNAL_JEMALLOC_LIBRARY_FOUND OR NOT EXTERNAL_JEMALLOC_LIBRARY_WORKS) + set(USE_INTERNAL_JEMALLOC_LIBRARY 1) + + if (OS_LINUX) + # ThreadPool select job randomly, and there can be some threads that had been + # performed some memory heavy task before and will be inactive for some time, + # but until it will became active again, the memory will not be freed since by + # default each thread has it's own arena, but there should be not more then + # 4*CPU arenas (see opt.nareans description). + # + # By enabling percpu_arena number of arenas limited to number of CPUs and hence + # this problem should go away. + # + # muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to + # avoid spurious latencies and additional work associated with + # MADV_DONTNEED. See + # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation. + set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:10000") + else() + set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:10000") + endif() + # CACHE variable is empty, to allow changing defaults without necessity + # to purge cache + set (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE "" CACHE STRING "Change default configuration string of JEMalloc" ) + if (JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE) + set (JEMALLOC_CONFIG_MALLOC_CONF "${JEMALLOC_CONFIG_MALLOC_CONF_OVERRIDE}") + endif() + message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}") + + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/jemalloc") + + set (SRCS + ${LIBRARY_DIR}/src/arena.c + ${LIBRARY_DIR}/src/background_thread.c + ${LIBRARY_DIR}/src/base.c + ${LIBRARY_DIR}/src/bin.c + ${LIBRARY_DIR}/src/bitmap.c + ${LIBRARY_DIR}/src/ckh.c + ${LIBRARY_DIR}/src/ctl.c + ${LIBRARY_DIR}/src/div.c + ${LIBRARY_DIR}/src/extent.c + ${LIBRARY_DIR}/src/extent_dss.c + ${LIBRARY_DIR}/src/extent_mmap.c + ${LIBRARY_DIR}/src/hash.c + ${LIBRARY_DIR}/src/hook.c + ${LIBRARY_DIR}/src/jemalloc.c + ${LIBRARY_DIR}/src/large.c + ${LIBRARY_DIR}/src/log.c + ${LIBRARY_DIR}/src/malloc_io.c + ${LIBRARY_DIR}/src/mutex.c + ${LIBRARY_DIR}/src/mutex_pool.c + ${LIBRARY_DIR}/src/nstime.c + ${LIBRARY_DIR}/src/pages.c + ${LIBRARY_DIR}/src/prng.c + ${LIBRARY_DIR}/src/prof.c + ${LIBRARY_DIR}/src/rtree.c + ${LIBRARY_DIR}/src/sc.c + ${LIBRARY_DIR}/src/stats.c + ${LIBRARY_DIR}/src/sz.c + ${LIBRARY_DIR}/src/tcache.c + ${LIBRARY_DIR}/src/test_hooks.c + ${LIBRARY_DIR}/src/ticker.c + ${LIBRARY_DIR}/src/tsd.c + ${LIBRARY_DIR}/src/witness.c + ${LIBRARY_DIR}/src/safety_check.c + ) + if (OS_DARWIN) + list(APPEND SRCS ${LIBRARY_DIR}/src/zone.c) + endif () + + add_library(jemalloc ${SRCS}) + target_include_directories(jemalloc PRIVATE ${LIBRARY_DIR}/include) + target_include_directories(jemalloc SYSTEM PUBLIC include) + + set (JEMALLOC_INCLUDE_PREFIX) + # OS_ + if (OS_LINUX) + set (JEMALLOC_INCLUDE_PREFIX "include_linux") + elseif (OS_FREEBSD) + set (JEMALLOC_INCLUDE_PREFIX "include_freebsd") + elseif (OS_DARWIN) + set (JEMALLOC_INCLUDE_PREFIX "include_darwin") + else () + message (FATAL_ERROR "internal jemalloc: This OS is not supported") + endif () + # ARCH_ + if (ARCH_AMD64) + set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_x86_64") + elseif (ARCH_ARM) + set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_aarch64") + else () + message (FATAL_ERROR "internal jemalloc: This arch is not supported") + endif () + + configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in + ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h) + target_include_directories(jemalloc SYSTEM PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal) + + target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE) + + if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") + target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_DEBUG=1 -DJEMALLOC_PROF=1) + + if (USE_UNWIND) + target_compile_definitions (jemalloc PRIVATE -DJEMALLOC_PROF_LIBUNWIND=1) + target_link_libraries (jemalloc PRIVATE unwind) endif () endif () - set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) - if (MAKE_STATIC_LIBRARIES) - # To detect whether we need to register jemalloc for osx as default zone. - set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS BUNDLED_STATIC_JEMALLOC=1) - endif() + target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) + # for RTLD_NEXT + target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) - message (STATUS "Using jemalloc") -else () - add_library(jemalloc INTERFACE) - target_compile_definitions(jemalloc INTERFACE USE_JEMALLOC=0) - - message (STATUS "Not using jemalloc") + set (USE_INTERNAL_JEMALLOC_LIBRARY 1) endif () + +set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) +if (MAKE_STATIC_LIBRARIES) + # To detect whether we need to register jemalloc for osx as default zone. + set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS BUNDLED_STATIC_JEMALLOC=1) +endif() + +message (STATUS "Using jemalloc") diff --git a/contrib/libcpuid-cmake/CMakeLists.txt b/contrib/libcpuid-cmake/CMakeLists.txt index cb28cbd21da..564e67994bf 100644 --- a/contrib/libcpuid-cmake/CMakeLists.txt +++ b/contrib/libcpuid-cmake/CMakeLists.txt @@ -1,35 +1,38 @@ option (ENABLE_CPUID "Enable libcpuid library (only internal)" ${ENABLE_LIBRARIES}) -if (ARCH_ARM) +if (ARCH_ARM AND ENABLE_CPUID) + message (${RECONFIGURE_MESSAGE_LEVEL} "cpuid is not supported on ARM") set (ENABLE_CPUID 0) endif () -if (ENABLE_CPUID) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/libcpuid) - - set (SRCS - ${LIBRARY_DIR}/libcpuid/asm-bits.c - ${LIBRARY_DIR}/libcpuid/cpuid_main.c - ${LIBRARY_DIR}/libcpuid/libcpuid_util.c - ${LIBRARY_DIR}/libcpuid/msrdriver.c - ${LIBRARY_DIR}/libcpuid/rdmsr.c - ${LIBRARY_DIR}/libcpuid/rdtsc.c - ${LIBRARY_DIR}/libcpuid/recog_amd.c - ${LIBRARY_DIR}/libcpuid/recog_intel.c - ) - - add_library (cpuid ${SRCS}) - - target_include_directories (cpuid SYSTEM PUBLIC ${LIBRARY_DIR}) - target_compile_definitions (cpuid PUBLIC USE_CPUID=1) - target_compile_definitions (cpuid PRIVATE VERSION="v0.4.1") - if (COMPILER_CLANG) - target_compile_options (cpuid PRIVATE -Wno-reserved-id-macro) - endif () - - message (STATUS "Using cpuid") -else () +if (NOT ENABLE_CPUID) add_library (cpuid INTERFACE) target_compile_definitions (cpuid INTERFACE USE_CPUID=0) + + return() +endif() + +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libcpuid") + +set (SRCS + ${LIBRARY_DIR}/libcpuid/asm-bits.c + ${LIBRARY_DIR}/libcpuid/cpuid_main.c + ${LIBRARY_DIR}/libcpuid/libcpuid_util.c + ${LIBRARY_DIR}/libcpuid/msrdriver.c + ${LIBRARY_DIR}/libcpuid/rdmsr.c + ${LIBRARY_DIR}/libcpuid/rdtsc.c + ${LIBRARY_DIR}/libcpuid/recog_amd.c + ${LIBRARY_DIR}/libcpuid/recog_intel.c +) + +add_library (cpuid ${SRCS}) + +target_include_directories (cpuid SYSTEM PUBLIC ${LIBRARY_DIR}) +target_compile_definitions (cpuid PUBLIC USE_CPUID=1) +target_compile_definitions (cpuid PRIVATE VERSION="v0.4.1") +if (COMPILER_CLANG) + target_compile_options (cpuid PRIVATE -Wno-reserved-id-macro) endif () + +message (STATUS "Using cpuid") diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 4c71770f5b6..ff07a0213c0 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -1,4 +1,14 @@ -if (NOT USE_INTERNAL_PROTOBUF_LIBRARY) +if (ENABLE_PROTOBUF AND NOT USE_INTERNAL_PROTOBUF_LIBRARY) + option(PROTOBUF_OLD_ABI_COMPAT "Set to ON for compatiability with external protobuf which was compiled old C++ ABI" ON) +endif() + +if (PROTOBUF_OLD_ABI_COMPAT) + if (NOT ENABLE_PROTOBUF OR USE_INTERNAL_PROTOBUF_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "PROTOBUF_OLD_ABI_COMPAT option is ignored") + endif() +endif() + +if (NOT USE_INTERNAL_PROTOBUF_LIBRARY AND PROTOBUF_OLD_ABI_COMPAT) # compatiable with protobuf which was compiled old C++ ABI set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") set(CMAKE_C_FLAGS "") @@ -7,7 +17,7 @@ if (NOT USE_INTERNAL_PROTOBUF_LIBRARY) endif () endif() -SET(WITH_KERBEROS false) +set(WITH_KERBEROS false) # project and source dir set(HDFS3_ROOT_DIR ${ClickHouse_SOURCE_DIR}/contrib/libhdfs3) set(HDFS3_SOURCE_DIR ${HDFS3_ROOT_DIR}/src) diff --git a/contrib/lz4-cmake/CMakeLists.txt b/contrib/lz4-cmake/CMakeLists.txt index b8121976213..374837fbe58 100644 --- a/contrib/lz4-cmake/CMakeLists.txt +++ b/contrib/lz4-cmake/CMakeLists.txt @@ -1,13 +1,30 @@ option (USE_INTERNAL_LZ4_LIBRARY "Use internal lz4 library" ${NOT_UNBUNDLED}) -if (USE_INTERNAL_LZ4_LIBRARY) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/lz4) +if (NOT USE_INTERNAL_LZ4_LIBRARY) + find_library (LIBRARY_LZ4 lz4) + find_path (INCLUDE_LZ4 lz4.h) + + if (LIBRARY_LZ4 AND INCLUDE_LZ4) + set(EXTERNAL_LZ4_LIBRARY_FOUND 1) + add_library (lz4 UNKNOWN IMPORTED) + set_property (TARGET lz4 PROPERTY IMPORTED_LOCATION ${LIBRARY_LZ4}) + set_property (TARGET lz4 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_LZ4}) + set_property (TARGET lz4 APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_XXHASH=0) + else() + set(EXTERNAL_LZ4_LIBRARY_FOUND 0) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system lz4") + endif() +endif() + +if (NOT EXTERNAL_LZ4_LIBRARY_FOUND) + set (USE_INTERNAL_LZ4_LIBRARY 1) + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lz4") set (SRCS - ${LIBRARY_DIR}/lib/lz4.c - ${LIBRARY_DIR}/lib/lz4hc.c - ${LIBRARY_DIR}/lib/lz4frame.c - ${LIBRARY_DIR}/lib/xxhash.c + "${LIBRARY_DIR}/lib/lz4.c" + "${LIBRARY_DIR}/lib/lz4hc.c" + "${LIBRARY_DIR}/lib/lz4frame.c" + "${LIBRARY_DIR}/lib/xxhash.c" ) add_library (lz4 ${SRCS}) @@ -17,12 +34,4 @@ if (USE_INTERNAL_LZ4_LIBRARY) target_compile_options (lz4 PRIVATE -fno-sanitize=undefined) endif () target_include_directories(lz4 PUBLIC ${LIBRARY_DIR}/lib) -else () - find_library (LIBRARY_LZ4 lz4) - find_path (INCLUDE_LZ4 lz4.h) - - add_library (lz4 UNKNOWN IMPORTED) - set_property (TARGET lz4 PROPERTY IMPORTED_LOCATION ${LIBRARY_LZ4}) - set_property (TARGET lz4 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_LZ4}) - set_property (TARGET lz4 APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_XXHASH=0) endif () diff --git a/contrib/poco-cmake/CMakeLists.txt b/contrib/poco-cmake/CMakeLists.txt index 59b6c84a1d1..a96b07085ba 100644 --- a/contrib/poco-cmake/CMakeLists.txt +++ b/contrib/poco-cmake/CMakeLists.txt @@ -4,6 +4,9 @@ if (USE_INTERNAL_POCO_LIBRARY) set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/poco) else () find_path (ROOT_DIR NAMES Foundation/include/Poco/Poco.h include/Poco/Poco.h) + if (NOT ROOT_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system poco") + endif() endif () add_subdirectory (Crypto) diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index 48d7e8bb36b..ed6bdaa25bc 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -1,35 +1,26 @@ option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES}) -if (ENABLE_REPLXX) - option (USE_INTERNAL_REPLXX "Use internal replxx library" ${NOT_UNBUNDLED}) +if (NOT ENABLE_REPLXX) + if (USE_INTERNAL_REPLXX_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal replxx with ENABLE_REPLXX=OFF") + endif() - if (USE_INTERNAL_REPLXX) - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") + add_library(replxx INTERFACE) + target_compile_definitions(replxx INTERFACE USE_REPLXX=0) - set(SRCS - ${LIBRARY_DIR}/src/conversion.cxx - ${LIBRARY_DIR}/src/ConvertUTF.cpp - ${LIBRARY_DIR}/src/escape.cxx - ${LIBRARY_DIR}/src/history.cxx - ${LIBRARY_DIR}/src/io.cxx - ${LIBRARY_DIR}/src/prompt.cxx - ${LIBRARY_DIR}/src/replxx_impl.cxx - ${LIBRARY_DIR}/src/replxx.cxx - ${LIBRARY_DIR}/src/util.cxx - ${LIBRARY_DIR}/src/wcwidth.cpp - ) + message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)") + return() +endif() - add_library (replxx ${SRCS}) - target_include_directories(replxx SYSTEM PUBLIC ${LIBRARY_DIR}/include) - else () - find_library(LIBRARY_REPLXX NAMES replxx replxx-static) - find_path(INCLUDE_REPLXX replxx.hxx) +option (USE_INTERNAL_REPLXX_LIBRARY "Use internal replxx library" ${NOT_UNBUNDLED}) - add_library(replxx UNKNOWN IMPORTED) - set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX}) - target_include_directories(replxx SYSTEM PUBLIC ${INCLUDE_REPLXX}) +if (NOT USE_INTERNAL_REPLXX_LIBRARY) + find_library(LIBRARY_REPLXX NAMES replxx replxx-static) + find_path(INCLUDE_REPLXX replxx.hxx) - set(CMAKE_REQUIRED_LIBRARIES replxx) + if (LIBRARY_REPLXX AND INCLUDE_REPLXX) + set(CMAKE_REQUIRED_LIBRARIES ${LIBRARY_REPLXX}) + set(CMAKE_REQUIRED_INCLUDES ${INCLUDE_REPLXX}) check_cxx_source_compiles( " #include @@ -41,20 +32,43 @@ if (ENABLE_REPLXX) ) if (NOT EXTERNAL_REPLXX_WORKS) - message (FATAL_ERROR "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}") - endif () - endif () + message (${RECONFIGURE_MESSAGE_LEVEL} "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}") + else() + add_library(replxx UNKNOWN IMPORTED) + set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX}) + target_include_directories(replxx SYSTEM PUBLIC ${INCLUDE_REPLXX}) + endif() + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system replxx") + endif() +endif() - if (COMPILER_CLANG) - target_compile_options(replxx PRIVATE -Wno-documentation) - endif () - target_compile_definitions(replxx PUBLIC USE_REPLXX=1) +if (NOT LIBRARY_REPLXX OR NOT INCLUDE_REPLXX OR NOT EXTERNAL_REPLXX_WORKS) + set(USE_INTERNAL_REPLXX_LIBRARY 1) + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") - message (STATUS "Using replxx") -else () - add_library(replxx INTERFACE) - target_compile_definitions(replxx INTERFACE USE_REPLXX=0) + set(SRCS + ${LIBRARY_DIR}/src/conversion.cxx + ${LIBRARY_DIR}/src/ConvertUTF.cpp + ${LIBRARY_DIR}/src/escape.cxx + ${LIBRARY_DIR}/src/history.cxx + ${LIBRARY_DIR}/src/io.cxx + ${LIBRARY_DIR}/src/prompt.cxx + ${LIBRARY_DIR}/src/replxx_impl.cxx + ${LIBRARY_DIR}/src/replxx.cxx + ${LIBRARY_DIR}/src/util.cxx + ${LIBRARY_DIR}/src/wcwidth.cpp + ) - message (STATUS "Not using replxx (Beware! Runtime fallback to readline is possible!)") + add_library (replxx ${SRCS}) + target_include_directories(replxx SYSTEM PUBLIC ${LIBRARY_DIR}/include) endif () + +if (COMPILER_CLANG) + target_compile_options(replxx PRIVATE -Wno-documentation) +endif () + +target_compile_definitions(replxx PUBLIC USE_REPLXX=1) + +message (STATUS "Using replxx") diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt index 658fa3329d3..7a585161cc1 100644 --- a/contrib/unixodbc-cmake/CMakeLists.txt +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -1,318 +1,336 @@ option (ENABLE_ODBC "Enable ODBC library" ${ENABLE_LIBRARIES}) if (NOT OS_LINUX) + if (ENABLE_ODBC) + message(STATUS "ODBC is only supported on Linux") + endif() set (ENABLE_ODBC OFF CACHE INTERNAL "") endif () -if (ENABLE_ODBC) - option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ${NOT_UNBUNDLED}) - +if (NOT ENABLE_ODBC) if (USE_INTERNAL_ODBC_LIBRARY) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/unixodbc) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal ODBC with ENABLE_ODBC=OFF") + endif() - # ltdl - - set (SRCS_LTDL - # This file is generated by 'libtool' inside libltdl directory and then removed. - linux_x86_64/libltdl/libltdlcS.c - - ${LIBRARY_DIR}/libltdl/lt__alloc.c - ${LIBRARY_DIR}/libltdl/lt__strl.c - ${LIBRARY_DIR}/libltdl/ltdl.c - ${LIBRARY_DIR}/libltdl/lt_dlloader.c - ${LIBRARY_DIR}/libltdl/slist.c - ${LIBRARY_DIR}/libltdl/lt_error.c - ${LIBRARY_DIR}/libltdl/loaders/dlopen.c - ${LIBRARY_DIR}/libltdl/loaders/preopen.c - ) - - add_library (ltdl ${SRCS_LTDL}) - - target_include_directories(ltdl - PRIVATE - linux_x86_64/libltdl - PUBLIC - ${LIBRARY_DIR}/libltdl - ${LIBRARY_DIR}/libltdl/libltdl - ) - target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc) - target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2) - - # odbc - - set (SRCS - ${LIBRARY_DIR}/DriverManager/__attribute.c - ${LIBRARY_DIR}/DriverManager/__connection.c - ${LIBRARY_DIR}/DriverManager/__handles.c - ${LIBRARY_DIR}/DriverManager/__info.c - ${LIBRARY_DIR}/DriverManager/__stats.c - ${LIBRARY_DIR}/DriverManager/SQLAllocConnect.c - ${LIBRARY_DIR}/DriverManager/SQLAllocEnv.c - ${LIBRARY_DIR}/DriverManager/SQLAllocHandle.c - ${LIBRARY_DIR}/DriverManager/SQLAllocHandleStd.c - ${LIBRARY_DIR}/DriverManager/SQLAllocStmt.c - ${LIBRARY_DIR}/DriverManager/SQLBindCol.c - ${LIBRARY_DIR}/DriverManager/SQLBindParam.c - ${LIBRARY_DIR}/DriverManager/SQLBindParameter.c - ${LIBRARY_DIR}/DriverManager/SQLBrowseConnect.c - ${LIBRARY_DIR}/DriverManager/SQLBrowseConnectW.c - ${LIBRARY_DIR}/DriverManager/SQLBulkOperations.c - ${LIBRARY_DIR}/DriverManager/SQLCancel.c - ${LIBRARY_DIR}/DriverManager/SQLCancelHandle.c - ${LIBRARY_DIR}/DriverManager/SQLCloseCursor.c - ${LIBRARY_DIR}/DriverManager/SQLColAttribute.c - ${LIBRARY_DIR}/DriverManager/SQLColAttributes.c - ${LIBRARY_DIR}/DriverManager/SQLColAttributesW.c - ${LIBRARY_DIR}/DriverManager/SQLColAttributeW.c - ${LIBRARY_DIR}/DriverManager/SQLColumnPrivileges.c - ${LIBRARY_DIR}/DriverManager/SQLColumnPrivilegesW.c - ${LIBRARY_DIR}/DriverManager/SQLColumns.c - ${LIBRARY_DIR}/DriverManager/SQLColumnsW.c - ${LIBRARY_DIR}/DriverManager/SQLConnect.c - ${LIBRARY_DIR}/DriverManager/SQLConnectW.c - ${LIBRARY_DIR}/DriverManager/SQLCopyDesc.c - ${LIBRARY_DIR}/DriverManager/SQLDataSources.c - ${LIBRARY_DIR}/DriverManager/SQLDataSourcesW.c - ${LIBRARY_DIR}/DriverManager/SQLDescribeCol.c - ${LIBRARY_DIR}/DriverManager/SQLDescribeColW.c - ${LIBRARY_DIR}/DriverManager/SQLDescribeParam.c - ${LIBRARY_DIR}/DriverManager/SQLDisconnect.c - ${LIBRARY_DIR}/DriverManager/SQLDriverConnect.c - ${LIBRARY_DIR}/DriverManager/SQLDriverConnectW.c - ${LIBRARY_DIR}/DriverManager/SQLDrivers.c - ${LIBRARY_DIR}/DriverManager/SQLDriversW.c - ${LIBRARY_DIR}/DriverManager/SQLEndTran.c - ${LIBRARY_DIR}/DriverManager/SQLError.c - ${LIBRARY_DIR}/DriverManager/SQLErrorW.c - ${LIBRARY_DIR}/DriverManager/SQLExecDirect.c - ${LIBRARY_DIR}/DriverManager/SQLExecDirectW.c - ${LIBRARY_DIR}/DriverManager/SQLExecute.c - ${LIBRARY_DIR}/DriverManager/SQLExtendedFetch.c - ${LIBRARY_DIR}/DriverManager/SQLFetch.c - ${LIBRARY_DIR}/DriverManager/SQLFetchScroll.c - ${LIBRARY_DIR}/DriverManager/SQLForeignKeys.c - ${LIBRARY_DIR}/DriverManager/SQLForeignKeysW.c - ${LIBRARY_DIR}/DriverManager/SQLFreeConnect.c - ${LIBRARY_DIR}/DriverManager/SQLFreeEnv.c - ${LIBRARY_DIR}/DriverManager/SQLFreeHandle.c - ${LIBRARY_DIR}/DriverManager/SQLFreeStmt.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttr.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectOption.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectOptionW.c - ${LIBRARY_DIR}/DriverManager/SQLGetCursorName.c - ${LIBRARY_DIR}/DriverManager/SQLGetCursorNameW.c - ${LIBRARY_DIR}/DriverManager/SQLGetData.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescField.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescFieldW.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescRec.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescRecW.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagField.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagFieldW.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagRec.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagRecW.c - ${LIBRARY_DIR}/DriverManager/SQLGetEnvAttr.c - ${LIBRARY_DIR}/DriverManager/SQLGetFunctions.c - ${LIBRARY_DIR}/DriverManager/SQLGetInfo.c - ${LIBRARY_DIR}/DriverManager/SQLGetInfoW.c - ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttr.c - ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLGetStmtOption.c - ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfo.c - ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfoW.c - ${LIBRARY_DIR}/DriverManager/SQLMoreResults.c - ${LIBRARY_DIR}/DriverManager/SQLNativeSql.c - ${LIBRARY_DIR}/DriverManager/SQLNativeSqlW.c - ${LIBRARY_DIR}/DriverManager/SQLNumParams.c - ${LIBRARY_DIR}/DriverManager/SQLNumResultCols.c - ${LIBRARY_DIR}/DriverManager/SQLParamData.c - ${LIBRARY_DIR}/DriverManager/SQLParamOptions.c - ${LIBRARY_DIR}/DriverManager/SQLPrepare.c - ${LIBRARY_DIR}/DriverManager/SQLPrepareW.c - ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeys.c - ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeysW.c - ${LIBRARY_DIR}/DriverManager/SQLProcedureColumns.c - ${LIBRARY_DIR}/DriverManager/SQLProcedureColumnsW.c - ${LIBRARY_DIR}/DriverManager/SQLProcedures.c - ${LIBRARY_DIR}/DriverManager/SQLProceduresW.c - ${LIBRARY_DIR}/DriverManager/SQLPutData.c - ${LIBRARY_DIR}/DriverManager/SQLRowCount.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttr.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectOption.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectOptionW.c - ${LIBRARY_DIR}/DriverManager/SQLSetCursorName.c - ${LIBRARY_DIR}/DriverManager/SQLSetCursorNameW.c - ${LIBRARY_DIR}/DriverManager/SQLSetDescField.c - ${LIBRARY_DIR}/DriverManager/SQLSetDescFieldW.c - ${LIBRARY_DIR}/DriverManager/SQLSetDescRec.c - ${LIBRARY_DIR}/DriverManager/SQLSetEnvAttr.c - ${LIBRARY_DIR}/DriverManager/SQLSetParam.c - ${LIBRARY_DIR}/DriverManager/SQLSetPos.c - ${LIBRARY_DIR}/DriverManager/SQLSetScrollOptions.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttr.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtOption.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtOptionW.c - ${LIBRARY_DIR}/DriverManager/SQLSpecialColumns.c - ${LIBRARY_DIR}/DriverManager/SQLSpecialColumnsW.c - ${LIBRARY_DIR}/DriverManager/SQLStatistics.c - ${LIBRARY_DIR}/DriverManager/SQLStatisticsW.c - ${LIBRARY_DIR}/DriverManager/SQLTablePrivileges.c - ${LIBRARY_DIR}/DriverManager/SQLTablePrivilegesW.c - ${LIBRARY_DIR}/DriverManager/SQLTables.c - ${LIBRARY_DIR}/DriverManager/SQLTablesW.c - ${LIBRARY_DIR}/DriverManager/SQLTransact.c - ${LIBRARY_DIR}/ini/_iniDump.c - ${LIBRARY_DIR}/ini/_iniObjectRead.c - ${LIBRARY_DIR}/ini/_iniPropertyRead.c - ${LIBRARY_DIR}/ini/_iniScanUntilObject.c - ${LIBRARY_DIR}/ini/iniAllTrim.c - ${LIBRARY_DIR}/ini/iniAppend.c - ${LIBRARY_DIR}/ini/iniClose.c - ${LIBRARY_DIR}/ini/iniCommit.c - ${LIBRARY_DIR}/ini/iniCursor.c - ${LIBRARY_DIR}/ini/iniDelete.c - ${LIBRARY_DIR}/ini/iniElement.c - ${LIBRARY_DIR}/ini/iniElementCount.c - ${LIBRARY_DIR}/ini/iniGetBookmark.c - ${LIBRARY_DIR}/ini/iniGotoBookmark.c - ${LIBRARY_DIR}/ini/iniObject.c - ${LIBRARY_DIR}/ini/iniObjectDelete.c - ${LIBRARY_DIR}/ini/iniObjectEOL.c - ${LIBRARY_DIR}/ini/iniObjectFirst.c - ${LIBRARY_DIR}/ini/iniObjectInsert.c - ${LIBRARY_DIR}/ini/iniObjectLast.c - ${LIBRARY_DIR}/ini/iniObjectNext.c - ${LIBRARY_DIR}/ini/iniObjectSeek.c - ${LIBRARY_DIR}/ini/iniObjectSeekSure.c - ${LIBRARY_DIR}/ini/iniObjectUpdate.c - ${LIBRARY_DIR}/ini/iniOpen.c - ${LIBRARY_DIR}/ini/iniProperty.c - ${LIBRARY_DIR}/ini/iniPropertyDelete.c - ${LIBRARY_DIR}/ini/iniPropertyEOL.c - ${LIBRARY_DIR}/ini/iniPropertyFirst.c - ${LIBRARY_DIR}/ini/iniPropertyInsert.c - ${LIBRARY_DIR}/ini/iniPropertyLast.c - ${LIBRARY_DIR}/ini/iniPropertyNext.c - ${LIBRARY_DIR}/ini/iniPropertySeek.c - ${LIBRARY_DIR}/ini/iniPropertySeekSure.c - ${LIBRARY_DIR}/ini/iniPropertyUpdate.c - ${LIBRARY_DIR}/ini/iniPropertyValue.c - ${LIBRARY_DIR}/ini/iniToUpper.c - ${LIBRARY_DIR}/ini/iniValue.c - ${LIBRARY_DIR}/log/_logFreeMsg.c - ${LIBRARY_DIR}/log/logClear.c - ${LIBRARY_DIR}/log/logClose.c - ${LIBRARY_DIR}/log/logOn.c - ${LIBRARY_DIR}/log/logOpen.c - ${LIBRARY_DIR}/log/logPeekMsg.c - ${LIBRARY_DIR}/log/logPopMsg.c - ${LIBRARY_DIR}/log/logPushMsg.c - ${LIBRARY_DIR}/lst/_lstAdjustCurrent.c - ${LIBRARY_DIR}/lst/_lstDump.c - ${LIBRARY_DIR}/lst/_lstFreeItem.c - ${LIBRARY_DIR}/lst/_lstNextValidItem.c - ${LIBRARY_DIR}/lst/_lstPrevValidItem.c - ${LIBRARY_DIR}/lst/_lstVisible.c - ${LIBRARY_DIR}/lst/lstAppend.c - ${LIBRARY_DIR}/lst/lstClose.c - ${LIBRARY_DIR}/lst/lstDelete.c - ${LIBRARY_DIR}/lst/lstEOL.c - ${LIBRARY_DIR}/lst/lstFirst.c - ${LIBRARY_DIR}/lst/lstGet.c - ${LIBRARY_DIR}/lst/lstGetBookMark.c - ${LIBRARY_DIR}/lst/lstGoto.c - ${LIBRARY_DIR}/lst/lstGotoBookMark.c - ${LIBRARY_DIR}/lst/lstInsert.c - ${LIBRARY_DIR}/lst/lstLast.c - ${LIBRARY_DIR}/lst/lstNext.c - ${LIBRARY_DIR}/lst/lstOpen.c - ${LIBRARY_DIR}/lst/lstOpenCursor.c - ${LIBRARY_DIR}/lst/lstPrev.c - ${LIBRARY_DIR}/lst/lstSeek.c - ${LIBRARY_DIR}/lst/lstSeekItem.c - ${LIBRARY_DIR}/lst/lstSet.c - ${LIBRARY_DIR}/lst/lstSetFreeFunc.c - ${LIBRARY_DIR}/odbcinst/_logging.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_ConfigModeINI.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_GetEntries.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_GetSections.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_SystemINI.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_UserINI.c - ${LIBRARY_DIR}/odbcinst/_SQLDriverConnectPrompt.c - ${LIBRARY_DIR}/odbcinst/_SQLGetInstalledDrivers.c - ${LIBRARY_DIR}/odbcinst/_SQLWriteInstalledDrivers.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTConstructProperties.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTDestructProperties.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTSetProperty.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperties.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperty.c - ${LIBRARY_DIR}/odbcinst/SQLConfigDataSource.c - ${LIBRARY_DIR}/odbcinst/SQLConfigDriver.c - ${LIBRARY_DIR}/odbcinst/SQLCreateDataSource.c - ${LIBRARY_DIR}/odbcinst/SQLGetAvailableDrivers.c - ${LIBRARY_DIR}/odbcinst/SQLGetConfigMode.c - ${LIBRARY_DIR}/odbcinst/SQLGetInstalledDrivers.c - ${LIBRARY_DIR}/odbcinst/SQLGetPrivateProfileString.c - ${LIBRARY_DIR}/odbcinst/SQLGetTranslator.c - ${LIBRARY_DIR}/odbcinst/SQLInstallDriverEx.c - ${LIBRARY_DIR}/odbcinst/SQLInstallDriverManager.c - ${LIBRARY_DIR}/odbcinst/SQLInstallerError.c - ${LIBRARY_DIR}/odbcinst/SQLInstallODBC.c - ${LIBRARY_DIR}/odbcinst/SQLInstallTranslatorEx.c - ${LIBRARY_DIR}/odbcinst/SQLManageDataSources.c - ${LIBRARY_DIR}/odbcinst/SQLPostInstallerError.c - ${LIBRARY_DIR}/odbcinst/SQLReadFileDSN.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveDriver.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveDriverManager.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveDSNFromIni.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveTranslator.c - ${LIBRARY_DIR}/odbcinst/SQLSetConfigMode.c - ${LIBRARY_DIR}/odbcinst/SQLValidDSN.c - ${LIBRARY_DIR}/odbcinst/SQLWriteDSNToIni.c - ${LIBRARY_DIR}/odbcinst/SQLWriteFileDSN.c - ${LIBRARY_DIR}/odbcinst/SQLWritePrivateProfileString.c - ) - - add_library (unixodbc ${SRCS}) - - target_link_libraries (unixodbc PRIVATE ltdl) - - # SYSTEM_FILE_PATH was changed to /etc - - target_include_directories (unixodbc - PRIVATE - linux_x86_64/private - PUBLIC - linux_x86_64 - ${LIBRARY_DIR}/include - ) - target_compile_definitions (unixodbc PRIVATE -DHAVE_CONFIG_H) - target_compile_options (unixodbc - PRIVATE - -Wno-dangling-else - -Wno-parentheses - -Wno-misleading-indentation - -Wno-unknown-warning-option - -Wno-reserved-id-macro - -O2 - ) - else () - add_library (unixodbc UNKNOWN IMPORTED) - - find_library (LIBRARY_ODBC unixodbc) - find_path (INCLUDE_ODBC sql.h) - set_target_properties (unixodbc PROPERTIES IMPORTED_LOCATION ${LIBRARY_ODBC}) - set_target_properties (unixodbc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_ODBC}) - endif () - - target_compile_definitions (unixodbc INTERFACE USE_ODBC=1) - - message (STATUS "Using unixodbc") -else () add_library (unixodbc INTERFACE) target_compile_definitions (unixodbc INTERFACE USE_ODBC=0) message (STATUS "Not using unixodbc") + return() +endif() + +option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ${NOT_UNBUNDLED}) + +if (NOT USE_INTERNAL_ODBC_LIBRARY) + find_library (LIBRARY_ODBC unixodbc) + find_path (INCLUDE_ODBC sql.h) + + if(LIBRARY_ODBC AND INCLUDE_ODBC) + add_library (unixodbc UNKNOWN IMPORTED) + set_target_properties (unixodbc PROPERTIES IMPORTED_LOCATION ${LIBRARY_ODBC}) + set_target_properties (unixodbc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_ODBC}) + set(EXTERNAL_ODBC_LIBRARY_FOUND 1) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ODBC library") + set(EXTERNAL_ODBC_LIBRARY_FOUND 0) + endif() +endif() + +if (NOT EXTERNAL_ODBC_LIBRARY_FOUND) + set (USE_INTERNAL_ODBC_LIBRARY 1) + + set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/unixodbc) + + # ltdl + + set (SRCS_LTDL + # This file is generated by 'libtool' inside libltdl directory and then removed. + linux_x86_64/libltdl/libltdlcS.c + + ${LIBRARY_DIR}/libltdl/lt__alloc.c + ${LIBRARY_DIR}/libltdl/lt__strl.c + ${LIBRARY_DIR}/libltdl/ltdl.c + ${LIBRARY_DIR}/libltdl/lt_dlloader.c + ${LIBRARY_DIR}/libltdl/slist.c + ${LIBRARY_DIR}/libltdl/lt_error.c + ${LIBRARY_DIR}/libltdl/loaders/dlopen.c + ${LIBRARY_DIR}/libltdl/loaders/preopen.c + ) + + add_library (ltdl ${SRCS_LTDL}) + + target_include_directories(ltdl + PRIVATE + linux_x86_64/libltdl + PUBLIC + ${LIBRARY_DIR}/libltdl + ${LIBRARY_DIR}/libltdl/libltdl + ) + target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc) + target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2) + + # odbc + + set (SRCS + ${LIBRARY_DIR}/DriverManager/__attribute.c + ${LIBRARY_DIR}/DriverManager/__connection.c + ${LIBRARY_DIR}/DriverManager/__handles.c + ${LIBRARY_DIR}/DriverManager/__info.c + ${LIBRARY_DIR}/DriverManager/__stats.c + ${LIBRARY_DIR}/DriverManager/SQLAllocConnect.c + ${LIBRARY_DIR}/DriverManager/SQLAllocEnv.c + ${LIBRARY_DIR}/DriverManager/SQLAllocHandle.c + ${LIBRARY_DIR}/DriverManager/SQLAllocHandleStd.c + ${LIBRARY_DIR}/DriverManager/SQLAllocStmt.c + ${LIBRARY_DIR}/DriverManager/SQLBindCol.c + ${LIBRARY_DIR}/DriverManager/SQLBindParam.c + ${LIBRARY_DIR}/DriverManager/SQLBindParameter.c + ${LIBRARY_DIR}/DriverManager/SQLBrowseConnect.c + ${LIBRARY_DIR}/DriverManager/SQLBrowseConnectW.c + ${LIBRARY_DIR}/DriverManager/SQLBulkOperations.c + ${LIBRARY_DIR}/DriverManager/SQLCancel.c + ${LIBRARY_DIR}/DriverManager/SQLCancelHandle.c + ${LIBRARY_DIR}/DriverManager/SQLCloseCursor.c + ${LIBRARY_DIR}/DriverManager/SQLColAttribute.c + ${LIBRARY_DIR}/DriverManager/SQLColAttributes.c + ${LIBRARY_DIR}/DriverManager/SQLColAttributesW.c + ${LIBRARY_DIR}/DriverManager/SQLColAttributeW.c + ${LIBRARY_DIR}/DriverManager/SQLColumnPrivileges.c + ${LIBRARY_DIR}/DriverManager/SQLColumnPrivilegesW.c + ${LIBRARY_DIR}/DriverManager/SQLColumns.c + ${LIBRARY_DIR}/DriverManager/SQLColumnsW.c + ${LIBRARY_DIR}/DriverManager/SQLConnect.c + ${LIBRARY_DIR}/DriverManager/SQLConnectW.c + ${LIBRARY_DIR}/DriverManager/SQLCopyDesc.c + ${LIBRARY_DIR}/DriverManager/SQLDataSources.c + ${LIBRARY_DIR}/DriverManager/SQLDataSourcesW.c + ${LIBRARY_DIR}/DriverManager/SQLDescribeCol.c + ${LIBRARY_DIR}/DriverManager/SQLDescribeColW.c + ${LIBRARY_DIR}/DriverManager/SQLDescribeParam.c + ${LIBRARY_DIR}/DriverManager/SQLDisconnect.c + ${LIBRARY_DIR}/DriverManager/SQLDriverConnect.c + ${LIBRARY_DIR}/DriverManager/SQLDriverConnectW.c + ${LIBRARY_DIR}/DriverManager/SQLDrivers.c + ${LIBRARY_DIR}/DriverManager/SQLDriversW.c + ${LIBRARY_DIR}/DriverManager/SQLEndTran.c + ${LIBRARY_DIR}/DriverManager/SQLError.c + ${LIBRARY_DIR}/DriverManager/SQLErrorW.c + ${LIBRARY_DIR}/DriverManager/SQLExecDirect.c + ${LIBRARY_DIR}/DriverManager/SQLExecDirectW.c + ${LIBRARY_DIR}/DriverManager/SQLExecute.c + ${LIBRARY_DIR}/DriverManager/SQLExtendedFetch.c + ${LIBRARY_DIR}/DriverManager/SQLFetch.c + ${LIBRARY_DIR}/DriverManager/SQLFetchScroll.c + ${LIBRARY_DIR}/DriverManager/SQLForeignKeys.c + ${LIBRARY_DIR}/DriverManager/SQLForeignKeysW.c + ${LIBRARY_DIR}/DriverManager/SQLFreeConnect.c + ${LIBRARY_DIR}/DriverManager/SQLFreeEnv.c + ${LIBRARY_DIR}/DriverManager/SQLFreeHandle.c + ${LIBRARY_DIR}/DriverManager/SQLFreeStmt.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttr.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectOption.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectOptionW.c + ${LIBRARY_DIR}/DriverManager/SQLGetCursorName.c + ${LIBRARY_DIR}/DriverManager/SQLGetCursorNameW.c + ${LIBRARY_DIR}/DriverManager/SQLGetData.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescField.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescFieldW.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescRec.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescRecW.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagField.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagFieldW.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagRec.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagRecW.c + ${LIBRARY_DIR}/DriverManager/SQLGetEnvAttr.c + ${LIBRARY_DIR}/DriverManager/SQLGetFunctions.c + ${LIBRARY_DIR}/DriverManager/SQLGetInfo.c + ${LIBRARY_DIR}/DriverManager/SQLGetInfoW.c + ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttr.c + ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLGetStmtOption.c + ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfo.c + ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfoW.c + ${LIBRARY_DIR}/DriverManager/SQLMoreResults.c + ${LIBRARY_DIR}/DriverManager/SQLNativeSql.c + ${LIBRARY_DIR}/DriverManager/SQLNativeSqlW.c + ${LIBRARY_DIR}/DriverManager/SQLNumParams.c + ${LIBRARY_DIR}/DriverManager/SQLNumResultCols.c + ${LIBRARY_DIR}/DriverManager/SQLParamData.c + ${LIBRARY_DIR}/DriverManager/SQLParamOptions.c + ${LIBRARY_DIR}/DriverManager/SQLPrepare.c + ${LIBRARY_DIR}/DriverManager/SQLPrepareW.c + ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeys.c + ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeysW.c + ${LIBRARY_DIR}/DriverManager/SQLProcedureColumns.c + ${LIBRARY_DIR}/DriverManager/SQLProcedureColumnsW.c + ${LIBRARY_DIR}/DriverManager/SQLProcedures.c + ${LIBRARY_DIR}/DriverManager/SQLProceduresW.c + ${LIBRARY_DIR}/DriverManager/SQLPutData.c + ${LIBRARY_DIR}/DriverManager/SQLRowCount.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttr.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectOption.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectOptionW.c + ${LIBRARY_DIR}/DriverManager/SQLSetCursorName.c + ${LIBRARY_DIR}/DriverManager/SQLSetCursorNameW.c + ${LIBRARY_DIR}/DriverManager/SQLSetDescField.c + ${LIBRARY_DIR}/DriverManager/SQLSetDescFieldW.c + ${LIBRARY_DIR}/DriverManager/SQLSetDescRec.c + ${LIBRARY_DIR}/DriverManager/SQLSetEnvAttr.c + ${LIBRARY_DIR}/DriverManager/SQLSetParam.c + ${LIBRARY_DIR}/DriverManager/SQLSetPos.c + ${LIBRARY_DIR}/DriverManager/SQLSetScrollOptions.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttr.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtOption.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtOptionW.c + ${LIBRARY_DIR}/DriverManager/SQLSpecialColumns.c + ${LIBRARY_DIR}/DriverManager/SQLSpecialColumnsW.c + ${LIBRARY_DIR}/DriverManager/SQLStatistics.c + ${LIBRARY_DIR}/DriverManager/SQLStatisticsW.c + ${LIBRARY_DIR}/DriverManager/SQLTablePrivileges.c + ${LIBRARY_DIR}/DriverManager/SQLTablePrivilegesW.c + ${LIBRARY_DIR}/DriverManager/SQLTables.c + ${LIBRARY_DIR}/DriverManager/SQLTablesW.c + ${LIBRARY_DIR}/DriverManager/SQLTransact.c + ${LIBRARY_DIR}/ini/_iniDump.c + ${LIBRARY_DIR}/ini/_iniObjectRead.c + ${LIBRARY_DIR}/ini/_iniPropertyRead.c + ${LIBRARY_DIR}/ini/_iniScanUntilObject.c + ${LIBRARY_DIR}/ini/iniAllTrim.c + ${LIBRARY_DIR}/ini/iniAppend.c + ${LIBRARY_DIR}/ini/iniClose.c + ${LIBRARY_DIR}/ini/iniCommit.c + ${LIBRARY_DIR}/ini/iniCursor.c + ${LIBRARY_DIR}/ini/iniDelete.c + ${LIBRARY_DIR}/ini/iniElement.c + ${LIBRARY_DIR}/ini/iniElementCount.c + ${LIBRARY_DIR}/ini/iniGetBookmark.c + ${LIBRARY_DIR}/ini/iniGotoBookmark.c + ${LIBRARY_DIR}/ini/iniObject.c + ${LIBRARY_DIR}/ini/iniObjectDelete.c + ${LIBRARY_DIR}/ini/iniObjectEOL.c + ${LIBRARY_DIR}/ini/iniObjectFirst.c + ${LIBRARY_DIR}/ini/iniObjectInsert.c + ${LIBRARY_DIR}/ini/iniObjectLast.c + ${LIBRARY_DIR}/ini/iniObjectNext.c + ${LIBRARY_DIR}/ini/iniObjectSeek.c + ${LIBRARY_DIR}/ini/iniObjectSeekSure.c + ${LIBRARY_DIR}/ini/iniObjectUpdate.c + ${LIBRARY_DIR}/ini/iniOpen.c + ${LIBRARY_DIR}/ini/iniProperty.c + ${LIBRARY_DIR}/ini/iniPropertyDelete.c + ${LIBRARY_DIR}/ini/iniPropertyEOL.c + ${LIBRARY_DIR}/ini/iniPropertyFirst.c + ${LIBRARY_DIR}/ini/iniPropertyInsert.c + ${LIBRARY_DIR}/ini/iniPropertyLast.c + ${LIBRARY_DIR}/ini/iniPropertyNext.c + ${LIBRARY_DIR}/ini/iniPropertySeek.c + ${LIBRARY_DIR}/ini/iniPropertySeekSure.c + ${LIBRARY_DIR}/ini/iniPropertyUpdate.c + ${LIBRARY_DIR}/ini/iniPropertyValue.c + ${LIBRARY_DIR}/ini/iniToUpper.c + ${LIBRARY_DIR}/ini/iniValue.c + ${LIBRARY_DIR}/log/_logFreeMsg.c + ${LIBRARY_DIR}/log/logClear.c + ${LIBRARY_DIR}/log/logClose.c + ${LIBRARY_DIR}/log/logOn.c + ${LIBRARY_DIR}/log/logOpen.c + ${LIBRARY_DIR}/log/logPeekMsg.c + ${LIBRARY_DIR}/log/logPopMsg.c + ${LIBRARY_DIR}/log/logPushMsg.c + ${LIBRARY_DIR}/lst/_lstAdjustCurrent.c + ${LIBRARY_DIR}/lst/_lstDump.c + ${LIBRARY_DIR}/lst/_lstFreeItem.c + ${LIBRARY_DIR}/lst/_lstNextValidItem.c + ${LIBRARY_DIR}/lst/_lstPrevValidItem.c + ${LIBRARY_DIR}/lst/_lstVisible.c + ${LIBRARY_DIR}/lst/lstAppend.c + ${LIBRARY_DIR}/lst/lstClose.c + ${LIBRARY_DIR}/lst/lstDelete.c + ${LIBRARY_DIR}/lst/lstEOL.c + ${LIBRARY_DIR}/lst/lstFirst.c + ${LIBRARY_DIR}/lst/lstGet.c + ${LIBRARY_DIR}/lst/lstGetBookMark.c + ${LIBRARY_DIR}/lst/lstGoto.c + ${LIBRARY_DIR}/lst/lstGotoBookMark.c + ${LIBRARY_DIR}/lst/lstInsert.c + ${LIBRARY_DIR}/lst/lstLast.c + ${LIBRARY_DIR}/lst/lstNext.c + ${LIBRARY_DIR}/lst/lstOpen.c + ${LIBRARY_DIR}/lst/lstOpenCursor.c + ${LIBRARY_DIR}/lst/lstPrev.c + ${LIBRARY_DIR}/lst/lstSeek.c + ${LIBRARY_DIR}/lst/lstSeekItem.c + ${LIBRARY_DIR}/lst/lstSet.c + ${LIBRARY_DIR}/lst/lstSetFreeFunc.c + ${LIBRARY_DIR}/odbcinst/_logging.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_ConfigModeINI.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_GetEntries.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_GetSections.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_SystemINI.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_UserINI.c + ${LIBRARY_DIR}/odbcinst/_SQLDriverConnectPrompt.c + ${LIBRARY_DIR}/odbcinst/_SQLGetInstalledDrivers.c + ${LIBRARY_DIR}/odbcinst/_SQLWriteInstalledDrivers.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTConstructProperties.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTDestructProperties.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTSetProperty.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperties.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperty.c + ${LIBRARY_DIR}/odbcinst/SQLConfigDataSource.c + ${LIBRARY_DIR}/odbcinst/SQLConfigDriver.c + ${LIBRARY_DIR}/odbcinst/SQLCreateDataSource.c + ${LIBRARY_DIR}/odbcinst/SQLGetAvailableDrivers.c + ${LIBRARY_DIR}/odbcinst/SQLGetConfigMode.c + ${LIBRARY_DIR}/odbcinst/SQLGetInstalledDrivers.c + ${LIBRARY_DIR}/odbcinst/SQLGetPrivateProfileString.c + ${LIBRARY_DIR}/odbcinst/SQLGetTranslator.c + ${LIBRARY_DIR}/odbcinst/SQLInstallDriverEx.c + ${LIBRARY_DIR}/odbcinst/SQLInstallDriverManager.c + ${LIBRARY_DIR}/odbcinst/SQLInstallerError.c + ${LIBRARY_DIR}/odbcinst/SQLInstallODBC.c + ${LIBRARY_DIR}/odbcinst/SQLInstallTranslatorEx.c + ${LIBRARY_DIR}/odbcinst/SQLManageDataSources.c + ${LIBRARY_DIR}/odbcinst/SQLPostInstallerError.c + ${LIBRARY_DIR}/odbcinst/SQLReadFileDSN.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveDriver.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveDriverManager.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveDSNFromIni.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveTranslator.c + ${LIBRARY_DIR}/odbcinst/SQLSetConfigMode.c + ${LIBRARY_DIR}/odbcinst/SQLValidDSN.c + ${LIBRARY_DIR}/odbcinst/SQLWriteDSNToIni.c + ${LIBRARY_DIR}/odbcinst/SQLWriteFileDSN.c + ${LIBRARY_DIR}/odbcinst/SQLWritePrivateProfileString.c + ) + + add_library (unixodbc ${SRCS}) + + target_link_libraries (unixodbc PRIVATE ltdl) + + # SYSTEM_FILE_PATH was changed to /etc + + target_include_directories (unixodbc + PRIVATE + linux_x86_64/private + PUBLIC + linux_x86_64 + ${LIBRARY_DIR}/include + ) + target_compile_definitions (unixodbc PRIVATE -DHAVE_CONFIG_H) + target_compile_options (unixodbc + PRIVATE + -Wno-dangling-else + -Wno-parentheses + -Wno-misleading-indentation + -Wno-unknown-warning-option + -Wno-reserved-id-macro + -O2 + ) endif () + +target_compile_definitions (unixodbc INTERFACE USE_ODBC=1) + +message (STATUS "Using unixodbc") From 528f9d587fc1ee7839fd0e9e2ea2bb5b780f6b76 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Wed, 19 Aug 2020 11:55:04 +0300 Subject: [PATCH 39/89] cmake: find/s3: use FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION Signed-off-by: Konstantin Podshumok --- cmake/find/s3.cmake | 51 ++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/cmake/find/s3.cmake b/cmake/find/s3.cmake index af53dc80feb..bf15c4944bb 100644 --- a/cmake/find/s3.cmake +++ b/cmake/find/s3.cmake @@ -1,26 +1,39 @@ if(NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_ARM) option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES}) +elseif(ENABLE_S3 OR USE_INTERNAL_AWS_S3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use S3 on ARM, Apple or FreeBSD") endif() -if(ENABLE_S3) - option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled" ${NOT_UNBUNDLED}) - - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3") - message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive") - set (MISSING_AWS_S3 1) - endif () - - if (USE_INTERNAL_AWS_S3_LIBRARY AND NOT MISSING_AWS_S3) - set(AWS_S3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3/include") - set(AWS_S3_CORE_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core/include") - set(AWS_S3_LIBRARY aws_s3) - set(USE_INTERNAL_AWS_S3_LIBRARY 1) - set(USE_AWS_S3 1) - else() - set(USE_INTERNAL_AWS_S3_LIBRARY 0) - set(USE_AWS_S3 0) - endif () - +if(NOT ENABLE_S3) + if(USE_INTERNAL_AWS_S3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal S3 library with ENABLE_S3=OFF") + endif() + return() endif() +option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled" ${NOT_UNBUNDLED}) +if (NOT USE_INTERNAL_AWS_S3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Compilation with external S3 library is not supported yet") +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3") + message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init --recursive") + if (USE_INTERNAL_AWS_S3_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal S3 library") + endif () + set (MISSING_AWS_S3 1) +endif () + +if (USE_INTERNAL_AWS_S3_LIBRARY AND NOT MISSING_AWS_S3) + set(AWS_S3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3/include") + set(AWS_S3_CORE_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core/include") + set(AWS_S3_LIBRARY aws_s3) + set(USE_INTERNAL_AWS_S3_LIBRARY 1) + set(USE_AWS_S3 1) +else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable S3") + set(USE_INTERNAL_AWS_S3_LIBRARY 0) + set(USE_AWS_S3 0) +endif () + message (STATUS "Using aws_s3=${USE_AWS_S3}: ${AWS_S3_INCLUDE_DIR} : ${AWS_S3_LIBRARY}") From 87a2ae18208c5afdd4c0f1eedfa0aefb5411312d Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 17 Aug 2020 23:28:05 +0300 Subject: [PATCH 40/89] cmake: contrib/curl-cmake: use FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION Signed-off-by: Konstantin Podshumok --- contrib/curl-cmake/CMakeLists.txt | 363 ++++++++++++++++-------------- 1 file changed, 188 insertions(+), 175 deletions(-) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 3c3226cae9e..eb97cfa5f14 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -1,187 +1,200 @@ option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES}) -if (ENABLE_CURL) - option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED}) - +if (NOT ENABLE_CURL) if (USE_INTERNAL_CURL) - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal curl with ENABLE_CURL=OFF") + endif() + return() +endif() - set (SRCS - ${LIBRARY_DIR}/lib/file.c - ${LIBRARY_DIR}/lib/timeval.c - ${LIBRARY_DIR}/lib/base64.c - ${LIBRARY_DIR}/lib/hostip.c - ${LIBRARY_DIR}/lib/progress.c - ${LIBRARY_DIR}/lib/formdata.c - ${LIBRARY_DIR}/lib/cookie.c - ${LIBRARY_DIR}/lib/http.c - ${LIBRARY_DIR}/lib/sendf.c - ${LIBRARY_DIR}/lib/url.c - ${LIBRARY_DIR}/lib/dict.c - ${LIBRARY_DIR}/lib/if2ip.c - ${LIBRARY_DIR}/lib/speedcheck.c - ${LIBRARY_DIR}/lib/ldap.c - ${LIBRARY_DIR}/lib/version.c - ${LIBRARY_DIR}/lib/getenv.c - ${LIBRARY_DIR}/lib/escape.c - ${LIBRARY_DIR}/lib/mprintf.c - ${LIBRARY_DIR}/lib/telnet.c - ${LIBRARY_DIR}/lib/netrc.c - ${LIBRARY_DIR}/lib/getinfo.c - ${LIBRARY_DIR}/lib/transfer.c - ${LIBRARY_DIR}/lib/strcase.c - ${LIBRARY_DIR}/lib/easy.c - ${LIBRARY_DIR}/lib/security.c - ${LIBRARY_DIR}/lib/curl_fnmatch.c - ${LIBRARY_DIR}/lib/fileinfo.c - ${LIBRARY_DIR}/lib/wildcard.c - ${LIBRARY_DIR}/lib/krb5.c - ${LIBRARY_DIR}/lib/memdebug.c - ${LIBRARY_DIR}/lib/http_chunks.c - ${LIBRARY_DIR}/lib/strtok.c - ${LIBRARY_DIR}/lib/connect.c - ${LIBRARY_DIR}/lib/llist.c - ${LIBRARY_DIR}/lib/hash.c - ${LIBRARY_DIR}/lib/multi.c - ${LIBRARY_DIR}/lib/content_encoding.c - ${LIBRARY_DIR}/lib/share.c - ${LIBRARY_DIR}/lib/http_digest.c - ${LIBRARY_DIR}/lib/md4.c - ${LIBRARY_DIR}/lib/md5.c - ${LIBRARY_DIR}/lib/http_negotiate.c - ${LIBRARY_DIR}/lib/inet_pton.c - ${LIBRARY_DIR}/lib/strtoofft.c - ${LIBRARY_DIR}/lib/strerror.c - ${LIBRARY_DIR}/lib/amigaos.c - ${LIBRARY_DIR}/lib/hostasyn.c - ${LIBRARY_DIR}/lib/hostip4.c - ${LIBRARY_DIR}/lib/hostip6.c - ${LIBRARY_DIR}/lib/hostsyn.c - ${LIBRARY_DIR}/lib/inet_ntop.c - ${LIBRARY_DIR}/lib/parsedate.c - ${LIBRARY_DIR}/lib/select.c - ${LIBRARY_DIR}/lib/splay.c - ${LIBRARY_DIR}/lib/strdup.c - ${LIBRARY_DIR}/lib/socks.c - ${LIBRARY_DIR}/lib/curl_addrinfo.c - ${LIBRARY_DIR}/lib/socks_gssapi.c - ${LIBRARY_DIR}/lib/socks_sspi.c - ${LIBRARY_DIR}/lib/curl_sspi.c - ${LIBRARY_DIR}/lib/slist.c - ${LIBRARY_DIR}/lib/nonblock.c - ${LIBRARY_DIR}/lib/curl_memrchr.c - ${LIBRARY_DIR}/lib/imap.c - ${LIBRARY_DIR}/lib/pop3.c - ${LIBRARY_DIR}/lib/smtp.c - ${LIBRARY_DIR}/lib/pingpong.c - ${LIBRARY_DIR}/lib/rtsp.c - ${LIBRARY_DIR}/lib/curl_threads.c - ${LIBRARY_DIR}/lib/warnless.c - ${LIBRARY_DIR}/lib/hmac.c - ${LIBRARY_DIR}/lib/curl_rtmp.c - ${LIBRARY_DIR}/lib/openldap.c - ${LIBRARY_DIR}/lib/curl_gethostname.c - ${LIBRARY_DIR}/lib/gopher.c - ${LIBRARY_DIR}/lib/idn_win32.c - ${LIBRARY_DIR}/lib/http_proxy.c - ${LIBRARY_DIR}/lib/non-ascii.c - ${LIBRARY_DIR}/lib/asyn-thread.c - ${LIBRARY_DIR}/lib/curl_gssapi.c - ${LIBRARY_DIR}/lib/http_ntlm.c - ${LIBRARY_DIR}/lib/curl_ntlm_wb.c - ${LIBRARY_DIR}/lib/curl_ntlm_core.c - ${LIBRARY_DIR}/lib/curl_sasl.c - ${LIBRARY_DIR}/lib/rand.c - ${LIBRARY_DIR}/lib/curl_multibyte.c - ${LIBRARY_DIR}/lib/hostcheck.c - ${LIBRARY_DIR}/lib/conncache.c - ${LIBRARY_DIR}/lib/dotdot.c - ${LIBRARY_DIR}/lib/x509asn1.c - ${LIBRARY_DIR}/lib/http2.c - ${LIBRARY_DIR}/lib/smb.c - ${LIBRARY_DIR}/lib/curl_endian.c - ${LIBRARY_DIR}/lib/curl_des.c - ${LIBRARY_DIR}/lib/system_win32.c - ${LIBRARY_DIR}/lib/mime.c - ${LIBRARY_DIR}/lib/sha256.c - ${LIBRARY_DIR}/lib/setopt.c - ${LIBRARY_DIR}/lib/curl_path.c - ${LIBRARY_DIR}/lib/curl_ctype.c - ${LIBRARY_DIR}/lib/curl_range.c - ${LIBRARY_DIR}/lib/psl.c - ${LIBRARY_DIR}/lib/doh.c - ${LIBRARY_DIR}/lib/urlapi.c - ${LIBRARY_DIR}/lib/curl_get_line.c - ${LIBRARY_DIR}/lib/altsvc.c - ${LIBRARY_DIR}/lib/socketpair.c - ${LIBRARY_DIR}/lib/vauth/vauth.c - ${LIBRARY_DIR}/lib/vauth/cleartext.c - ${LIBRARY_DIR}/lib/vauth/cram.c - ${LIBRARY_DIR}/lib/vauth/digest.c - ${LIBRARY_DIR}/lib/vauth/digest_sspi.c - ${LIBRARY_DIR}/lib/vauth/krb5_gssapi.c - ${LIBRARY_DIR}/lib/vauth/krb5_sspi.c - ${LIBRARY_DIR}/lib/vauth/ntlm.c - ${LIBRARY_DIR}/lib/vauth/ntlm_sspi.c - ${LIBRARY_DIR}/lib/vauth/oauth2.c - ${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c - ${LIBRARY_DIR}/lib/vauth/spnego_sspi.c - ${LIBRARY_DIR}/lib/vtls/openssl.c - ${LIBRARY_DIR}/lib/vtls/gtls.c - ${LIBRARY_DIR}/lib/vtls/vtls.c - ${LIBRARY_DIR}/lib/vtls/nss.c - ${LIBRARY_DIR}/lib/vtls/polarssl.c - ${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c - ${LIBRARY_DIR}/lib/vtls/wolfssl.c - ${LIBRARY_DIR}/lib/vtls/schannel.c - ${LIBRARY_DIR}/lib/vtls/schannel_verify.c - ${LIBRARY_DIR}/lib/vtls/sectransp.c - ${LIBRARY_DIR}/lib/vtls/gskit.c - ${LIBRARY_DIR}/lib/vtls/mbedtls.c - ${LIBRARY_DIR}/lib/vtls/mesalink.c - ${LIBRARY_DIR}/lib/vtls/bearssl.c - ${LIBRARY_DIR}/lib/vquic/ngtcp2.c - ${LIBRARY_DIR}/lib/vquic/quiche.c - ${LIBRARY_DIR}/lib/vssh/libssh2.c - ${LIBRARY_DIR}/lib/vssh/libssh.c - ) +option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED}) - add_library (curl ${SRCS}) +if (NOT USE_INTERNAL_CURL) + find_package (CURL) + if (NOT CURL_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system curl") + endif() +endif() - target_compile_definitions (curl PRIVATE - HAVE_CONFIG_H - BUILDING_LIBCURL - CURL_HIDDEN_SYMBOLS - libcurl_EXPORTS - OS="${CMAKE_SYSTEM_NAME}" - ) - target_include_directories (curl PUBLIC - ${LIBRARY_DIR}/include - ${LIBRARY_DIR}/lib - . # curl_config.h - ) +if (NOT CURL_FOUND) + set (USE_INTERNAL_CURL 1) + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") - target_link_libraries (curl PRIVATE ssl) + set (SRCS + ${LIBRARY_DIR}/lib/file.c + ${LIBRARY_DIR}/lib/timeval.c + ${LIBRARY_DIR}/lib/base64.c + ${LIBRARY_DIR}/lib/hostip.c + ${LIBRARY_DIR}/lib/progress.c + ${LIBRARY_DIR}/lib/formdata.c + ${LIBRARY_DIR}/lib/cookie.c + ${LIBRARY_DIR}/lib/http.c + ${LIBRARY_DIR}/lib/sendf.c + ${LIBRARY_DIR}/lib/url.c + ${LIBRARY_DIR}/lib/dict.c + ${LIBRARY_DIR}/lib/if2ip.c + ${LIBRARY_DIR}/lib/speedcheck.c + ${LIBRARY_DIR}/lib/ldap.c + ${LIBRARY_DIR}/lib/version.c + ${LIBRARY_DIR}/lib/getenv.c + ${LIBRARY_DIR}/lib/escape.c + ${LIBRARY_DIR}/lib/mprintf.c + ${LIBRARY_DIR}/lib/telnet.c + ${LIBRARY_DIR}/lib/netrc.c + ${LIBRARY_DIR}/lib/getinfo.c + ${LIBRARY_DIR}/lib/transfer.c + ${LIBRARY_DIR}/lib/strcase.c + ${LIBRARY_DIR}/lib/easy.c + ${LIBRARY_DIR}/lib/security.c + ${LIBRARY_DIR}/lib/curl_fnmatch.c + ${LIBRARY_DIR}/lib/fileinfo.c + ${LIBRARY_DIR}/lib/wildcard.c + ${LIBRARY_DIR}/lib/krb5.c + ${LIBRARY_DIR}/lib/memdebug.c + ${LIBRARY_DIR}/lib/http_chunks.c + ${LIBRARY_DIR}/lib/strtok.c + ${LIBRARY_DIR}/lib/connect.c + ${LIBRARY_DIR}/lib/llist.c + ${LIBRARY_DIR}/lib/hash.c + ${LIBRARY_DIR}/lib/multi.c + ${LIBRARY_DIR}/lib/content_encoding.c + ${LIBRARY_DIR}/lib/share.c + ${LIBRARY_DIR}/lib/http_digest.c + ${LIBRARY_DIR}/lib/md4.c + ${LIBRARY_DIR}/lib/md5.c + ${LIBRARY_DIR}/lib/http_negotiate.c + ${LIBRARY_DIR}/lib/inet_pton.c + ${LIBRARY_DIR}/lib/strtoofft.c + ${LIBRARY_DIR}/lib/strerror.c + ${LIBRARY_DIR}/lib/amigaos.c + ${LIBRARY_DIR}/lib/hostasyn.c + ${LIBRARY_DIR}/lib/hostip4.c + ${LIBRARY_DIR}/lib/hostip6.c + ${LIBRARY_DIR}/lib/hostsyn.c + ${LIBRARY_DIR}/lib/inet_ntop.c + ${LIBRARY_DIR}/lib/parsedate.c + ${LIBRARY_DIR}/lib/select.c + ${LIBRARY_DIR}/lib/splay.c + ${LIBRARY_DIR}/lib/strdup.c + ${LIBRARY_DIR}/lib/socks.c + ${LIBRARY_DIR}/lib/curl_addrinfo.c + ${LIBRARY_DIR}/lib/socks_gssapi.c + ${LIBRARY_DIR}/lib/socks_sspi.c + ${LIBRARY_DIR}/lib/curl_sspi.c + ${LIBRARY_DIR}/lib/slist.c + ${LIBRARY_DIR}/lib/nonblock.c + ${LIBRARY_DIR}/lib/curl_memrchr.c + ${LIBRARY_DIR}/lib/imap.c + ${LIBRARY_DIR}/lib/pop3.c + ${LIBRARY_DIR}/lib/smtp.c + ${LIBRARY_DIR}/lib/pingpong.c + ${LIBRARY_DIR}/lib/rtsp.c + ${LIBRARY_DIR}/lib/curl_threads.c + ${LIBRARY_DIR}/lib/warnless.c + ${LIBRARY_DIR}/lib/hmac.c + ${LIBRARY_DIR}/lib/curl_rtmp.c + ${LIBRARY_DIR}/lib/openldap.c + ${LIBRARY_DIR}/lib/curl_gethostname.c + ${LIBRARY_DIR}/lib/gopher.c + ${LIBRARY_DIR}/lib/idn_win32.c + ${LIBRARY_DIR}/lib/http_proxy.c + ${LIBRARY_DIR}/lib/non-ascii.c + ${LIBRARY_DIR}/lib/asyn-thread.c + ${LIBRARY_DIR}/lib/curl_gssapi.c + ${LIBRARY_DIR}/lib/http_ntlm.c + ${LIBRARY_DIR}/lib/curl_ntlm_wb.c + ${LIBRARY_DIR}/lib/curl_ntlm_core.c + ${LIBRARY_DIR}/lib/curl_sasl.c + ${LIBRARY_DIR}/lib/rand.c + ${LIBRARY_DIR}/lib/curl_multibyte.c + ${LIBRARY_DIR}/lib/hostcheck.c + ${LIBRARY_DIR}/lib/conncache.c + ${LIBRARY_DIR}/lib/dotdot.c + ${LIBRARY_DIR}/lib/x509asn1.c + ${LIBRARY_DIR}/lib/http2.c + ${LIBRARY_DIR}/lib/smb.c + ${LIBRARY_DIR}/lib/curl_endian.c + ${LIBRARY_DIR}/lib/curl_des.c + ${LIBRARY_DIR}/lib/system_win32.c + ${LIBRARY_DIR}/lib/mime.c + ${LIBRARY_DIR}/lib/sha256.c + ${LIBRARY_DIR}/lib/setopt.c + ${LIBRARY_DIR}/lib/curl_path.c + ${LIBRARY_DIR}/lib/curl_ctype.c + ${LIBRARY_DIR}/lib/curl_range.c + ${LIBRARY_DIR}/lib/psl.c + ${LIBRARY_DIR}/lib/doh.c + ${LIBRARY_DIR}/lib/urlapi.c + ${LIBRARY_DIR}/lib/curl_get_line.c + ${LIBRARY_DIR}/lib/altsvc.c + ${LIBRARY_DIR}/lib/socketpair.c + ${LIBRARY_DIR}/lib/vauth/vauth.c + ${LIBRARY_DIR}/lib/vauth/cleartext.c + ${LIBRARY_DIR}/lib/vauth/cram.c + ${LIBRARY_DIR}/lib/vauth/digest.c + ${LIBRARY_DIR}/lib/vauth/digest_sspi.c + ${LIBRARY_DIR}/lib/vauth/krb5_gssapi.c + ${LIBRARY_DIR}/lib/vauth/krb5_sspi.c + ${LIBRARY_DIR}/lib/vauth/ntlm.c + ${LIBRARY_DIR}/lib/vauth/ntlm_sspi.c + ${LIBRARY_DIR}/lib/vauth/oauth2.c + ${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c + ${LIBRARY_DIR}/lib/vauth/spnego_sspi.c + ${LIBRARY_DIR}/lib/vtls/openssl.c + ${LIBRARY_DIR}/lib/vtls/gtls.c + ${LIBRARY_DIR}/lib/vtls/vtls.c + ${LIBRARY_DIR}/lib/vtls/nss.c + ${LIBRARY_DIR}/lib/vtls/polarssl.c + ${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c + ${LIBRARY_DIR}/lib/vtls/wolfssl.c + ${LIBRARY_DIR}/lib/vtls/schannel.c + ${LIBRARY_DIR}/lib/vtls/schannel_verify.c + ${LIBRARY_DIR}/lib/vtls/sectransp.c + ${LIBRARY_DIR}/lib/vtls/gskit.c + ${LIBRARY_DIR}/lib/vtls/mbedtls.c + ${LIBRARY_DIR}/lib/vtls/mesalink.c + ${LIBRARY_DIR}/lib/vtls/bearssl.c + ${LIBRARY_DIR}/lib/vquic/ngtcp2.c + ${LIBRARY_DIR}/lib/vquic/quiche.c + ${LIBRARY_DIR}/lib/vssh/libssh2.c + ${LIBRARY_DIR}/lib/vssh/libssh.c + ) - # The library is large - avoid bloat (XXX: is it?) - target_compile_options (curl PRIVATE -g0) + add_library (curl ${SRCS}) - # find_package(CURL) compatibility for the following packages that uses - # find_package(CURL)/include(FindCURL): - # - mariadb-connector-c - # - aws-s3-cmake - # - sentry-native - set (CURL_FOUND ON CACHE BOOL "") - set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "") - set (CURL_INCLUDE_DIR ${LIBRARY_DIR}/include CACHE PATH "") - set (CURL_INCLUDE_DIRS ${LIBRARY_DIR}/include CACHE PATH "") - set (CURL_LIBRARY curl CACHE STRING "") - set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") - set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") - add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) - else () - find_package (CURL REQUIRED) - endif () + target_compile_definitions (curl PRIVATE + HAVE_CONFIG_H + BUILDING_LIBCURL + CURL_HIDDEN_SYMBOLS + libcurl_EXPORTS + OS="${CMAKE_SYSTEM_NAME}" + ) + target_include_directories (curl PUBLIC + ${LIBRARY_DIR}/include + ${LIBRARY_DIR}/lib + . # curl_config.h + ) + + target_link_libraries (curl PRIVATE ssl) + + # The library is large - avoid bloat (XXX: is it?) + target_compile_options (curl PRIVATE -g0) + + # find_package(CURL) compatibility for the following packages that uses + # find_package(CURL)/include(FindCURL): + # - mariadb-connector-c + # - aws-s3-cmake + # - sentry-native + set (CURL_FOUND ON CACHE BOOL "") + set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "") + set (CURL_INCLUDE_DIR ${LIBRARY_DIR}/include CACHE PATH "") + set (CURL_INCLUDE_DIRS ${LIBRARY_DIR}/include CACHE PATH "") + set (CURL_LIBRARY curl CACHE STRING "") + set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") + set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") + add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) + + set (USE_INTERNAL_CURL 1) endif () message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}") From ad23bf46b607c1f75e63c72ecafd901972869f2d Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 14 Aug 2020 02:07:39 +0300 Subject: [PATCH 41/89] cmake: message when ThinLTO is enabled without clang Squashed: - ENABLE_THINLTO shouldn't be enabled by default when tests or sanitizers are enabled Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 32edc1b7f9c..cdc23d8aac8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -263,7 +263,9 @@ if (COMPILER_CLANG) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths") - option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) + if (NOT ENABLE_TESTS AND NOT SANITIZE) + option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON) + endif() # We cannot afford to use LTO when compiling unitests, and it's not enough # to only supply -fno-lto at the final linking stage. So we disable it @@ -294,6 +296,8 @@ if (COMPILER_CLANG) else () message(WARNING "Cannot find llvm-ranlib. System ranlib will be used instead. It does not work with ThinLTO.") endif () +elseif (ENABLE_THINLTO) + message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang") endif () option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON) From 2fc37bd6946ca3e1663eeea910d0d5f8ede5d17e Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 14 Aug 2020 02:10:49 +0300 Subject: [PATCH 42/89] cmake: enable FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION by default Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cdc23d8aac8..189ca4416d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,9 @@ endforeach() project(ClickHouse) option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION - "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" OFF) + "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" + ON +) if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR) else() From 191352223380bacae69900605994b16996d099b1 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 17 Aug 2020 15:44:41 +0300 Subject: [PATCH 43/89] cmake: always use internal poco library as it went too far from upstream Squashed: - cmake: move USE_INTERNAL_POCO_LIBRARY option to separate file - point contrib/poco to "add to TCPServerDispatcher" Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 1 + cmake/find/poco.cmake | 10 ++++++++++ cmake/find/s3.cmake | 8 +++++++- contrib/poco | 2 +- contrib/poco-cmake/CMakeLists.txt | 11 ----------- 5 files changed, 19 insertions(+), 13 deletions(-) create mode 100644 cmake/find/poco.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 189ca4416d6..52ae3c9a59b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -411,6 +411,7 @@ include (cmake/find/protobuf.cmake) include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf +include (cmake/find/poco.cmake) include (cmake/find/s3.cmake) include (cmake/find/base64.cmake) include (cmake/find/parquet.cmake) diff --git a/cmake/find/poco.cmake b/cmake/find/poco.cmake new file mode 100644 index 00000000000..e749fc27925 --- /dev/null +++ b/cmake/find/poco.cmake @@ -0,0 +1,10 @@ +option (USE_INTERNAL_POCO_LIBRARY "Use internal Poco library" ON) + +if (USE_INTERNAL_POCO_LIBRARY) + set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/poco) +else () + find_path (ROOT_DIR NAMES Foundation/include/Poco/Poco.h include/Poco/Poco.h) + if (NOT ROOT_DIR) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system poco") + endif() +endif () diff --git a/cmake/find/s3.cmake b/cmake/find/s3.cmake index bf15c4944bb..1bbf48fd6b0 100644 --- a/cmake/find/s3.cmake +++ b/cmake/find/s3.cmake @@ -11,7 +11,13 @@ if(NOT ENABLE_S3) return() endif() -option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled" ${NOT_UNBUNDLED}) +option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk)" + ON) + +if (NOT USE_INTERNAL_POCO_LIBRARY AND USE_INTERNAL_AWS_S3_LIBRARY) + message (FATAL_ERROR "Currently S3 support can be built only with internal POCO library") +endif() + if (NOT USE_INTERNAL_AWS_S3_LIBRARY) message (${RECONFIGURE_MESSAGE_LEVEL} "Compilation with external S3 library is not supported yet") endif() diff --git a/contrib/poco b/contrib/poco index 74c93443342..297fc905e16 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 74c93443342f6028fa6402057684733b316aa737 +Subproject commit 297fc905e166392156f83b96aaa5f44e8a6a35c4 diff --git a/contrib/poco-cmake/CMakeLists.txt b/contrib/poco-cmake/CMakeLists.txt index a96b07085ba..fff9051f40f 100644 --- a/contrib/poco-cmake/CMakeLists.txt +++ b/contrib/poco-cmake/CMakeLists.txt @@ -1,14 +1,3 @@ -option (USE_INTERNAL_POCO_LIBRARY "Use internal Poco library" ${NOT_UNBUNDLED}) - -if (USE_INTERNAL_POCO_LIBRARY) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/poco) -else () - find_path (ROOT_DIR NAMES Foundation/include/Poco/Poco.h include/Poco/Poco.h) - if (NOT ROOT_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system poco") - endif() -endif () - add_subdirectory (Crypto) add_subdirectory (Data) add_subdirectory (Data/ODBC) From 986bdf86ee7b8151e0f7d8d5f29f3ed41d61dff2 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 14 Aug 2020 03:36:38 +0300 Subject: [PATCH 44/89] cmake: don't enable internal llvm with disabled embedded compiler Even if UNBUNDLED is off Squashed: - cmake: fix misleading error message about internal llvm on unsupported arches - cmake: re-do some checks for LLVM options Signed-off-by: Konstantin Podshumok --- cmake/find/llvm.cmake | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index e46b8c9335e..9542b8da6fc 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -1,10 +1,12 @@ -# Broken in macos. TODO: update clang, re-test, enable -if (NOT APPLE) - option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile_expressions' option for query execution" ${ENABLE_LIBRARIES}) +if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64) + set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "") +endif() + +option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile_expressions' option for query execution" ${ENABLE_LIBRARIES}) +# Broken in macos. TODO: update clang, re-test, enable on Apple +if (ENABLE_EMBEDDED_COMPILER AND NOT SPLIT_SHARED_LIBRARIES AND ARCH_AMD64 AND NOT (SANITIZE STREQUAL "undefined")) option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library." ${NOT_UNBUNDLED}) -elseif(ENABLE_EMBEDDED_COMPILER OR USE_INTERNAL_LLVM_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Embedded compiler is not currently supported on Apple") -endif () +endif() if (NOT ENABLE_EMBEDDED_COMPILER) if(USE_INTERNAL_LLVM_LIBRARY) @@ -13,10 +15,12 @@ if (NOT ENABLE_EMBEDDED_COMPILER) return() endif() -if (USE_INTERNAL_LLVM_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt") - message (WARNING "submodule contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't fidd internal LLVM library") - set (USE_INTERNAL_LLVM_LIBRARY 0) +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt") + if (USE_INTERNAL_LLVM_LIBRARY) + message (WARNING "submodule contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't fidd internal LLVM library") + endif() + set (MISSING_INTERNAL_LLVM_LIBRARY 1) endif () if (NOT USE_INTERNAL_LLVM_LIBRARY) @@ -46,35 +50,38 @@ if (NOT USE_INTERNAL_LLVM_LIBRARY) set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) endif () -else() +endif() + +if(NOT LLVM_FOUND AND NOT MISSING_INTERNAL_LLVM_LIBRARY) if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) - message(WARNING "Option ENABLE_EMBEDDED_COMPILER is set but LLVM library cannot build if build directory is the same as source directory.") + message(WARNING "Option ENABLE_EMBEDDED_COMPILER is set but internal LLVM library cannot build if build directory is the same as source directory.") set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) elseif (SPLIT_SHARED_LIBRARIES) # llvm-tablegen cannot find shared libraries that we build. Probably can be easily fixed. - message(WARNING "Option ENABLE_EMBEDDED_COMPILER is not compatible with SPLIT_SHARED_LIBRARIES. Build of LLVM will be disabled.") + message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not compatible with SPLIT_SHARED_LIBRARIES. Build of LLVM will be disabled.") set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) elseif (NOT ARCH_AMD64) # It's not supported yet, but you can help. - message(WARNING "Option ENABLE_EMBEDDED_COMPILER is only available for x86_64. Build of LLVM will be disabled.") + message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is only available for x86_64. Build of LLVM will be disabled.") set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) elseif (SANITIZE STREQUAL "undefined") # llvm-tblgen, that is used during LLVM build, doesn't work with UBSan. - message(WARNING "Option ENABLE_EMBEDDED_COMPILER does not work with UBSan, because 'llvm-tblgen' tool from LLVM has undefined behaviour. Build of LLVM will be disabled.") + message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY does not work with UBSan, because 'llvm-tblgen' tool from LLVM has undefined behaviour. Build of LLVM will be disabled.") set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) else () + set (USE_INTERNAL_LLVM_LIBRARY ON) set (LLVM_FOUND 1) set (USE_EMBEDDED_COMPILER 1) set (LLVM_VERSION "9.0.0bundled") set (LLVM_INCLUDE_DIRS - ${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include - ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include + "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include" + "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include" ) - set (LLVM_LIBRARY_DIRS ${ClickHouse_BINARY_DIR}/contrib/llvm/llvm) + set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm") endif() endif() From 3ec7a4bc53b693d24025dc2b35d674b54d47f8b8 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 14 Aug 2020 13:29:51 +0300 Subject: [PATCH 45/89] cmake: make sure ENABLE_GRPC is off by default when PROTOBUF is OFF by default Squashed: - cmake: make sure protobuf compiler and snappy are there for unbundled build - cmake: Turn ON internal GRPC library by default until relevant features will be implemented - cmake: allow unbundled grpc Signed-off-by: Konstantin Podshumok --- cmake/find/grpc.cmake | 9 ++++++++- cmake/find/protobuf.cmake | 16 +++++++++------- cmake/target.cmake | 3 +++ contrib/protobuf-cmake/CMakeLists.txt | 6 +++--- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/cmake/find/grpc.cmake b/cmake/find/grpc.cmake index 216f8ae1878..52d0e042573 100644 --- a/cmake/find/grpc.cmake +++ b/cmake/find/grpc.cmake @@ -7,10 +7,17 @@ if (NOT ENABLE_GRPC) return() endif() -option (USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled" ${NOT_UNBUNDLED}) +option (USE_INTERNAL_GRPC_LIBRARY + "Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk)" + ${NOT_UNBUNDLED}) if (NOT USE_INTERNAL_GRPC_LIBRARY) find_package(grpc) + if (NOT GRPC_FOUND) + find_path(GRPC_INCLUDE_DIR grpcpp/grpcpp.h) + find_library(GRPC_LIBRARY grpc++) + endif () + if (GRPC_INCLUDE_DIR AND GRPC_LIBRARY) set (USE_GRPC ON) else() diff --git a/cmake/find/protobuf.cmake b/cmake/find/protobuf.cmake index 49d09299f27..a7769206e9f 100644 --- a/cmake/find/protobuf.cmake +++ b/cmake/find/protobuf.cmake @@ -20,15 +20,17 @@ endif() if(NOT USE_INTERNAL_PROTOBUF_LIBRARY) find_package(Protobuf) - if (NOT Protobuf_LIBRARY OR NOT Protobuf_INCLUDE_DIR) + if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR AND Protobuf_PROTOC_EXECUTABLE) + set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 1) + set(USE_PROTOBUF 1) + else() message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf") + set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0) endif() endif() -if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR) - set(USE_PROTOBUF 1) -elseif(NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) - set(Protobuf_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf/src) +if (NOT EXTERNAL_PROTOBUF_LIBRARY_FOUND AND NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) + set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") set(USE_PROTOBUF 1) set(USE_INTERNAL_PROTOBUF_LIBRARY 1) @@ -43,13 +45,13 @@ if(OS_FREEBSD AND SANITIZE STREQUAL "address") # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found # #include if(LLVM_INCLUDE_DIRS) - set(Protobuf_INCLUDE_DIR ${Protobuf_INCLUDE_DIR} ${LLVM_INCLUDE_DIRS}) + set(Protobuf_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}" ${LLVM_INCLUDE_DIRS}) else() message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use protobuf on FreeBSD with address sanitizer without LLVM") set(USE_PROTOBUF 0) endif() endif() -include (${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake) +include ("${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake") message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE}") diff --git a/cmake/target.cmake b/cmake/target.cmake index 35040e48956..5646d60df5e 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -19,6 +19,7 @@ if (CMAKE_CROSSCOMPILING) # FIXME: broken dependencies set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") + set (ENABLE_GRPC OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") @@ -26,12 +27,14 @@ if (CMAKE_CROSSCOMPILING) if (ARCH_AARCH64) # FIXME: broken dependencies set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") + set (ENABLE_GRPC OFF CACHE INTERNAL "") set (ENABLE_PARQUET OFF CACHE INTERNAL "") set (ENABLE_MYSQL OFF CACHE INTERNAL "") endif () elseif (OS_FREEBSD) # FIXME: broken dependencies set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") + set (ENABLE_GRPC OFF CACHE INTERNAL "") set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "") else () message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!") diff --git a/contrib/protobuf-cmake/CMakeLists.txt b/contrib/protobuf-cmake/CMakeLists.txt index 3cb9053d647..683429194fc 100644 --- a/contrib/protobuf-cmake/CMakeLists.txt +++ b/contrib/protobuf-cmake/CMakeLists.txt @@ -1,5 +1,5 @@ -set(protobuf_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf) -set(protobuf_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/protobuf) +set(protobuf_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf") +set(protobuf_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/protobuf") set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE) @@ -10,4 +10,4 @@ else () set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE) endif () -add_subdirectory(${protobuf_SOURCE_DIR}/cmake ${protobuf_BINARY_DIR}) +add_subdirectory("${protobuf_SOURCE_DIR}/cmake" "${protobuf_BINARY_DIR}") From 0ad4b5dea69e63c82dd4cce93547b139c8e68ad7 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 14 Aug 2020 18:33:17 +0300 Subject: [PATCH 46/89] cmake: disable jemalloc on alternative arches and with sanitizers Signed-off-by: Konstantin Podshumok --- contrib/jemalloc-cmake/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index e8a690ff149..dd7f9f3e2bb 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -1,9 +1,11 @@ -option (ENABLE_JEMALLOC "Enable jemalloc allocator" ${ENABLE_LIBRARIES}) - if (SANITIZE OR NOT (ARCH_AMD64 OR ARCH_ARM) OR NOT (OS_LINUX OR OS_FREEBSD OR OS_DARWIN)) + if (ENABLE_JEMALLOC) + message (${RECONFIGURE_MESSAGE_LEVEL} + "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64 or aarch64 on linux or freebsd.") + endif() set (ENABLE_JEMALLOC OFF) - message (${RECONFIGURE_MESSAGE_LEVEL} - "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64 or aarch64 on linux or freebsd.") +else() + option (ENABLE_JEMALLOC "Enable jemalloc allocator" ${ENABLE_LIBRARIES}) endif () if (NOT ENABLE_JEMALLOC) From 0c30721b9164c4077922f52657bf539a8773e1cc Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 15 Aug 2020 00:45:37 +0300 Subject: [PATCH 47/89] cmake: set ENABLE_ORC to OFF by default on platforms that don't support protobuf or parquet Squashed: - cmake: Better diagnostic message about broken ORC dependencies Signed-off-by: Konstantin Podshumok --- cmake/find/orc.cmake | 4 +++- cmake/target.cmake | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index c9474a095c4..e91f0e95819 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -43,7 +43,9 @@ elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY AND SNAPPY_LIBRARY) # set(USE_ORC 1) set(USE_INTERNAL_ORC_LIBRARY 1) else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable ORC support - missing dependencies") + message (${RECONFIGURE_MESSAGE_LEVEL} + "Can't enable ORC support - missing dependencies. Missing internal orc=${MISSING_INTERNAL_ORC_LIBRARY}. " + "arrow=${ARROW_LIBRARY} snappy=${SNAPPY_LIBRARY}") set(USE_INTERNAL_ORC_LIBRARY 0) endif() diff --git a/cmake/target.cmake b/cmake/target.cmake index 5646d60df5e..7174ca3c2a9 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -17,10 +17,13 @@ endif () if (CMAKE_CROSSCOMPILING) if (OS_DARWIN) # FIXME: broken dependencies - set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") - set (ENABLE_GRPC OFF CACHE INTERNAL "") - set (ENABLE_PARQUET OFF CACHE INTERNAL "") + set (ENABLE_GRPC OFF CACHE INTERNAL "") # no protobuf -> no grpc + + set (USE_SNAPPY OFF CACHE INTERNAL "") + set (ENABLE_PARQUET OFF CACHE INTERNAL "") # no snappy and protobuf -> no parquet + set (ENABLE_ORC OFF CACHE INTERNAL "") # no arrow (parquet) -> no orc + set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") elseif (OS_LINUX OR OS_ANDROID) @@ -28,13 +31,19 @@ if (CMAKE_CROSSCOMPILING) # FIXME: broken dependencies set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") + set (ENABLE_PARQUET OFF CACHE INTERNAL "") + set (ENABLE_ORC OFF CACHE INTERNAL "") + set (ENABLE_MYSQL OFF CACHE INTERNAL "") endif () elseif (OS_FREEBSD) # FIXME: broken dependencies set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") + + set (ENABLE_ORC OFF CACHE INTERNAL "") # no protobuf -> no parquet -> no orc + set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "") else () message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!") From 18fd3e7357c3e20b2b2f337889004efa4449b1b5 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 15 Aug 2020 02:22:46 +0300 Subject: [PATCH 48/89] cmake: ARM: make sure cpuid is not enabled by default Signed-off-by: Konstantin Podshumok --- contrib/libcpuid-cmake/CMakeLists.txt | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/contrib/libcpuid-cmake/CMakeLists.txt b/contrib/libcpuid-cmake/CMakeLists.txt index 564e67994bf..8c1be50b4e6 100644 --- a/contrib/libcpuid-cmake/CMakeLists.txt +++ b/contrib/libcpuid-cmake/CMakeLists.txt @@ -1,4 +1,6 @@ -option (ENABLE_CPUID "Enable libcpuid library (only internal)" ${ENABLE_LIBRARIES}) +if (NOT ARCH_ARM) + option (ENABLE_CPUID "Enable libcpuid library (only internal)" ${ENABLE_LIBRARIES}) +endif() if (ARCH_ARM AND ENABLE_CPUID) message (${RECONFIGURE_MESSAGE_LEVEL} "cpuid is not supported on ARM") @@ -16,19 +18,19 @@ endif() set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libcpuid") set (SRCS - ${LIBRARY_DIR}/libcpuid/asm-bits.c - ${LIBRARY_DIR}/libcpuid/cpuid_main.c - ${LIBRARY_DIR}/libcpuid/libcpuid_util.c - ${LIBRARY_DIR}/libcpuid/msrdriver.c - ${LIBRARY_DIR}/libcpuid/rdmsr.c - ${LIBRARY_DIR}/libcpuid/rdtsc.c - ${LIBRARY_DIR}/libcpuid/recog_amd.c - ${LIBRARY_DIR}/libcpuid/recog_intel.c + "${LIBRARY_DIR}/libcpuid/asm-bits.c" + "${LIBRARY_DIR}/libcpuid/cpuid_main.c" + "${LIBRARY_DIR}/libcpuid/libcpuid_util.c" + "${LIBRARY_DIR}/libcpuid/msrdriver.c" + "${LIBRARY_DIR}/libcpuid/rdmsr.c" + "${LIBRARY_DIR}/libcpuid/rdtsc.c" + "${LIBRARY_DIR}/libcpuid/recog_amd.c" + "${LIBRARY_DIR}/libcpuid/recog_intel.c" ) add_library (cpuid ${SRCS}) -target_include_directories (cpuid SYSTEM PUBLIC ${LIBRARY_DIR}) +target_include_directories (cpuid SYSTEM PUBLIC "${LIBRARY_DIR}") target_compile_definitions (cpuid PUBLIC USE_CPUID=1) target_compile_definitions (cpuid PRIVATE VERSION="v0.4.1") if (COMPILER_CLANG) From 139ba529449c8cecc6b9a7e24296c4b4e0c5ccfa Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 15 Aug 2020 02:39:16 +0300 Subject: [PATCH 49/89] cmake: unbundled + use_static_libs = no ldap Signed-off-by: Konstantin Podshumok --- cmake/find/ldap.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index 3f7871617d4..369c1e42e8d 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -1,3 +1,7 @@ +if (UNBUNDLED AND USE_STATIC_LIBRARIES) + set (ENABLE_LDAP OFF CACHE INTERNAL "") +endif() + option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES}) if (NOT ENABLE_LDAP) From 4e9b5bd31bd3e3f19ff655716b19bfe36b5b04e1 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 17 Aug 2020 23:33:50 +0300 Subject: [PATCH 50/89] docker/packager/deb: new dependencies for unbundled Squashed: - docker/packager/deb: add arrow repo - docker/packager/deb: cleanup - docker/packager/deb: Add thrift - docker/packager/deb: add myariadbclient - docker/packager/deb: remove poco-dev as we always use contrib/ version now - docker/packager/deb: add libssl-dev (unbundled) and restructure - docker/packager/deb: fix repo url - docker/packager/deb: fix tzdata not updated in build.sh - docker/packager/deb: add mysqlclient-dev Signed-off-by: Konstantin Podshumok --- docker/packager/deb/Dockerfile | 58 ++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index c1260b5c7ff..dc1040f6a41 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,46 +1,52 @@ # docker build -t yandex/clickhouse-deb-builder . FROM ubuntu:19.10 -RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive + +RUN \ + apt update && \ + apt install ca-certificates lsb-release wget gnupg apt-transport-https --yes --no-install-recommends --verbose-versions && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" && \ + echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-10 main" >> /etc/apt/sources.list && \ + wget "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" && \ + apt install "./apache-arrow-archive-keyring-latest-${CODENAME}.deb" --yes --no-install-recommends --verbose-versions && \ + rm -v "./apache-arrow-archive-keyring-latest-${CODENAME}.deb" # initial packages -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ +RUN apt update && \ + apt install --no-install-recommends \ bash \ fakeroot \ ccache \ curl \ software-properties-common \ - gnupg \ - apt-transport-https \ - ca-certificates + --yes # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able # to compress files using pigz (https://zlib.net/pigz/) instead of gzip. # Significantly increase deb packaging speed and compatible with old systems -RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb -RUN chmod +x dpkg-deb -RUN cp dpkg-deb /usr/bin +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb && \ + chmod +x dpkg-deb && \ + cp dpkg-deb /usr/bin -# Libraries from OS are only needed to test the "unbundled" build (that is not used in production). -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ +# Libraries from OS are only needed to test the "unbundled" build (this is not used in production). +RUN apt update && \ + apt install --no-install-recommends \ gcc-9 \ g++-9 \ llvm-10 \ + llvm-10-dev \ + llvm-9-dev \ clang-10 \ lld-10 \ clang-tidy-10 \ clang-9 \ lld-9 \ clang-tidy-9 \ + libssl-dev \ libicu-dev \ - libreadline-dev \ gperf \ ninja-build \ perl \ @@ -59,9 +65,8 @@ RUN apt-get --allow-unauthenticated update -y \ zlib1g-dev \ liblz4-dev \ libdouble-conversion-dev \ + libxml2-dev \ librdkafka-dev \ - libpoconetssl62 \ - libpoco-dev \ libgoogle-perftools-dev \ libzstd-dev \ libltdl-dev \ @@ -85,8 +90,21 @@ RUN apt-get --allow-unauthenticated update -y \ libcctz-dev \ libldap2-dev \ libsasl2-dev \ + libgsasl7-dev \ heimdal-multidev \ - libhyperscan-dev + libhyperscan-dev \ + libbrotli-dev \ + protobuf-compiler \ + libprotoc-dev \ + libgrpc++-dev \ + rapidjson-dev \ + libsnappy-dev \ + libparquet-dev \ + libthrift-dev \ + libavro-dev \ + libfarmhash-dev \ + libmysqlclient-dev \ + --yes # This symlink required by gcc to find lld compiler From f40a9f5b20e0f357b944a68e2d4e8e16ebe7f23d Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Wed, 19 Aug 2020 12:06:18 +0300 Subject: [PATCH 51/89] cmake: set USE_INTERNAL_ORC and USE_INTERNAL_HDFS3 to OFF by default even in unbundled Signed-off-by: Konstantin Podshumok --- cmake/find/hdfs3.cmake | 3 ++- cmake/find/orc.cmake | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmake/find/hdfs3.cmake b/cmake/find/hdfs3.cmake index e8e9fb359c9..3f7afd3f6b3 100644 --- a/cmake/find/hdfs3.cmake +++ b/cmake/find/hdfs3.cmake @@ -11,7 +11,8 @@ if(NOT ENABLE_HDFS) return() endif() -option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ${NOT_UNBUNDLED}) +option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk)" + ON) # We don't know any linux distribution with package for it if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") if(USE_INTERNAL_HDFS3_LIBRARY) diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index e91f0e95819..3ab67a98644 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -7,7 +7,8 @@ if(NOT ENABLE_ORC) return() endif() -option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) +option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk)" + ON) include(cmake/find/snappy.cmake) From e101db600cf1940c329a401f8bb8dcd3010e1b86 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 15 Aug 2020 19:13:23 +0300 Subject: [PATCH 52/89] cmake: set USE_INTERNAL_SPARSEHASH_LIBRARY=ON by default as we don't know any distribution with sparsehash-c11 package Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 4 ++++ cmake/find/sparsehash.cmake | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 52ae3c9a59b..35db796b775 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -425,8 +425,12 @@ include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/stats.cmake) +set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "") find_contrib_lib(cityhash) + find_contrib_lib(farmhash) + +set (USE_INTERNAL_BTRIE_LIBRARY ON CACHE INTERNAL "") find_contrib_lib(btrie) if (ENABLE_TESTS) diff --git a/cmake/find/sparsehash.cmake b/cmake/find/sparsehash.cmake index f8b6d293705..f258f6c1c5b 100644 --- a/cmake/find/sparsehash.cmake +++ b/cmake/find/sparsehash.cmake @@ -1,4 +1,5 @@ -option (USE_INTERNAL_SPARSEHASH_LIBRARY "Set to FALSE to use system sparsehash library instead of bundled" ${NOT_UNBUNDLED}) +option (USE_INTERNAL_SPARSEHASH_LIBRARY "Set to FALSE to use system sparsehash library instead of bundled" + ON) # ON by default as we are not aware of any system providing package for sparsehash-c11 if (NOT USE_INTERNAL_SPARSEHASH_LIBRARY) find_path (SPARSEHASH_INCLUDE_DIR NAMES sparsehash/sparse_hash_map PATHS ${SPARSEHASH_INCLUDE_PATHS}) From 2c9c7c9e9535f70bcaf5a798ad47a44e46889d85 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sun, 16 Aug 2020 06:43:36 +0300 Subject: [PATCH 53/89] cmake: USE_INTERNAL_H3_LIBRARY is always ON by default as I don't know any system with package for it Signed-off-by: Konstantin Podshumok --- cmake/find/h3.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/find/h3.cmake b/cmake/find/h3.cmake index 7ca070b45fe..03b6f32fc3c 100644 --- a/cmake/find/h3.cmake +++ b/cmake/find/h3.cmake @@ -6,7 +6,8 @@ if(NOT ENABLE_H3) return() endif() -option(USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of bundled" ${NOT_UNBUNDLED}) +option(USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of bundled" + ON) # we are not aware of any distribution that provides h3 package if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include/h3Index.h") if(USE_INTERNAL_H3_LIBRARY) From 9af119ea358686fadf5694e280553bc21514e950 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 18 Aug 2020 23:01:29 +0300 Subject: [PATCH 54/89] cmake: USE_INTERNAL_REPLXX_LIBRARY is ON by default now As we don't now any system that ships it as a package Signed-off-by: Konstantin Podshumok --- contrib/replxx-cmake/CMakeLists.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index ed6bdaa25bc..2c0ad86e583 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -12,7 +12,7 @@ if (NOT ENABLE_REPLXX) return() endif() -option (USE_INTERNAL_REPLXX_LIBRARY "Use internal replxx library" ${NOT_UNBUNDLED}) +option (USE_INTERNAL_REPLXX_LIBRARY "Use internal replxx library (Experimental: set to OFF on your own risk)" ON) if (NOT USE_INTERNAL_REPLXX_LIBRARY) find_library(LIBRARY_REPLXX NAMES replxx replxx-static) @@ -49,16 +49,16 @@ if (NOT LIBRARY_REPLXX OR NOT INCLUDE_REPLXX OR NOT EXTERNAL_REPLXX_WORKS) set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") set(SRCS - ${LIBRARY_DIR}/src/conversion.cxx - ${LIBRARY_DIR}/src/ConvertUTF.cpp - ${LIBRARY_DIR}/src/escape.cxx - ${LIBRARY_DIR}/src/history.cxx - ${LIBRARY_DIR}/src/io.cxx - ${LIBRARY_DIR}/src/prompt.cxx - ${LIBRARY_DIR}/src/replxx_impl.cxx - ${LIBRARY_DIR}/src/replxx.cxx - ${LIBRARY_DIR}/src/util.cxx - ${LIBRARY_DIR}/src/wcwidth.cpp + "${LIBRARY_DIR}/src/conversion.cxx" + "${LIBRARY_DIR}/src/ConvertUTF.cpp" + "${LIBRARY_DIR}/src/escape.cxx" + "${LIBRARY_DIR}/src/history.cxx" + "${LIBRARY_DIR}/src/io.cxx" + "${LIBRARY_DIR}/src/prompt.cxx" + "${LIBRARY_DIR}/src/replxx_impl.cxx" + "${LIBRARY_DIR}/src/replxx.cxx" + "${LIBRARY_DIR}/src/util.cxx" + "${LIBRARY_DIR}/src/wcwidth.cpp" ) add_library (replxx ${SRCS}) From 2b867cb2dbe0b7140780c0b7c0003650e1f94d5d Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 17 Aug 2020 00:04:42 +0300 Subject: [PATCH 55/89] cmake: disable external avro as we don't support it yet Signed-off-by: Konstantin Podshumok --- cmake/find/avro.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake index a07e6353212..e0f73d99111 100644 --- a/cmake/find/avro.cmake +++ b/cmake/find/avro.cmake @@ -7,7 +7,8 @@ if (NOT ENABLE_AVRO) return() endif() -option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED}) +option (USE_INTERNAL_AVRO_LIBRARY + "Set to FALSE to use system avro library instead of bundled" ON) # TODO: provide unbundled support if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt") if (USE_INTERNAL_AVRO_LIBRARY) From 3e3b67601f0ea66bbdb2fcd9be5534d8de7c606a Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 21 Aug 2020 14:35:35 +0300 Subject: [PATCH 56/89] cmake: USE_INTERNAL_GSASL_LIBRARY by default when USE_STATIC_LIBRARIES Signed-off-by: Konstantin Podshumok --- cmake/find/libgsasl.cmake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/find/libgsasl.cmake b/cmake/find/libgsasl.cmake index 3b580a4ddb2..3c742af2566 100644 --- a/cmake/find/libgsasl.cmake +++ b/cmake/find/libgsasl.cmake @@ -7,7 +7,12 @@ if (NOT ENABLE_GSASL_LIBRARY) return() endif() -option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${NOT_UNBUNDLED}) +if (UNBUNDLED) + # when USE_STATIC_LIBRARIES we usually need to pick up hell a lot of dependencies for libgsasl + option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ${USE_STATIC_LIBRARIES}) +else() + option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ON) +endif() if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h") if (USE_INTERNAL_LIBGSASL_LIBRARY) From 17c46faaa7f16b10101d1482a80b6779f57455b0 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 17 Aug 2020 00:38:31 +0300 Subject: [PATCH 57/89] cmake: fix parquet/arrow variables squashed: - sync FindArrow.cmake and FindParquet.cmake with arrow repo - unbundled: add arrow to dbms link libraries - cmake: fix-up unbundled Arrow support Signed-off-by: Konstantin Podshumok --- cmake/Modules/FindArrow.cmake | 524 ++++++++++++++++++++++++-------- cmake/Modules/FindParquet.cmake | 207 ++++++------- cmake/find/parquet.cmake | 88 +++++- docker/packager/deb/Dockerfile | 2 + src/CMakeLists.txt | 3 + 5 files changed, 582 insertions(+), 242 deletions(-) diff --git a/cmake/Modules/FindArrow.cmake b/cmake/Modules/FindArrow.cmake index 4043a474988..5bd111de1e3 100644 --- a/cmake/Modules/FindArrow.cmake +++ b/cmake/Modules/FindArrow.cmake @@ -17,143 +17,417 @@ # specific language governing permissions and limitations # under the License. -# - Find ARROW (arrow/api.h, libarrow.a, libarrow.so) +# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so) # This module defines +# ARROW_FOUND, whether Arrow has been found +# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0" +# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only) # ARROW_INCLUDE_DIR, directory containing headers -# ARROW_LIBS, directory containing arrow libraries -# ARROW_STATIC_LIB, path to libarrow.a +# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead +# ARROW_LIB_DIR, directory containing Arrow libraries +# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead # ARROW_SHARED_LIB, path to libarrow's shared library -# ARROW_SHARED_IMP_LIB, path to libarrow's import library (MSVC only) -# ARROW_FOUND, whether arrow has been found +# ARROW_SO_VERSION, shared object version of found Arrow such as "100" +# ARROW_STATIC_LIB, path to libarrow.a +# ARROW_VERSION, version of found Arrow +# ARROW_VERSION_MAJOR, major version of found Arrow +# ARROW_VERSION_MINOR, minor version of found Arrow +# ARROW_VERSION_PATCH, patch version of found Arrow + +if(DEFINED ARROW_FOUND) + return() +endif() include(FindPkgConfig) -include(GNUInstallDirs) +include(FindPackageHandleStandardArgs) -if ("$ENV{ARROW_HOME}" STREQUAL "") - pkg_check_modules(ARROW arrow) - if (ARROW_FOUND) - pkg_get_variable(ARROW_SO_VERSION arrow so_version) - set(ARROW_ABI_VERSION ${ARROW_SO_VERSION}) - message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}") - pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version) - message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}") - if ("${ARROW_INCLUDE_DIRS}" STREQUAL "") - set(ARROW_INCLUDE_DIRS "/usr/${CMAKE_INSTALL_INCLUDEDIR}") - endif() - if ("${ARROW_LIBRARY_DIRS}" STREQUAL "") - set(ARROW_LIBRARY_DIRS "/usr/${CMAKE_INSTALL_LIBDIR}") - if (EXISTS "/etc/debian_version" AND CMAKE_LIBRARY_ARCHITECTURE) - set(ARROW_LIBRARY_DIRS - "${ARROW_LIBRARY_DIRS}/${CMAKE_LIBRARY_ARCHITECTURE}") - endif() - endif() - set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS}) - set(ARROW_LIBS ${ARROW_LIBRARY_DIRS}) - set(ARROW_SEARCH_LIB_PATH ${ARROW_LIBRARY_DIRS}) - endif() -else() - set(ARROW_HOME "$ENV{ARROW_HOME}") - - set(ARROW_SEARCH_HEADER_PATHS - ${ARROW_HOME}/include - ) - - set(ARROW_SEARCH_LIB_PATH - ${ARROW_HOME}/lib - ) - - find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS - ${ARROW_SEARCH_HEADER_PATHS} - # make sure we don't accidentally pick up a different version - NO_DEFAULT_PATH - ) +set(ARROW_SEARCH_LIB_PATH_SUFFIXES) +if(CMAKE_LIBRARY_ARCHITECTURE) + list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}") +endif() +list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES + "lib64" + "lib32" + "lib" + "bin") +set(ARROW_CONFIG_SUFFIXES + "_RELEASE" + "_RELWITHDEBINFO" + "_MINSIZEREL" + "_DEBUG" + "") +if(CMAKE_BUILD_TYPE) + string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED) + set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}") + list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}") endif() -find_library(ARROW_LIB_PATH NAMES arrow - PATHS - ${ARROW_SEARCH_LIB_PATH} - NO_DEFAULT_PATH) -get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY) - -find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python - PATHS - ${ARROW_SEARCH_LIB_PATH} - NO_DEFAULT_PATH) -get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY) - -if (MSVC) - SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll") - - if (MSVC AND NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX) +if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX) + if(MSVC) set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static") - endif() - - find_library(ARROW_SHARED_LIBRARIES NAMES arrow - PATHS ${ARROW_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "bin" ) - - find_library(ARROW_PYTHON_SHARED_LIBRARIES NAMES arrow_python - PATHS ${ARROW_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "bin" ) - get_filename_component(ARROW_SHARED_LIBS ${ARROW_SHARED_LIBRARIES} PATH ) - get_filename_component(ARROW_PYTHON_SHARED_LIBS ${ARROW_PYTHON_SHARED_LIBRARIES} PATH ) -endif () - -if (ARROW_INCLUDE_DIR AND ARROW_LIBS) - set(ARROW_FOUND TRUE) - set(ARROW_LIB_NAME arrow) - set(ARROW_PYTHON_LIB_NAME arrow_python) - if (MSVC) - set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(ARROW_SHARED_LIB ${ARROW_SHARED_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_SHARED_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib) - set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib) else() - set(ARROW_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}.a) - set(ARROW_PYTHON_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}.a) - - set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(ARROW_MSVC_STATIC_LIB_SUFFIX "") endif() endif() -if (ARROW_FOUND) - if (NOT Arrow_FIND_QUIETLY) - message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}") - message(STATUS "Found the Arrow Python library: ${ARROW_PYTHON_LIB_PATH}") - endif () -else () - if (NOT Arrow_FIND_QUIETLY) - set(ARROW_ERR_MSG "Could not find the Arrow library. Looked for headers") - set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_HEADER_PATHS}, and for libs") - set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_LIB_PATH}") - if (Arrow_FIND_REQUIRED) - message(FATAL_ERROR "${ARROW_ERR_MSG}") - else (Arrow_FIND_REQUIRED) - message(STATUS "${ARROW_ERR_MSG}") - endif (Arrow_FIND_REQUIRED) - endif () - set(ARROW_FOUND FALSE) -endif () +# Internal function. +# +# Set shared library name for ${base_name} to ${output_variable}. +# +# Example: +# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow) +# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux +# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS +# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows +# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows +function(arrow_build_shared_library_name output_variable base_name) + set(${output_variable} + "${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}" + PARENT_SCOPE) +endfunction() -if (MSVC) - mark_as_advanced( - ARROW_INCLUDE_DIR - ARROW_STATIC_LIB - ARROW_SHARED_LIB - ARROW_SHARED_IMP_LIB - ARROW_PYTHON_STATIC_LIB - ARROW_PYTHON_SHARED_LIB - ARROW_PYTHON_SHARED_IMP_LIB - ) -else() - mark_as_advanced( - ARROW_INCLUDE_DIR - ARROW_STATIC_LIB - ARROW_SHARED_LIB - ARROW_PYTHON_STATIC_LIB - ARROW_PYTHON_SHARED_LIB - ) +# Internal function. +# +# Set import library name for ${base_name} to ${output_variable}. +# This is useful only for MSVC build. Import library is used only +# with MSVC build. +# +# Example: +# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow) +# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless) +# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless) +# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows +# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows +function(arrow_build_import_library_name output_variable base_name) + set(${output_variable} + "${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}" + PARENT_SCOPE) +endfunction() + +# Internal function. +# +# Set static library name for ${base_name} to ${output_variable}. +# +# Example: +# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow) +# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux +# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS +# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows +# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows +function(arrow_build_static_library_name output_variable base_name) + set( + ${output_variable} + "${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" + PARENT_SCOPE) +endfunction() + +# Internal function. +# +# Set macro value for ${macro_name} in ${header_content} to ${output_variable}. +# +# Example: +# arrow_extract_macro_value(version_major +# "ARROW_VERSION_MAJOR" +# "#define ARROW_VERSION_MAJOR 1.0.0") +# # -> version_major=1.0.0 +function(arrow_extract_macro_value output_variable macro_name header_content) + string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition + "${header_content}") + string(REGEX + REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}") + set(${output_variable} "${macro_value}" PARENT_SCOPE) +endfunction() + +# Internal macro only for arrow_find_package. +# +# Find package in HOME. +macro(arrow_find_package_home) + find_path(${prefix}_include_dir "${header_path}" + PATHS "${home}" + PATH_SUFFIXES "include" + NO_DEFAULT_PATH) + set(include_dir "${${prefix}_include_dir}") + set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) + + if(MSVC) + set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES}) + # .dll isn't found by find_library with MSVC because .dll isn't included in + # CMAKE_FIND_LIBRARY_SUFFIXES. + list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}") + endif() + find_library(${prefix}_shared_lib + NAMES "${shared_lib_name}" + PATHS "${home}" + PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} + NO_DEFAULT_PATH) + if(MSVC) + set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL}) + endif() + set(shared_lib "${${prefix}_shared_lib}") + set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE) + if(shared_lib) + add_library(${target_shared} SHARED IMPORTED) + set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}") + if(include_dir) + set_target_properties(${target_shared} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}") + endif() + find_library(${prefix}_import_lib + NAMES "${import_lib_name}" + PATHS "${home}" + PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} + NO_DEFAULT_PATH) + set(import_lib "${${prefix}_import_lib}") + set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE) + if(import_lib) + set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}") + endif() + endif() + + find_library(${prefix}_static_lib + NAMES "${static_lib_name}" + PATHS "${home}" + PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} + NO_DEFAULT_PATH) + set(static_lib "${${prefix}_static_lib}") + set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) + if(static_lib) + add_library(${target_static} STATIC IMPORTED) + set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}") + if(include_dir) + set_target_properties(${target_static} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}") + endif() + endif() +endmacro() + +# Internal macro only for arrow_find_package. +# +# Find package by CMake package configuration. +macro(arrow_find_package_cmake_package_configuration) + find_package(${cmake_package_name} CONFIG) + if(${cmake_package_name}_FOUND) + set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE) + if(TARGET ${target_shared}) + foreach(suffix ${ARROW_CONFIG_SUFFIXES}) + get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix}) + if(shared_lib) + # Remove shared library version: + # libarrow.so.100.0.0 -> libarrow.so + # Because ARROW_HOME and pkg-config approaches don't add + # shared library version. + string(REGEX + REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib + "${shared_lib}") + set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE) + break() + endif() + endforeach() + endif() + if(TARGET ${target_static}) + foreach(suffix ${ARROW_CONFIG_SUFFIXES}) + get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix}) + if(static_lib) + set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) + break() + endif() + endforeach() + endif() + endif() +endmacro() + +# Internal macro only for arrow_find_package. +# +# Find package by pkg-config. +macro(arrow_find_package_pkg_config) + pkg_check_modules(${prefix}_PC ${pkg_config_name}) + if(${prefix}_PC_FOUND) + set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE) + + set(include_dir "${${prefix}_PC_INCLUDEDIR}") + set(lib_dir "${${prefix}_PC_LIBDIR}") + set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}") + # Use the first shared library path as the IMPORTED_LOCATION + # for ${target_shared}. This assumes that the first shared library + # path is the shared library path for this module. + list(GET shared_lib_paths 0 first_shared_lib_path) + # Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES + # for ${target_shared}. This assumes that the rest shared library + # paths are dependency library paths for this module. + list(LENGTH shared_lib_paths n_shared_lib_paths) + if(n_shared_lib_paths LESS_EQUAL 1) + set(rest_shared_lib_paths) + else() + list(SUBLIST + shared_lib_paths + 1 + -1 + rest_shared_lib_paths) + endif() + + set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE) + set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) + set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE) + + add_library(${target_shared} SHARED IMPORTED) + set_target_properties(${target_shared} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES + "${include_dir}" + INTERFACE_LINK_LIBRARIES + "${rest_shared_lib_paths}" + IMPORTED_LOCATION + "${first_shared_lib_path}") + get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION) + + find_library(${prefix}_static_lib + NAMES "${static_lib_name}" + PATHS "${lib_dir}" + NO_DEFAULT_PATH) + set(static_lib "${${prefix}_static_lib}") + set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) + if(static_lib) + add_library(${target_static} STATIC IMPORTED) + set_target_properties(${target_static} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}" + IMPORTED_LOCATION "${static_lib}") + endif() + endif() +endmacro() + +function(arrow_find_package + prefix + home + base_name + header_path + cmake_package_name + pkg_config_name) + arrow_build_shared_library_name(shared_lib_name ${base_name}) + arrow_build_import_library_name(import_lib_name ${base_name}) + arrow_build_static_library_name(static_lib_name ${base_name}) + + set(target_shared ${base_name}_shared) + set(target_static ${base_name}_static) + + if(home) + arrow_find_package_home() + set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE) + else() + arrow_find_package_cmake_package_configuration() + if(${cmake_package_name}_FOUND) + set(${prefix}_FIND_APPROACH + "CMake package configuration: ${cmake_package_name}" + PARENT_SCOPE) + else() + arrow_find_package_pkg_config() + set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE) + endif() + endif() + + if(NOT include_dir) + if(TARGET ${target_shared}) + get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES) + elseif(TARGET ${target_static}) + get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES) + endif() + endif() + if(include_dir) + set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) + endif() + + if(shared_lib) + get_filename_component(lib_dir "${shared_lib}" DIRECTORY) + elseif(static_lib) + get_filename_component(lib_dir "${static_lib}" DIRECTORY) + else() + set(lib_dir NOTFOUND) + endif() + set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE) + # For backward compatibility + set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE) +endfunction() + +if(NOT "$ENV{ARROW_HOME}" STREQUAL "") + file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME) +endif() +arrow_find_package(ARROW + "${ARROW_HOME}" + arrow + arrow/api.h + Arrow + arrow) + +if(ARROW_HOME) + if(ARROW_INCLUDE_DIR) + file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT) + arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR" + "${ARROW_CONFIG_H_CONTENT}") + arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR" + "${ARROW_CONFIG_H_CONTENT}") + arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH" + "${ARROW_CONFIG_H_CONTENT}") + if("${ARROW_VERSION_MAJOR}" STREQUAL "" + OR "${ARROW_VERSION_MINOR}" STREQUAL "" + OR "${ARROW_VERSION_PATCH}" STREQUAL "") + set(ARROW_VERSION "0.0.0") + else() + set(ARROW_VERSION + "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}") + endif() + + arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION" + "${ARROW_CONFIG_H_CONTENT}") + string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}") + arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION" + "${ARROW_CONFIG_H_CONTENT}") + string(REGEX + REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION + "${ARROW_FULL_SO_VERSION_QUOTED}") + endif() +else() + if(ARROW_USE_CMAKE_PACKAGE_CONFIG) + find_package(Arrow CONFIG) + elseif(ARROW_USE_PKG_CONFIG) + pkg_get_variable(ARROW_SO_VERSION arrow so_version) + pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version) + endif() +endif() + +set(ARROW_ABI_VERSION ${ARROW_SO_VERSION}) + +mark_as_advanced(ARROW_ABI_VERSION + ARROW_CONFIG_SUFFIXES + ARROW_FULL_SO_VERSION + ARROW_IMPORT_LIB + ARROW_INCLUDE_DIR + ARROW_LIBS + ARROW_LIB_DIR + ARROW_SEARCH_LIB_PATH_SUFFIXES + ARROW_SHARED_IMP_LIB + ARROW_SHARED_LIB + ARROW_SO_VERSION + ARROW_STATIC_LIB + ARROW_VERSION + ARROW_VERSION_MAJOR + ARROW_VERSION_MINOR + ARROW_VERSION_PATCH) + +find_package_handle_standard_args(Arrow REQUIRED_VARS + # The first required variable is shown + # in the found message. So this list is + # not sorted alphabetically. + ARROW_INCLUDE_DIR + ARROW_LIB_DIR + ARROW_FULL_SO_VERSION + ARROW_SO_VERSION + VERSION_VAR + ARROW_VERSION) +set(ARROW_FOUND ${Arrow_FOUND}) + +if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY) + message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})") + message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}") + message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}") + message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}") + message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}") + message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}") endif() diff --git a/cmake/Modules/FindParquet.cmake b/cmake/Modules/FindParquet.cmake index ab9c31efe2d..654020c0b87 100644 --- a/cmake/Modules/FindParquet.cmake +++ b/cmake/Modules/FindParquet.cmake @@ -17,131 +17,116 @@ # specific language governing permissions and limitations # under the License. -# - Find PARQUET (parquet/parquet.h, libparquet.a, libparquet.so) +# - Find Parquet (parquet/api/reader.h, libparquet.a, libparquet.so) +# +# This module requires Arrow from which it uses +# arrow_find_package() +# # This module defines +# PARQUET_FOUND, whether Parquet has been found +# PARQUET_IMPORT_LIB, path to libparquet's import library (Windows only) # PARQUET_INCLUDE_DIR, directory containing headers -# PARQUET_LIBS, directory containing parquet libraries -# PARQUET_STATIC_LIB, path to libparquet.a +# PARQUET_LIBS, deprecated. Use PARQUET_LIB_DIR instead +# PARQUET_LIB_DIR, directory containing Parquet libraries +# PARQUET_SHARED_IMP_LIB, deprecated. Use PARQUET_IMPORT_LIB instead # PARQUET_SHARED_LIB, path to libparquet's shared library -# PARQUET_SHARED_IMP_LIB, path to libparquet's import library (MSVC only) -# PARQUET_FOUND, whether parquet has been found +# PARQUET_SO_VERSION, shared object version of found Parquet such as "100" +# PARQUET_STATIC_LIB, path to libparquet.a -include(FindPkgConfig) +if(DEFINED PARQUET_FOUND) + return() +endif() + +set(find_package_arguments) +if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION) + list(APPEND find_package_arguments "${${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION}") +endif() +if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED) + list(APPEND find_package_arguments REQUIRED) +endif() +if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY) + list(APPEND find_package_arguments QUIET) +endif() +find_package(Arrow ${find_package_arguments}) if(NOT "$ENV{PARQUET_HOME}" STREQUAL "") - set(PARQUET_HOME "$ENV{PARQUET_HOME}") + file(TO_CMAKE_PATH "$ENV{PARQUET_HOME}" PARQUET_HOME) endif() -if (MSVC) - SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll") +if((NOT PARQUET_HOME) AND ARROW_HOME) + set(PARQUET_HOME ${ARROW_HOME}) +endif() - if (MSVC AND NOT DEFINED PARQUET_MSVC_STATIC_LIB_SUFFIX) - set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static") - endif() - - find_library(PARQUET_SHARED_LIBRARIES NAMES parquet - PATHS ${PARQUET_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "bin" ) - - get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH ) -endif () - -if(PARQUET_HOME) - set(PARQUET_SEARCH_HEADER_PATHS - ${PARQUET_HOME}/include - ) - set(PARQUET_SEARCH_LIB_PATH - ${PARQUET_HOME}/lib - ) - find_path(PARQUET_INCLUDE_DIR parquet/api/reader.h PATHS - ${PARQUET_SEARCH_HEADER_PATHS} - # make sure we don't accidentally pick up a different version - NO_DEFAULT_PATH - ) - find_library(PARQUET_LIBRARIES NAMES parquet - PATHS ${PARQUET_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") - get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) - - # Try to autodiscover the Parquet ABI version - get_filename_component(PARQUET_LIB_REALPATH ${PARQUET_LIBRARIES} REALPATH) - get_filename_component(PARQUET_EXT_REALPATH ${PARQUET_LIB_REALPATH} EXT) - string(REGEX MATCH ".([0-9]+.[0-9]+.[0-9]+)" HAS_ABI_VERSION ${PARQUET_EXT_REALPATH}) - if (HAS_ABI_VERSION) - if (APPLE) - string(REGEX REPLACE ".([0-9]+.[0-9]+.[0-9]+).dylib" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH}) +if(ARROW_FOUND) + arrow_find_package(PARQUET + "${PARQUET_HOME}" + parquet + parquet/api/reader.h + Parquet + parquet) + if(PARQUET_HOME) + if(PARQUET_INCLUDE_DIR) + file(READ "${PARQUET_INCLUDE_DIR}/parquet/parquet_version.h" + PARQUET_VERSION_H_CONTENT) + arrow_extract_macro_value(PARQUET_VERSION_MAJOR "PARQUET_VERSION_MAJOR" + "${PARQUET_VERSION_H_CONTENT}") + arrow_extract_macro_value(PARQUET_VERSION_MINOR "PARQUET_VERSION_MINOR" + "${PARQUET_VERSION_H_CONTENT}") + arrow_extract_macro_value(PARQUET_VERSION_PATCH "PARQUET_VERSION_PATCH" + "${PARQUET_VERSION_H_CONTENT}") + if("${PARQUET_VERSION_MAJOR}" STREQUAL "" + OR "${PARQUET_VERSION_MINOR}" STREQUAL "" + OR "${PARQUET_VERSION_PATCH}" STREQUAL "") + set(PARQUET_VERSION "0.0.0") else() - string(REGEX REPLACE ".so.([0-9]+.[0-9]+.[0-9]+)" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH}) + set(PARQUET_VERSION + "${PARQUET_VERSION_MAJOR}.${PARQUET_VERSION_MINOR}.${PARQUET_VERSION_PATCH}") endif() - string(REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" PARQUET_SO_VERSION ${PARQUET_ABI_VERSION}) - else() - set(PARQUET_ABI_VERSION "1.0.0") - set(PARQUET_SO_VERSION "1") + + arrow_extract_macro_value(PARQUET_SO_VERSION_QUOTED "PARQUET_SO_VERSION" + "${PARQUET_VERSION_H_CONTENT}") + string(REGEX + REPLACE "^\"(.+)\"$" "\\1" PARQUET_SO_VERSION "${PARQUET_SO_VERSION_QUOTED}") + arrow_extract_macro_value(PARQUET_FULL_SO_VERSION_QUOTED "PARQUET_FULL_SO_VERSION" + "${PARQUET_VERSION_H_CONTENT}") + string(REGEX + REPLACE "^\"(.+)\"$" "\\1" PARQUET_FULL_SO_VERSION + "${PARQUET_FULL_SO_VERSION_QUOTED}") endif() -else() - pkg_check_modules(PARQUET parquet) - if (PARQUET_FOUND) - pkg_get_variable(PARQUET_ABI_VERSION parquet abi_version) - message(STATUS "Parquet C++ ABI version: ${PARQUET_ABI_VERSION}") - pkg_get_variable(PARQUET_SO_VERSION parquet so_version) - message(STATUS "Parquet C++ SO version: ${PARQUET_SO_VERSION}") - set(PARQUET_INCLUDE_DIR ${PARQUET_INCLUDE_DIRS}) - set(PARQUET_LIBS ${PARQUET_LIBRARY_DIRS}) - set(PARQUET_SEARCH_LIB_PATH ${PARQUET_LIBRARY_DIRS}) - message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}") - find_library(PARQUET_LIBRARIES NAMES parquet - PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) - else() - find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h ) - find_library(PARQUET_LIBRARIES NAMES parquet) - get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) + else() + if(PARQUET_USE_CMAKE_PACKAGE_CONFIG) + find_package(Parquet CONFIG) + elseif(PARQUET_USE_PKG_CONFIG) + pkg_get_variable(PARQUET_SO_VERSION parquet so_version) + pkg_get_variable(PARQUET_FULL_SO_VERSION parquet full_so_version) endif() + endif() + set(PARQUET_ABI_VERSION "${PARQUET_SO_VERSION}") endif() -if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES) - set(PARQUET_FOUND TRUE) - set(PARQUET_LIB_NAME parquet) - if (MSVC) - set(PARQUET_STATIC_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(PARQUET_SHARED_LIB "${PARQUET_SHARED_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(PARQUET_SHARED_IMP_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}.lib") - else() - set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_LIB_NAME}.a) - set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - endif() -else () - set(PARQUET_FOUND FALSE) -endif () +mark_as_advanced(PARQUET_ABI_VERSION + PARQUET_IMPORT_LIB + PARQUET_INCLUDE_DIR + PARQUET_LIBS + PARQUET_LIB_DIR + PARQUET_SHARED_IMP_LIB + PARQUET_SHARED_LIB + PARQUET_SO_VERSION + PARQUET_STATIC_LIB + PARQUET_VERSION) -if (PARQUET_FOUND) - if (NOT Parquet_FIND_QUIETLY) - message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}") - endif () -else () - if (NOT Parquet_FIND_QUIETLY) - if (NOT PARQUET_FOUND) - set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.") - endif() +find_package_handle_standard_args(Parquet + REQUIRED_VARS + PARQUET_INCLUDE_DIR + PARQUET_LIB_DIR + PARQUET_SO_VERSION + VERSION_VAR + PARQUET_VERSION) +set(PARQUET_FOUND ${Parquet_FOUND}) - set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ") - if ( _parquet_roots ) - set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.") - else () - set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} system search paths.") - endif () - if (Parquet_FIND_REQUIRED) - message(FATAL_ERROR "${PARQUET_ERR_MSG}") - else (Parquet_FIND_REQUIRED) - message(STATUS "${PARQUET_ERR_MSG}") - endif (Parquet_FIND_REQUIRED) - endif () -endif () - -mark_as_advanced( - PARQUET_FOUND - PARQUET_INCLUDE_DIR - PARQUET_LIBS - PARQUET_LIBRARIES - PARQUET_STATIC_LIB - PARQUET_SHARED_LIB -) +if(Parquet_FOUND AND NOT Parquet_FIND_QUIETLY) + message(STATUS "Parquet version: ${PARQUET_VERSION} (${PARQUET_FIND_APPROACH})") + message(STATUS "Found the Parquet shared library: ${PARQUET_SHARED_LIB}") + message(STATUS "Found the Parquet import library: ${PARQUET_IMPORT_LIB}") + message(STATUS "Found the Parquet static library: ${PARQUET_STATIC_LIB}") +endif() diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index 3cb6be38c3f..d9b1cdcf70f 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -27,18 +27,91 @@ if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/CMakeLists.txt") set(MISSING_INTERNAL_PARQUET_LIBRARY 1) endif() +if (NOT SNAPPY_LIBRARY) + include(cmake/find/snappy.cmake) +endif() + if(NOT USE_INTERNAL_PARQUET_LIBRARY) find_package(Arrow) find_package(Parquet) + find_library(THRIFT_LIBRARY thrift) + find_library(UTF8_PROC_LIBRARY utf8proc) + find_package(BZip2) - if(NOT ARROW_INCLUDE_DIR OR PARQUET_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system parquet: arrow=${ARROW_INCLUDE_DIR} parquet=${PARQUET_INCLUDE_DIR}" ) + if(USE_STATIC_LIBRARIES) + find_library(ARROW_DEPS_LIBRARY arrow_bundled_dependencies) + + if (ARROW_DEPS_LIBRARY) + set(ARROW_IMPORT_OBJ_DIR "${CMAKE_CURRENT_BINARY_DIR}/contrib/arrow-cmake/imported-objects") + set(ARROW_OTHER_OBJS + "${ARROW_IMPORT_OBJ_DIR}/jemalloc.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/arena.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/background_thread.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/base.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/bin.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/bitmap.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/ckh.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/ctl.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/div.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/extent.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/extent_dss.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/extent_mmap.pic.o" + # skip hash + "${ARROW_IMPORT_OBJ_DIR}/hook.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/large.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/log.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/malloc_io.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/mutex.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/mutex_pool.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/nstime.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/pages.pic.o" + # skip prng + "${ARROW_IMPORT_OBJ_DIR}/prof.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/rtree.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/stats.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/sc.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/sz.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/tcache.pic.o" + # skip ticker + "${ARROW_IMPORT_OBJ_DIR}/tsd.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/test_hooks.pic.o" + "${ARROW_IMPORT_OBJ_DIR}/witness.pic.o" + ) + add_custom_command(OUTPUT ${ARROW_OTHER_OBJS} + COMMAND + mkdir -p "${ARROW_IMPORT_OBJ_DIR}" && + cd "${ARROW_IMPORT_OBJ_DIR}" && + "${CMAKE_AR}" x "${ARROW_DEPS_LIBRARY}" + ) + set_source_files_properties(jemalloc.pic.o PROPERTIES EXTERNAL_OBJECT true GENERATED true) + add_library(imported_arrow_deps STATIC ${ARROW_OTHER_OBJS}) + + set(ARROW_LIBRARY ${ARROW_STATIC_LIB} + imported_arrow_deps ${THRIFT_LIBRARY} ${UTF8_PROC_LIBRARY} ${BZIP2_LIBRARIES} ${SNAPPY_LIBRARY}) + else() + message(WARNING "Using external static Arrow does not always work. " + "Could not find arrow_bundled_dependencies.a. If compilation fails, " + "Try: -D\"USE_INTERNAL_PARQUET_LIBRARY\"=ON or -D\"ENABLE_PARQUET\"=OFF or " + "-D\"USE_STATIC_LIBRARIES\"=OFF") + set(ARROW_LIBRARY ${ARROW_STATIC_LIB}) + endif() + set(PARQUET_LIBRARY ${PARQUET_STATIC_LIB}) + else() + set(ARROW_LIBRARY ${ARROW_SHARED_LIB}) + set(PARQUET_LIBRARY ${PARQUET_SHARED_LIB}) + endif() + + if(ARROW_INCLUDE_DIR AND ARROW_LIBRARY AND PARQUET_INCLUDE_DIR AND PARQUET_LIBRARY AND THRIFT_LIBRARY AND UTF8_PROC_LIBRARY AND BZIP2_FOUND) + set(USE_PARQUET 1) + set(EXTERNAL_PARQUET_FOUND 1) + else() + message (${RECONFIGURE_MESSAGE_LEVEL} + "Can't find system parquet: arrow=${ARROW_INCLUDE_DIR} parquet=${PARQUET_INCLUDE_DIR}" ) + set(EXTERNAL_PARQUET_FOUND 0) endif() endif() -if(ARROW_INCLUDE_DIR AND PARQUET_INCLUDE_DIR) -elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) - include(cmake/find/snappy.cmake) +if(NOT EXTERNAL_PARQUET_FOUND AND NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) if(SNAPPY_LIBRARY) set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) else() @@ -95,7 +168,10 @@ elseif(OS_FREEBSD) endif() if(USE_PARQUET) - message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY} ; ${FLATBUFFERS_LIBRARY}") + message(STATUS "Using Parquet: arrow=${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ;" + " parquet=${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ;" + " thrift=${THRIFT_LIBRARY} ;" + " flatbuffers=${FLATBUFFERS_LIBRARY}") else() message(STATUS "Building without Parquet support") endif() diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index dc1040f6a41..013dc513494 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -101,6 +101,8 @@ RUN apt update && \ libsnappy-dev \ libparquet-dev \ libthrift-dev \ + libutf8proc-dev \ + libbz2-dev \ libavro-dev \ libfarmhash-dev \ libmysqlclient-dev \ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 509d7a2150b..843dd8c2615 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -331,6 +331,9 @@ if (USE_PARQUET) dbms_target_link_libraries(PRIVATE ${PARQUET_LIBRARY}) if (NOT USE_INTERNAL_PARQUET_LIBRARY OR USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE) dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${PARQUET_INCLUDE_DIR} ${ARROW_INCLUDE_DIR}) + if (USE_STATIC_LIBRARIES) + dbms_target_link_libraries(PRIVATE ${ARROW_LIBRARY}) + endif() endif () endif () From e0939a8666c00466e9a1a102f0403f615e30518e Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 18 Aug 2020 21:20:28 +0300 Subject: [PATCH 58/89] unbundled: add thrift when using external parquet Signed-off-by: Konstantin Podshumok --- cmake/find/parquet.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index d9b1cdcf70f..6d05fa17aec 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -106,7 +106,9 @@ if(NOT USE_INTERNAL_PARQUET_LIBRARY) set(EXTERNAL_PARQUET_FOUND 1) else() message (${RECONFIGURE_MESSAGE_LEVEL} - "Can't find system parquet: arrow=${ARROW_INCLUDE_DIR} parquet=${PARQUET_INCLUDE_DIR}" ) + "Can't find system parquet: arrow=${ARROW_INCLUDE_DIR}:${ARROW_LIBRARY} ;" + " parquet=${PARQUET_INCLUDE_DIR}:${PARQUET_LIBRARY} ;" + " thrift=${THRIFT_LIBRARY} ;") set(EXTERNAL_PARQUET_FOUND 0) endif() endif() From cc6a59d59b3df3b36697397c592f5b95871bb657 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Mon, 17 Aug 2020 21:47:16 +0300 Subject: [PATCH 59/89] cmake: disable internal ORC when using external Parquet Squashed: - fix internal poco linking with `zlib` Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 5 +++++ cmake/find/orc.cmake | 6 +++++- cmake/find/zlib.cmake | 5 ++++- contrib/boost-cmake/CMakeLists.txt | 2 +- contrib/poco-cmake/Foundation/CMakeLists.txt | 2 +- 5 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 35db796b775..0d93d833693 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -418,7 +418,12 @@ include (cmake/find/parquet.cmake) include (cmake/find/simdjson.cmake) include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) + +if(NOT USE_INTERNAL_PARQUET_LIBRARY) + set (ENABLE_ORC OFF CACHE INTERNAL "") +endif() include (cmake/find/orc.cmake) + include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index 3ab67a98644..86a72e65093 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -7,8 +7,12 @@ if(NOT ENABLE_ORC) return() endif() -option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk)" +if (USE_INTERNAL_PARQUET_LIBRARY) + option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk)" ON) +elseif(USE_INTERNAL_ORC_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Currently internal ORC can be build only with bundled Parquet") +endif() include(cmake/find/snappy.cmake) diff --git a/cmake/find/zlib.cmake b/cmake/find/zlib.cmake index 33b317a9a5e..9a82699dc3a 100644 --- a/cmake/find/zlib.cmake +++ b/cmake/find/zlib.cmake @@ -23,6 +23,8 @@ if (NOT USE_INTERNAL_ZLIB_LIBRARY) find_package (ZLIB) if (NOT ZLIB_FOUND) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system zlib library") + else() + set (ZLIB_NAME "libz") endif() endif () @@ -33,6 +35,7 @@ if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY) set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf set (ZLIB_FOUND 1) # for poco set (ZLIB_LIBRARIES zlib CACHE INTERNAL "") + set (ZLIB_NAME "${INTERNAL_ZLIB_NAME}") endif () -message (STATUS "Using ${INTERNAL_ZLIB_NAME}: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}") +message (STATUS "Using ${ZLIB_NAME}: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}") diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 4d6a2ff0e62..62379f8c2dc 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -84,7 +84,7 @@ if (NOT EXTERNAL_BOOST_FOUND) add_library (_boost_iostreams ${SRCS_IOSTREAMS}) add_library (boost::iostreams ALIAS _boost_iostreams) target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR}) - target_link_libraries (_boost_iostreams PRIVATE zlib) + target_link_libraries (_boost_iostreams PRIVATE ${ZLIB_LIBRARIES}) # program_options diff --git a/contrib/poco-cmake/Foundation/CMakeLists.txt b/contrib/poco-cmake/Foundation/CMakeLists.txt index 740fe53db1b..f4647461ec0 100644 --- a/contrib/poco-cmake/Foundation/CMakeLists.txt +++ b/contrib/poco-cmake/Foundation/CMakeLists.txt @@ -222,7 +222,7 @@ if (USE_INTERNAL_POCO_LIBRARY) POCO_OS_FAMILY_UNIX ) target_include_directories (_poco_foundation SYSTEM PUBLIC ${LIBRARY_DIR}/Foundation/include) - target_link_libraries (_poco_foundation PRIVATE Poco::Foundation::PCRE zlib) + target_link_libraries (_poco_foundation PRIVATE Poco::Foundation::PCRE ${ZLIB_LIBRARIES}) else () add_library (Poco::Foundation UNKNOWN IMPORTED GLOBAL) From d739c6a38fc35d02e8a1f2c9cd212510d3da7f1c Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 18 Aug 2020 20:40:04 +0300 Subject: [PATCH 60/89] Revert "Merge pull request #13689 from ClickHouse/remove-termcap" This reverts commit da8d8f35ddcbd7071ce6ae1e521e447aa4a169d2, reversing changes made to c3d0a87b8bf80665dbf529f2d18a6c847934ed7d. --- CMakeLists.txt | 1 + cmake/find/termcap.cmake | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 cmake/find/termcap.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d93d833693..88092681e35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -393,6 +393,7 @@ include (cmake/find/icu.cmake) include (cmake/find/zlib.cmake) include (cmake/find/zstd.cmake) include (cmake/find/ltdl.cmake) # for odbc +include (cmake/find/termcap.cmake) # openssl, zlib before poco include (cmake/find/sparsehash.cmake) include (cmake/find/re2.cmake) diff --git a/cmake/find/termcap.cmake b/cmake/find/termcap.cmake new file mode 100644 index 00000000000..7564b7134e7 --- /dev/null +++ b/cmake/find/termcap.cmake @@ -0,0 +1,8 @@ +find_library (TERMCAP_LIBRARY tinfo) +if (NOT TERMCAP_LIBRARY) + find_library (TERMCAP_LIBRARY ncurses) +endif() +if (NOT TERMCAP_LIBRARY) + find_library (TERMCAP_LIBRARY termcap) +endif() +message (STATUS "Using termcap: ${TERMCAP_LIBRARY}") From d989e596998d89752df619d7ccb134223e03dd55 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 18 Aug 2020 21:03:37 +0300 Subject: [PATCH 61/89] unbundled: link with termcap when using static llvm ld.lld: error: undefined symbol: setupterm referenced by Process.cpp.o:(llvm::sys::Process::FileDescriptorHasColors(int)) in archive /usr/lib/llvm-9/lib/libLLVMSupport.a Squashed: - fixup termcap + external llvm Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 2 +- cmake/find/llvm.cmake | 5 +++-- cmake/find/termcap.cmake | 23 ++++++++++++++++------- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 88092681e35..e956ffbfb40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -393,7 +393,6 @@ include (cmake/find/icu.cmake) include (cmake/find/zlib.cmake) include (cmake/find/zstd.cmake) include (cmake/find/ltdl.cmake) # for odbc -include (cmake/find/termcap.cmake) # openssl, zlib before poco include (cmake/find/sparsehash.cmake) include (cmake/find/re2.cmake) @@ -404,6 +403,7 @@ include (cmake/find/rdkafka.cmake) include (cmake/find/amqpcpp.cmake) include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) +include (cmake/find/termcap.cmake) # for external static llvm include (cmake/find/opencl.cmake) include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index 9542b8da6fc..70d0e356c39 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -44,8 +44,9 @@ if (NOT USE_INTERNAL_LLVM_LIBRARY) set (USE_EMBEDDED_COMPILER 0) endif() - if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX) - message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages in Linux is incompatible with libc++ ABI. LLVM Will be disabled.") + if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX AND NOT FORCE_LLVM_WITH_LIBCXX) + message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages " + "in Linux is incompatible with libc++ ABI. LLVM Will be disabled. Force: -DFORCE_LLVM_WITH_LIBCXX=ON") message (${RECONFIGURE_MESSAGE_LEVEL} "Unsupported LLVM configuration, cannot enable LLVM") set (LLVM_FOUND 0) set (USE_EMBEDDED_COMPILER 0) diff --git a/cmake/find/termcap.cmake b/cmake/find/termcap.cmake index 7564b7134e7..58454165785 100644 --- a/cmake/find/termcap.cmake +++ b/cmake/find/termcap.cmake @@ -1,8 +1,17 @@ -find_library (TERMCAP_LIBRARY tinfo) -if (NOT TERMCAP_LIBRARY) - find_library (TERMCAP_LIBRARY ncurses) +if (ENABLE_EMBEDDED_COMPILER AND NOT USE_INTERNAL_LLVM_LIBRARY AND USE_STATIC_LIBRARIES) + find_library (TERMCAP_LIBRARY tinfo) + if (NOT TERMCAP_LIBRARY) + find_library (TERMCAP_LIBRARY ncurses) + endif() + if (NOT TERMCAP_LIBRARY) + find_library (TERMCAP_LIBRARY termcap) + endif() + + if (NOT TERMCAP_LIBRARY) + message (FATAL_ERROR "Statically Linking external LLVM requires termcap") + endif() + + target_link_libraries(LLVMSupport INTERFACE ${TERMCAP_LIBRARY}) + + message (STATUS "Using termcap: ${TERMCAP_LIBRARY}") endif() -if (NOT TERMCAP_LIBRARY) - find_library (TERMCAP_LIBRARY termcap) -endif() -message (STATUS "Using termcap: ${TERMCAP_LIBRARY}") From b3180437dad18e2e0d44efb8c81815d134d13b02 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 18 Aug 2020 21:30:39 +0300 Subject: [PATCH 62/89] test builds: docker/packager/packager: re-enable mysql Signed-off-by: Konstantin Podshumok --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index bc97429336c..2d49447e996 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -142,7 +142,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if unbundled: # TODO: fix build with ENABLE_RDKAFKA - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_MYSQL=0 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') From 45e089e0c33af2faa0896492fe692f02725dcd3d Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Wed, 19 Aug 2020 00:39:35 +0300 Subject: [PATCH 63/89] re-enable replxx in packager for unbundled Signed-off-by: Konstantin Podshumok --- docker/packager/packager | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/packager b/docker/packager/packager index 2d49447e996..974f2de0b26 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -142,7 +142,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if unbundled: # TODO: fix build with ENABLE_RDKAFKA - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_ODBC=0 -DENABLE_REPLXX=0 -DENABLE_RDKAFKA=0') + cmake_flags.append('-DUNBUNDLED=1 -DENABLE_ODBC=0 -DENABLE_RDKAFKA=0') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') From ef858027d42c448b4104c58c4c797bc63c11bfbd Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Wed, 19 Aug 2020 05:22:30 +0300 Subject: [PATCH 64/89] unbundled: fix detection of mariadb headers Squashed: - unbundled: add warning about mariadbclient dependencies Signed-off-by: Konstantin Podshumok --- base/mysqlxx/CMakeLists.txt | 8 ++++++++ cmake/find/mysqlclient.cmake | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/base/mysqlxx/CMakeLists.txt b/base/mysqlxx/CMakeLists.txt index 7d35c1bd31d..b410c38cfad 100644 --- a/base/mysqlxx/CMakeLists.txt +++ b/base/mysqlxx/CMakeLists.txt @@ -50,6 +50,14 @@ if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR) target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) endif () +if (NOT USE_INTERNAL_MYSQL_LIBRARY AND USE_STATIC_LIBRARIES) + message(WARNING "Statically linking with system mysql/mariadb only works " + "if mysql client libraries are built with same openssl version as " + "we are going to use now. It wouldn't work if GnuTLS is used. " + "Try -D\"USE_INTERNAL_MYSQL_LIBRARY\"=ON or -D\"ENABLE_MYSQL\"=OFF or " + "-D\"USE_STATIC_LIBRARIES\"=OFF") +endif () + if (ENABLE_TESTS) add_subdirectory (tests) endif () diff --git a/cmake/find/mysqlclient.cmake b/cmake/find/mysqlclient.cmake index b0bfd02fabc..634681d98f6 100644 --- a/cmake/find/mysqlclient.cmake +++ b/cmake/find/mysqlclient.cmake @@ -40,9 +40,11 @@ if (NOT USE_INTERNAL_MYSQL_LIBRARY) "/usr/local/opt/mysql/include" "/usr/mysql/include" "/usr/local/include" + "/usr/include/mariadb" + "/usr/include/mysql" "/usr/include") - find_path (MYSQL_INCLUDE_DIR NAMES mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) + find_path (MYSQL_INCLUDE_DIR NAMES mysql.h mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) if (USE_STATIC_LIBRARIES) find_library (STATIC_MYSQLCLIENT_LIB NAMES mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) From 5d13c1f03f1446f9f8ffe39e621f6354c69b9033 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 21 Aug 2020 14:36:50 +0300 Subject: [PATCH 65/89] cmake: unbundled: libxml2 requires liblzma when building static Squashed: - fixup hdfs3-libxml2 Signed-off-by: Konstantin Podshumok --- cmake/find/hdfs3.cmake | 2 +- cmake/find/libxml2.cmake | 9 +++++++-- contrib/libhdfs3-cmake/CMakeLists.txt | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmake/find/hdfs3.cmake b/cmake/find/hdfs3.cmake index 3f7afd3f6b3..7b385f24e1e 100644 --- a/cmake/find/hdfs3.cmake +++ b/cmake/find/hdfs3.cmake @@ -33,7 +33,7 @@ endif() if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) set(USE_HDFS 1) -elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) +elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARIES) set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") set(HDFS3_LIBRARY hdfs3) set(USE_INTERNAL_HDFS3_LIBRARY 1) diff --git a/cmake/find/libxml2.cmake b/cmake/find/libxml2.cmake index 04609e497a8..cdf079c33d2 100644 --- a/cmake/find/libxml2.cmake +++ b/cmake/find/libxml2.cmake @@ -17,13 +17,18 @@ if (NOT USE_INTERNAL_LIBXML2_LIBRARY) if (NOT LIBXML2_LIBRARY OR NOT LIBXML2_INCLUDE_DIR) message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libxml2") endif () + + if (USE_STATIC_LIBRARIES) + find_package(LibLZMA) + set (LIBXML2_LIBRARIES ${LIBXML2_LIBRARIES} ${LIBLZMA_LIBRARIES}) + endif () endif () if (LIBXML2_LIBRARY AND LIBXML2_INCLUDE_DIR) elseif (NOT MISSING_INTERNAL_LIBXML2_LIBRARY) set (LIBXML2_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libxml2/include ${ClickHouse_SOURCE_DIR}/contrib/libxml2-cmake/linux_x86_64/include) set (USE_INTERNAL_LIBXML2_LIBRARY 1) - set (LIBXML2_LIBRARY libxml2) + set (LIBXML2_LIBRARIES libxml2) endif () -message (STATUS "Using libxml2: ${LIBXML2_INCLUDE_DIR} : ${LIBXML2_LIBRARY}") +message (STATUS "Using libxml2: ${LIBXML2_INCLUDE_DIR} : ${LIBXML2_LIBRARIES}") diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index ff07a0213c0..de69ccdf9c6 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -216,7 +216,7 @@ target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) if (WITH_KERBEROS) target_link_libraries(hdfs3 PRIVATE ${KERBEROS_LIBRARIES}) endif() -target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) # inherit from parent cmake target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) From 8479da3033ce4af09e0c8adb0fcbe2deda46924a Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Fri, 21 Aug 2020 14:39:05 +0300 Subject: [PATCH 66/89] cmake: unbundled protobuf: don't expect old c++ API by default Signed-off-by: Konstantin Podshumok --- contrib/libhdfs3-cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index de69ccdf9c6..ab4857f0387 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -1,5 +1,5 @@ if (ENABLE_PROTOBUF AND NOT USE_INTERNAL_PROTOBUF_LIBRARY) - option(PROTOBUF_OLD_ABI_COMPAT "Set to ON for compatiability with external protobuf which was compiled old C++ ABI" ON) + option(PROTOBUF_OLD_ABI_COMPAT "Set to ON for compatiability with external protobuf which was compiled old C++ ABI" OFF) endif() if (PROTOBUF_OLD_ABI_COMPAT) From 50977938721c8733c0d238f77384883c24897e92 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 22 Aug 2020 01:09:24 +0300 Subject: [PATCH 67/89] cmake: Fix odbc in unbudled Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 1 + cmake/find/odbc.cmake | 53 ++ contrib/poco-cmake/Data/ODBC/CMakeLists.txt | 4 + contrib/unixodbc-cmake/CMakeLists.txt | 601 +++++++++----------- 4 files changed, 338 insertions(+), 321 deletions(-) create mode 100644 cmake/find/odbc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e956ffbfb40..45e9c83ac41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -419,6 +419,7 @@ include (cmake/find/parquet.cmake) include (cmake/find/simdjson.cmake) include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) +include (cmake/find/odbc.cmake) if(NOT USE_INTERNAL_PARQUET_LIBRARY) set (ENABLE_ORC OFF CACHE INTERNAL "") diff --git a/cmake/find/odbc.cmake b/cmake/find/odbc.cmake new file mode 100644 index 00000000000..392d0d47390 --- /dev/null +++ b/cmake/find/odbc.cmake @@ -0,0 +1,53 @@ +option (ENABLE_ODBC "Enable ODBC library" ${ENABLE_LIBRARIES}) + +if (NOT OS_LINUX) + if (ENABLE_ODBC) + message(STATUS "ODBC is only supported on Linux") + endif() + set (ENABLE_ODBC OFF CACHE INTERNAL "") +endif () + +if (NOT ENABLE_ODBC) + if (USE_INTERNAL_ODBC_LIBRARY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal ODBC with ENABLE_ODBC=OFF") + endif() + + add_library (unixodbc INTERFACE) + target_compile_definitions (unixodbc INTERFACE USE_ODBC=0) + + message (STATUS "Not using unixodbc") + return() +endif() + +option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ${NOT_UNBUNDLED}) + +if (NOT USE_INTERNAL_ODBC_LIBRARY) + find_library (LIBRARY_ODBC NAMES unixodbc odbc) + find_path (INCLUDE_ODBC sql.h) + + if(LIBRARY_ODBC AND INCLUDE_ODBC) + add_library (unixodbc UNKNOWN IMPORTED) + set_target_properties (unixodbc PROPERTIES IMPORTED_LOCATION ${LIBRARY_ODBC}) + set_target_properties (unixodbc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_ODBC}) + set_target_properties (unixodbc PROPERTIES INTERFACE_COMPILE_DEFINITIONS USE_ODBC=1) + + if (USE_STATIC_LIBRARIES) + find_library(LTDL_LIBRARY ltdl) + if (LTDL_LIBRARY) + target_link_libraries(unixodbc INTERFACE ${LTDL_LIBRARY}) + endif() + endif() + + set(EXTERNAL_ODBC_LIBRARY_FOUND 1) + message (STATUS "Found odbc: ${LIBRARY_ODBC}") + else() + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ODBC library") + set(EXTERNAL_ODBC_LIBRARY_FOUND 0) + endif() +endif() + +if (NOT EXTERNAL_ODBC_LIBRARY_FOUND) + set (USE_INTERNAL_ODBC_LIBRARY 1) +endif () + +message (STATUS "Using unixodbc") diff --git a/contrib/poco-cmake/Data/ODBC/CMakeLists.txt b/contrib/poco-cmake/Data/ODBC/CMakeLists.txt index b53b58b0d54..cd7c5ef2863 100644 --- a/contrib/poco-cmake/Data/ODBC/CMakeLists.txt +++ b/contrib/poco-cmake/Data/ODBC/CMakeLists.txt @@ -1,4 +1,8 @@ if (ENABLE_ODBC) + if (NOT TARGET unixodbc) + message(FATAL_ERROR "Configuration error: unixodbc is not a target") + endif() + if (USE_INTERNAL_POCO_LIBRARY) set (SRCS ${LIBRARY_DIR}/Data/ODBC/src/Binder.cpp diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt index 7a585161cc1..c971c4bdd89 100644 --- a/contrib/unixodbc-cmake/CMakeLists.txt +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -1,336 +1,295 @@ -option (ENABLE_ODBC "Enable ODBC library" ${ENABLE_LIBRARIES}) - -if (NOT OS_LINUX) - if (ENABLE_ODBC) - message(STATUS "ODBC is only supported on Linux") - endif() - set (ENABLE_ODBC OFF CACHE INTERNAL "") -endif () - -if (NOT ENABLE_ODBC) - if (USE_INTERNAL_ODBC_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal ODBC with ENABLE_ODBC=OFF") - endif() - - add_library (unixodbc INTERFACE) - target_compile_definitions (unixodbc INTERFACE USE_ODBC=0) - - message (STATUS "Not using unixodbc") +if (NOT USE_INTERNAL_ODBC_LIBRARY) return() endif() -option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ${NOT_UNBUNDLED}) +set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/unixodbc) -if (NOT USE_INTERNAL_ODBC_LIBRARY) - find_library (LIBRARY_ODBC unixodbc) - find_path (INCLUDE_ODBC sql.h) +# ltdl - if(LIBRARY_ODBC AND INCLUDE_ODBC) - add_library (unixodbc UNKNOWN IMPORTED) - set_target_properties (unixodbc PROPERTIES IMPORTED_LOCATION ${LIBRARY_ODBC}) - set_target_properties (unixodbc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_ODBC}) - set(EXTERNAL_ODBC_LIBRARY_FOUND 1) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ODBC library") - set(EXTERNAL_ODBC_LIBRARY_FOUND 0) - endif() -endif() +set (SRCS_LTDL + # This file is generated by 'libtool' inside libltdl directory and then removed. + linux_x86_64/libltdl/libltdlcS.c -if (NOT EXTERNAL_ODBC_LIBRARY_FOUND) - set (USE_INTERNAL_ODBC_LIBRARY 1) + ${LIBRARY_DIR}/libltdl/lt__alloc.c + ${LIBRARY_DIR}/libltdl/lt__strl.c + ${LIBRARY_DIR}/libltdl/ltdl.c + ${LIBRARY_DIR}/libltdl/lt_dlloader.c + ${LIBRARY_DIR}/libltdl/slist.c + ${LIBRARY_DIR}/libltdl/lt_error.c + ${LIBRARY_DIR}/libltdl/loaders/dlopen.c + ${LIBRARY_DIR}/libltdl/loaders/preopen.c +) - set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/unixodbc) +add_library (ltdl ${SRCS_LTDL}) - # ltdl +target_include_directories(ltdl + PRIVATE + linux_x86_64/libltdl + PUBLIC + ${LIBRARY_DIR}/libltdl + ${LIBRARY_DIR}/libltdl/libltdl +) +target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc) +target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2) - set (SRCS_LTDL - # This file is generated by 'libtool' inside libltdl directory and then removed. - linux_x86_64/libltdl/libltdlcS.c +# odbc - ${LIBRARY_DIR}/libltdl/lt__alloc.c - ${LIBRARY_DIR}/libltdl/lt__strl.c - ${LIBRARY_DIR}/libltdl/ltdl.c - ${LIBRARY_DIR}/libltdl/lt_dlloader.c - ${LIBRARY_DIR}/libltdl/slist.c - ${LIBRARY_DIR}/libltdl/lt_error.c - ${LIBRARY_DIR}/libltdl/loaders/dlopen.c - ${LIBRARY_DIR}/libltdl/loaders/preopen.c - ) +set (SRCS + ${LIBRARY_DIR}/DriverManager/__attribute.c + ${LIBRARY_DIR}/DriverManager/__connection.c + ${LIBRARY_DIR}/DriverManager/__handles.c + ${LIBRARY_DIR}/DriverManager/__info.c + ${LIBRARY_DIR}/DriverManager/__stats.c + ${LIBRARY_DIR}/DriverManager/SQLAllocConnect.c + ${LIBRARY_DIR}/DriverManager/SQLAllocEnv.c + ${LIBRARY_DIR}/DriverManager/SQLAllocHandle.c + ${LIBRARY_DIR}/DriverManager/SQLAllocHandleStd.c + ${LIBRARY_DIR}/DriverManager/SQLAllocStmt.c + ${LIBRARY_DIR}/DriverManager/SQLBindCol.c + ${LIBRARY_DIR}/DriverManager/SQLBindParam.c + ${LIBRARY_DIR}/DriverManager/SQLBindParameter.c + ${LIBRARY_DIR}/DriverManager/SQLBrowseConnect.c + ${LIBRARY_DIR}/DriverManager/SQLBrowseConnectW.c + ${LIBRARY_DIR}/DriverManager/SQLBulkOperations.c + ${LIBRARY_DIR}/DriverManager/SQLCancel.c + ${LIBRARY_DIR}/DriverManager/SQLCancelHandle.c + ${LIBRARY_DIR}/DriverManager/SQLCloseCursor.c + ${LIBRARY_DIR}/DriverManager/SQLColAttribute.c + ${LIBRARY_DIR}/DriverManager/SQLColAttributes.c + ${LIBRARY_DIR}/DriverManager/SQLColAttributesW.c + ${LIBRARY_DIR}/DriverManager/SQLColAttributeW.c + ${LIBRARY_DIR}/DriverManager/SQLColumnPrivileges.c + ${LIBRARY_DIR}/DriverManager/SQLColumnPrivilegesW.c + ${LIBRARY_DIR}/DriverManager/SQLColumns.c + ${LIBRARY_DIR}/DriverManager/SQLColumnsW.c + ${LIBRARY_DIR}/DriverManager/SQLConnect.c + ${LIBRARY_DIR}/DriverManager/SQLConnectW.c + ${LIBRARY_DIR}/DriverManager/SQLCopyDesc.c + ${LIBRARY_DIR}/DriverManager/SQLDataSources.c + ${LIBRARY_DIR}/DriverManager/SQLDataSourcesW.c + ${LIBRARY_DIR}/DriverManager/SQLDescribeCol.c + ${LIBRARY_DIR}/DriverManager/SQLDescribeColW.c + ${LIBRARY_DIR}/DriverManager/SQLDescribeParam.c + ${LIBRARY_DIR}/DriverManager/SQLDisconnect.c + ${LIBRARY_DIR}/DriverManager/SQLDriverConnect.c + ${LIBRARY_DIR}/DriverManager/SQLDriverConnectW.c + ${LIBRARY_DIR}/DriverManager/SQLDrivers.c + ${LIBRARY_DIR}/DriverManager/SQLDriversW.c + ${LIBRARY_DIR}/DriverManager/SQLEndTran.c + ${LIBRARY_DIR}/DriverManager/SQLError.c + ${LIBRARY_DIR}/DriverManager/SQLErrorW.c + ${LIBRARY_DIR}/DriverManager/SQLExecDirect.c + ${LIBRARY_DIR}/DriverManager/SQLExecDirectW.c + ${LIBRARY_DIR}/DriverManager/SQLExecute.c + ${LIBRARY_DIR}/DriverManager/SQLExtendedFetch.c + ${LIBRARY_DIR}/DriverManager/SQLFetch.c + ${LIBRARY_DIR}/DriverManager/SQLFetchScroll.c + ${LIBRARY_DIR}/DriverManager/SQLForeignKeys.c + ${LIBRARY_DIR}/DriverManager/SQLForeignKeysW.c + ${LIBRARY_DIR}/DriverManager/SQLFreeConnect.c + ${LIBRARY_DIR}/DriverManager/SQLFreeEnv.c + ${LIBRARY_DIR}/DriverManager/SQLFreeHandle.c + ${LIBRARY_DIR}/DriverManager/SQLFreeStmt.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttr.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectOption.c + ${LIBRARY_DIR}/DriverManager/SQLGetConnectOptionW.c + ${LIBRARY_DIR}/DriverManager/SQLGetCursorName.c + ${LIBRARY_DIR}/DriverManager/SQLGetCursorNameW.c + ${LIBRARY_DIR}/DriverManager/SQLGetData.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescField.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescFieldW.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescRec.c + ${LIBRARY_DIR}/DriverManager/SQLGetDescRecW.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagField.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagFieldW.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagRec.c + ${LIBRARY_DIR}/DriverManager/SQLGetDiagRecW.c + ${LIBRARY_DIR}/DriverManager/SQLGetEnvAttr.c + ${LIBRARY_DIR}/DriverManager/SQLGetFunctions.c + ${LIBRARY_DIR}/DriverManager/SQLGetInfo.c + ${LIBRARY_DIR}/DriverManager/SQLGetInfoW.c + ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttr.c + ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLGetStmtOption.c + ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfo.c + ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfoW.c + ${LIBRARY_DIR}/DriverManager/SQLMoreResults.c + ${LIBRARY_DIR}/DriverManager/SQLNativeSql.c + ${LIBRARY_DIR}/DriverManager/SQLNativeSqlW.c + ${LIBRARY_DIR}/DriverManager/SQLNumParams.c + ${LIBRARY_DIR}/DriverManager/SQLNumResultCols.c + ${LIBRARY_DIR}/DriverManager/SQLParamData.c + ${LIBRARY_DIR}/DriverManager/SQLParamOptions.c + ${LIBRARY_DIR}/DriverManager/SQLPrepare.c + ${LIBRARY_DIR}/DriverManager/SQLPrepareW.c + ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeys.c + ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeysW.c + ${LIBRARY_DIR}/DriverManager/SQLProcedureColumns.c + ${LIBRARY_DIR}/DriverManager/SQLProcedureColumnsW.c + ${LIBRARY_DIR}/DriverManager/SQLProcedures.c + ${LIBRARY_DIR}/DriverManager/SQLProceduresW.c + ${LIBRARY_DIR}/DriverManager/SQLPutData.c + ${LIBRARY_DIR}/DriverManager/SQLRowCount.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttr.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectOption.c + ${LIBRARY_DIR}/DriverManager/SQLSetConnectOptionW.c + ${LIBRARY_DIR}/DriverManager/SQLSetCursorName.c + ${LIBRARY_DIR}/DriverManager/SQLSetCursorNameW.c + ${LIBRARY_DIR}/DriverManager/SQLSetDescField.c + ${LIBRARY_DIR}/DriverManager/SQLSetDescFieldW.c + ${LIBRARY_DIR}/DriverManager/SQLSetDescRec.c + ${LIBRARY_DIR}/DriverManager/SQLSetEnvAttr.c + ${LIBRARY_DIR}/DriverManager/SQLSetParam.c + ${LIBRARY_DIR}/DriverManager/SQLSetPos.c + ${LIBRARY_DIR}/DriverManager/SQLSetScrollOptions.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttr.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttrW.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtOption.c + ${LIBRARY_DIR}/DriverManager/SQLSetStmtOptionW.c + ${LIBRARY_DIR}/DriverManager/SQLSpecialColumns.c + ${LIBRARY_DIR}/DriverManager/SQLSpecialColumnsW.c + ${LIBRARY_DIR}/DriverManager/SQLStatistics.c + ${LIBRARY_DIR}/DriverManager/SQLStatisticsW.c + ${LIBRARY_DIR}/DriverManager/SQLTablePrivileges.c + ${LIBRARY_DIR}/DriverManager/SQLTablePrivilegesW.c + ${LIBRARY_DIR}/DriverManager/SQLTables.c + ${LIBRARY_DIR}/DriverManager/SQLTablesW.c + ${LIBRARY_DIR}/DriverManager/SQLTransact.c + ${LIBRARY_DIR}/ini/_iniDump.c + ${LIBRARY_DIR}/ini/_iniObjectRead.c + ${LIBRARY_DIR}/ini/_iniPropertyRead.c + ${LIBRARY_DIR}/ini/_iniScanUntilObject.c + ${LIBRARY_DIR}/ini/iniAllTrim.c + ${LIBRARY_DIR}/ini/iniAppend.c + ${LIBRARY_DIR}/ini/iniClose.c + ${LIBRARY_DIR}/ini/iniCommit.c + ${LIBRARY_DIR}/ini/iniCursor.c + ${LIBRARY_DIR}/ini/iniDelete.c + ${LIBRARY_DIR}/ini/iniElement.c + ${LIBRARY_DIR}/ini/iniElementCount.c + ${LIBRARY_DIR}/ini/iniGetBookmark.c + ${LIBRARY_DIR}/ini/iniGotoBookmark.c + ${LIBRARY_DIR}/ini/iniObject.c + ${LIBRARY_DIR}/ini/iniObjectDelete.c + ${LIBRARY_DIR}/ini/iniObjectEOL.c + ${LIBRARY_DIR}/ini/iniObjectFirst.c + ${LIBRARY_DIR}/ini/iniObjectInsert.c + ${LIBRARY_DIR}/ini/iniObjectLast.c + ${LIBRARY_DIR}/ini/iniObjectNext.c + ${LIBRARY_DIR}/ini/iniObjectSeek.c + ${LIBRARY_DIR}/ini/iniObjectSeekSure.c + ${LIBRARY_DIR}/ini/iniObjectUpdate.c + ${LIBRARY_DIR}/ini/iniOpen.c + ${LIBRARY_DIR}/ini/iniProperty.c + ${LIBRARY_DIR}/ini/iniPropertyDelete.c + ${LIBRARY_DIR}/ini/iniPropertyEOL.c + ${LIBRARY_DIR}/ini/iniPropertyFirst.c + ${LIBRARY_DIR}/ini/iniPropertyInsert.c + ${LIBRARY_DIR}/ini/iniPropertyLast.c + ${LIBRARY_DIR}/ini/iniPropertyNext.c + ${LIBRARY_DIR}/ini/iniPropertySeek.c + ${LIBRARY_DIR}/ini/iniPropertySeekSure.c + ${LIBRARY_DIR}/ini/iniPropertyUpdate.c + ${LIBRARY_DIR}/ini/iniPropertyValue.c + ${LIBRARY_DIR}/ini/iniToUpper.c + ${LIBRARY_DIR}/ini/iniValue.c + ${LIBRARY_DIR}/log/_logFreeMsg.c + ${LIBRARY_DIR}/log/logClear.c + ${LIBRARY_DIR}/log/logClose.c + ${LIBRARY_DIR}/log/logOn.c + ${LIBRARY_DIR}/log/logOpen.c + ${LIBRARY_DIR}/log/logPeekMsg.c + ${LIBRARY_DIR}/log/logPopMsg.c + ${LIBRARY_DIR}/log/logPushMsg.c + ${LIBRARY_DIR}/lst/_lstAdjustCurrent.c + ${LIBRARY_DIR}/lst/_lstDump.c + ${LIBRARY_DIR}/lst/_lstFreeItem.c + ${LIBRARY_DIR}/lst/_lstNextValidItem.c + ${LIBRARY_DIR}/lst/_lstPrevValidItem.c + ${LIBRARY_DIR}/lst/_lstVisible.c + ${LIBRARY_DIR}/lst/lstAppend.c + ${LIBRARY_DIR}/lst/lstClose.c + ${LIBRARY_DIR}/lst/lstDelete.c + ${LIBRARY_DIR}/lst/lstEOL.c + ${LIBRARY_DIR}/lst/lstFirst.c + ${LIBRARY_DIR}/lst/lstGet.c + ${LIBRARY_DIR}/lst/lstGetBookMark.c + ${LIBRARY_DIR}/lst/lstGoto.c + ${LIBRARY_DIR}/lst/lstGotoBookMark.c + ${LIBRARY_DIR}/lst/lstInsert.c + ${LIBRARY_DIR}/lst/lstLast.c + ${LIBRARY_DIR}/lst/lstNext.c + ${LIBRARY_DIR}/lst/lstOpen.c + ${LIBRARY_DIR}/lst/lstOpenCursor.c + ${LIBRARY_DIR}/lst/lstPrev.c + ${LIBRARY_DIR}/lst/lstSeek.c + ${LIBRARY_DIR}/lst/lstSeekItem.c + ${LIBRARY_DIR}/lst/lstSet.c + ${LIBRARY_DIR}/lst/lstSetFreeFunc.c + ${LIBRARY_DIR}/odbcinst/_logging.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_ConfigModeINI.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_GetEntries.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_GetSections.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_SystemINI.c + ${LIBRARY_DIR}/odbcinst/_odbcinst_UserINI.c + ${LIBRARY_DIR}/odbcinst/_SQLDriverConnectPrompt.c + ${LIBRARY_DIR}/odbcinst/_SQLGetInstalledDrivers.c + ${LIBRARY_DIR}/odbcinst/_SQLWriteInstalledDrivers.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTConstructProperties.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTDestructProperties.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTSetProperty.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperties.c + ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperty.c + ${LIBRARY_DIR}/odbcinst/SQLConfigDataSource.c + ${LIBRARY_DIR}/odbcinst/SQLConfigDriver.c + ${LIBRARY_DIR}/odbcinst/SQLCreateDataSource.c + ${LIBRARY_DIR}/odbcinst/SQLGetAvailableDrivers.c + ${LIBRARY_DIR}/odbcinst/SQLGetConfigMode.c + ${LIBRARY_DIR}/odbcinst/SQLGetInstalledDrivers.c + ${LIBRARY_DIR}/odbcinst/SQLGetPrivateProfileString.c + ${LIBRARY_DIR}/odbcinst/SQLGetTranslator.c + ${LIBRARY_DIR}/odbcinst/SQLInstallDriverEx.c + ${LIBRARY_DIR}/odbcinst/SQLInstallDriverManager.c + ${LIBRARY_DIR}/odbcinst/SQLInstallerError.c + ${LIBRARY_DIR}/odbcinst/SQLInstallODBC.c + ${LIBRARY_DIR}/odbcinst/SQLInstallTranslatorEx.c + ${LIBRARY_DIR}/odbcinst/SQLManageDataSources.c + ${LIBRARY_DIR}/odbcinst/SQLPostInstallerError.c + ${LIBRARY_DIR}/odbcinst/SQLReadFileDSN.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveDriver.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveDriverManager.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveDSNFromIni.c + ${LIBRARY_DIR}/odbcinst/SQLRemoveTranslator.c + ${LIBRARY_DIR}/odbcinst/SQLSetConfigMode.c + ${LIBRARY_DIR}/odbcinst/SQLValidDSN.c + ${LIBRARY_DIR}/odbcinst/SQLWriteDSNToIni.c + ${LIBRARY_DIR}/odbcinst/SQLWriteFileDSN.c + ${LIBRARY_DIR}/odbcinst/SQLWritePrivateProfileString.c +) - add_library (ltdl ${SRCS_LTDL}) +add_library (unixodbc ${SRCS}) - target_include_directories(ltdl - PRIVATE - linux_x86_64/libltdl - PUBLIC - ${LIBRARY_DIR}/libltdl - ${LIBRARY_DIR}/libltdl/libltdl - ) - target_compile_definitions(ltdl PRIVATE -DHAVE_CONFIG_H -DLTDL -DLTDLOPEN=libltdlc) - target_compile_options(ltdl PRIVATE -Wno-constant-logical-operand -Wno-unknown-warning-option -O2) +target_link_libraries (unixodbc PRIVATE ltdl) - # odbc - - set (SRCS - ${LIBRARY_DIR}/DriverManager/__attribute.c - ${LIBRARY_DIR}/DriverManager/__connection.c - ${LIBRARY_DIR}/DriverManager/__handles.c - ${LIBRARY_DIR}/DriverManager/__info.c - ${LIBRARY_DIR}/DriverManager/__stats.c - ${LIBRARY_DIR}/DriverManager/SQLAllocConnect.c - ${LIBRARY_DIR}/DriverManager/SQLAllocEnv.c - ${LIBRARY_DIR}/DriverManager/SQLAllocHandle.c - ${LIBRARY_DIR}/DriverManager/SQLAllocHandleStd.c - ${LIBRARY_DIR}/DriverManager/SQLAllocStmt.c - ${LIBRARY_DIR}/DriverManager/SQLBindCol.c - ${LIBRARY_DIR}/DriverManager/SQLBindParam.c - ${LIBRARY_DIR}/DriverManager/SQLBindParameter.c - ${LIBRARY_DIR}/DriverManager/SQLBrowseConnect.c - ${LIBRARY_DIR}/DriverManager/SQLBrowseConnectW.c - ${LIBRARY_DIR}/DriverManager/SQLBulkOperations.c - ${LIBRARY_DIR}/DriverManager/SQLCancel.c - ${LIBRARY_DIR}/DriverManager/SQLCancelHandle.c - ${LIBRARY_DIR}/DriverManager/SQLCloseCursor.c - ${LIBRARY_DIR}/DriverManager/SQLColAttribute.c - ${LIBRARY_DIR}/DriverManager/SQLColAttributes.c - ${LIBRARY_DIR}/DriverManager/SQLColAttributesW.c - ${LIBRARY_DIR}/DriverManager/SQLColAttributeW.c - ${LIBRARY_DIR}/DriverManager/SQLColumnPrivileges.c - ${LIBRARY_DIR}/DriverManager/SQLColumnPrivilegesW.c - ${LIBRARY_DIR}/DriverManager/SQLColumns.c - ${LIBRARY_DIR}/DriverManager/SQLColumnsW.c - ${LIBRARY_DIR}/DriverManager/SQLConnect.c - ${LIBRARY_DIR}/DriverManager/SQLConnectW.c - ${LIBRARY_DIR}/DriverManager/SQLCopyDesc.c - ${LIBRARY_DIR}/DriverManager/SQLDataSources.c - ${LIBRARY_DIR}/DriverManager/SQLDataSourcesW.c - ${LIBRARY_DIR}/DriverManager/SQLDescribeCol.c - ${LIBRARY_DIR}/DriverManager/SQLDescribeColW.c - ${LIBRARY_DIR}/DriverManager/SQLDescribeParam.c - ${LIBRARY_DIR}/DriverManager/SQLDisconnect.c - ${LIBRARY_DIR}/DriverManager/SQLDriverConnect.c - ${LIBRARY_DIR}/DriverManager/SQLDriverConnectW.c - ${LIBRARY_DIR}/DriverManager/SQLDrivers.c - ${LIBRARY_DIR}/DriverManager/SQLDriversW.c - ${LIBRARY_DIR}/DriverManager/SQLEndTran.c - ${LIBRARY_DIR}/DriverManager/SQLError.c - ${LIBRARY_DIR}/DriverManager/SQLErrorW.c - ${LIBRARY_DIR}/DriverManager/SQLExecDirect.c - ${LIBRARY_DIR}/DriverManager/SQLExecDirectW.c - ${LIBRARY_DIR}/DriverManager/SQLExecute.c - ${LIBRARY_DIR}/DriverManager/SQLExtendedFetch.c - ${LIBRARY_DIR}/DriverManager/SQLFetch.c - ${LIBRARY_DIR}/DriverManager/SQLFetchScroll.c - ${LIBRARY_DIR}/DriverManager/SQLForeignKeys.c - ${LIBRARY_DIR}/DriverManager/SQLForeignKeysW.c - ${LIBRARY_DIR}/DriverManager/SQLFreeConnect.c - ${LIBRARY_DIR}/DriverManager/SQLFreeEnv.c - ${LIBRARY_DIR}/DriverManager/SQLFreeHandle.c - ${LIBRARY_DIR}/DriverManager/SQLFreeStmt.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttr.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectOption.c - ${LIBRARY_DIR}/DriverManager/SQLGetConnectOptionW.c - ${LIBRARY_DIR}/DriverManager/SQLGetCursorName.c - ${LIBRARY_DIR}/DriverManager/SQLGetCursorNameW.c - ${LIBRARY_DIR}/DriverManager/SQLGetData.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescField.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescFieldW.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescRec.c - ${LIBRARY_DIR}/DriverManager/SQLGetDescRecW.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagField.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagFieldW.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagRec.c - ${LIBRARY_DIR}/DriverManager/SQLGetDiagRecW.c - ${LIBRARY_DIR}/DriverManager/SQLGetEnvAttr.c - ${LIBRARY_DIR}/DriverManager/SQLGetFunctions.c - ${LIBRARY_DIR}/DriverManager/SQLGetInfo.c - ${LIBRARY_DIR}/DriverManager/SQLGetInfoW.c - ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttr.c - ${LIBRARY_DIR}/DriverManager/SQLGetStmtAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLGetStmtOption.c - ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfo.c - ${LIBRARY_DIR}/DriverManager/SQLGetTypeInfoW.c - ${LIBRARY_DIR}/DriverManager/SQLMoreResults.c - ${LIBRARY_DIR}/DriverManager/SQLNativeSql.c - ${LIBRARY_DIR}/DriverManager/SQLNativeSqlW.c - ${LIBRARY_DIR}/DriverManager/SQLNumParams.c - ${LIBRARY_DIR}/DriverManager/SQLNumResultCols.c - ${LIBRARY_DIR}/DriverManager/SQLParamData.c - ${LIBRARY_DIR}/DriverManager/SQLParamOptions.c - ${LIBRARY_DIR}/DriverManager/SQLPrepare.c - ${LIBRARY_DIR}/DriverManager/SQLPrepareW.c - ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeys.c - ${LIBRARY_DIR}/DriverManager/SQLPrimaryKeysW.c - ${LIBRARY_DIR}/DriverManager/SQLProcedureColumns.c - ${LIBRARY_DIR}/DriverManager/SQLProcedureColumnsW.c - ${LIBRARY_DIR}/DriverManager/SQLProcedures.c - ${LIBRARY_DIR}/DriverManager/SQLProceduresW.c - ${LIBRARY_DIR}/DriverManager/SQLPutData.c - ${LIBRARY_DIR}/DriverManager/SQLRowCount.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttr.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectOption.c - ${LIBRARY_DIR}/DriverManager/SQLSetConnectOptionW.c - ${LIBRARY_DIR}/DriverManager/SQLSetCursorName.c - ${LIBRARY_DIR}/DriverManager/SQLSetCursorNameW.c - ${LIBRARY_DIR}/DriverManager/SQLSetDescField.c - ${LIBRARY_DIR}/DriverManager/SQLSetDescFieldW.c - ${LIBRARY_DIR}/DriverManager/SQLSetDescRec.c - ${LIBRARY_DIR}/DriverManager/SQLSetEnvAttr.c - ${LIBRARY_DIR}/DriverManager/SQLSetParam.c - ${LIBRARY_DIR}/DriverManager/SQLSetPos.c - ${LIBRARY_DIR}/DriverManager/SQLSetScrollOptions.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttr.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtAttrW.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtOption.c - ${LIBRARY_DIR}/DriverManager/SQLSetStmtOptionW.c - ${LIBRARY_DIR}/DriverManager/SQLSpecialColumns.c - ${LIBRARY_DIR}/DriverManager/SQLSpecialColumnsW.c - ${LIBRARY_DIR}/DriverManager/SQLStatistics.c - ${LIBRARY_DIR}/DriverManager/SQLStatisticsW.c - ${LIBRARY_DIR}/DriverManager/SQLTablePrivileges.c - ${LIBRARY_DIR}/DriverManager/SQLTablePrivilegesW.c - ${LIBRARY_DIR}/DriverManager/SQLTables.c - ${LIBRARY_DIR}/DriverManager/SQLTablesW.c - ${LIBRARY_DIR}/DriverManager/SQLTransact.c - ${LIBRARY_DIR}/ini/_iniDump.c - ${LIBRARY_DIR}/ini/_iniObjectRead.c - ${LIBRARY_DIR}/ini/_iniPropertyRead.c - ${LIBRARY_DIR}/ini/_iniScanUntilObject.c - ${LIBRARY_DIR}/ini/iniAllTrim.c - ${LIBRARY_DIR}/ini/iniAppend.c - ${LIBRARY_DIR}/ini/iniClose.c - ${LIBRARY_DIR}/ini/iniCommit.c - ${LIBRARY_DIR}/ini/iniCursor.c - ${LIBRARY_DIR}/ini/iniDelete.c - ${LIBRARY_DIR}/ini/iniElement.c - ${LIBRARY_DIR}/ini/iniElementCount.c - ${LIBRARY_DIR}/ini/iniGetBookmark.c - ${LIBRARY_DIR}/ini/iniGotoBookmark.c - ${LIBRARY_DIR}/ini/iniObject.c - ${LIBRARY_DIR}/ini/iniObjectDelete.c - ${LIBRARY_DIR}/ini/iniObjectEOL.c - ${LIBRARY_DIR}/ini/iniObjectFirst.c - ${LIBRARY_DIR}/ini/iniObjectInsert.c - ${LIBRARY_DIR}/ini/iniObjectLast.c - ${LIBRARY_DIR}/ini/iniObjectNext.c - ${LIBRARY_DIR}/ini/iniObjectSeek.c - ${LIBRARY_DIR}/ini/iniObjectSeekSure.c - ${LIBRARY_DIR}/ini/iniObjectUpdate.c - ${LIBRARY_DIR}/ini/iniOpen.c - ${LIBRARY_DIR}/ini/iniProperty.c - ${LIBRARY_DIR}/ini/iniPropertyDelete.c - ${LIBRARY_DIR}/ini/iniPropertyEOL.c - ${LIBRARY_DIR}/ini/iniPropertyFirst.c - ${LIBRARY_DIR}/ini/iniPropertyInsert.c - ${LIBRARY_DIR}/ini/iniPropertyLast.c - ${LIBRARY_DIR}/ini/iniPropertyNext.c - ${LIBRARY_DIR}/ini/iniPropertySeek.c - ${LIBRARY_DIR}/ini/iniPropertySeekSure.c - ${LIBRARY_DIR}/ini/iniPropertyUpdate.c - ${LIBRARY_DIR}/ini/iniPropertyValue.c - ${LIBRARY_DIR}/ini/iniToUpper.c - ${LIBRARY_DIR}/ini/iniValue.c - ${LIBRARY_DIR}/log/_logFreeMsg.c - ${LIBRARY_DIR}/log/logClear.c - ${LIBRARY_DIR}/log/logClose.c - ${LIBRARY_DIR}/log/logOn.c - ${LIBRARY_DIR}/log/logOpen.c - ${LIBRARY_DIR}/log/logPeekMsg.c - ${LIBRARY_DIR}/log/logPopMsg.c - ${LIBRARY_DIR}/log/logPushMsg.c - ${LIBRARY_DIR}/lst/_lstAdjustCurrent.c - ${LIBRARY_DIR}/lst/_lstDump.c - ${LIBRARY_DIR}/lst/_lstFreeItem.c - ${LIBRARY_DIR}/lst/_lstNextValidItem.c - ${LIBRARY_DIR}/lst/_lstPrevValidItem.c - ${LIBRARY_DIR}/lst/_lstVisible.c - ${LIBRARY_DIR}/lst/lstAppend.c - ${LIBRARY_DIR}/lst/lstClose.c - ${LIBRARY_DIR}/lst/lstDelete.c - ${LIBRARY_DIR}/lst/lstEOL.c - ${LIBRARY_DIR}/lst/lstFirst.c - ${LIBRARY_DIR}/lst/lstGet.c - ${LIBRARY_DIR}/lst/lstGetBookMark.c - ${LIBRARY_DIR}/lst/lstGoto.c - ${LIBRARY_DIR}/lst/lstGotoBookMark.c - ${LIBRARY_DIR}/lst/lstInsert.c - ${LIBRARY_DIR}/lst/lstLast.c - ${LIBRARY_DIR}/lst/lstNext.c - ${LIBRARY_DIR}/lst/lstOpen.c - ${LIBRARY_DIR}/lst/lstOpenCursor.c - ${LIBRARY_DIR}/lst/lstPrev.c - ${LIBRARY_DIR}/lst/lstSeek.c - ${LIBRARY_DIR}/lst/lstSeekItem.c - ${LIBRARY_DIR}/lst/lstSet.c - ${LIBRARY_DIR}/lst/lstSetFreeFunc.c - ${LIBRARY_DIR}/odbcinst/_logging.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_ConfigModeINI.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_GetEntries.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_GetSections.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_SystemINI.c - ${LIBRARY_DIR}/odbcinst/_odbcinst_UserINI.c - ${LIBRARY_DIR}/odbcinst/_SQLDriverConnectPrompt.c - ${LIBRARY_DIR}/odbcinst/_SQLGetInstalledDrivers.c - ${LIBRARY_DIR}/odbcinst/_SQLWriteInstalledDrivers.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTConstructProperties.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTDestructProperties.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTSetProperty.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperties.c - ${LIBRARY_DIR}/odbcinst/ODBCINSTValidateProperty.c - ${LIBRARY_DIR}/odbcinst/SQLConfigDataSource.c - ${LIBRARY_DIR}/odbcinst/SQLConfigDriver.c - ${LIBRARY_DIR}/odbcinst/SQLCreateDataSource.c - ${LIBRARY_DIR}/odbcinst/SQLGetAvailableDrivers.c - ${LIBRARY_DIR}/odbcinst/SQLGetConfigMode.c - ${LIBRARY_DIR}/odbcinst/SQLGetInstalledDrivers.c - ${LIBRARY_DIR}/odbcinst/SQLGetPrivateProfileString.c - ${LIBRARY_DIR}/odbcinst/SQLGetTranslator.c - ${LIBRARY_DIR}/odbcinst/SQLInstallDriverEx.c - ${LIBRARY_DIR}/odbcinst/SQLInstallDriverManager.c - ${LIBRARY_DIR}/odbcinst/SQLInstallerError.c - ${LIBRARY_DIR}/odbcinst/SQLInstallODBC.c - ${LIBRARY_DIR}/odbcinst/SQLInstallTranslatorEx.c - ${LIBRARY_DIR}/odbcinst/SQLManageDataSources.c - ${LIBRARY_DIR}/odbcinst/SQLPostInstallerError.c - ${LIBRARY_DIR}/odbcinst/SQLReadFileDSN.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveDriver.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveDriverManager.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveDSNFromIni.c - ${LIBRARY_DIR}/odbcinst/SQLRemoveTranslator.c - ${LIBRARY_DIR}/odbcinst/SQLSetConfigMode.c - ${LIBRARY_DIR}/odbcinst/SQLValidDSN.c - ${LIBRARY_DIR}/odbcinst/SQLWriteDSNToIni.c - ${LIBRARY_DIR}/odbcinst/SQLWriteFileDSN.c - ${LIBRARY_DIR}/odbcinst/SQLWritePrivateProfileString.c - ) - - add_library (unixodbc ${SRCS}) - - target_link_libraries (unixodbc PRIVATE ltdl) - - # SYSTEM_FILE_PATH was changed to /etc - - target_include_directories (unixodbc - PRIVATE - linux_x86_64/private - PUBLIC - linux_x86_64 - ${LIBRARY_DIR}/include - ) - target_compile_definitions (unixodbc PRIVATE -DHAVE_CONFIG_H) - target_compile_options (unixodbc - PRIVATE - -Wno-dangling-else - -Wno-parentheses - -Wno-misleading-indentation - -Wno-unknown-warning-option - -Wno-reserved-id-macro - -O2 - ) -endif () +# SYSTEM_FILE_PATH was changed to /etc +target_include_directories (unixodbc + PRIVATE + linux_x86_64/private + PUBLIC + linux_x86_64 + ${LIBRARY_DIR}/include +) +target_compile_definitions (unixodbc PRIVATE -DHAVE_CONFIG_H) +target_compile_options (unixodbc + PRIVATE + -Wno-dangling-else + -Wno-parentheses + -Wno-misleading-indentation + -Wno-unknown-warning-option + -Wno-reserved-id-macro + -O2 +) target_compile_definitions (unixodbc INTERFACE USE_ODBC=1) - -message (STATUS "Using unixodbc") From 68a921cdead9858d4e478a428bab0029f787385c Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 22 Aug 2020 01:10:29 +0300 Subject: [PATCH 68/89] cmake: use bundled rdkafaka for in unbundled tests for now Signed-off-by: Konstantin Podshumok --- cmake/find/rdkafka.cmake | 6 +++++- contrib/librdkafka-cmake/CMakeLists.txt | 2 +- docker/packager/packager | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake index 32a78320e12..d9f815dbcdd 100644 --- a/cmake/find/rdkafka.cmake +++ b/cmake/find/rdkafka.cmake @@ -56,7 +56,11 @@ endif () if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR) set (USE_RDKAFKA 1) - set (RDKAFKA_LIBRARY ${RDKAFKA_LIB} ${OPENSSL_LIBRARIES}) + add_library (rdkafka_imp UNKNOWN IMPORTED) + set_target_properties (rdkafka_imp PROPERTIES IMPORTED_LOCATION ${RDKAFKA_LIB}) + set_target_properties (rdkafka_imp PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RDKAFKA_INCLUDE_DIR}) + + set (RDKAFKA_LIBRARY rdkafka_imp ${OPENSSL_LIBRARIES}) set (CPPKAFKA_LIBRARY cppkafka) if (SASL2_LIBRARY) list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY}) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 4a67ebadba6..c8d8d2070b0 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -6,7 +6,7 @@ set(SRCS # ${RDKAFKA_SOURCE_DIR}/lz4.c # ${RDKAFKA_SOURCE_DIR}/lz4frame.c # ${RDKAFKA_SOURCE_DIR}/lz4hc.c -# ${RDKAFKA_SOURCE_DIR}/rdxxhash.c + ${RDKAFKA_SOURCE_DIR}/rdxxhash.c # ${RDKAFKA_SOURCE_DIR}/regexp.c ${RDKAFKA_SOURCE_DIR}/rdaddr.c ${RDKAFKA_SOURCE_DIR}/rdavl.c diff --git a/docker/packager/packager b/docker/packager/packager index 974f2de0b26..3182987d090 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -142,7 +142,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if unbundled: # TODO: fix build with ENABLE_RDKAFKA - cmake_flags.append('-DUNBUNDLED=1 -DENABLE_ODBC=0 -DENABLE_RDKAFKA=0') + cmake_flags.append('-DUNBUNDLED=1 -DUSE_INTERNAL_RDKAFKA_LIBRARY=1') # too old version in ubuntu 19.10 if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') From e296997b6c0aaa21a28f5176fd8dba7736fbf95c Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 22 Aug 2020 01:20:18 +0300 Subject: [PATCH 69/89] cmake: fix potential curl linking problems Signed-off-by: Konstantin Podshumok --- CMakeLists.txt | 1 + cmake/find/curl.cmake | 37 +++ contrib/curl-cmake/CMakeLists.txt | 361 ++++++++++++++---------------- 3 files changed, 208 insertions(+), 191 deletions(-) create mode 100644 cmake/find/curl.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 45e9c83ac41..cf97b2c40ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -413,6 +413,7 @@ include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf include (cmake/find/poco.cmake) +include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/base64.cmake) include (cmake/find/parquet.cmake) diff --git a/cmake/find/curl.cmake b/cmake/find/curl.cmake new file mode 100644 index 00000000000..0a9fcc05ccf --- /dev/null +++ b/cmake/find/curl.cmake @@ -0,0 +1,37 @@ +option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_CURL) + if (USE_INTERNAL_CURL) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal curl with ENABLE_CURL=OFF") + endif() + return() +endif() + +option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED}) + +if (NOT USE_INTERNAL_CURL) + find_package (CURL) + if (NOT CURL_FOUND) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system curl") + endif() +endif() + +if (NOT CURL_FOUND) + set (USE_INTERNAL_CURL 1) + set (CURL_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") + + # find_package(CURL) compatibility for the following packages that uses + # find_package(CURL)/include(FindCURL): + # - mariadb-connector-c + # - aws-s3-cmake + # - sentry-native + set (CURL_FOUND ON CACHE BOOL "") + set (CURL_ROOT_DIR ${CURL_LIBRARY_DIR} CACHE PATH "") + set (CURL_INCLUDE_DIR ${CURL_LIBRARY_DIR}/include CACHE PATH "") + set (CURL_INCLUDE_DIRS ${CURL_LIBRARY_DIR}/include CACHE PATH "") + set (CURL_LIBRARY curl CACHE STRING "") + set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") + set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") +endif () + +message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}") diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index eb97cfa5f14..a24c9fa8765 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -1,200 +1,179 @@ -option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_CURL) - if (USE_INTERNAL_CURL) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal curl with ENABLE_CURL=OFF") - endif() +if (NOT USE_INTERNAL_CURL) return() endif() -option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED}) +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") -if (NOT USE_INTERNAL_CURL) - find_package (CURL) - if (NOT CURL_FOUND) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system curl") - endif() -endif() +set (SRCS + ${LIBRARY_DIR}/lib/file.c + ${LIBRARY_DIR}/lib/timeval.c + ${LIBRARY_DIR}/lib/base64.c + ${LIBRARY_DIR}/lib/hostip.c + ${LIBRARY_DIR}/lib/progress.c + ${LIBRARY_DIR}/lib/formdata.c + ${LIBRARY_DIR}/lib/cookie.c + ${LIBRARY_DIR}/lib/http.c + ${LIBRARY_DIR}/lib/sendf.c + ${LIBRARY_DIR}/lib/url.c + ${LIBRARY_DIR}/lib/dict.c + ${LIBRARY_DIR}/lib/if2ip.c + ${LIBRARY_DIR}/lib/speedcheck.c + ${LIBRARY_DIR}/lib/ldap.c + ${LIBRARY_DIR}/lib/version.c + ${LIBRARY_DIR}/lib/getenv.c + ${LIBRARY_DIR}/lib/escape.c + ${LIBRARY_DIR}/lib/mprintf.c + ${LIBRARY_DIR}/lib/telnet.c + ${LIBRARY_DIR}/lib/netrc.c + ${LIBRARY_DIR}/lib/getinfo.c + ${LIBRARY_DIR}/lib/transfer.c + ${LIBRARY_DIR}/lib/strcase.c + ${LIBRARY_DIR}/lib/easy.c + ${LIBRARY_DIR}/lib/security.c + ${LIBRARY_DIR}/lib/curl_fnmatch.c + ${LIBRARY_DIR}/lib/fileinfo.c + ${LIBRARY_DIR}/lib/wildcard.c + ${LIBRARY_DIR}/lib/krb5.c + ${LIBRARY_DIR}/lib/memdebug.c + ${LIBRARY_DIR}/lib/http_chunks.c + ${LIBRARY_DIR}/lib/strtok.c + ${LIBRARY_DIR}/lib/connect.c + ${LIBRARY_DIR}/lib/llist.c + ${LIBRARY_DIR}/lib/hash.c + ${LIBRARY_DIR}/lib/multi.c + ${LIBRARY_DIR}/lib/content_encoding.c + ${LIBRARY_DIR}/lib/share.c + ${LIBRARY_DIR}/lib/http_digest.c + ${LIBRARY_DIR}/lib/md4.c + ${LIBRARY_DIR}/lib/md5.c + ${LIBRARY_DIR}/lib/http_negotiate.c + ${LIBRARY_DIR}/lib/inet_pton.c + ${LIBRARY_DIR}/lib/strtoofft.c + ${LIBRARY_DIR}/lib/strerror.c + ${LIBRARY_DIR}/lib/amigaos.c + ${LIBRARY_DIR}/lib/hostasyn.c + ${LIBRARY_DIR}/lib/hostip4.c + ${LIBRARY_DIR}/lib/hostip6.c + ${LIBRARY_DIR}/lib/hostsyn.c + ${LIBRARY_DIR}/lib/inet_ntop.c + ${LIBRARY_DIR}/lib/parsedate.c + ${LIBRARY_DIR}/lib/select.c + ${LIBRARY_DIR}/lib/splay.c + ${LIBRARY_DIR}/lib/strdup.c + ${LIBRARY_DIR}/lib/socks.c + ${LIBRARY_DIR}/lib/curl_addrinfo.c + ${LIBRARY_DIR}/lib/socks_gssapi.c + ${LIBRARY_DIR}/lib/socks_sspi.c + ${LIBRARY_DIR}/lib/curl_sspi.c + ${LIBRARY_DIR}/lib/slist.c + ${LIBRARY_DIR}/lib/nonblock.c + ${LIBRARY_DIR}/lib/curl_memrchr.c + ${LIBRARY_DIR}/lib/imap.c + ${LIBRARY_DIR}/lib/pop3.c + ${LIBRARY_DIR}/lib/smtp.c + ${LIBRARY_DIR}/lib/pingpong.c + ${LIBRARY_DIR}/lib/rtsp.c + ${LIBRARY_DIR}/lib/curl_threads.c + ${LIBRARY_DIR}/lib/warnless.c + ${LIBRARY_DIR}/lib/hmac.c + ${LIBRARY_DIR}/lib/curl_rtmp.c + ${LIBRARY_DIR}/lib/openldap.c + ${LIBRARY_DIR}/lib/curl_gethostname.c + ${LIBRARY_DIR}/lib/gopher.c + ${LIBRARY_DIR}/lib/idn_win32.c + ${LIBRARY_DIR}/lib/http_proxy.c + ${LIBRARY_DIR}/lib/non-ascii.c + ${LIBRARY_DIR}/lib/asyn-thread.c + ${LIBRARY_DIR}/lib/curl_gssapi.c + ${LIBRARY_DIR}/lib/http_ntlm.c + ${LIBRARY_DIR}/lib/curl_ntlm_wb.c + ${LIBRARY_DIR}/lib/curl_ntlm_core.c + ${LIBRARY_DIR}/lib/curl_sasl.c + ${LIBRARY_DIR}/lib/rand.c + ${LIBRARY_DIR}/lib/curl_multibyte.c + ${LIBRARY_DIR}/lib/hostcheck.c + ${LIBRARY_DIR}/lib/conncache.c + ${LIBRARY_DIR}/lib/dotdot.c + ${LIBRARY_DIR}/lib/x509asn1.c + ${LIBRARY_DIR}/lib/http2.c + ${LIBRARY_DIR}/lib/smb.c + ${LIBRARY_DIR}/lib/curl_endian.c + ${LIBRARY_DIR}/lib/curl_des.c + ${LIBRARY_DIR}/lib/system_win32.c + ${LIBRARY_DIR}/lib/mime.c + ${LIBRARY_DIR}/lib/sha256.c + ${LIBRARY_DIR}/lib/setopt.c + ${LIBRARY_DIR}/lib/curl_path.c + ${LIBRARY_DIR}/lib/curl_ctype.c + ${LIBRARY_DIR}/lib/curl_range.c + ${LIBRARY_DIR}/lib/psl.c + ${LIBRARY_DIR}/lib/doh.c + ${LIBRARY_DIR}/lib/urlapi.c + ${LIBRARY_DIR}/lib/curl_get_line.c + ${LIBRARY_DIR}/lib/altsvc.c + ${LIBRARY_DIR}/lib/socketpair.c + ${LIBRARY_DIR}/lib/vauth/vauth.c + ${LIBRARY_DIR}/lib/vauth/cleartext.c + ${LIBRARY_DIR}/lib/vauth/cram.c + ${LIBRARY_DIR}/lib/vauth/digest.c + ${LIBRARY_DIR}/lib/vauth/digest_sspi.c + ${LIBRARY_DIR}/lib/vauth/krb5_gssapi.c + ${LIBRARY_DIR}/lib/vauth/krb5_sspi.c + ${LIBRARY_DIR}/lib/vauth/ntlm.c + ${LIBRARY_DIR}/lib/vauth/ntlm_sspi.c + ${LIBRARY_DIR}/lib/vauth/oauth2.c + ${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c + ${LIBRARY_DIR}/lib/vauth/spnego_sspi.c + ${LIBRARY_DIR}/lib/vtls/openssl.c + ${LIBRARY_DIR}/lib/vtls/gtls.c + ${LIBRARY_DIR}/lib/vtls/vtls.c + ${LIBRARY_DIR}/lib/vtls/nss.c + ${LIBRARY_DIR}/lib/vtls/polarssl.c + ${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c + ${LIBRARY_DIR}/lib/vtls/wolfssl.c + ${LIBRARY_DIR}/lib/vtls/schannel.c + ${LIBRARY_DIR}/lib/vtls/schannel_verify.c + ${LIBRARY_DIR}/lib/vtls/sectransp.c + ${LIBRARY_DIR}/lib/vtls/gskit.c + ${LIBRARY_DIR}/lib/vtls/mbedtls.c + ${LIBRARY_DIR}/lib/vtls/mesalink.c + ${LIBRARY_DIR}/lib/vtls/bearssl.c + ${LIBRARY_DIR}/lib/vquic/ngtcp2.c + ${LIBRARY_DIR}/lib/vquic/quiche.c + ${LIBRARY_DIR}/lib/vssh/libssh2.c + ${LIBRARY_DIR}/lib/vssh/libssh.c +) -if (NOT CURL_FOUND) - set (USE_INTERNAL_CURL 1) - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") +add_library (curl ${SRCS}) - set (SRCS - ${LIBRARY_DIR}/lib/file.c - ${LIBRARY_DIR}/lib/timeval.c - ${LIBRARY_DIR}/lib/base64.c - ${LIBRARY_DIR}/lib/hostip.c - ${LIBRARY_DIR}/lib/progress.c - ${LIBRARY_DIR}/lib/formdata.c - ${LIBRARY_DIR}/lib/cookie.c - ${LIBRARY_DIR}/lib/http.c - ${LIBRARY_DIR}/lib/sendf.c - ${LIBRARY_DIR}/lib/url.c - ${LIBRARY_DIR}/lib/dict.c - ${LIBRARY_DIR}/lib/if2ip.c - ${LIBRARY_DIR}/lib/speedcheck.c - ${LIBRARY_DIR}/lib/ldap.c - ${LIBRARY_DIR}/lib/version.c - ${LIBRARY_DIR}/lib/getenv.c - ${LIBRARY_DIR}/lib/escape.c - ${LIBRARY_DIR}/lib/mprintf.c - ${LIBRARY_DIR}/lib/telnet.c - ${LIBRARY_DIR}/lib/netrc.c - ${LIBRARY_DIR}/lib/getinfo.c - ${LIBRARY_DIR}/lib/transfer.c - ${LIBRARY_DIR}/lib/strcase.c - ${LIBRARY_DIR}/lib/easy.c - ${LIBRARY_DIR}/lib/security.c - ${LIBRARY_DIR}/lib/curl_fnmatch.c - ${LIBRARY_DIR}/lib/fileinfo.c - ${LIBRARY_DIR}/lib/wildcard.c - ${LIBRARY_DIR}/lib/krb5.c - ${LIBRARY_DIR}/lib/memdebug.c - ${LIBRARY_DIR}/lib/http_chunks.c - ${LIBRARY_DIR}/lib/strtok.c - ${LIBRARY_DIR}/lib/connect.c - ${LIBRARY_DIR}/lib/llist.c - ${LIBRARY_DIR}/lib/hash.c - ${LIBRARY_DIR}/lib/multi.c - ${LIBRARY_DIR}/lib/content_encoding.c - ${LIBRARY_DIR}/lib/share.c - ${LIBRARY_DIR}/lib/http_digest.c - ${LIBRARY_DIR}/lib/md4.c - ${LIBRARY_DIR}/lib/md5.c - ${LIBRARY_DIR}/lib/http_negotiate.c - ${LIBRARY_DIR}/lib/inet_pton.c - ${LIBRARY_DIR}/lib/strtoofft.c - ${LIBRARY_DIR}/lib/strerror.c - ${LIBRARY_DIR}/lib/amigaos.c - ${LIBRARY_DIR}/lib/hostasyn.c - ${LIBRARY_DIR}/lib/hostip4.c - ${LIBRARY_DIR}/lib/hostip6.c - ${LIBRARY_DIR}/lib/hostsyn.c - ${LIBRARY_DIR}/lib/inet_ntop.c - ${LIBRARY_DIR}/lib/parsedate.c - ${LIBRARY_DIR}/lib/select.c - ${LIBRARY_DIR}/lib/splay.c - ${LIBRARY_DIR}/lib/strdup.c - ${LIBRARY_DIR}/lib/socks.c - ${LIBRARY_DIR}/lib/curl_addrinfo.c - ${LIBRARY_DIR}/lib/socks_gssapi.c - ${LIBRARY_DIR}/lib/socks_sspi.c - ${LIBRARY_DIR}/lib/curl_sspi.c - ${LIBRARY_DIR}/lib/slist.c - ${LIBRARY_DIR}/lib/nonblock.c - ${LIBRARY_DIR}/lib/curl_memrchr.c - ${LIBRARY_DIR}/lib/imap.c - ${LIBRARY_DIR}/lib/pop3.c - ${LIBRARY_DIR}/lib/smtp.c - ${LIBRARY_DIR}/lib/pingpong.c - ${LIBRARY_DIR}/lib/rtsp.c - ${LIBRARY_DIR}/lib/curl_threads.c - ${LIBRARY_DIR}/lib/warnless.c - ${LIBRARY_DIR}/lib/hmac.c - ${LIBRARY_DIR}/lib/curl_rtmp.c - ${LIBRARY_DIR}/lib/openldap.c - ${LIBRARY_DIR}/lib/curl_gethostname.c - ${LIBRARY_DIR}/lib/gopher.c - ${LIBRARY_DIR}/lib/idn_win32.c - ${LIBRARY_DIR}/lib/http_proxy.c - ${LIBRARY_DIR}/lib/non-ascii.c - ${LIBRARY_DIR}/lib/asyn-thread.c - ${LIBRARY_DIR}/lib/curl_gssapi.c - ${LIBRARY_DIR}/lib/http_ntlm.c - ${LIBRARY_DIR}/lib/curl_ntlm_wb.c - ${LIBRARY_DIR}/lib/curl_ntlm_core.c - ${LIBRARY_DIR}/lib/curl_sasl.c - ${LIBRARY_DIR}/lib/rand.c - ${LIBRARY_DIR}/lib/curl_multibyte.c - ${LIBRARY_DIR}/lib/hostcheck.c - ${LIBRARY_DIR}/lib/conncache.c - ${LIBRARY_DIR}/lib/dotdot.c - ${LIBRARY_DIR}/lib/x509asn1.c - ${LIBRARY_DIR}/lib/http2.c - ${LIBRARY_DIR}/lib/smb.c - ${LIBRARY_DIR}/lib/curl_endian.c - ${LIBRARY_DIR}/lib/curl_des.c - ${LIBRARY_DIR}/lib/system_win32.c - ${LIBRARY_DIR}/lib/mime.c - ${LIBRARY_DIR}/lib/sha256.c - ${LIBRARY_DIR}/lib/setopt.c - ${LIBRARY_DIR}/lib/curl_path.c - ${LIBRARY_DIR}/lib/curl_ctype.c - ${LIBRARY_DIR}/lib/curl_range.c - ${LIBRARY_DIR}/lib/psl.c - ${LIBRARY_DIR}/lib/doh.c - ${LIBRARY_DIR}/lib/urlapi.c - ${LIBRARY_DIR}/lib/curl_get_line.c - ${LIBRARY_DIR}/lib/altsvc.c - ${LIBRARY_DIR}/lib/socketpair.c - ${LIBRARY_DIR}/lib/vauth/vauth.c - ${LIBRARY_DIR}/lib/vauth/cleartext.c - ${LIBRARY_DIR}/lib/vauth/cram.c - ${LIBRARY_DIR}/lib/vauth/digest.c - ${LIBRARY_DIR}/lib/vauth/digest_sspi.c - ${LIBRARY_DIR}/lib/vauth/krb5_gssapi.c - ${LIBRARY_DIR}/lib/vauth/krb5_sspi.c - ${LIBRARY_DIR}/lib/vauth/ntlm.c - ${LIBRARY_DIR}/lib/vauth/ntlm_sspi.c - ${LIBRARY_DIR}/lib/vauth/oauth2.c - ${LIBRARY_DIR}/lib/vauth/spnego_gssapi.c - ${LIBRARY_DIR}/lib/vauth/spnego_sspi.c - ${LIBRARY_DIR}/lib/vtls/openssl.c - ${LIBRARY_DIR}/lib/vtls/gtls.c - ${LIBRARY_DIR}/lib/vtls/vtls.c - ${LIBRARY_DIR}/lib/vtls/nss.c - ${LIBRARY_DIR}/lib/vtls/polarssl.c - ${LIBRARY_DIR}/lib/vtls/polarssl_threadlock.c - ${LIBRARY_DIR}/lib/vtls/wolfssl.c - ${LIBRARY_DIR}/lib/vtls/schannel.c - ${LIBRARY_DIR}/lib/vtls/schannel_verify.c - ${LIBRARY_DIR}/lib/vtls/sectransp.c - ${LIBRARY_DIR}/lib/vtls/gskit.c - ${LIBRARY_DIR}/lib/vtls/mbedtls.c - ${LIBRARY_DIR}/lib/vtls/mesalink.c - ${LIBRARY_DIR}/lib/vtls/bearssl.c - ${LIBRARY_DIR}/lib/vquic/ngtcp2.c - ${LIBRARY_DIR}/lib/vquic/quiche.c - ${LIBRARY_DIR}/lib/vssh/libssh2.c - ${LIBRARY_DIR}/lib/vssh/libssh.c - ) +target_compile_definitions (curl PRIVATE + HAVE_CONFIG_H + BUILDING_LIBCURL + CURL_HIDDEN_SYMBOLS + libcurl_EXPORTS + OS="${CMAKE_SYSTEM_NAME}" +) +target_include_directories (curl PUBLIC + ${LIBRARY_DIR}/include + ${LIBRARY_DIR}/lib + . # curl_config.h +) - add_library (curl ${SRCS}) +target_link_libraries (curl PRIVATE ssl) - target_compile_definitions (curl PRIVATE - HAVE_CONFIG_H - BUILDING_LIBCURL - CURL_HIDDEN_SYMBOLS - libcurl_EXPORTS - OS="${CMAKE_SYSTEM_NAME}" - ) - target_include_directories (curl PUBLIC - ${LIBRARY_DIR}/include - ${LIBRARY_DIR}/lib - . # curl_config.h - ) +# The library is large - avoid bloat (XXX: is it?) +target_compile_options (curl PRIVATE -g0) - target_link_libraries (curl PRIVATE ssl) - - # The library is large - avoid bloat (XXX: is it?) - target_compile_options (curl PRIVATE -g0) - - # find_package(CURL) compatibility for the following packages that uses - # find_package(CURL)/include(FindCURL): - # - mariadb-connector-c - # - aws-s3-cmake - # - sentry-native - set (CURL_FOUND ON CACHE BOOL "") - set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "") - set (CURL_INCLUDE_DIR ${LIBRARY_DIR}/include CACHE PATH "") - set (CURL_INCLUDE_DIRS ${LIBRARY_DIR}/include CACHE PATH "") - set (CURL_LIBRARY curl CACHE STRING "") - set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") - set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") - add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) - - set (USE_INTERNAL_CURL 1) -endif () - -message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}") +# find_package(CURL) compatibility for the following packages that uses +# find_package(CURL)/include(FindCURL): +# - mariadb-connector-c +# - aws-s3-cmake +# - sentry-native +set (CURL_FOUND ON CACHE BOOL "") +set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "") +set (CURL_INCLUDE_DIR ${LIBRARY_DIR}/include CACHE PATH "") +set (CURL_INCLUDE_DIRS ${LIBRARY_DIR}/include CACHE PATH "") +set (CURL_LIBRARY curl CACHE STRING "") +set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") +set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") +add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) From 8ce844a40d6c77aa3c0bf86c27619d1f62cd413e Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Sat, 22 Aug 2020 01:25:38 +0300 Subject: [PATCH 70/89] cmake: when USE_STATIC_LIBRARIES prefer bundled brotli even if UNBUNDLED=ON because libbrotli*.a and similar are not available in most of the systems Signed-off-by: Konstantin Podshumok --- cmake/find/brotli.cmake | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/find/brotli.cmake b/cmake/find/brotli.cmake index eca3df0d2c8..bf498802922 100644 --- a/cmake/find/brotli.cmake +++ b/cmake/find/brotli.cmake @@ -7,7 +7,12 @@ if (NOT ENABLE_BROTLI) return() endif() -option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${NOT_UNBUNDLED}) +if (UNBUNDLED) + # Many system ship only dynamic brotly libraries, so we back off to bundled by default + option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${USE_STATIC_LIBRARIES}) +else() + option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ON) +endif() if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h") if (USE_INTERNAL_BROTLI_LIBRARY) From cda4472ebc62b3ab37f3beef9caa2ce5005913b1 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 25 Aug 2020 06:11:01 +0300 Subject: [PATCH 71/89] dockerfiles: verify 3rd party apt repos, disable LLVM in PVS check Signed-off-by: Konstantin Podshumok --- docker/builder/Dockerfile | 62 +++++---- docker/images.json | 7 +- docker/packager/binary/Dockerfile | 109 ++++++++------- docker/packager/deb/Dockerfile | 201 ++++++++++++++------------- docker/packager/unbundled/Dockerfile | 109 ++++++++------- docker/test/base/Dockerfile | 77 +++++----- docker/test/fasttest/Dockerfile | 116 +++++++++------- docker/test/pvs/Dockerfile | 35 +++-- 8 files changed, 393 insertions(+), 323 deletions(-) diff --git a/docker/builder/Dockerfile b/docker/builder/Dockerfile index 53a591ca4da..b7dadc3ec6d 100644 --- a/docker/builder/Dockerfile +++ b/docker/builder/Dockerfile @@ -1,33 +1,41 @@ FROM ubuntu:19.10 -RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 -RUN apt-get update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get install --yes --no-install-recommends \ - bash \ - ccache \ - cmake \ - curl \ - expect \ - g++ \ - gcc \ - ninja-build \ - perl \ - pkg-config \ - python \ - python-lxml \ - python-requests \ - python-termcolor \ - sudo \ - tzdata \ - llvm-10 \ - clang-10 \ - clang-tidy-10 \ - lld-10 \ - lldb-10 +RUN apt-get update \ + && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ + --yes --no-install-recommends --verbose-versions \ + && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ + && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ + && apt-key add /tmp/llvm-snapshot.gpg.key \ + && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + /etc/apt/sources.list + +RUN apt-get update \ + && apt-get install \ + bash \ + ccache \ + cmake \ + curl \ + expect \ + g++ \ + gcc \ + ninja-build \ + perl \ + pkg-config \ + python \ + python-lxml \ + python-requests \ + python-termcolor \ + tzdata \ + llvm-${LLVM_VERSION} \ + clang-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + lld-${LLVM_VERSION} \ + lldb-${LLVM_VERSION} \ + --yes --no-install-recommends COPY build.sh / diff --git a/docker/images.json b/docker/images.json index 723d5fc3e7d..2aced3d17bd 100644 --- a/docker/images.json +++ b/docker/images.json @@ -5,7 +5,8 @@ "docker/test/stateless", "docker/test/stateless_with_coverage", "docker/test/stateless_pytest", - "docker/test/coverage" + "docker/test/coverage", + "docker/packager/unbundled" ] }, "docker/packager/binary": { @@ -39,6 +40,10 @@ "name": "yandex/clickhouse-performance-comparison", "dependent": [] }, + "docker/test/pvs": { + "name": "yandex/clickhouse-pvs-test", + "dependent": [] + }, "docker/test/stateful": { "name": "yandex/clickhouse-stateful-test", "dependent": [ diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 8bdc7e116e5..b8650b945e1 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,62 +1,69 @@ # docker build -t yandex/clickhouse-binary-builder . FROM ubuntu:19.10 -RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - bash \ - fakeroot \ - ccache \ - curl \ - software-properties-common \ - gnupg \ - apt-transport-https \ - ca-certificates +RUN apt-get update \ + && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ + --yes --no-install-recommends --verbose-versions \ + && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ + && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ + && apt-key add /tmp/llvm-snapshot.gpg.key \ + && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + /etc/apt/sources.list -RUN apt-get update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get install --yes --no-install-recommends \ - bash \ - cmake \ - ccache \ - curl \ - gcc-9 \ - g++-9 \ - llvm-10 \ - clang-10 \ - lld-10 \ - clang-tidy-10 \ - clang-9 \ - lld-9 \ - clang-tidy-9 \ - clang-8 \ - lld-8 \ - clang-tidy-8 \ - libicu-dev \ - libreadline-dev \ - ninja-build \ - gperf \ - git \ - opencl-headers \ - ocl-icd-libopencl1 \ - intel-opencl-icd \ - tzdata \ - gperf \ - cmake \ - gdb \ - rename \ - wget \ - build-essential +# initial packages +RUN apt-get update \ + && apt-get install \ + bash \ + fakeroot \ + ccache \ + curl \ + software-properties-common \ + --yes --no-install-recommends + +RUN apt-get update \ + && apt-get install \ + bash \ + cmake \ + ccache \ + curl \ + gcc-9 \ + g++-9 \ + llvm-${LLVM_VERSION} \ + clang-${LLVM_VERSION} \ + lld-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + clang-9 \ + lld-9 \ + clang-tidy-9 \ + clang-8 \ + lld-8 \ + clang-tidy-8 \ + libicu-dev \ + libreadline-dev \ + ninja-build \ + gperf \ + git \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ + tzdata \ + gperf \ + cmake \ + gdb \ + rename \ + wget \ + build-essential \ + --yes --no-install-recommends # This symlink required by gcc to find lld compiler -RUN ln -s /usr/bin/lld-10 /usr/bin/ld.lld +RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld -ENV CC=clang-10 -ENV CXX=clang++-10 +ENV CC=clang-${LLVM_VERSION} +ENV CXX=clang++-${LLVM_VERSION} # libtapi is required to support .tbh format from recent MacOS SDKs RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \ diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 013dc513494..26f589e559d 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -1,116 +1,121 @@ # docker build -t yandex/clickhouse-deb-builder . FROM ubuntu:19.10 -ENV DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 -RUN \ - apt update && \ - apt install ca-certificates lsb-release wget gnupg apt-transport-https --yes --no-install-recommends --verbose-versions && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" && \ - echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-10 main" >> /etc/apt/sources.list && \ - wget "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" && \ - apt install "./apache-arrow-archive-keyring-latest-${CODENAME}.deb" --yes --no-install-recommends --verbose-versions && \ - rm -v "./apache-arrow-archive-keyring-latest-${CODENAME}.deb" +RUN apt-get update \ + && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ + --yes --no-install-recommends --verbose-versions \ + && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ + && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ + && apt-key add /tmp/llvm-snapshot.gpg.key \ + && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + /etc/apt/sources.list # initial packages -RUN apt update && \ - apt install --no-install-recommends \ - bash \ - fakeroot \ - ccache \ - curl \ - software-properties-common \ - --yes +RUN apt-get update \ + && apt-get install \ + bash \ + fakeroot \ + ccache \ + curl \ + software-properties-common \ + --yes --no-install-recommends # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able # to compress files using pigz (https://zlib.net/pigz/) instead of gzip. # Significantly increase deb packaging speed and compatible with old systems -RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb && \ - chmod +x dpkg-deb && \ - cp dpkg-deb /usr/bin +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ + && chmod +x dpkg-deb \ + && cp dpkg-deb /usr/bin +ENV APACHE_PUBKEY_HASH="ec52a838a734b9d5e09d95c0167b09bf1af9565f814d4d5b3785f66fd5b53dd01c918d9ec008cb58e843324dec523d26" + +RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && wget -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ + && echo "${APACHE_PUBKEY_HASH} /tmp/arrow-keyring.deb" | sha384sum -c \ + && dpkg -i /tmp/arrow-keyring.deb # Libraries from OS are only needed to test the "unbundled" build (this is not used in production). -RUN apt update && \ - apt install --no-install-recommends \ - gcc-9 \ - g++-9 \ - llvm-10 \ - llvm-10-dev \ - llvm-9-dev \ - clang-10 \ - lld-10 \ - clang-tidy-10 \ - clang-9 \ - lld-9 \ - clang-tidy-9 \ - libssl-dev \ - libicu-dev \ - gperf \ - ninja-build \ - perl \ - pkg-config \ - devscripts \ - debhelper \ - git \ - libc++-dev \ - libc++abi-dev \ - libboost-program-options-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-thread-dev \ - libboost-iostreams-dev \ - libboost-regex-dev \ - zlib1g-dev \ - liblz4-dev \ - libdouble-conversion-dev \ - libxml2-dev \ - librdkafka-dev \ - libgoogle-perftools-dev \ - libzstd-dev \ - libltdl-dev \ - libre2-dev \ - libjemalloc-dev \ - libmsgpack-dev \ - libcurl4-openssl-dev \ - opencl-headers \ - ocl-icd-libopencl1 \ - intel-opencl-icd \ - unixodbc-dev \ - odbcinst \ - tzdata \ - gperf \ - alien \ - libcapnp-dev \ - cmake \ - gdb \ - pigz \ - moreutils \ - libcctz-dev \ - libldap2-dev \ - libsasl2-dev \ - libgsasl7-dev \ - heimdal-multidev \ - libhyperscan-dev \ - libbrotli-dev \ - protobuf-compiler \ - libprotoc-dev \ - libgrpc++-dev \ - rapidjson-dev \ - libsnappy-dev \ - libparquet-dev \ - libthrift-dev \ - libutf8proc-dev \ - libbz2-dev \ - libavro-dev \ - libfarmhash-dev \ - libmysqlclient-dev \ - --yes - +RUN apt-get update \ + && apt-get install \ + gcc-9 \ + g++-9 \ + llvm-${LLVM_VERSION} \ + llvm-9-dev \ + llvm-${LLVM_VERSION}-dev \ + lld-9 \ + lld-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + clang-9 \ + clang-${LLVM_VERSION} \ + clang-tidy-9 \ + libssl-dev \ + libicu-dev \ + gperf \ + ninja-build \ + perl \ + pkg-config \ + devscripts \ + debhelper \ + git \ + libc++-dev \ + libc++abi-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-thread-dev \ + libboost-iostreams-dev \ + libboost-regex-dev \ + zlib1g-dev \ + liblz4-dev \ + libdouble-conversion-dev \ + libxml2-dev \ + librdkafka-dev \ + libgoogle-perftools-dev \ + libzstd-dev \ + libltdl-dev \ + libre2-dev \ + libjemalloc-dev \ + libmsgpack-dev \ + libcurl4-openssl-dev \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ + unixodbc-dev \ + odbcinst \ + tzdata \ + alien \ + libcapnp-dev \ + cmake \ + gdb \ + pigz \ + moreutils \ + libcctz-dev \ + libldap2-dev \ + libsasl2-dev \ + libgsasl7-dev \ + heimdal-multidev \ + libhyperscan-dev \ + libbrotli-dev \ + protobuf-compiler \ + libprotoc-dev \ + libgrpc++-dev \ + rapidjson-dev \ + libsnappy-dev \ + libparquet-dev \ + libthrift-dev \ + libutf8proc-dev \ + libbz2-dev \ + libavro-dev \ + libfarmhash-dev \ + libmysqlclient-dev \ + --yes --no-install-recommends # This symlink required by gcc to find lld compiler -RUN ln -s /usr/bin/lld-10 /usr/bin/ld.lld +RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld COPY build.sh / diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index b52c1d47dd2..4e2b6e3f2db 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -2,55 +2,66 @@ FROM yandex/clickhouse-deb-builder # Libraries from OS are only needed to test the "unbundled" build (that is not used in production). -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - libicu-dev \ - libreadline-dev \ - gperf \ - perl \ - pkg-config \ - devscripts \ - libc++-dev \ - libc++abi-dev \ - libboost-program-options-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libboost-thread-dev \ - libboost-iostreams-dev \ - libboost-regex-dev \ - zlib1g-dev \ - liblz4-dev \ - libdouble-conversion-dev \ - librdkafka-dev \ - libpoconetssl62 \ - libpoco-dev \ - libgoogle-perftools-dev \ - libzstd-dev \ - libltdl-dev \ - libre2-dev \ - libjemalloc-dev \ - libmsgpack-dev \ - libcurl4-openssl-dev \ - opencl-headers \ - ocl-icd-libopencl1 \ - intel-opencl-icd \ - unixodbc-dev \ - odbcinst \ - tzdata \ - gperf \ - alien \ - libcapnp-dev \ - cmake \ - gdb \ - pigz \ - moreutils \ - libcctz-dev \ - libldap2-dev \ - libsasl2-dev \ - heimdal-multidev \ - libhyperscan-dev +RUN apt-get update \ + && apt-get install \ + libicu-dev \ + gperf \ + perl \ + pkg-config \ + devscripts \ + libc++-dev \ + libc++abi-dev \ + libboost-program-options-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-thread-dev \ + libboost-iostreams-dev \ + libboost-regex-dev \ + zlib1g-dev \ + liblz4-dev \ + libdouble-conversion-dev \ + libxml2-dev \ + librdkafka-dev \ + libgoogle-perftools-dev \ + libzstd-dev \ + libltdl-dev \ + libre2-dev \ + libjemalloc-dev \ + libmsgpack-dev \ + libcurl4-openssl-dev \ + opencl-headers \ + ocl-icd-libopencl1 \ + intel-opencl-icd \ + unixodbc-dev \ + odbcinst \ + tzdata \ + alien \ + libcapnp-dev \ + cmake \ + gdb \ + pigz \ + moreutils \ + libcctz-dev \ + libldap2-dev \ + libsasl2-dev \ + libgsasl7-dev \ + heimdal-multidev \ + libhyperscan-dev \ + libbrotli-dev \ + protobuf-compiler \ + libprotoc-dev \ + libgrpc++-dev \ + rapidjson-dev \ + libsnappy-dev \ + libparquet-dev \ + libthrift-dev \ + libutf8proc-dev \ + libbz2-dev \ + libavro-dev \ + libfarmhash-dev \ + libmysqlclient-dev \ + --yes --no-install-recommends COPY build.sh / -CMD ["/bin/bash", "/build.sh"] \ No newline at end of file +CMD ["/bin/bash", "/build.sh"] diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile index 851ec40a038..c9b0700ecfc 100644 --- a/docker/test/base/Dockerfile +++ b/docker/test/base/Dockerfile @@ -1,51 +1,58 @@ # docker build -t yandex/clickhouse-test-base . FROM ubuntu:19.10 -RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 + +RUN apt-get update \ + && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ + --yes --no-install-recommends --verbose-versions \ + && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ + && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ + && apt-key add /tmp/llvm-snapshot.gpg.key \ + && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + /etc/apt/sources.list # initial packages -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - apt-transport-https \ - bash \ - ca-certificates \ - curl \ - fakeroot \ - gnupg \ - software-properties-common +RUN apt-get update \ + && apt-get install \ + bash \ + fakeroot \ + ccache \ + curl \ + software-properties-common \ + --yes --no-install-recommends # Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able # to compress files using pigz (https://zlib.net/pigz/) instead of gzip. # Significantly increase deb packaging speed and compatible with old systems -RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb -RUN chmod +x dpkg-deb -RUN cp dpkg-deb /usr/bin +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ + && chmod +x dpkg-deb \ + && cp dpkg-deb /usr/bin -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - clang-10 \ - debhelper \ - devscripts \ - gdb \ - git \ - gperf \ - lcov \ - llvm-10 \ - moreutils \ - perl \ - perl \ - pigz \ - pkg-config \ - tzdata +RUN apt-get update \ + && apt-get install \ + clang-${LLVM_VERSION} \ + debhelper \ + devscripts \ + gdb \ + git \ + gperf \ + lcov \ + llvm-${LLVM_VERSION} \ + moreutils \ + perl \ + perl \ + pigz \ + pkg-config \ + tzdata \ + --yes --no-install-recommends # Sanitizer options RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 history_size=7'" >> /etc/environment; \ echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment; \ echo "MSAN_OPTIONS='abort_on_error=1'" >> /etc/environment; \ - ln -s /usr/lib/llvm-10/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; + ln -s /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-symbolizer /usr/bin/llvm-symbolizer; -CMD sleep 1 \ No newline at end of file +CMD sleep 1 diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile index f8bb3683f60..49845d72f1d 100644 --- a/docker/test/fasttest/Dockerfile +++ b/docker/test/fasttest/Dockerfile @@ -1,52 +1,73 @@ # docker build -t yandex/clickhouse-fasttest . FROM ubuntu:19.10 +ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=10 + +RUN apt-get update \ + && apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \ + --yes --no-install-recommends --verbose-versions \ + && export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \ + && wget -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \ + && echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \ + && apt-key add /tmp/llvm-snapshot.gpg.key \ + && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \ + /etc/apt/sources.list + +# initial packages +RUN apt-get update \ + && apt-get install \ + bash \ + fakeroot \ + ccache \ + curl \ + software-properties-common \ + --yes --no-install-recommends + +# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able +# to compress files using pigz (https://zlib.net/pigz/) instead of gzip. +# Significantly increase deb packaging speed and compatible with old systems +RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/1/dpkg-deb \ + && chmod +x dpkg-deb \ + && cp dpkg-deb /usr/bin + +RUN apt-get update \ + && apt-get install \ + apt-transport-https \ + bash \ + brotli \ + build-essential \ + ca-certificates \ + ccache \ + clang-${LLVM_VERSION} \ + clang-tidy-${LLVM_VERSION} \ + cmake \ + curl \ + expect \ + fakeroot \ + git \ + gperf \ + lld-${LLVM_VERSION} \ + llvm-${LLVM_VERSION} \ + moreutils \ + ninja-build \ + psmisc \ + python \ + python-lxml \ + python-requests \ + python-termcolor \ + qemu-user-static \ + rename \ + software-properties-common \ + tzdata \ + unixodbc \ + wget \ + --yes --no-install-recommends + +# This symlink required by gcc to find lld compiler +RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld + ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz" -ENV COMMIT_SHA='' -ENV PULL_REQUEST_NUMBER='' - -RUN apt-get --allow-unauthenticated update -y && apt-get install --yes wget gnupg -RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 main" >> /etc/apt/sources.list - - -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - apt-transport-https \ - bash \ - bash \ - brotli \ - build-essential \ - ca-certificates \ - ccache \ - ccache \ - clang-10 \ - clang-tidy-10 \ - cmake \ - curl \ - expect \ - fakeroot \ - fakeroot \ - git \ - gperf \ - gperf \ - lld-10 \ - llvm-10 \ - moreutils \ - ninja-build \ - psmisc \ - python \ - python-lxml \ - python-requests \ - python-termcolor \ - qemu-user-static \ - rename \ - software-properties-common \ - sudo \ - tzdata \ - unixodbc \ - wget RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \ @@ -55,12 +76,11 @@ RUN mkdir -p /tmp/clickhouse-odbc-tmp \ && odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \ && rm -rf /tmp/clickhouse-odbc-tmp -# This symlink required by gcc to find lld compiler -RUN ln -s /usr/bin/lld-10 /usr/bin/ld.lld - ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ENV COMMIT_SHA='' +ENV PULL_REQUEST_NUMBER='' COPY run.sh / CMD ["/bin/bash", "/run.sh"] diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile index 8d9fb784a3e..ebd9c105705 100644 --- a/docker/test/pvs/Dockerfile +++ b/docker/test/pvs/Dockerfile @@ -2,15 +2,15 @@ FROM yandex/clickhouse-binary-builder -RUN apt-get --allow-unauthenticated update -y \ - && env DEBIAN_FRONTEND=noninteractive \ - apt-get --allow-unauthenticated install --yes --no-install-recommends \ - bash \ - sudo \ - wget \ - software-properties-common \ - gpg-agent \ - strace +RUN apt-get update --yes \ + && apt-get install \ + bash \ + wget \ + software-properties-common \ + gpg-agent \ + debsig-verify \ + strace \ + --yes --no-install-recommends #RUN wget -q -O - http://files.viva64.com/etc/pubkey.txt | sudo apt-key add - #RUN sudo wget -O /etc/apt/sources.list.d/viva64.list http://files.viva64.com/etc/viva64.list @@ -20,14 +20,21 @@ RUN apt-get --allow-unauthenticated update -y \ # apt-get --allow-unauthenticated install --yes --no-install-recommends \ # pvs-studio -ENV PKG_VERSION="pvs-studio-7.08.39365.50-amd64.deb" +ENV PKG_VERSION="pvs-studio-latest" -RUN wget "https://files.viva64.com/$PKG_VERSION" -RUN sudo dpkg -i "$PKG_VERSION" +RUN set -x \ + && export PUBKEY_HASHSUM="486a0694c7f92e96190bbfac01c3b5ac2cb7823981db510a28f744c99eabbbf17a7bcee53ca42dc6d84d4323c2742761" \ + && wget https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \ + && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \ + && apt-key add /tmp/pubkey.txt \ + && wget "https://files.viva64.com/${PKG_VERSION}.deb" \ + && { debsig-verify ${PKG_VERSION}.deb \ + || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \ + && dpkg -i "${PKG_VERSION}.deb" CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \ - && cmake . && ninja re2_st && \ - pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \ + && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF && ninja re2_st \ + && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \ plog-converter -a GA:1,2 -t fullhtml -o /test_output/pvs-studio-html-report pvs-studio.log; \ plog-converter -a GA:1,2 -t tasklist -o /test_output/pvs-studio-task-report.txt pvs-studio.log From f4383389d584c6cfd0cf8d0e46e3a25844871aee Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 25 Aug 2020 06:12:55 +0300 Subject: [PATCH 72/89] sync unbundled/build.sh with a0ae1b2b42950df47a677525621047f2eb9eeaec Signed-off-by: Konstantin Podshumok --- docker/packager/unbundled/build.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/packager/unbundled/build.sh b/docker/packager/unbundled/build.sh index 8b26bbb19cb..fbaa0151c6b 100755 --- a/docker/packager/unbundled/build.sh +++ b/docker/packager/unbundled/build.sh @@ -2,9 +2,6 @@ set -x -e -# Update tzdata to the latest version. It is embedded into clickhouse binary. -sudo apt-get update && sudo apt-get install tzdata - ccache --show-stats ||: ccache --zero-stats ||: build/release --no-pbuilder $ALIEN_PKGS | ts '%Y-%m-%d %H:%M:%S' From cbdedc39f2164ecbd614aa99521a2e7ec7fc6849 Mon Sep 17 00:00:00 2001 From: Konstantin Podshumok Date: Tue, 25 Aug 2020 20:34:36 +0300 Subject: [PATCH 73/89] REVERSE ME: workaround for docker/packager/unbundled Please, reverse this commit after docker/packager/deb image update Signed-off-by: Konstantin Podshumok --- docker/packager/unbundled/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker/packager/unbundled/Dockerfile b/docker/packager/unbundled/Dockerfile index 4e2b6e3f2db..18342deda7e 100644 --- a/docker/packager/unbundled/Dockerfile +++ b/docker/packager/unbundled/Dockerfile @@ -1,6 +1,13 @@ # docker build -t yandex/clickhouse-unbundled-builder . FROM yandex/clickhouse-deb-builder +ENV APACHE_PUBKEY_HASH="ec52a838a734b9d5e09d95c0167b09bf1af9565f814d4d5b3785f66fd5b53dd01c918d9ec008cb58e843324dec523d26" + +RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \ + && wget -O /tmp/arrow-keyring.deb "https://apache.bintray.com/arrow/ubuntu/apache-arrow-archive-keyring-latest-${CODENAME}.deb" \ + && echo "${APACHE_PUBKEY_HASH} /tmp/arrow-keyring.deb" | sha384sum -c \ + && dpkg -i /tmp/arrow-keyring.deb + # Libraries from OS are only needed to test the "unbundled" build (that is not used in production). RUN apt-get update \ && apt-get install \ From d6ae1a80bf543e86f8ca0acbbb60a216913f6dac Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:14:59 +0500 Subject: [PATCH 74/89] fixed formatting --- docs/en/interfaces/third-party/client-libraries.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/interfaces/third-party/client-libraries.md b/docs/en/interfaces/third-party/client-libraries.md index aa13ebd436b..9dbbe0a0022 100644 --- a/docs/en/interfaces/third-party/client-libraries.md +++ b/docs/en/interfaces/third-party/client-libraries.md @@ -46,7 +46,7 @@ toc_title: Client Libraries - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# - - [Octonica.ClickHouseClient] (https://github.com/Octonica/ClickHouseClient) + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From 0a34ae85a33a8e490acca365bddf5db5146dc1c5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 25 Aug 2020 21:15:36 +0300 Subject: [PATCH 75/89] Use terminal width for the formatted help text of the clickhouse-copier Like other utils, clickhouse-client and so on (since copier uses Poco, while other uses boost). Otherwise clickhouse-copier --help is unreadable. --- programs/copier/ClusterCopierApp.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/programs/copier/ClusterCopierApp.cpp b/programs/copier/ClusterCopierApp.cpp index d6ec5a7e8ad..ec64e118f45 100644 --- a/programs/copier/ClusterCopierApp.cpp +++ b/programs/copier/ClusterCopierApp.cpp @@ -1,5 +1,7 @@ #include "ClusterCopierApp.h" #include +#include +#include namespace DB @@ -52,7 +54,13 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) void ClusterCopierApp::handleHelp(const std::string &, const std::string &) { + uint16_t terminal_width = 0; + if (isatty(STDIN_FILENO)) + terminal_width = getTerminalWidth(); + Poco::Util::HelpFormatter help_formatter(options()); + if (terminal_width) + help_formatter.setWidth(terminal_width); help_formatter.setCommand(commandName()); help_formatter.setHeader("Copies tables from one cluster to another"); help_formatter.setUsage("--config-file --task-path "); From e7a3ae3e4a21cc9266f56a01902f1a26b36a4da6 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:17:46 +0500 Subject: [PATCH 76/89] updated client libraries in Russian docs --- docs/ru/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ru/interfaces/third-party/client-libraries.md b/docs/ru/interfaces/third-party/client-libraries.md index 928c1393c26..52ff782b790 100644 --- a/docs/ru/interfaces/third-party/client-libraries.md +++ b/docs/ru/interfaces/third-party/client-libraries.md @@ -40,6 +40,7 @@ - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From e8d24ae204af65c673a96e24c8012e6be361444f Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:18:34 +0500 Subject: [PATCH 77/89] updated client libraries in es docs --- docs/es/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/es/interfaces/third-party/client-libraries.md b/docs/es/interfaces/third-party/client-libraries.md index 44186ec9135..9dbbe0a0022 100644 --- a/docs/es/interfaces/third-party/client-libraries.md +++ b/docs/es/interfaces/third-party/client-libraries.md @@ -46,6 +46,7 @@ toc_title: Client Libraries - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From e0adbf5d6ad8fa80f7088c59fd3af9b58e757125 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:19:19 +0500 Subject: [PATCH 78/89] updated client libraries in fa docs --- docs/fa/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/fa/interfaces/third-party/client-libraries.md b/docs/fa/interfaces/third-party/client-libraries.md index 60829aded12..4d35089e1d3 100644 --- a/docs/fa/interfaces/third-party/client-libraries.md +++ b/docs/fa/interfaces/third-party/client-libraries.md @@ -49,6 +49,7 @@ toc_title: "\u06A9\u062A\u0627\u0628\u062E\u0627\u0646\u0647 \u0647\u0627\u06CC - کوتلین - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [فاحشه خانه.ادو](https://github.com/killwort/ClickHouse-Net) - [فاحشه خانه.کارگیر](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From 9071457d3f885bedaad7ce8c2ae29b65e34aa112 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 19 Aug 2020 21:04:31 +0300 Subject: [PATCH 79/89] Fix C++20 comparison calls recursively with reversed arguments in UInt128.h (over.match.oper#3.4.4 in gcc10) Due to [1], gcc10 reports: ../src/Common/UInt128.h:92:81: error: in C++20 this comparison calls the current function recursively with reversed arguments [-Werror] 92 | template bool inline operator == (T a, const UInt128 b) { return b == a; } [1]: http://eel.is/c++draft/over.match.oper#3.4.4 --- src/Common/UInt128.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/UInt128.h b/src/Common/UInt128.h index 497f49c8ef0..77bb60bef2b 100644 --- a/src/Common/UInt128.h +++ b/src/Common/UInt128.h @@ -89,8 +89,8 @@ struct UInt128 UInt128 & operator= (const UInt64 rhs) { low = rhs; high = 0; return *this; } }; -template bool inline operator == (T a, const UInt128 b) { return b == a; } -template bool inline operator != (T a, const UInt128 b) { return b != a; } +template bool inline operator == (T a, const UInt128 b) { return b.operator==(a); } +template bool inline operator != (T a, const UInt128 b) { return b.operator!=(a); } template bool inline operator >= (T a, const UInt128 b) { return b <= a; } template bool inline operator > (T a, const UInt128 b) { return b < a; } template bool inline operator <= (T a, const UInt128 b) { return b >= a; } From a792129723d54000786950f3cd00c13f58766bc6 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:20:02 +0500 Subject: [PATCH 80/89] updated client libraries in fr docs --- docs/fr/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/fr/interfaces/third-party/client-libraries.md b/docs/fr/interfaces/third-party/client-libraries.md index f07a27aec6c..c3385d207a3 100644 --- a/docs/fr/interfaces/third-party/client-libraries.md +++ b/docs/fr/interfaces/third-party/client-libraries.md @@ -48,6 +48,7 @@ toc_title: "Biblioth\xE8ques Clientes" - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [ClickHouse.Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Client](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From 99db9341a200c7abb3c5b03fa423bd8fbb80129c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 19 Aug 2020 22:36:28 +0300 Subject: [PATCH 81/89] Fix -Werror=type-limits in AggregateFunctionTimeSeriesGroupSum.h (size_t() >= 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gcc10 reports: In file included from ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.cpp:1: ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h: In instantiation of ‘void DB::AggregateFunctionTimeSeriesGroupSumData::add(DB::UInt64, DB::Int64, DB::Float64) [with bool rate = true; DB::UInt64 = long unsigned int; DB::Int64 = long int; DB::Float64 = double]’: ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h:246:34: required from ‘void DB::AggregateFunctionTimeSeriesGroupSum::add(DB::AggregateDataPtr, const DB::IColumn**, size_t, DB::Arena*) const [with bool rate = true; DB::AggregateDataPtr = char*; size_t = long unsigned int]’ ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h:239:10: required from here ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h:113:71: error: comparison of unsigned expression in ‘>= 0’ is always true [-Werror=type-limits] 113 | while (result[i].first > it_ss->second.dps.front().first && i >= 0) | ~~^~~~ ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h: In instantiation of ‘void DB::AggregateFunctionTimeSeriesGroupSumData::add(DB::UInt64, DB::Int64, DB::Float64) [with bool rate = false; DB::UInt64 = long unsigned int; DB::Int64 = long int; DB::Float64 = double]’: ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h:246:34: required from ‘void DB::AggregateFunctionTimeSeriesGroupSum::add(DB::AggregateDataPtr, const DB::IColumn**, size_t, DB::Arena*) const [with bool rate = false; DB::AggregateDataPtr = char*; size_t = long unsigned int]’ ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h:239:10: required from here ../src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h:113:71: error: comparison of unsigned expression in ‘>= 0’ is always true [-Werror=type-limits] --- src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h index 3ec40455cf3..be0a3eb4af5 100644 --- a/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h +++ b/src/AggregateFunctions/AggregateFunctionTimeSeriesGroupSum.h @@ -108,13 +108,13 @@ struct AggregateFunctionTimeSeriesGroupSumData else result.emplace_back(std::make_pair(t, v)); } - size_t i = result.size() - 1; + ssize_t i = result.size() - 1; //reverse find out the index of timestamp that more than previous timestamp of t while (result[i].first > it_ss->second.dps.front().first && i >= 0) i--; i++; - while (i < result.size() - 1) + while (i < ssize_t(result.size()) - 1) { result[i].second += it_ss->second.getval(result[i].first); i++; From 0286974f6eb0e24ac2b452d1a306f7c60a254e41 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:20:52 +0500 Subject: [PATCH 82/89] updated client libraries in tr docs --- docs/tr/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/tr/interfaces/third-party/client-libraries.md b/docs/tr/interfaces/third-party/client-libraries.md index b9940b557b5..7fba28f662a 100644 --- a/docs/tr/interfaces/third-party/client-libraries.md +++ b/docs/tr/interfaces/third-party/client-libraries.md @@ -48,6 +48,7 @@ toc_title: "\u0130stemci Kitapl\u0131klar\u0131" - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [ClickHouse.Gürültü](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Müşteri](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From 8e50bdd6c0ab9ccd48f0d4a3faed4588201c7aa1 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:21:32 +0500 Subject: [PATCH 83/89] updated client libraries in zh docs --- docs/zh/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/interfaces/third-party/client-libraries.md b/docs/zh/interfaces/third-party/client-libraries.md index 007bc986452..7bc4ae0aa27 100644 --- a/docs/zh/interfaces/third-party/client-libraries.md +++ b/docs/zh/interfaces/third-party/client-libraries.md @@ -39,6 +39,7 @@ - Kotlin - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [克莱克豪斯Ado](https://github.com/killwort/ClickHouse-Net) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) - [克莱克豪斯客户](https://github.com/DarkWanderer/ClickHouse.Client) From 40c9e87aa22a7108eb31233cc99d3eb7a9e05696 Mon Sep 17 00:00:00 2001 From: Sergey Mirvoda Date: Tue, 25 Aug 2020 23:22:11 +0500 Subject: [PATCH 84/89] updated client libraries in ja docs --- docs/ja/interfaces/third-party/client-libraries.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ja/interfaces/third-party/client-libraries.md b/docs/ja/interfaces/third-party/client-libraries.md index 4a68c25281b..b88fe0c7389 100644 --- a/docs/ja/interfaces/third-party/client-libraries.md +++ b/docs/ja/interfaces/third-party/client-libraries.md @@ -48,6 +48,7 @@ toc_title: "\u30AF\u30E9\u30A4\u30A2\u30F3\u30C8" - コトリン - [AORM](https://github.com/TanVD/AORM) - C\# + - [Octonica.ClickHouseClient](https://github.com/Octonica/ClickHouseClient) - [クリックハウスAdo](https://github.com/killwort/ClickHouse-Net) - [クリックハウスクライアン](https://github.com/DarkWanderer/ClickHouse.Client) - [ClickHouse.Net](https://github.com/ilyabreev/ClickHouse.Net) From 9971cf2eff7ff2b64d6fff4b1024b728866669a6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 25 Aug 2020 22:29:54 +0300 Subject: [PATCH 85/89] Bump zlib-ng (to fix gcc10 build) [1] is required for gcc10 build, due to incorrect `static_ltree` declaration: ld.lld: error: duplicate symbol: static_ltree >>> defined at trees.h:6 (../contrib/zlib-ng/trees.h:6) >>> trees.c.o:(static_ltree) in archive contrib/zlib-ng/libzlib.a >>> defined at deflate_quick.c:168 (../contrib/zlib-ng/arch/x86/deflate_quick.c:168) >>> deflate_quick.c.o:(.rodata+0x0) in archive contrib/zlib-ng/libzlib.a collect2: error: ld returned 1 exit status ninja: build stopped: subcommand failed. [1]: https://github.com/ClickHouse-Extras/zlib-ng/pull/2 --- contrib/zlib-ng | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/zlib-ng b/contrib/zlib-ng index bba56a73be2..6fd1846c8b8 160000 --- a/contrib/zlib-ng +++ b/contrib/zlib-ng @@ -1 +1 @@ -Subproject commit bba56a73be249514acfbc7d49aa2a68994dad8ab +Subproject commit 6fd1846c8b8f59436fe2dd752d0f316ddbb64df6 From 8864356f33f7f0e24455d1176f28899fc4d1935b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 25 Aug 2020 23:09:23 +0300 Subject: [PATCH 86/89] Kludge --- .../01056_prepared_statements_null_and_escaping.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01056_prepared_statements_null_and_escaping.sh b/tests/queries/0_stateless/01056_prepared_statements_null_and_escaping.sh index 527f1970c85..f4bf616d2ed 100755 --- a/tests/queries/0_stateless/01056_prepared_statements_null_and_escaping.sh +++ b/tests/queries/0_stateless/01056_prepared_statements_null_and_escaping.sh @@ -22,7 +22,7 @@ ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}¶m_x=%5CN" \ -d "SELECT {x:Nullable(String)}"; ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}¶m_x=Hello,%09World" \ - -d "SELECT {x:Nullable(String)}" 2>&1 | grep -oF '457'; + -d "SELECT {x:Nullable(String)}" 2>&1 | grep -oF '457' | head -n1; ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}¶m_x=Hello,%0AWorld" \ - -d "SELECT {x:Nullable(String)}" 2>&1 | grep -oF '457'; + -d "SELECT {x:Nullable(String)}" 2>&1 | grep -oF '457' | head -n1; From f77e2149fc5e78cddf1ea5cc8472a3d4d4179ac3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 25 Aug 2020 23:28:38 +0300 Subject: [PATCH 87/89] Adjust test time --- .../0_stateless/01396_inactive_replica_cleanup_nodes.sh | 8 +++++--- tests/queries/skip_list.json | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes.sh b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes.sh index 4b55bb29c22..2ab3e6dd24f 100755 --- a/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes.sh +++ b/tests/queries/0_stateless/01396_inactive_replica_cleanup_nodes.sh @@ -6,6 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # Check that if we have one inactive replica and a huge number of INSERTs to active replicas, # the number of nodes in ZooKeeper does not grow unbounded. +SCALE=5000 + $CLICKHOUSE_CLIENT -n --query " DROP TABLE IF EXISTS r1; DROP TABLE IF EXISTS r2; @@ -14,19 +16,19 @@ $CLICKHOUSE_CLIENT -n --query " DETACH TABLE r2; " -$CLICKHOUSE_CLIENT --max_block_size 1 --min_insert_block_size_rows 1 --min_insert_block_size_bytes 1 --max_insert_threads 16 --query "INSERT INTO r1 SELECT * FROM numbers_mt(10000)" +$CLICKHOUSE_CLIENT --max_block_size 1 --min_insert_block_size_rows 1 --min_insert_block_size_bytes 1 --max_insert_threads 16 --query "INSERT INTO r1 SELECT * FROM numbers_mt(${SCALE})" # Now wait for cleanup thread for _ in {1..60}; do $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS" - [[ $($CLICKHOUSE_CLIENT --query "SELECT sum(toUInt32(extract(message, 'Removed (\d+) old log entries'))) FROM system.text_log WHERE event_date >= yesterday() AND logger_name LIKE '%' || currentDatabase() || '%r1%(ReplicatedMergeTreeCleanupThread)%' AND message LIKE '%Removed % old log entries%'") -gt 9900 ]] && break; + [[ $($CLICKHOUSE_CLIENT --query "SELECT sum(toUInt32(extract(message, 'Removed (\d+) old log entries'))) FROM system.text_log WHERE event_date >= yesterday() AND logger_name LIKE '%' || currentDatabase() || '%r1%(ReplicatedMergeTreeCleanupThread)%' AND message LIKE '%Removed % old log entries%'") -gt $((SCALE - 100)) ]] && break; sleep 1 done -$CLICKHOUSE_CLIENT --query "SELECT numChildren < 2500 FROM system.zookeeper WHERE path = '/clickhouse/tables/r' AND name = 'log'"; +$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/r' AND name = 'log'"; echo -e '\n---\n'; $CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/r/replicas/1' AND name = 'is_lost'"; $CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/r/replicas/2' AND name = 'is_lost'"; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index c0355f60bd5..a024dd8430a 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -61,7 +61,8 @@ "01103_check_cpu_instructions_at_startup", "01037_polygon_dicts_", "hyperscan", - "01193_metadata_loading" + "01193_metadata_loading", + "01396_inactive_replica_cleanup_nodes" ], "unbundled-build": [ "00429", From e05ad381cedaa896815211ec1c6ec9dc333e1869 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 25 Aug 2020 23:33:03 +0300 Subject: [PATCH 88/89] Fix error in clickhouse-benchmark test --- tests/queries/0_stateless/01393_benchmark_secure_port.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01393_benchmark_secure_port.sh b/tests/queries/0_stateless/01393_benchmark_secure_port.sh index 5a540e50a45..4ec220efa2e 100755 --- a/tests/queries/0_stateless/01393_benchmark_secure_port.sh +++ b/tests/queries/0_stateless/01393_benchmark_secure_port.sh @@ -3,4 +3,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh -$CLICKHOUSE_BENCHMARK --secure -i 100 <<< 'SELECT 1' 2>&1 | grep -F 'Queries executed' +$CLICKHOUSE_BENCHMARK --secure -i 100 <<< 'SELECT 1' 2>&1 | grep -F 'Queries executed' | tail -n1 From 86d99f7c7f5e679dd53402ed0373e3b5f579acc7 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 26 Aug 2020 01:33:31 +0300 Subject: [PATCH 89/89] Disable "optimize_trivial_insert_select" for one perf test --- tests/performance/point_in_polygon.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml index a1ef4891577..403c2d62cba 100644 --- a/tests/performance/point_in_polygon.xml +++ b/tests/performance/point_in_polygon.xml @@ -1,4 +1,8 @@ + + 0 + + CREATE TABLE polygons (polygon Array(Array(Tuple(Float64, Float64)))) ENGINE = Memory INSERT INTO polygons