diff --git a/CMakeLists.txt b/CMakeLists.txt index 09227fead49..7b77687c645 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,11 +25,6 @@ endif () # Write compile_commands.json set(CMAKE_EXPORT_COMPILE_COMMANDS 1) - -set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "") -set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "") -include (cmake/limit_jobs.cmake) - include (cmake/find_ccache.cmake) if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None") diff --git a/cmake/find_zlib.cmake b/cmake/find_zlib.cmake index 90fe674ab36..fb6b8c7971d 100644 --- a/cmake/find_zlib.cmake +++ b/cmake/find_zlib.cmake @@ -8,23 +8,24 @@ endif () if (NOT ZLIB_FOUND) if (NOT MSVC) - set (INTERNAL_ZLIB_NAME "zlib-ng") + set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "") else () - set (INTERNAL_ZLIB_NAME "zlib") + set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "") if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") endif () endif () set (USE_INTERNAL_ZLIB_LIBRARY 1) - set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}") # generated zconf.h + set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco + set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf set (ZLIB_FOUND 1) # for poco if (USE_STATIC_LIBRARIES) - set (ZLIB_LIBRARIES zlibstatic) + set (ZLIB_LIBRARIES zlibstatic CACHE INTERNAL "") else () - set (ZLIB_LIBRARIES zlib) + set (ZLIB_LIBRARIES zlib CACHE INTERNAL "") endif () endif () -message (STATUS "Using zlib: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}") +message (STATUS "Using ${INTERNAL_ZLIB_NAME}: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}") diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake index d71c5260240..1f05dad912b 100644 --- a/cmake/limit_jobs.cmake +++ b/cmake/limit_jobs.cmake @@ -6,30 +6,32 @@ cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd option(PARALLEL_COMPILE_JOBS "Define the maximum number of concurrent compilation jobs" "") -if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY) - math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/2500) # ~2.5gb max per one compiler +if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY) + math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_COMPILER_MEMORY}) if (NOT PARALLEL_COMPILE_JOBS) set (PARALLEL_COMPILE_JOBS 1) endif () endif () if (PARALLEL_COMPILE_JOBS) - set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${PARALLEL_COMPILE_JOBS}) - set(CMAKE_JOB_POOL_COMPILE compile_job_pool) + set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR}) + string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE}) + set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS}) endif () option(PARALLEL_LINK_JOBS "Define the maximum number of concurrent link jobs" "") -if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY) - math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/4000) # ~4gb max per one linker +if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY) + math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY}) if (NOT PARALLEL_LINK_JOBS) set (PARALLEL_LINK_JOBS 1) endif () endif () if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS) - message(STATUS "Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}") + message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}") endif () if (LLVM_PARALLEL_LINK_JOBS) - set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${PARALLEL_LINK_JOBS}) - set(CMAKE_JOB_POOL_LINK link_job_pool) + set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR}) + string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK}) + set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_LINK}=${PARALLEL_LINK_JOBS}) endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 514ed752529..58a0f80b435 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -206,6 +206,7 @@ if (USE_INTERNAL_HDFS3_LIBRARY) if (USE_INTERNAL_PROTOBUF_LIBRARY) set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE) set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) + set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find add_subdirectory(protobuf/cmake) endif () add_subdirectory(libhdfs3-cmake) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index b7f11731662..d5ac533b0cd 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -2,15 +2,23 @@ if (USE_INCLUDE_WHAT_YOU_USE) set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH}) endif () -include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_vectorclass.cmake) +set (MAX_COMPILER_MEMORY 2500 CACHE INTERNAL "") +if (MAKE_STATIC_LIBRARIES) + set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "") +else() + set (MAX_LINKER_MEMORY 2500 CACHE INTERNAL "") +endif () +include (../cmake/limit_jobs.cmake) + +include(cmake/find_vectorclass.cmake) set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h) set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h) include (cmake/version.cmake) message (STATUS "Will build ${VERSION_FULL}") -configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config.h.in ${CONFIG_COMMON}) -configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_version.h.in ${CONFIG_VERSION}) +configure_file (src/Common/config.h.in ${CONFIG_COMMON}) +configure_file (src/Common/config_version.h.in ${CONFIG_VERSION}) if (NOT MSVC) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") @@ -53,7 +61,7 @@ add_subdirectory (src) set(dbms_headers) set(dbms_sources) -include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake) +include(../cmake/dbms_glob_sources.cmake) add_headers_and_sources(clickhouse_common_io src/Common) add_headers_and_sources(clickhouse_common_io src/Common/HashTable) diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 90153da9074..d5a354973e2 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -666,6 +666,12 @@ private: const bool test_mode = config().has("testmode"); if (config().has("multiquery")) { + { /// disable logs if expects errors + TestHint test_hint(test_mode, text); + if (test_hint.clientError() || test_hint.serverError()) + process("SET send_logs_level = 'none'"); + } + /// Several queries separated by ';'. /// INSERT data is ended by the end of line, not ';'. diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h index 790e58ee7fe..7ddbfbb59f8 100644 --- a/dbms/programs/client/TestHint.h +++ b/dbms/programs/client/TestHint.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -27,25 +28,26 @@ public: if (!enabled_) return; - /// TODO: This is absolutely wrong. Fragment may be contained inside string literal. - size_t pos = query.find("--"); + String full_comment; + Lexer lexer(query.data(), query.data() + query.size()); - if (pos != String::npos && query.find("--", pos + 2) != String::npos) - return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query. - - if (pos != String::npos) + for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { - /// TODO: This is also wrong. Comment may already have ended by line break. - pos = query.find('{', pos + 2); + if (token.type == TokenType::Comment) + full_comment += String(token.begin, token.begin + token.size()) + ' '; + } - if (pos != String::npos) + if (!full_comment.empty()) + { + size_t pos_start = full_comment.find('{', 0); + if (pos_start != String::npos) { - String hint = query.substr(pos + 1); - - /// TODO: And this is wrong for the same reason. - pos = hint.find('}'); - hint.resize(pos); - parse(hint); + size_t pos_end = full_comment.find('}', pos_start); + if (pos_end != String::npos) + { + String hint(full_comment.begin() + pos_start + 1, full_comment.begin() + pos_end); + parse(hint); + } } } } diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index 7e9579366b1..36ff7a99529 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -447,6 +447,11 @@ XMLDocumentPtr ConfigProcessor::processConfig( merge(config, with); contributing_files.push_back(merge_file); } + catch (Exception & e) + { + e.addMessage("while merging config '" + path + "' with '" + merge_file + "'"); + throw; + } catch (Poco::Exception & e) { throw Poco::Exception("Failed to merge config with '" + merge_file + "': " + e.displayText()); @@ -479,6 +484,11 @@ XMLDocumentPtr ConfigProcessor::processConfig( doIncludesRecursive(config, include_from, getRootNode(config.get()), zk_node_cache, zk_changed_event, contributing_zk_paths); } + catch (Exception & e) + { + e.addMessage("while preprocessing config '" + path + "'"); + throw; + } catch (Poco::Exception & e) { throw Poco::Exception("Failed to preprocess config '" + path + "': " + e.displayText(), e); diff --git a/dbms/src/Common/Config/ConfigReloader.cpp b/dbms/src/Common/Config/ConfigReloader.cpp index 60447638f46..f6edd3f16fb 100644 --- a/dbms/src/Common/Config/ConfigReloader.cpp +++ b/dbms/src/Common/Config/ConfigReloader.cpp @@ -81,7 +81,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac std::lock_guard lock(reload_mutex); FilesChangesTracker new_files = getNewFileList(); - if (force || new_files.isDifferOrNewerThan(files)) + if (force || need_reload_from_zk || new_files.isDifferOrNewerThan(files)) { ConfigProcessor config_processor(path); ConfigProcessor::LoadedConfig loaded_config; @@ -94,6 +94,17 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac loaded_config = config_processor.loadConfigWithZooKeeperIncludes( zk_node_cache, zk_changed_event, fallback_to_preprocessed); } + catch (const Coordination::Exception & e) + { + if (Coordination::isHardwareError(e.code)) + need_reload_from_zk = true; + + if (throw_on_error) + throw; + + tryLogCurrentException(log, "ZooKeeper error when loading config from `" + path + "'"); + return; + } catch (...) { if (throw_on_error) @@ -110,7 +121,10 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac * When file has been written (and contain valid data), we don't load new data since modification time remains the same. */ if (!loaded_config.loaded_from_preprocessed) + { files = std::move(new_files); + need_reload_from_zk = false; + } try { diff --git a/dbms/src/Common/Config/ConfigReloader.h b/dbms/src/Common/Config/ConfigReloader.h index 9ceaa981109..ca4c97c5aee 100644 --- a/dbms/src/Common/Config/ConfigReloader.h +++ b/dbms/src/Common/Config/ConfigReloader.h @@ -75,6 +75,7 @@ private: std::string preprocessed_dir; FilesChangesTracker files; zkutil::ZooKeeperNodeCache zk_node_cache; + bool need_reload_from_zk = false; zkutil::EventPtr zk_changed_event = std::make_shared(); Updater updater; diff --git a/dbms/src/IO/ReadBufferFromHDFS.h b/dbms/src/IO/ReadBufferFromHDFS.h index 21705c1ef30..a97ad5ece2f 100644 --- a/dbms/src/IO/ReadBufferFromHDFS.h +++ b/dbms/src/IO/ReadBufferFromHDFS.h @@ -7,6 +7,7 @@ #include #include #include +#include #ifndef O_DIRECT #define O_DIRECT 00040000 @@ -52,7 +53,7 @@ namespace DB if (fs == nullptr) { - throw Exception("Unable to connect to HDFS: " + String(hdfsGetLastError()), ErrorCodes::NETWORK_ERROR); + throw Exception("Unable to connect to HDFS: " + std::string(hdfsGetLastError()), ErrorCodes::NETWORK_ERROR); } fin = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0); @@ -77,7 +78,7 @@ namespace DB int bytes_read = hdfsRead(fs, fin, internal_buffer.begin(), internal_buffer.size()); if (bytes_read < 0) { - throw Exception("Fail to read HDFS file: " + hdfs_uri + " " + String(hdfsGetLastError()), ErrorCodes::NETWORK_ERROR); + throw Exception("Fail to read HDFS file: " + hdfs_uri + " " + std::string(hdfsGetLastError()), ErrorCodes::NETWORK_ERROR); } if (bytes_read) diff --git a/dbms/src/Interpreters/Users.cpp b/dbms/src/Interpreters/Users.cpp index f3165e26497..0ac4e4aff16 100644 --- a/dbms/src/Interpreters/Users.cpp +++ b/dbms/src/Interpreters/Users.cpp @@ -70,18 +70,20 @@ public: else { String addr(str, 0, pos - str.c_str()); - mask_address = toIPv6(Poco::Net::IPAddress(addr)); + auto real_address = Poco::Net::IPAddress(addr); String str_mask(str, addr.length() + 1, str.length() - addr.length() - 1); if (isDigits(str_mask)) { UInt8 prefix_bits = parse(pos + 1); - construct(prefix_bits); + construct(prefix_bits, real_address.family() == Poco::Net::AddressFamily::IPv4); } else { subnet_mask = netmaskToIPv6(Poco::Net::IPAddress(str_mask)); } + + mask_address = toIPv6(real_address); } } @@ -97,9 +99,9 @@ private: subnet_mask = Poco::Net::IPAddress(128, Poco::Net::IPAddress::IPv6); } - void construct(UInt8 prefix_bits) + void construct(UInt8 prefix_bits, bool is_ipv4) { - prefix_bits = mask_address.family() == Poco::Net::IPAddress::IPv4 ? prefix_bits + 96 : prefix_bits; + prefix_bits = is_ipv4 ? prefix_bits + 96 : prefix_bits; subnet_mask = Poco::Net::IPAddress(prefix_bits, Poco::Net::IPAddress::IPv6); } diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py index 22d34d05844..329ea631bfc 100644 --- a/dbms/tests/integration/helpers/cluster.py +++ b/dbms/tests/integration/helpers/cluster.py @@ -97,7 +97,7 @@ class ClickHouseCluster: cmd += " client" return cmd - def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False): + def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False, ipv4_address=None, ipv6_address=None): """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -116,7 +116,8 @@ class ClickHouseCluster: instance = ClickHouseInstance( self, self.base_dir, name, config_dir, main_configs, user_configs, macros, with_zookeeper, self.zookeeper_config_path, with_mysql, with_kafka, self.base_configs_dir, self.server_bin_path, - clickhouse_path_dir, with_odbc_drivers, hostname=hostname, env_variables=env_variables, image=image, stay_alive=stay_alive) + clickhouse_path_dir, with_odbc_drivers, hostname=hostname, env_variables=env_variables, image=image, + stay_alive=stay_alive, ipv4_address=ipv4_address, ipv6_address=ipv6_address) self.instances[name] = instance self.base_cmd.extend(['--file', instance.docker_compose_path]) @@ -332,7 +333,7 @@ CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-serv CLICKHOUSE_STAY_ALIVE_COMMAND = 'bash -c "{} --daemon; tail -f /dev/null"'.format(CLICKHOUSE_START_COMMAND) DOCKER_COMPOSE_TEMPLATE = ''' -version: '2' +version: '2.2' services: {name}: image: {image} @@ -344,9 +345,27 @@ services: - {logs_dir}:/var/log/clickhouse-server/ {odbc_ini_path} entrypoint: {entrypoint_cmd} + cap_add: + - SYS_PTRACE depends_on: {depends_on} env_file: - {env_file} + {networks} + {app_net} + {ipv4_address} + {ipv6_address} + +networks: + app_net: + driver: bridge + enable_ipv6: true + ipam: + driver: default + config: + - subnet: 10.5.0.0/12 + gateway: 10.5.1.1 + - subnet: 2001:3984:3989::/64 + gateway: 2001:3984:3989::1 ''' class ClickHouseInstance: @@ -354,7 +373,8 @@ class ClickHouseInstance: def __init__( self, cluster, base_path, name, custom_config_dir, custom_main_configs, custom_user_configs, macros, with_zookeeper, zookeeper_config_path, with_mysql, with_kafka, base_configs_dir, server_bin_path, - clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False): + clickhouse_path_dir, with_odbc_drivers, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", + stay_alive=False, ipv4_address=None, ipv6_address=None): self.name = name self.base_cmd = cluster.base_cmd[:] @@ -391,6 +411,8 @@ class ClickHouseInstance: self.default_timeout = 20.0 # 20 sec self.image = image self.stay_alive = stay_alive + self.ipv4_address = ipv4_address + self.ipv6_address = ipv6_address # Connects to the instance via clickhouse-client, sends a query (1st argument) and returns the answer def query(self, sql, stdin=None, timeout=None, settings=None, user=None, ignore_error=False): @@ -609,6 +631,18 @@ class ClickHouseInstance: if self.stay_alive: entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND + ipv4_address = ipv6_address = "" + if self.ipv4_address is None and self.ipv6_address is None: + networks = "" + app_net = "" + else: + networks = "networks:" + app_net = "app_net:" + if self.ipv4_address is not None: + ipv4_address = "ipv4_address: " + self.ipv4_address + if self.ipv6_address is not None: + ipv6_address = "ipv6_address: " + self.ipv6_address + with open(self.docker_compose_path, 'w') as docker_compose: docker_compose.write(DOCKER_COMPOSE_TEMPLATE.format( image=self.image, @@ -623,6 +657,10 @@ class ClickHouseInstance: env_file=env_file, odbc_ini_path=odbc_ini_path, entrypoint_cmd=entrypoint_cmd, + networks=networks, + app_net=app_net, + ipv4_address=ipv4_address, + ipv6_address=ipv6_address, )) diff --git a/dbms/tests/integration/helpers/docker_compose_hdfs.yml b/dbms/tests/integration/helpers/docker_compose_hdfs.yml index ecfb0c329b3..010a6cd37c1 100644 --- a/dbms/tests/integration/helpers/docker_compose_hdfs.yml +++ b/dbms/tests/integration/helpers/docker_compose_hdfs.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: hdfs1: image: sequenceiq/hadoop-docker:2.7.0 diff --git a/dbms/tests/integration/helpers/docker_compose_kafka.yml b/dbms/tests/integration/helpers/docker_compose_kafka.yml index 42dd154b1e8..461d4626a81 100644 --- a/dbms/tests/integration/helpers/docker_compose_kafka.yml +++ b/dbms/tests/integration/helpers/docker_compose_kafka.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: kafka_zookeeper: diff --git a/dbms/tests/integration/helpers/docker_compose_mysql.yml b/dbms/tests/integration/helpers/docker_compose_mysql.yml index 6106b588f76..30c76ef1981 100644 --- a/dbms/tests/integration/helpers/docker_compose_mysql.yml +++ b/dbms/tests/integration/helpers/docker_compose_mysql.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: mysql1: image: mysql:5.7 diff --git a/dbms/tests/integration/helpers/docker_compose_postgres.yml b/dbms/tests/integration/helpers/docker_compose_postgres.yml index 74049d9f0b3..0745fe827c0 100644 --- a/dbms/tests/integration/helpers/docker_compose_postgres.yml +++ b/dbms/tests/integration/helpers/docker_compose_postgres.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: postgres1: image: postgres diff --git a/dbms/tests/integration/helpers/docker_compose_zookeeper.yml b/dbms/tests/integration/helpers/docker_compose_zookeeper.yml index 365744f106d..22c45cc5960 100644 --- a/dbms/tests/integration/helpers/docker_compose_zookeeper.yml +++ b/dbms/tests/integration/helpers/docker_compose_zookeeper.yml @@ -1,4 +1,4 @@ -version: '2' +version: '2.2' services: zoo1: image: zookeeper diff --git a/dbms/tests/integration/test_user_ip_restrictions/__init__.py b/dbms/tests/integration/test_user_ip_restrictions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_user_ip_restrictions/configs/config_ipv6.xml b/dbms/tests/integration/test_user_ip_restrictions/configs/config_ipv6.xml new file mode 100644 index 00000000000..675225f9575 --- /dev/null +++ b/dbms/tests/integration/test_user_ip_restrictions/configs/config_ipv6.xml @@ -0,0 +1 @@ +:: diff --git a/dbms/tests/integration/test_user_ip_restrictions/configs/users_ipv4.xml b/dbms/tests/integration/test_user_ip_restrictions/configs/users_ipv4.xml new file mode 100644 index 00000000000..f505af55a31 --- /dev/null +++ b/dbms/tests/integration/test_user_ip_restrictions/configs/users_ipv4.xml @@ -0,0 +1,27 @@ + + + + 10000000000 + 0 + random + + + + + + + + 10.5.173.1 + 10.5.172.0/24 + 10.5.175.0/255.255.255.0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_user_ip_restrictions/configs/users_ipv6.xml b/dbms/tests/integration/test_user_ip_restrictions/configs/users_ipv6.xml new file mode 100644 index 00000000000..eeacb72fde5 --- /dev/null +++ b/dbms/tests/integration/test_user_ip_restrictions/configs/users_ipv6.xml @@ -0,0 +1,26 @@ + + + + 10000000000 + 0 + random + + + + + + + + 2001:3984:3989:0:0:0:1:1111 + 2001:3984:3989:0:0:0:0:0/112 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_user_ip_restrictions/test.py b/dbms/tests/integration/test_user_ip_restrictions/test.py new file mode 100644 index 00000000000..731f2bd7fa8 --- /dev/null +++ b/dbms/tests/integration/test_user_ip_restrictions/test.py @@ -0,0 +1,74 @@ +import time +import pytest + +from helpers.cluster import ClickHouseCluster + +from helpers.test_tools import assert_eq_with_retry + +cluster = ClickHouseCluster(__file__) + +node_ipv4 = cluster.add_instance('node_ipv4', config_dir="configs", user_configs=['configs/users_ipv4.xml'], ipv4_address='10.5.172.77') +client_ipv4_ok = cluster.add_instance('client_ipv4_ok', config_dir="configs", ipv4_address='10.5.172.10') +client_ipv4_ok_direct = cluster.add_instance('client_ipv4_ok_direct', config_dir="configs", ipv4_address='10.5.173.1') +client_ipv4_ok_full_mask = cluster.add_instance('client_ipv4_ok_full_mask', config_dir="configs", ipv4_address='10.5.175.77') +client_ipv4_bad = cluster.add_instance('client_ipv4_bad', config_dir="configs", ipv4_address='10.5.173.10') + +node_ipv6 = cluster.add_instance('node_ipv6', config_dir="configs", main_configs=["configs/config_ipv6.xml"], user_configs=['configs/users_ipv6.xml'], ipv6_address='2001:3984:3989::1:1000') +client_ipv6_ok = cluster.add_instance('client_ipv6_ok', config_dir="configs", ipv6_address='2001:3984:3989::5555') +client_ipv6_ok_direct = cluster.add_instance('client_ipv6_ok_direct', config_dir="configs", ipv6_address='2001:3984:3989::1:1111') +client_ipv6_bad = cluster.add_instance('client_ipv6_bad', config_dir="configs", ipv6_address='2001:3984:3989::1:1112') + + +@pytest.fixture(scope="module") +def setup_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_ipv4(setup_cluster): + try: + client_ipv4_ok.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + except Exception as ex: + assert False, "allowed client with 10.5.172.10 cannot connect to server with allowed mask '10.5.172.0/24'" + + try: + client_ipv4_ok_direct.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + except Exception as ex: + assert False, "allowed client with 10.5.173.1 cannot connect to server with allowed ip '10.5.173.1'" + + try: + client_ipv4_ok_full_mask.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + except Exception as ex: + assert False, "allowed client with 10.5.175.77 cannot connect to server with allowed ip '10.5.175.0/255.255.255.0'" + + try: + client_ipv4_bad.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 10.5.172.77 --query 'select 1'"], privileged=True, user='root') + assert False, "restricted client with 10.5.173.10 can connect to server with allowed mask '10.5.172.0/24'" + except AssertionError: + raise + except Exception as ex: + print ex + +def test_ipv6(setup_cluster): + try: + client_ipv6_ok.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989::1:1000 --query 'select 1'"], privileged=True, user='root') + except Exception as ex: + print ex + assert False, "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" + + try: + client_ipv6_ok_direct.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], privileged=True, user='root') + except Exception as ex: + assert False, "allowed client with 2001:3984:3989:0:0:0:1:1111 cannot connect to server with allowed ip '2001:3984:3989:0:0:0:1:1111'" + + try: + client_ipv6_bad.exec_in_container(["bash", "-c", "/usr/bin/clickhouse client --host 2001:3984:3989:0:0:0:1:1000 --query 'select 1'"], privileged=True, user='root') + assert False, "restricted client with 2001:3984:3989:0:0:0:1:1112 can connect to server with allowed mask '2001:3984:3989:0:0:0:0:0/112'" + except AssertionError: + raise + except Exception as ex: + print ex diff --git a/dbms/tests/queries/0_stateless/00700_decimal_aggregates.sql b/dbms/tests/queries/0_stateless/00700_decimal_aggregates.sql index 951e1384e93..0e482c9b109 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_aggregates.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_aggregates.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql b/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql index d2cdfc8d18a..fa0a3598c65 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_arithm.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql b/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql index 4aa9c38b94a..13f62d2da18 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_bounds.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_casts.sql b/dbms/tests/queries/0_stateless/00700_decimal_casts.sql index 14593b50b5e..b1bb0c62d78 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_casts.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_casts.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - SELECT toDecimal32('1.1', 1), toDecimal32('1.1', 2), toDecimal32('1.1', 8); SELECT toDecimal32('1.1', 0); -- { serverError 69 } SELECT toDecimal32(1.1, 0), toDecimal32(1.1, 1), toDecimal32(1.1, 2), toDecimal32(1.1, 8); diff --git a/dbms/tests/queries/0_stateless/00700_decimal_compare.sql b/dbms/tests/queries/0_stateless/00700_decimal_compare.sql index a0d0c3407a4..767bd0e3232 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_compare.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_compare.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_complex_types.sql b/dbms/tests/queries/0_stateless/00700_decimal_complex_types.sql index 4afb09975ba..7cf763b43ea 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_complex_types.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_complex_types.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_defaults.sql b/dbms/tests/queries/0_stateless/00700_decimal_defaults.sql index 2c9cc29d708..e61d7ef9df9 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_defaults.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_defaults.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/dbms/tests/queries/0_stateless/00700_decimal_math.sql b/dbms/tests/queries/0_stateless/00700_decimal_math.sql index fdc3eabfe71..a3a175777ed 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_math.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_math.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - SELECT toDecimal32('42.42', 4) AS x, toDecimal32(log(x), 4) AS y, exp(y); SELECT toDecimal32('42.42', 4) AS x, toDecimal32(log2(x), 4) AS y, exp2(y); SELECT toDecimal32('42.42', 4) AS x, toDecimal32(log10(x), 4) AS y, exp10(y); diff --git a/dbms/tests/queries/0_stateless/00700_decimal_null.sql b/dbms/tests/queries/0_stateless/00700_decimal_null.sql index 24571213241..1cc389b29c9 100644 --- a/dbms/tests/queries/0_stateless/00700_decimal_null.sql +++ b/dbms/tests/queries/0_stateless/00700_decimal_null.sql @@ -1,5 +1,3 @@ -SET send_logs_level = 'none'; - CREATE DATABASE IF NOT EXISTS test; DROP TABLE IF EXISTS test.decimal; diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index ba202d6bddc..08a60cdf19c 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -3,6 +3,13 @@ if (NOT NO_WERROR) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror") endif () +if (MAKE_STATIC_LIBRARIES) + set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "") +else() + set (MAX_LINKER_MEMORY 2500 CACHE INTERNAL "") +endif () +include (../cmake/limit_jobs.cmake) + # Utils used in package add_subdirectory (config-processor) add_subdirectory (report) diff --git a/utils/check-style/check-include b/utils/check-style/check-include index eede9940dcd..4a02f488278 100755 --- a/utils/check-style/check-include +++ b/utils/check-style/check-include @@ -52,6 +52,9 @@ inc="-I. \ -I./libs/libdaemon/include \ -I./libs/consistent-hashing \ -I./libs/consistent-hashing-sumbur \ +-I./contrib/libhdfs3/include \ +-I./contrib/base64/include \ +-I./contrib/lz4/lib \ -I./dbms/src \ -I${BUILD_DIR}/dbms/src"