diff --git a/.gitmodules b/.gitmodules index 66a2370f0da..2ccce88e5e4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -228,3 +228,7 @@ [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp url = https://github.com/ClickHouse-Extras/datasketches-cpp.git + +[submodule "contrib/yaml-cpp"] + path = contrib/yaml-cpp + url = https://github.com/ClickHouse-Extras/yaml-cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c3fa088995..866d9f542e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -527,6 +527,7 @@ include (cmake/find/nanodbc.cmake) include (cmake/find/rocksdb.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) +include (cmake/find/yaml-cpp.cmake) if(NOT USE_INTERNAL_PARQUET_LIBRARY) diff --git a/base/bridge/CMakeLists.txt b/base/bridge/CMakeLists.txt index 20b0b651677..bcba43e8c2e 100644 --- a/base/bridge/CMakeLists.txt +++ b/base/bridge/CMakeLists.txt @@ -3,5 +3,11 @@ add_library (bridge ) target_include_directories (daemon PUBLIC ..) -target_link_libraries (bridge PRIVATE daemon dbms Poco::Data Poco::Data::ODBC) +target_link_libraries (bridge + PRIVATE + daemon + dbms + Poco::Data + Poco::Data::ODBC +) diff --git a/cmake/find/yaml-cpp.cmake b/cmake/find/yaml-cpp.cmake new file mode 100644 index 00000000000..9b9d9bd39d6 --- /dev/null +++ b/cmake/find/yaml-cpp.cmake @@ -0,0 +1,9 @@ +option(USE_YAML_CPP "Enable yaml-cpp" ${ENABLE_LIBRARIES}) + +if (NOT USE_YAML_CPP) + return() +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp") + message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init --recursive") +endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9eafec23f51..a9438aa4b76 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -50,6 +50,10 @@ add_subdirectory (replxx-cmake) add_subdirectory (unixodbc-cmake) add_subdirectory (nanodbc-cmake) +if (USE_YAML_CPP) + add_subdirectory (yaml-cpp-cmake) +endif() + if (USE_INTERNAL_XZ_LIBRARY) add_subdirectory (xz) endif() diff --git a/contrib/yaml-cpp b/contrib/yaml-cpp new file mode 160000 index 00000000000..0c86adac6d1 --- /dev/null +++ b/contrib/yaml-cpp @@ -0,0 +1 @@ +Subproject commit 0c86adac6d117ee2b4afcedb8ade19036ca0327d diff --git a/contrib/yaml-cpp-cmake/CMakeLists.txt b/contrib/yaml-cpp-cmake/CMakeLists.txt new file mode 100644 index 00000000000..ed0287de110 --- /dev/null +++ b/contrib/yaml-cpp-cmake/CMakeLists.txt @@ -0,0 +1,39 @@ +set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp) + +set (SRCS + ${LIBRARY_DIR}/src/binary.cpp + ${LIBRARY_DIR}/src/emitterutils.cpp + ${LIBRARY_DIR}/src/null.cpp + ${LIBRARY_DIR}/src/scantoken.cpp + ${LIBRARY_DIR}/src/convert.cpp + ${LIBRARY_DIR}/src/exceptions.cpp + ${LIBRARY_DIR}/src/ostream_wrapper.cpp + ${LIBRARY_DIR}/src/simplekey.cpp + ${LIBRARY_DIR}/src/depthguard.cpp + ${LIBRARY_DIR}/src/exp.cpp + ${LIBRARY_DIR}/src/parse.cpp + ${LIBRARY_DIR}/src/singledocparser.cpp + ${LIBRARY_DIR}/src/directives.cpp + ${LIBRARY_DIR}/src/memory.cpp + ${LIBRARY_DIR}/src/parser.cpp + ${LIBRARY_DIR}/src/stream.cpp + ${LIBRARY_DIR}/src/emit.cpp + ${LIBRARY_DIR}/src/nodebuilder.cpp + ${LIBRARY_DIR}/src/regex_yaml.cpp + ${LIBRARY_DIR}/src/tag.cpp + ${LIBRARY_DIR}/src/emitfromevents.cpp + ${LIBRARY_DIR}/src/node.cpp + ${LIBRARY_DIR}/src/scanner.cpp + ${LIBRARY_DIR}/src/emitter.cpp + ${LIBRARY_DIR}/src/node_data.cpp + ${LIBRARY_DIR}/src/scanscalar.cpp + ${LIBRARY_DIR}/src/emitterstate.cpp + ${LIBRARY_DIR}/src/nodeevents.cpp + ${LIBRARY_DIR}/src/scantag.cpp +) + +add_library (yaml-cpp ${SRCS}) + + +target_include_directories(yaml-cpp PRIVATE ${LIBRARY_DIR}/include/yaml-cpp) +target_include_directories(yaml-cpp SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include) diff --git a/programs/server/config-example.yaml.disabled b/programs/server/config-example.yaml.disabled new file mode 100644 index 00000000000..a83acf50de0 --- /dev/null +++ b/programs/server/config-example.yaml.disabled @@ -0,0 +1,86 @@ +# We can use 3 main node types in YAML: Scalar, Map and Sequence. + + + +# A Scalar is a simple key-value pair: + +scalar: 123 + +# Here we have a key "scalar" and value "123" +# If we rewrite this in XML, we will get 123 + +# We can also represent an empty value with '': + +key: '' + + + +# A Map is a node, which contains other nodes: + +map: + key1: value1 + key2: value2 + small_map: + key3: value3 + +# This map can be converted into: +# +# value1 +# value2 +# +# value3 +# +# + + + +# A Sequence is a node, which contains also other nodes. +# The main difference from Map is that Sequence can also contain simple values. + +sequence: + - val1 + - val2 + - key: 123 + - map: + mkey1: foo + mkey2: bar + +# We can represent it in XML this way: +# val1 +# val2 +# +# 123 +# +# +# +# foo +# bar +# +# + + + +# YAML does not have direct support for structures like XML attributes. +# We represent them as nodes with @ prefix in key. Note, that @ is reserved by YAML standard, +# so you will need to write double quotes around the key. Both Map and Sequence can have +# attributes as children nodes + +map: + "@attr1": value1 + "@attr2": value2 + key: 123 + +# This gives us: +# +# 123 +# + +sequence: + - "@attr1": value1 + - "@attr2": value2 + - 123 + - abc + +# And this gives us: +# 123 +# abc diff --git a/src/Common/Config/CMakeLists.txt b/src/Common/Config/CMakeLists.txt index a7914fb17ec..3da44be2af6 100644 --- a/src/Common/Config/CMakeLists.txt +++ b/src/Common/Config/CMakeLists.txt @@ -3,6 +3,7 @@ set (SRCS ConfigProcessor.cpp configReadClient.cpp ConfigReloader.cpp + YAMLParser.cpp ) add_library(clickhouse_common_config ${SRCS}) @@ -15,3 +16,10 @@ target_link_libraries(clickhouse_common_config PRIVATE string_utils ) + +if (USE_YAML_CPP) +target_link_libraries(clickhouse_common_config + PRIVATE + yaml-cpp +) +endif() diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index bc2a8a27943..fa9e9b72087 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -1,4 +1,8 @@ +#if !defined(ARCADIA_BUILD) + #include +#endif #include "ConfigProcessor.h" +#include "YAMLParser.h" #include #include @@ -20,10 +24,8 @@ #include #include - #define PREPROCESSED_SUFFIX "-preprocessed" - namespace fs = std::filesystem; using namespace Poco::XML; @@ -438,8 +440,10 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & std::string base_name = path.getBaseName(); // Skip non-config and temporary files - if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, ".")) - files.push_back(file.path()); + if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, ".")) + { + files.push_back(file.path()); + } } } @@ -453,12 +457,21 @@ XMLDocumentPtr ConfigProcessor::processConfig( zkutil::ZooKeeperNodeCache * zk_node_cache, const zkutil::EventPtr & zk_changed_event) { - XMLDocumentPtr config; LOG_DEBUG(log, "Processing configuration file '{}'.", path); + XMLDocumentPtr config; + if (fs::exists(path)) { - config = dom_parser.parse(path); + fs::path p(path); + if (p.extension() == ".xml") + { + config = dom_parser.parse(path); + } + else if (p.extension() == ".yaml" || p.extension() == ".yml") + { + config = YAMLParser::parse(path); + } } else { @@ -493,8 +506,20 @@ XMLDocumentPtr ConfigProcessor::processConfig( { LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file); - XMLDocumentPtr with = dom_parser.parse(merge_file); + XMLDocumentPtr with; + + fs::path p(merge_file); + if (p.extension() == ".yaml" || p.extension() == ".yml") + { + with = YAMLParser::parse(merge_file); + } + else + { + with = dom_parser.parse(merge_file); + } + merge(config, with); + contributing_files.push_back(merge_file); } catch (Exception & e) diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h index 7a4102140d9..5b16bc0cb1b 100644 --- a/src/Common/Config/ConfigProcessor.h +++ b/src/Common/Config/ConfigProcessor.h @@ -1,5 +1,9 @@ #pragma once +#if !defined(ARCADIA_BUILD) + #include +#endif + #include #include #include @@ -141,3 +145,4 @@ private: }; } + diff --git a/src/Common/Config/YAMLParser.cpp b/src/Common/Config/YAMLParser.cpp new file mode 100644 index 00000000000..9eaf1cdc1ad --- /dev/null +++ b/src/Common/Config/YAMLParser.cpp @@ -0,0 +1,166 @@ +#if !defined(ARCADIA_BUILD) + #include +#endif + +#if USE_YAML_CPP +#include "YAMLParser.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include // Y_IGNORE + +#include + +using namespace Poco::XML; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_PARSE_YAML; +} + +/// A prefix symbol in yaml key +/// We add attributes to nodes by using a prefix symbol in the key part. +/// Currently we use @ as a prefix symbol. Note, that @ is reserved +/// by YAML standard, so we need to write a key-value pair like this: "@attribute": attr_value +const char YAML_ATTRIBUTE_PREFIX = '@'; + +namespace +{ + +Poco::AutoPtr createCloneNode(Poco::XML::Element & original_node) +{ + Poco::AutoPtr clone_node = original_node.ownerDocument()->createElement(original_node.nodeName()); + original_node.parentNode()->appendChild(clone_node); + return clone_node; +} + +void processNode(const YAML::Node & node, Poco::XML::Element & parent_xml_element) +{ + auto * xml_document = parent_xml_element.ownerDocument(); + switch (node.Type()) + { + case YAML::NodeType::Scalar: + { + auto value = node.as(); + Poco::AutoPtr xml_value = xml_document->createTextNode(value); + parent_xml_element.appendChild(xml_value); + break; + } + + /// We process YAML Sequences as a + /// list of value tags with same key and different values. + /// For example, we translate this sequence + /// seq: + /// - val1 + /// - val2 + /// + /// into this: + /// val1 + /// val2 + case YAML::NodeType::Sequence: + { + for (const auto & child_node : node) + if (parent_xml_element.hasChildNodes()) + { + /// We want to process sequences like that: + /// seq: + /// - val1 + /// - k2: val2 + /// - val3 + /// - k4: val4 + /// - val5 + /// into xml like this: + /// val1 + /// + /// val2 + /// + /// val3 + /// + /// val4 + /// + /// val5 + /// So, we create a new parent node with same tag for each child node + processNode(child_node, *createCloneNode(parent_xml_element)); + } + else + { + processNode(child_node, parent_xml_element); + } + break; + } + case YAML::NodeType::Map: + { + for (const auto & key_value_pair : node) + { + const auto & key_node = key_value_pair.first; + const auto & value_node = key_value_pair.second; + auto key = key_node.as(); + bool is_attribute = (key.starts_with(YAML_ATTRIBUTE_PREFIX) && value_node.IsScalar()); + if (is_attribute) + { + /// we use substr(1) here to remove YAML_ATTRIBUTE_PREFIX from key + auto attribute_name = key.substr(1); + auto value = value_node.as(); + parent_xml_element.setAttribute(attribute_name, value); + } + else + { + Poco::AutoPtr xml_key = xml_document->createElement(key); + parent_xml_element.appendChild(xml_key); + processNode(value_node, *xml_key); + } + } + break; + } + case YAML::NodeType::Null: break; + case YAML::NodeType::Undefined: + { + throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "YAMLParser has encountered node with undefined type and cannot continue parsing of the file"); + } + } +} + +} + +Poco::AutoPtr YAMLParser::parse(const String& path) +{ + YAML::Node node_yml; + try + { + node_yml = YAML::LoadFile(path); + } + catch (const YAML::ParserException& e) + { + /// yaml-cpp cannot parse the file because its contents are incorrect + throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "Unable to parse YAML configuration file {}", path, e.what()); + } + catch (const YAML::BadFile&) + { + /// yaml-cpp cannot open the file even though it exists + throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Unable to open YAML configuration file {}", path); + } + Poco::AutoPtr xml = new Document; + Poco::AutoPtr root_node = xml->createElement("yandex"); + xml->appendChild(root_node); + processNode(node_yml, *root_node); + return xml; +} + +} +#endif diff --git a/src/Common/Config/YAMLParser.h b/src/Common/Config/YAMLParser.h new file mode 100644 index 00000000000..a716f3de9b1 --- /dev/null +++ b/src/Common/Config/YAMLParser.h @@ -0,0 +1,55 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) + #include +#endif + +#include + +#include +#include "Poco/DOM/AutoPtr.h" +#include + +#if USE_YAML_CPP + +namespace DB +{ + +/// Real YAML parser: loads yaml file into a YAML::Node +class YAMLParserImpl +{ +public: + static Poco::AutoPtr parse(const String& path); +}; + +using YAMLParser = YAMLParserImpl; + +} + +#else + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_PARSE_YAML; +} + +/// Fake YAML parser: throws an exception if we try to parse YAML configs in a build without yaml-cpp +class DummyYAMLParser +{ +public: + static Poco::AutoPtr parse(const String& path) + { + Poco::AutoPtr xml = new Poco::XML::Document; + throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "Unable to parse YAML configuration file {} without usage of yaml-cpp library", path); + return xml; + } +}; + +using YAMLParser = DummyYAMLParser; + +} + +#endif diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index b1efd791bf8..330a193212b 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -552,6 +552,7 @@ M(582, NO_SUCH_PROJECTION_IN_TABLE) \ M(583, ILLEGAL_PROJECTION) \ M(584, PROJECTION_NOT_USED) \ + M(585, CANNOT_PARSE_YAML) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 28a21ea7764..6844f0fa9e3 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -16,3 +16,4 @@ #cmakedefine01 USE_STATS #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_DATASKETCHES +#cmakedefine01 USE_YAML_CPP diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 5bd608ef758..6287064b616 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -126,7 +126,8 @@ class ClickHouseCluster: """ def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None, - odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None): + odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, + custom_dockerd_host=None): for param in list(os.environ.keys()): print("ENV %40s %s" % (param, os.environ[param])) self.base_dir = p.dirname(base_path) @@ -219,7 +220,9 @@ class ClickHouseCluster: with_redis=False, with_minio=False, with_cassandra=False, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None, stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None, - zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True): + zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True, + main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True): + """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -280,6 +283,9 @@ class ClickHouseCluster: ipv4_address=ipv4_address, ipv6_address=ipv6_address, with_installed_binary=with_installed_binary, + main_config_name=main_config_name, + users_config_name=users_config_name, + copy_common_configs=copy_common_configs, tmpfs=tmpfs or []) docker_compose_yml_dir = get_docker_compose_path() @@ -944,7 +950,7 @@ class ClickHouseCluster: subprocess_check_call(self.base_zookeeper_cmd + ["start", n]) -CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" +CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/{main_config_file} --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log" CLICKHOUSE_STAY_ALIVE_COMMAND = 'bash -c "{} --daemon; tail -f /dev/null"'.format(CLICKHOUSE_START_COMMAND) @@ -1000,6 +1006,8 @@ class ClickHouseInstance: macros, with_zookeeper, zookeeper_config_path, with_mysql, with_mysql_cluster, with_kafka, with_kerberized_kafka, with_rabbitmq, with_kerberized_hdfs, with_mongo, with_redis, with_minio, with_cassandra, server_bin_path, odbc_bridge_bin_path, library_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers, + clickhouse_start_command=CLICKHOUSE_START_COMMAND, + main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True, hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag="latest", stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None): @@ -1036,6 +1044,12 @@ class ClickHouseInstance: self.with_minio = with_minio self.with_cassandra = with_cassandra + self.main_config_name = main_config_name + self.users_config_name = users_config_name + self.copy_common_configs = copy_common_configs + + self.clickhouse_start_command = clickhouse_start_command.replace("{main_config_file}", self.main_config_name) + self.path = p.join(self.cluster.instances_dir, name) self.docker_compose_path = p.join(self.path, 'docker-compose.yml') self.env_variables = env_variables or {} @@ -1177,7 +1191,7 @@ class ClickHouseInstance: if not self.stay_alive: raise Exception("clickhouse can be started again only with stay_alive=True instance") - self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid())) + self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) # wait start from helpers.test_tools import assert_eq_with_retry assert_eq_with_retry(self, "select 1", "1", retry_count=int(start_wait_sec / 0.5), sleep_time=0.5) @@ -1263,7 +1277,7 @@ class ClickHouseInstance: self.exec_in_container(["bash", "-c", "cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"], user='root') - self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid())) + self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid())) from helpers.test_tools import assert_eq_with_retry # wait start assert_eq_with_retry(self, "select 1", "1", retry_count=retries) @@ -1404,8 +1418,10 @@ class ClickHouseInstance: os.makedirs(instance_config_dir) print("Copy common default production configuration from {}".format(self.base_config_dir)) - shutil.copyfile(p.join(self.base_config_dir, 'config.xml'), p.join(instance_config_dir, 'config.xml')) - shutil.copyfile(p.join(self.base_config_dir, 'users.xml'), p.join(instance_config_dir, 'users.xml')) + + shutil.copyfile(p.join(self.base_config_dir, self.main_config_name), p.join(instance_config_dir, self.main_config_name)) + + shutil.copyfile(p.join(self.base_config_dir, self.users_config_name), p.join(instance_config_dir, self.users_config_name)) print("Create directory for configuration generated in this helper") # used by all utils with any config @@ -1423,7 +1439,9 @@ class ClickHouseInstance: print("Copy common configuration from helpers") # The file is named with 0_ prefix to be processed before other configuration overloads. - shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir) + if self.copy_common_configs: + shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir) + shutil.copy(p.join(HELPERS_DIR, '0_common_instance_users.xml'), users_d_dir) if len(self.custom_dictionaries_paths): shutil.copy(p.join(HELPERS_DIR, '0_common_enable_dictionaries.xml'), self.config_d_dir) @@ -1502,11 +1520,11 @@ class ClickHouseInstance: self._create_odbc_config_file() odbc_ini_path = '- ' + self.odbc_ini_path - entrypoint_cmd = CLICKHOUSE_START_COMMAND + entrypoint_cmd = self.clickhouse_start_command if self.stay_alive: - entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND - + entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace("{main_config_file}", self.main_config_name) + print("Entrypoint cmd: {}".format(entrypoint_cmd)) networks = app_net = ipv4_address = ipv6_address = net_aliases = net_alias1 = "" diff --git a/tests/integration/runner b/tests/integration/runner index ee116c29aa5..16eb31dfd5e 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -70,11 +70,11 @@ def check_args_and_update_paths(args): if not os.path.exists(path): raise Exception("Path {} doesn't exist".format(path)) - if not os.path.exists(os.path.join(args.base_configs_dir, "config.xml")): - raise Exception("No configs.xml in {}".format(args.base_configs_dir)) + if (not os.path.exists(os.path.join(args.base_configs_dir, "config.xml"))) and (not os.path.exists(os.path.join(args.base_configs_dir, "config.yaml"))): + raise Exception("No configs.xml or configs.yaml in {}".format(args.base_configs_dir)) - if not os.path.exists(os.path.join(args.base_configs_dir, "users.xml")): - raise Exception("No users.xml in {}".format(args.base_configs_dir)) + if (not os.path.exists(os.path.join(args.base_configs_dir, "users.xml"))) and (not os.path.exists(os.path.join(args.base_configs_dir, "users.yaml"))): + raise Exception("No users.xml or users.yaml in {}".format(args.base_configs_dir)) def docker_kill_handler_handler(signum, frame): subprocess.check_call('docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format(name=CONTAINER_NAME), shell=True) diff --git a/tests/integration/test_config_xml_full/__init__.py b/tests/integration/test_config_xml_full/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_config_xml_full/configs/config.d/access_control.xml b/tests/integration/test_config_xml_full/configs/config.d/access_control.xml new file mode 100644 index 00000000000..6567c39f171 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/access_control.xml @@ -0,0 +1,13 @@ + + + + + + users.xml + + + + access/ + + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml b/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml new file mode 100644 index 00000000000..b21df47bc85 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml @@ -0,0 +1,23 @@ + + + 9181 + 1 + + + 10000 + 30000 + false + 60000 + + 1000000000000000 + + + + + 1 + localhost + 44444 + + + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml b/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml new file mode 100644 index 00000000000..227c53647f3 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml @@ -0,0 +1,7 @@ + + + true + + + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml new file mode 100644 index 00000000000..2c34585437b --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml @@ -0,0 +1,8 @@ + + + + never + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/macros.xml b/tests/integration/test_config_xml_full/configs/config.d/macros.xml new file mode 100644 index 00000000000..4902b12bc81 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/macros.xml @@ -0,0 +1,9 @@ + + + Hello, world! + s1 + r1 + /clickhouse/tables/{database}/{shard}/ + table_{table} + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml b/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml new file mode 100644 index 00000000000..0ca9f162416 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml @@ -0,0 +1,8 @@ + + + system + metric_log
+ 7500 + 1000 +
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml b/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml new file mode 100644 index 00000000000..aecbf9e0ba7 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml @@ -0,0 +1,49 @@ + + + + + + + + + localhost + 9001 + + + + + + + + localhost + 9001 + + + + + localhost + 9002 + + + + + + + + localhost + 9000 + + + + + localhost + 9001 + + + + + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/part_log.xml b/tests/integration/test_config_xml_full/configs/config.d/part_log.xml new file mode 100644 index 00000000000..6c6fc9c6982 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/part_log.xml @@ -0,0 +1,8 @@ + + + system + part_log
+ + 7500 +
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/path.xml b/tests/integration/test_config_xml_full/configs/config.d/path.xml new file mode 100644 index 00000000000..466ed0d1663 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/path.xml @@ -0,0 +1,8 @@ + + ./ + ./tmp/ + ./user_files/ + ./format_schemas/ + ./access/ + ./top_level_domains/ + diff --git a/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml new file mode 100644 index 00000000000..5a854848f3d --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml @@ -0,0 +1,10 @@ + + + + + + TOPSECRET.TOPSECRET + [hidden] + + + diff --git a/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml b/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml new file mode 100644 index 00000000000..19046054c16 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml @@ -0,0 +1,3 @@ + + 9010 + diff --git a/tests/integration/test_config_xml_full/configs/config.d/text_log.xml b/tests/integration/test_config_xml_full/configs/config.d/text_log.xml new file mode 100644 index 00000000000..3699a23578c --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/text_log.xml @@ -0,0 +1,7 @@ + + + system + text_log
+ 7500 +
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml b/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml new file mode 100644 index 00000000000..06ed7fcd39f --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml @@ -0,0 +1,8 @@ + + + + localhost + 9181 + + + diff --git a/tests/integration/test_config_xml_full/configs/config.xml b/tests/integration/test_config_xml_full/configs/config.xml new file mode 100644 index 00000000000..a7bb9d49e95 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/config.xml @@ -0,0 +1,1118 @@ + + + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + 8123 + + + 9000 + + + 9004 + + + 9005 + + + + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + deflate + + + medium + + + -1 + -1 + + + false + + + + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + 0.9 + + + 4194304 + + + 0 + + + + + + 8589934592 + + + 5368709120 + + + + 1000 + + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default + + + + + + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + + + + + + + + + + + + + + + + localhost + 9000 + + + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + + + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + + + + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + + + system + metric_log
+ 7500 + 1000 +
+ + + + system + asynchronous_metric_log
+ + 60000 +
+ + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 +
+ + + + + system + crash_log
+ + + 1000 +
+ + + + + + + + + + + + + + + + + + *_dictionary.xml + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + +
diff --git a/tests/integration/test_config_xml_full/configs/embedded.xml b/tests/integration/test_config_xml_full/configs/embedded.xml new file mode 100644 index 00000000000..a66f57d1eb7 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/embedded.xml @@ -0,0 +1,40 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + 8589934592 + 5368709120 + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml new file mode 100644 index 00000000000..b94e95bc043 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml @@ -0,0 +1,8 @@ + + + + + 1 + + + diff --git a/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml b/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml new file mode 100644 index 00000000000..25261072ade --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_config_xml_full/configs/users.xml b/tests/integration/test_config_xml_full/configs/users.xml new file mode 100644 index 00000000000..829748dafb1 --- /dev/null +++ b/tests/integration/test_config_xml_full/configs/users.xml @@ -0,0 +1,120 @@ + + + + + + + + 10000000000 + 64999 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/tests/integration/test_config_xml_full/test.py b/tests/integration/test_config_xml_full/test.py new file mode 100644 index 00000000000..a8650a0dc55 --- /dev/null +++ b/tests/integration/test_config_xml_full/test.py @@ -0,0 +1,40 @@ +import time +import threading +from os import path as p, unlink +from tempfile import NamedTemporaryFile + +import helpers +import pytest +from helpers.cluster import ClickHouseCluster + + +def test_xml_full_conf(): + # all configs are in XML + cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml') + + all_confd = ['configs/config.d/access_control.xml', + 'configs/config.d/keeper_port.xml', + 'configs/config.d/logging_no_rotate.xml', + 'configs/config.d/log_to_console.xml', + 'configs/config.d/macros.xml', + 'configs/config.d/metric_log.xml', + 'configs/config.d/more_clusters.xml', + 'configs/config.d/part_log.xml', + 'configs/config.d/path.xml', + 'configs/config.d/query_masking_rules.xml', + 'configs/config.d/tcp_with_proxy.xml', + 'configs/config.d/text_log.xml', + 'configs/config.d/zookeeper.xml'] + + all_userd = ['configs/users.d/allow_introspection_functions.xml', + 'configs/users.d/log_queries.xml'] + + node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False) + + try: + cluster.start() + assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") + assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + + finally: + cluster.shutdown() diff --git a/tests/integration/test_config_xml_main/__init__.py b/tests/integration/test_config_xml_main/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml b/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml new file mode 100644 index 00000000000..d8ead517c86 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml @@ -0,0 +1,7 @@ +user_directories: + users_xml: + path: users.xml + local_directory: + path: access/ + "@replace": replace + diff --git a/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml b/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml new file mode 100644 index 00000000000..91723bc372f --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml @@ -0,0 +1,15 @@ +keeper_server: + tcp_port: 9181 + server_id: 1 + coordination_settings: + operation_timeout_ms: 10000 + session_timeout_ms: 30000 + force_sync: false + startup_timeout: 60000 + reserved_log_items: 1000000000000000 + raft_configuration: + server: + id: 1 + hostname: localhost + port: 44444 + diff --git a/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml b/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml new file mode 100644 index 00000000000..7b59339800b --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml @@ -0,0 +1,7 @@ +logger: + console: true + log: + "@remove": remove + errorlog: + "@remove": remove + diff --git a/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml b/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml new file mode 100644 index 00000000000..513cd3f7dc5 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml @@ -0,0 +1,2 @@ +logger: + size: never diff --git a/tests/integration/test_config_xml_main/configs/config.d/macros.yaml b/tests/integration/test_config_xml_main/configs/config.d/macros.yaml new file mode 100644 index 00000000000..a9c61e270ab --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/macros.yaml @@ -0,0 +1,7 @@ +macros: + test: 'Hello, world!' + shard: s1 + replica: r1 + default_path_test: '/clickhouse/tables/{database}/{shard}/' + default_name_test: 'table_{table}' + diff --git a/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml new file mode 100644 index 00000000000..aafad939c37 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml @@ -0,0 +1,6 @@ +metric_log: + database: system + table: metric_log + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 + diff --git a/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml b/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml new file mode 100644 index 00000000000..7da07190894 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml @@ -0,0 +1,23 @@ +remote_servers: + single_remote_shard_at_port_9001: + shard: + replica: + host: localhost + port: 9001 + two_remote_shards_at_port_9001_9002: + shard: + - replica: + host: localhost + port: 9001 + - replica: + host: localhost + port: 9002 + two_shards_one_local_one_remote_at_port_9001: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 9001 + diff --git a/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml new file mode 100644 index 00000000000..43f60146dca --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml @@ -0,0 +1,5 @@ +part_log: + database: system + table: part_log + flush_interval_milliseconds: 7500 + diff --git a/tests/integration/test_config_xml_main/configs/config.d/path.yaml b/tests/integration/test_config_xml_main/configs/config.d/path.yaml new file mode 100644 index 00000000000..3e26e8906ee --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/path.yaml @@ -0,0 +1,18 @@ +path: + - ./ + - "@replace": replace +tmp_path: + - ./tmp/ + - "@replace": replace +user_files_path: + - ./user_files/ + - "@replace": replace +format_schema_path: + - ./format_schemas/ + - "@replace": replace +access_control_path: + - ./access/ + - "@replace": replace +top_level_domains_path: + - ./top_level_domains/ + - "@replace": replace diff --git a/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml b/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml new file mode 100644 index 00000000000..38163429d81 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml @@ -0,0 +1,4 @@ +query_masking_rules: + rule: + regexp: TOPSECRET.TOPSECRET + replace: '[hidden]' diff --git a/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml new file mode 100644 index 00000000000..b0349f5a9b9 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml @@ -0,0 +1 @@ +tcp_with_proxy_port: 9010 diff --git a/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml b/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml new file mode 100644 index 00000000000..309a7daa81d --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml @@ -0,0 +1,11 @@ +remote_servers: + test_cluster_with_incorrect_pw: + shard: + internal_replication: true + replica: + - host: 127.0.0.1 + port: 9000 + password: foo + - host: 127.0.0.2 + port: 9000 + password: foo diff --git a/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml new file mode 100644 index 00000000000..4d188020e37 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml @@ -0,0 +1,4 @@ +text_log: + database: system + table: text_log + flush_interval_milliseconds: 7500 diff --git a/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml b/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml new file mode 100644 index 00000000000..be02c516798 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml @@ -0,0 +1,5 @@ +zookeeper: + node: + host: localhost + port: 9181 + "@index": 1 diff --git a/tests/integration/test_config_xml_main/configs/config.xml b/tests/integration/test_config_xml_main/configs/config.xml new file mode 100644 index 00000000000..3b5dab50ffe --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/config.xml @@ -0,0 +1,277 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + 8123 + 9000 + 9004 + 9005 + 9009 + 4096 + 3 + + false + /path/to/ssl_cert_file + /path/to/ssl_key_file + false + /path/to/ssl_ca_cert_file + deflate + medium + -1 + -1 + false + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + RejectCertificateHandler + + + + 100 + 0 + 10000 + + 0.9 + + 4194304 + + 0 + + 8589934592 + + 5368709120 + + 1000 + + /var/lib/clickhouse/ + + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + + + + + users.xml + + + /var/lib/clickhouse/access/ + + + default + + default + true + + false + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + 3600 + + 3600 + + 60 + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + system + metric_log
+ 7500 + 1000 +
+ + + system + asynchronous_metric_log
+ + 60000 +
+ + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 +
+ + + system + crash_log
+ + + 1000 +
+ + + + + *_dictionary.xml + + /clickhouse/task_queue/ddl + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + /var/lib/clickhouse/format_schemas/ + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + \1(???) + + + + false + false + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + +
diff --git a/tests/integration/test_config_xml_main/configs/embedded.xml b/tests/integration/test_config_xml_main/configs/embedded.xml new file mode 100644 index 00000000000..a66f57d1eb7 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/embedded.xml @@ -0,0 +1,40 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + 8589934592 + 5368709120 + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml b/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml new file mode 100644 index 00000000000..84612c198c9 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml @@ -0,0 +1,3 @@ +profiles: + default: + allow_introspection_functions: 1 diff --git a/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml b/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml new file mode 100644 index 00000000000..88574e8f764 --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml @@ -0,0 +1,3 @@ +profiles: + default: + log_queries: 1 diff --git a/tests/integration/test_config_xml_main/configs/users.xml b/tests/integration/test_config_xml_main/configs/users.xml new file mode 100644 index 00000000000..b473413bdfa --- /dev/null +++ b/tests/integration/test_config_xml_main/configs/users.xml @@ -0,0 +1,19 @@ + + + + + 10000000000 + 64999 + + + + + + + + ::/0 + + default + + + diff --git a/tests/integration/test_config_xml_main/test.py b/tests/integration/test_config_xml_main/test.py new file mode 100644 index 00000000000..052f9adb01f --- /dev/null +++ b/tests/integration/test_config_xml_main/test.py @@ -0,0 +1,43 @@ + + +import time +import threading +from os import path as p, unlink +from tempfile import NamedTemporaryFile + +import helpers +import pytest +from helpers.cluster import ClickHouseCluster + + +def test_xml_main_conf(): + # main configs are in XML; config.d and users.d are in YAML + cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml') + + all_confd = ['configs/config.d/access_control.yaml', + 'configs/config.d/keeper_port.yaml', + 'configs/config.d/logging_no_rotate.yaml', + 'configs/config.d/log_to_console.yaml', + 'configs/config.d/macros.yaml', + 'configs/config.d/metric_log.yaml', + 'configs/config.d/more_clusters.yaml', + 'configs/config.d/part_log.yaml', + 'configs/config.d/path.yaml', + 'configs/config.d/query_masking_rules.yaml', + 'configs/config.d/tcp_with_proxy.yaml', + 'configs/config.d/test_cluster_with_incorrect_pw.yaml', + 'configs/config.d/text_log.yaml', + 'configs/config.d/zookeeper.yaml'] + + all_userd = ['configs/users.d/allow_introspection_functions.yaml', + 'configs/users.d/log_queries.yaml'] + + node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False) + + try: + cluster.start() + assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") + assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + + finally: + cluster.shutdown() diff --git a/tests/integration/test_config_xml_yaml_mix/__init__.py b/tests/integration/test_config_xml_yaml_mix/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml new file mode 100644 index 00000000000..62e4ba8c744 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml @@ -0,0 +1,6 @@ +timezone: Europe/Moscow +listen_host: 0.0.0.0 +custom_settings_prefixes: custom_ +path: /var/lib/clickhouse/ +tmp_path: /var/lib/clickhouse/tmp/ +users_config: users.yaml diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml new file mode 100644 index 00000000000..ce2e23839ef --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml @@ -0,0 +1,7 @@ +user_directories: + users_xml: + path: users.yaml + local_directory: + path: access/ + "@replace": replace + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml new file mode 100644 index 00000000000..b21df47bc85 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml @@ -0,0 +1,23 @@ + + + 9181 + 1 + + + 10000 + 30000 + false + 60000 + + 1000000000000000 + + + + + 1 + localhost + 44444 + + + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml new file mode 100644 index 00000000000..7b59339800b --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml @@ -0,0 +1,7 @@ +logger: + console: true + log: + "@remove": remove + errorlog: + "@remove": remove + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml new file mode 100644 index 00000000000..2c34585437b --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml @@ -0,0 +1,8 @@ + + + + never + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml new file mode 100644 index 00000000000..a9c61e270ab --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml @@ -0,0 +1,7 @@ +macros: + test: 'Hello, world!' + shard: s1 + replica: r1 + default_path_test: '/clickhouse/tables/{database}/{shard}/' + default_name_test: 'table_{table}' + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml new file mode 100644 index 00000000000..0ca9f162416 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml @@ -0,0 +1,8 @@ + + + system + metric_log
+ 7500 + 1000 +
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml new file mode 100644 index 00000000000..7da07190894 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml @@ -0,0 +1,23 @@ +remote_servers: + single_remote_shard_at_port_9001: + shard: + replica: + host: localhost + port: 9001 + two_remote_shards_at_port_9001_9002: + shard: + - replica: + host: localhost + port: 9001 + - replica: + host: localhost + port: 9002 + two_shards_one_local_one_remote_at_port_9001: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 9001 + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml new file mode 100644 index 00000000000..6c6fc9c6982 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml @@ -0,0 +1,8 @@ + + + system + part_log
+ + 7500 +
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml new file mode 100644 index 00000000000..3e26e8906ee --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml @@ -0,0 +1,18 @@ +path: + - ./ + - "@replace": replace +tmp_path: + - ./tmp/ + - "@replace": replace +user_files_path: + - ./user_files/ + - "@replace": replace +format_schema_path: + - ./format_schemas/ + - "@replace": replace +access_control_path: + - ./access/ + - "@replace": replace +top_level_domains_path: + - ./top_level_domains/ + - "@replace": replace diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml new file mode 100644 index 00000000000..5a854848f3d --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml @@ -0,0 +1,10 @@ + + + + + + TOPSECRET.TOPSECRET + [hidden] + + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml new file mode 100644 index 00000000000..b0349f5a9b9 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml @@ -0,0 +1 @@ +tcp_with_proxy_port: 9010 diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml new file mode 100644 index 00000000000..109e35afc37 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml @@ -0,0 +1,21 @@ + + + + + true + + 127.0.0.1 + 9000 + + foo + + + 127.0.0.2 + 9000 + + foo + + + + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml new file mode 100644 index 00000000000..4d188020e37 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml @@ -0,0 +1,4 @@ +text_log: + database: system + table: text_log + flush_interval_milliseconds: 7500 diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml new file mode 100644 index 00000000000..06ed7fcd39f --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml @@ -0,0 +1,8 @@ + + + + localhost + 9181 + + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.xml new file mode 100644 index 00000000000..e6a2b6d5324 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/config.xml @@ -0,0 +1,277 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + 8123 + 9000 + 9004 + 9005 + 9009 + 4096 + 3 + + false + /path/to/ssl_cert_file + /path/to/ssl_key_file + false + /path/to/ssl_ca_cert_file + deflate + medium + -1 + -1 + false + + + + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key + /etc/clickhouse-server/dhparam.pem + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + RejectCertificateHandler + + + + 100 + 0 + 10000 + + 0.9 + + 4194304 + + 0 + + 8589934592 + + 5368709120 + + 1000 + + /var/lib/clickhouse/ + + /var/lib/clickhouse/tmp/ + /var/lib/clickhouse/user_files/ + + + + + users.yaml + + + /var/lib/clickhouse/access/ + + + default + + default + true + + false + + + + + localhost + 9000 + + + + + + + localhost + 9000 + + + + + localhost + 9000 + + + + + + + 127.0.0.1 + 9000 + + + + + 127.0.0.2 + 9000 + + + + + + true + + 127.0.0.1 + 9000 + + + + true + + 127.0.0.2 + 9000 + + + + + + + localhost + 9440 + 1 + + + + + + + localhost + 9000 + + + + + localhost + 1 + + + + + + 3600 + + 3600 + + 60 + + + system + query_log
+ + toYYYYMM(event_date) + + 7500 +
+ + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + system + metric_log
+ 7500 + 1000 +
+ + + system + asynchronous_metric_log
+ + 60000 +
+ + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 +
+ + + system + crash_log
+ + + 1000 +
+ + + + + *_dictionary.xml + + /clickhouse/task_queue/ddl + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + /var/lib/clickhouse/format_schemas/ + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + \1(???) + + + + false + false + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + +
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml b/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml new file mode 100644 index 00000000000..a66f57d1eb7 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml @@ -0,0 +1,40 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + 8589934592 + 5368709120 + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml new file mode 100644 index 00000000000..b94e95bc043 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml @@ -0,0 +1,8 @@ + + + + + 1 + + + diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml b/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml new file mode 100644 index 00000000000..88574e8f764 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml @@ -0,0 +1,3 @@ +profiles: + default: + log_queries: 1 diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.yaml b/tests/integration/test_config_xml_yaml_mix/configs/users.yaml new file mode 100644 index 00000000000..a87a8c82819 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/configs/users.yaml @@ -0,0 +1,12 @@ +profiles: + default: + max_memory_usage: 10000000000 + max_block_size: 64999 +users: + default: + password: '' + networks: + "@replace": replace + ip: '::/0' + profile: default + diff --git a/tests/integration/test_config_xml_yaml_mix/test.py b/tests/integration/test_config_xml_yaml_mix/test.py new file mode 100644 index 00000000000..90ee8a2dea5 --- /dev/null +++ b/tests/integration/test_config_xml_yaml_mix/test.py @@ -0,0 +1,43 @@ +import time +import threading +from os import path as p, unlink +from tempfile import NamedTemporaryFile + +import helpers +import pytest +from helpers.cluster import ClickHouseCluster + + +def test_extra_yaml_mix(): + # some configs are written in XML, others are written in YAML + cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml') + + all_confd = ['configs/config.d/0_common_instance_config.yaml', + 'configs/config.d/access_control.yaml', + 'configs/config.d/keeper_port.xml', + 'configs/config.d/logging_no_rotate.xml', + 'configs/config.d/log_to_console.yaml', + 'configs/config.d/macros.yaml', + 'configs/config.d/metric_log.xml', + 'configs/config.d/more_clusters.yaml', + 'configs/config.d/part_log.xml', + 'configs/config.d/path.yaml', + 'configs/config.d/query_masking_rules.xml', + 'configs/config.d/tcp_with_proxy.yaml', + 'configs/config.d/test_cluster_with_incorrect_pw.xml', + 'configs/config.d/text_log.yaml', + 'configs/config.d/zookeeper.xml'] + + all_userd = ['configs/users.d/allow_introspection_functions.xml', + 'configs/users.d/log_queries.yaml'] + + node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False, + users_config_name="users.yaml", copy_common_configs=False) + + try: + cluster.start() + assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") + assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + + finally: + cluster.shutdown() diff --git a/tests/integration/test_config_yaml_full/__init__.py b/tests/integration/test_config_yaml_full/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml new file mode 100644 index 00000000000..62e4ba8c744 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml @@ -0,0 +1,6 @@ +timezone: Europe/Moscow +listen_host: 0.0.0.0 +custom_settings_prefixes: custom_ +path: /var/lib/clickhouse/ +tmp_path: /var/lib/clickhouse/tmp/ +users_config: users.yaml diff --git a/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml b/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml new file mode 100644 index 00000000000..ce2e23839ef --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml @@ -0,0 +1,7 @@ +user_directories: + users_xml: + path: users.yaml + local_directory: + path: access/ + "@replace": replace + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml b/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml new file mode 100644 index 00000000000..91723bc372f --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml @@ -0,0 +1,15 @@ +keeper_server: + tcp_port: 9181 + server_id: 1 + coordination_settings: + operation_timeout_ms: 10000 + session_timeout_ms: 30000 + force_sync: false + startup_timeout: 60000 + reserved_log_items: 1000000000000000 + raft_configuration: + server: + id: 1 + hostname: localhost + port: 44444 + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml b/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml new file mode 100644 index 00000000000..7b59339800b --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml @@ -0,0 +1,7 @@ +logger: + console: true + log: + "@remove": remove + errorlog: + "@remove": remove + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml b/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml new file mode 100644 index 00000000000..513cd3f7dc5 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml @@ -0,0 +1,2 @@ +logger: + size: never diff --git a/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml b/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml new file mode 100644 index 00000000000..a9c61e270ab --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml @@ -0,0 +1,7 @@ +macros: + test: 'Hello, world!' + shard: s1 + replica: r1 + default_path_test: '/clickhouse/tables/{database}/{shard}/' + default_name_test: 'table_{table}' + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml new file mode 100644 index 00000000000..aafad939c37 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml @@ -0,0 +1,6 @@ +metric_log: + database: system + table: metric_log + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml b/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml new file mode 100644 index 00000000000..7da07190894 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml @@ -0,0 +1,23 @@ +remote_servers: + single_remote_shard_at_port_9001: + shard: + replica: + host: localhost + port: 9001 + two_remote_shards_at_port_9001_9002: + shard: + - replica: + host: localhost + port: 9001 + - replica: + host: localhost + port: 9002 + two_shards_one_local_one_remote_at_port_9001: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 9001 + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml new file mode 100644 index 00000000000..43f60146dca --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml @@ -0,0 +1,5 @@ +part_log: + database: system + table: part_log + flush_interval_milliseconds: 7500 + diff --git a/tests/integration/test_config_yaml_full/configs/config.d/path.yaml b/tests/integration/test_config_yaml_full/configs/config.d/path.yaml new file mode 100644 index 00000000000..3e26e8906ee --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/path.yaml @@ -0,0 +1,18 @@ +path: + - ./ + - "@replace": replace +tmp_path: + - ./tmp/ + - "@replace": replace +user_files_path: + - ./user_files/ + - "@replace": replace +format_schema_path: + - ./format_schemas/ + - "@replace": replace +access_control_path: + - ./access/ + - "@replace": replace +top_level_domains_path: + - ./top_level_domains/ + - "@replace": replace diff --git a/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml b/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml new file mode 100644 index 00000000000..38163429d81 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml @@ -0,0 +1,4 @@ +query_masking_rules: + rule: + regexp: TOPSECRET.TOPSECRET + replace: '[hidden]' diff --git a/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml new file mode 100644 index 00000000000..b0349f5a9b9 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml @@ -0,0 +1 @@ +tcp_with_proxy_port: 9010 diff --git a/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml b/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml new file mode 100644 index 00000000000..309a7daa81d --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml @@ -0,0 +1,11 @@ +remote_servers: + test_cluster_with_incorrect_pw: + shard: + internal_replication: true + replica: + - host: 127.0.0.1 + port: 9000 + password: foo + - host: 127.0.0.2 + port: 9000 + password: foo diff --git a/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml new file mode 100644 index 00000000000..4d188020e37 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml @@ -0,0 +1,4 @@ +text_log: + database: system + table: text_log + flush_interval_milliseconds: 7500 diff --git a/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml b/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml new file mode 100644 index 00000000000..be02c516798 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml @@ -0,0 +1,5 @@ +zookeeper: + node: + host: localhost + port: 9181 + "@index": 1 diff --git a/tests/integration/test_config_yaml_full/configs/config.yaml b/tests/integration/test_config_yaml_full/configs/config.yaml new file mode 100644 index 00000000000..619a3735269 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/config.yaml @@ -0,0 +1,183 @@ +logger: + level: trace + log: /var/log/clickhouse-server/clickhouse-server.log + errorlog: /var/log/clickhouse-server/clickhouse-server.err.log + size: 1000M + count: 10 +http_port: 8123 +tcp_port: 9000 +mysql_port: 9004 +postgresql_port: 9005 +interserver_http_port: 9009 +max_connections: 4096 +keep_alive_timeout: 3 +grpc: + enable_ssl: false + ssl_cert_file: /path/to/ssl_cert_file + ssl_key_file: /path/to/ssl_key_file + ssl_require_client_auth: false + ssl_ca_cert_file: /path/to/ssl_ca_cert_file + compression: deflate + compression_level: medium + max_send_message_size: -1 + max_receive_message_size: -1 + verbose_logs: false +openSSL: + server: + certificateFile: /etc/clickhouse-server/server.crt + privateKeyFile: /etc/clickhouse-server/server.key + dhParamsFile: /etc/clickhouse-server/dhparam.pem + verificationMode: none + loadDefaultCAFile: true + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + client: + loadDefaultCAFile: true + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + invalidCertificateHandler: + name: RejectCertificateHandler +max_concurrent_queries: 100 +max_server_memory_usage: 0 +max_thread_pool_size: 10000 +max_server_memory_usage_to_ram_ratio: 0.9 +total_memory_profiler_step: 4194304 +total_memory_tracker_sample_probability: 0 +uncompressed_cache_size: 8589934592 +mark_cache_size: 5368709120 +mmap_cache_size: 1000 +path: /var/lib/clickhouse/ +tmp_path: /var/lib/clickhouse/tmp/ +user_files_path: /var/lib/clickhouse/user_files/ +ldap_servers: '' +user_directories: + users_xml: + path: users.yaml + local_directory: + path: /var/lib/clickhouse/access/ +default_profile: default +custom_settings_prefixes: '' +default_database: default +mlock_executable: true +remap_executable: false +remote_servers: + test_shard_localhost: + shard: + replica: + host: localhost + port: 9000 + test_cluster_two_shards_localhost: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 9000 + test_cluster_two_shards: + shard: + - replica: + host: 127.0.0.1 + port: 9000 + - replica: + host: 127.0.0.2 + port: 9000 + test_cluster_two_shards_internal_replication: + shard: + - internal_replication: true + replica: + host: 127.0.0.1 + port: 9000 + - internal_replication: true + replica: + host: 127.0.0.2 + port: 9000 + test_shard_localhost_secure: + shard: + replica: + host: localhost + port: 9440 + secure: 1 + test_unavailable_shard: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 1 +builtin_dictionaries_reload_interval: 3600 +max_session_timeout: 3600 +default_session_timeout: 60 +query_log: + database: system + table: query_log + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 +trace_log: + database: system + table: trace_log + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 +query_thread_log: + database: system + table: query_thread_log + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 +metric_log: + database: system + table: metric_log + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 +asynchronous_metric_log: + database: system + table: asynchronous_metric_log + flush_interval_milliseconds: 60000 +opentelemetry_span_log: + engine: |- + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + database: system + table: opentelemetry_span_log + flush_interval_milliseconds: 7500 +crash_log: + database: system + table: crash_log + partition_by: '' + flush_interval_milliseconds: 1000 +top_level_domains_lists: '' +dictionaries_config: '*_dictionary.xml' +distributed_ddl: + path: /clickhouse/task_queue/ddl +graphite_rollup_example: + pattern: + regexp: click_cost + function: any + retention: + - age: 0 + precision: 3600 + - age: 86400 + precision: 60 + default: + function: max + retention: + - age: 0 + precision: 60 + - age: 3600 + precision: 300 + - age: 86400 + precision: 3600 +format_schema_path: /var/lib/clickhouse/format_schemas/ +query_masking_rules: + rule: + name: hide encrypt/decrypt arguments + regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)' + replace: \1(???) +send_crash_reports: + enabled: false + anonymize: false + endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277' + diff --git a/tests/integration/test_config_yaml_full/configs/embedded.xml b/tests/integration/test_config_yaml_full/configs/embedded.xml new file mode 100644 index 00000000000..a66f57d1eb7 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/embedded.xml @@ -0,0 +1,40 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + 8589934592 + 5368709120 + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml b/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml new file mode 100644 index 00000000000..84612c198c9 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml @@ -0,0 +1,3 @@ +profiles: + default: + allow_introspection_functions: 1 diff --git a/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml b/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml new file mode 100644 index 00000000000..88574e8f764 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml @@ -0,0 +1,3 @@ +profiles: + default: + log_queries: 1 diff --git a/tests/integration/test_config_yaml_full/configs/users.yaml b/tests/integration/test_config_yaml_full/configs/users.yaml new file mode 100644 index 00000000000..a87a8c82819 --- /dev/null +++ b/tests/integration/test_config_yaml_full/configs/users.yaml @@ -0,0 +1,12 @@ +profiles: + default: + max_memory_usage: 10000000000 + max_block_size: 64999 +users: + default: + password: '' + networks: + "@replace": replace + ip: '::/0' + profile: default + diff --git a/tests/integration/test_config_yaml_full/test.py b/tests/integration/test_config_yaml_full/test.py new file mode 100644 index 00000000000..bc4fa40384c --- /dev/null +++ b/tests/integration/test_config_yaml_full/test.py @@ -0,0 +1,42 @@ +import time +import threading +from os import path as p, unlink +from tempfile import NamedTemporaryFile + +import helpers +import pytest +from helpers.cluster import ClickHouseCluster + +def test_yaml_full_conf(): + # all configs are in YAML + cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml') + + all_confd = ['configs/config.d/0_common_instance_config.yaml', + 'configs/config.d/access_control.yaml', + 'configs/config.d/keeper_port.yaml', + 'configs/config.d/logging_no_rotate.yaml', + 'configs/config.d/log_to_console.yaml', + 'configs/config.d/macros.yaml', + 'configs/config.d/metric_log.yaml', + 'configs/config.d/more_clusters.yaml', + 'configs/config.d/part_log.yaml', + 'configs/config.d/path.yaml', + 'configs/config.d/query_masking_rules.yaml', + 'configs/config.d/tcp_with_proxy.yaml', + 'configs/config.d/test_cluster_with_incorrect_pw.yaml', + 'configs/config.d/text_log.yaml', + 'configs/config.d/zookeeper.yaml'] + + all_userd = ['configs/users.d/allow_introspection_functions.yaml', + 'configs/users.d/log_queries.yaml'] + + node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, + with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False) + + try: + cluster.start() + assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") + assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + + finally: + cluster.shutdown() diff --git a/tests/integration/test_config_yaml_main/__init__.py b/tests/integration/test_config_yaml_main/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml new file mode 100644 index 00000000000..62e4ba8c744 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml @@ -0,0 +1,6 @@ +timezone: Europe/Moscow +listen_host: 0.0.0.0 +custom_settings_prefixes: custom_ +path: /var/lib/clickhouse/ +tmp_path: /var/lib/clickhouse/tmp/ +users_config: users.yaml diff --git a/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml b/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml new file mode 100644 index 00000000000..b61f89bd904 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml @@ -0,0 +1,13 @@ + + + + + + users.yaml + + + + access/ + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml b/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml new file mode 100644 index 00000000000..b21df47bc85 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml @@ -0,0 +1,23 @@ + + + 9181 + 1 + + + 10000 + 30000 + false + 60000 + + 1000000000000000 + + + + + 1 + localhost + 44444 + + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml b/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml new file mode 100644 index 00000000000..227c53647f3 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml @@ -0,0 +1,7 @@ + + + true + + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml new file mode 100644 index 00000000000..2c34585437b --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml @@ -0,0 +1,8 @@ + + + + never + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/macros.xml b/tests/integration/test_config_yaml_main/configs/config.d/macros.xml new file mode 100644 index 00000000000..4902b12bc81 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/macros.xml @@ -0,0 +1,9 @@ + + + Hello, world! + s1 + r1 + /clickhouse/tables/{database}/{shard}/ + table_{table} + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml new file mode 100644 index 00000000000..0ca9f162416 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml @@ -0,0 +1,8 @@ + + + system + metric_log
+ 7500 + 1000 +
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml b/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml new file mode 100644 index 00000000000..aecbf9e0ba7 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml @@ -0,0 +1,49 @@ + + + + + + + + + localhost + 9001 + + + + + + + + localhost + 9001 + + + + + localhost + 9002 + + + + + + + + localhost + 9000 + + + + + localhost + 9001 + + + + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml new file mode 100644 index 00000000000..6c6fc9c6982 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml @@ -0,0 +1,8 @@ + + + system + part_log
+ + 7500 +
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/path.xml b/tests/integration/test_config_yaml_main/configs/config.d/path.xml new file mode 100644 index 00000000000..466ed0d1663 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/path.xml @@ -0,0 +1,8 @@ + + ./ + ./tmp/ + ./user_files/ + ./format_schemas/ + ./access/ + ./top_level_domains/ + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml new file mode 100644 index 00000000000..5a854848f3d --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml @@ -0,0 +1,10 @@ + + + + + + TOPSECRET.TOPSECRET + [hidden] + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml b/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml new file mode 100644 index 00000000000..19046054c16 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml @@ -0,0 +1,3 @@ + + 9010 + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml b/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml new file mode 100644 index 00000000000..109e35afc37 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml @@ -0,0 +1,21 @@ + + + + + true + + 127.0.0.1 + 9000 + + foo + + + 127.0.0.2 + 9000 + + foo + + + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml new file mode 100644 index 00000000000..3699a23578c --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml @@ -0,0 +1,7 @@ + + + system + text_log
+ 7500 +
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml b/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml new file mode 100644 index 00000000000..06ed7fcd39f --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml @@ -0,0 +1,8 @@ + + + + localhost + 9181 + + + diff --git a/tests/integration/test_config_yaml_main/configs/config.yaml b/tests/integration/test_config_yaml_main/configs/config.yaml new file mode 100644 index 00000000000..e5a36b1e49b --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/config.yaml @@ -0,0 +1,183 @@ +logger: + level: trace + log: /var/log/clickhouse-server/clickhouse-server.log + errorlog: /var/log/clickhouse-server/clickhouse-server.err.log + size: 1000M + count: 10 +http_port: 8123 +tcp_port: 9000 +mysql_port: 9004 +postgresql_port: 9005 +interserver_http_port: 9009 +max_connections: 4096 +keep_alive_timeout: 3 +grpc: + enable_ssl: false + ssl_cert_file: /path/to/ssl_cert_file + ssl_key_file: /path/to/ssl_key_file + ssl_require_client_auth: false + ssl_ca_cert_file: /path/to/ssl_ca_cert_file + compression: deflate + compression_level: medium + max_send_message_size: -1 + max_receive_message_size: -1 + verbose_logs: false +openSSL: + server: + certificateFile: /etc/clickhouse-server/server.crt + privateKeyFile: /etc/clickhouse-server/server.key + dhParamsFile: /etc/clickhouse-server/dhparam.pem + verificationMode: none + loadDefaultCAFile: true + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + client: + loadDefaultCAFile: true + cacheSessions: true + disableProtocols: 'sslv2,sslv3' + preferServerCiphers: true + invalidCertificateHandler: + name: RejectCertificateHandler +max_concurrent_queries: 100 +max_server_memory_usage: 0 +max_thread_pool_size: 10000 +max_server_memory_usage_to_ram_ratio: 0.9 +total_memory_profiler_step: 4194304 +total_memory_tracker_sample_probability: 0 +uncompressed_cache_size: 8589934592 +mark_cache_size: 5368709120 +mmap_cache_size: 1000 +path: /var/lib/clickhouse/ +tmp_path: /var/lib/clickhouse/tmp/ +user_files_path: /var/lib/clickhouse/user_files/ +ldap_servers: '' +user_directories: + users_xml: + path: users.yml + local_directory: + path: /var/lib/clickhouse/access/ +default_profile: default +custom_settings_prefixes: '' +default_database: default +mlock_executable: true +remap_executable: false +remote_servers: + test_shard_localhost: + shard: + replica: + host: localhost + port: 9000 + test_cluster_two_shards_localhost: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 9000 + test_cluster_two_shards: + shard: + - replica: + host: 127.0.0.1 + port: 9000 + - replica: + host: 127.0.0.2 + port: 9000 + test_cluster_two_shards_internal_replication: + shard: + - internal_replication: true + replica: + host: 127.0.0.1 + port: 9000 + - internal_replication: true + replica: + host: 127.0.0.2 + port: 9000 + test_shard_localhost_secure: + shard: + replica: + host: localhost + port: 9440 + secure: 1 + test_unavailable_shard: + shard: + - replica: + host: localhost + port: 9000 + - replica: + host: localhost + port: 1 +builtin_dictionaries_reload_interval: 3600 +max_session_timeout: 3600 +default_session_timeout: 60 +query_log: + database: system + table: query_log + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 +trace_log: + database: system + table: trace_log + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 +query_thread_log: + database: system + table: query_thread_log + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 +metric_log: + database: system + table: metric_log + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 +asynchronous_metric_log: + database: system + table: asynchronous_metric_log + flush_interval_milliseconds: 60000 +opentelemetry_span_log: + engine: |- + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + database: system + table: opentelemetry_span_log + flush_interval_milliseconds: 7500 +crash_log: + database: system + table: crash_log + partition_by: '' + flush_interval_milliseconds: 1000 +top_level_domains_lists: '' +dictionaries_config: '*_dictionary.xml' +distributed_ddl: + path: /clickhouse/task_queue/ddl +graphite_rollup_example: + pattern: + regexp: click_cost + function: any + retention: + - age: 0 + precision: 3600 + - age: 86400 + precision: 60 + default: + function: max + retention: + - age: 0 + precision: 60 + - age: 3600 + precision: 300 + - age: 86400 + precision: 3600 +format_schema_path: /var/lib/clickhouse/format_schemas/ +query_masking_rules: + rule: + name: hide encrypt/decrypt arguments + regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)' + replace: \1(???) +send_crash_reports: + enabled: false + anonymize: false + endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277' + diff --git a/tests/integration/test_config_yaml_main/configs/embedded.xml b/tests/integration/test_config_yaml_main/configs/embedded.xml new file mode 100644 index 00000000000..a66f57d1eb7 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/embedded.xml @@ -0,0 +1,40 @@ + + + + + trace + true + + + 8123 + 9000 + 9004 + + ./ + + 8589934592 + 5368709120 + true + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml new file mode 100644 index 00000000000..b94e95bc043 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml @@ -0,0 +1,8 @@ + + + + + 1 + + + diff --git a/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml b/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml new file mode 100644 index 00000000000..25261072ade --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_config_yaml_main/configs/users.yaml b/tests/integration/test_config_yaml_main/configs/users.yaml new file mode 100644 index 00000000000..a87a8c82819 --- /dev/null +++ b/tests/integration/test_config_yaml_main/configs/users.yaml @@ -0,0 +1,12 @@ +profiles: + default: + max_memory_usage: 10000000000 + max_block_size: 64999 +users: + default: + password: '' + networks: + "@replace": replace + ip: '::/0' + profile: default + diff --git a/tests/integration/test_config_yaml_main/test.py b/tests/integration/test_config_yaml_main/test.py new file mode 100644 index 00000000000..f4de16c35a2 --- /dev/null +++ b/tests/integration/test_config_yaml_main/test.py @@ -0,0 +1,43 @@ +import time +import threading +from os import path as p, unlink +from tempfile import NamedTemporaryFile + +import helpers +import pytest +from helpers.cluster import ClickHouseCluster + + +def test_yaml_main_conf(): + # main configs are in YAML; config.d and users.d are in XML + cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml') + + all_confd = ['configs/config.d/0_common_instance_config.yaml', + 'configs/config.d/access_control.xml', + 'configs/config.d/keeper_port.xml', + 'configs/config.d/logging_no_rotate.xml', + 'configs/config.d/log_to_console.xml', + 'configs/config.d/macros.xml', + 'configs/config.d/metric_log.xml', + 'configs/config.d/more_clusters.xml', + 'configs/config.d/part_log.xml', + 'configs/config.d/path.xml', + 'configs/config.d/query_masking_rules.xml', + 'configs/config.d/tcp_with_proxy.xml', + 'configs/config.d/test_cluster_with_incorrect_pw.xml', + 'configs/config.d/text_log.xml', + 'configs/config.d/zookeeper.xml'] + + all_userd = ['configs/users.d/allow_introspection_functions.xml', + 'configs/users.d/log_queries.xml'] + + node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, + with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False) + + try: + cluster.start() + assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n") + assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n") + + finally: + cluster.shutdown()