diff --git a/.gitmodules b/.gitmodules
index 66a2370f0da..2ccce88e5e4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -228,3 +228,7 @@
[submodule "contrib/datasketches-cpp"]
path = contrib/datasketches-cpp
url = https://github.com/ClickHouse-Extras/datasketches-cpp.git
+
+[submodule "contrib/yaml-cpp"]
+ path = contrib/yaml-cpp
+ url = https://github.com/ClickHouse-Extras/yaml-cpp.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2c3fa088995..866d9f542e1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -527,6 +527,7 @@ include (cmake/find/nanodbc.cmake)
include (cmake/find/rocksdb.cmake)
include (cmake/find/libpqxx.cmake)
include (cmake/find/nuraft.cmake)
+include (cmake/find/yaml-cpp.cmake)
if(NOT USE_INTERNAL_PARQUET_LIBRARY)
diff --git a/base/bridge/CMakeLists.txt b/base/bridge/CMakeLists.txt
index 20b0b651677..bcba43e8c2e 100644
--- a/base/bridge/CMakeLists.txt
+++ b/base/bridge/CMakeLists.txt
@@ -3,5 +3,11 @@ add_library (bridge
)
target_include_directories (daemon PUBLIC ..)
-target_link_libraries (bridge PRIVATE daemon dbms Poco::Data Poco::Data::ODBC)
+target_link_libraries (bridge
+ PRIVATE
+ daemon
+ dbms
+ Poco::Data
+ Poco::Data::ODBC
+)
diff --git a/cmake/find/yaml-cpp.cmake b/cmake/find/yaml-cpp.cmake
new file mode 100644
index 00000000000..9b9d9bd39d6
--- /dev/null
+++ b/cmake/find/yaml-cpp.cmake
@@ -0,0 +1,9 @@
+option(USE_YAML_CPP "Enable yaml-cpp" ${ENABLE_LIBRARIES})
+
+if (NOT USE_YAML_CPP)
+ return()
+endif()
+
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp")
+ message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init --recursive")
+endif()
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 9eafec23f51..a9438aa4b76 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -50,6 +50,10 @@ add_subdirectory (replxx-cmake)
add_subdirectory (unixodbc-cmake)
add_subdirectory (nanodbc-cmake)
+if (USE_YAML_CPP)
+ add_subdirectory (yaml-cpp-cmake)
+endif()
+
if (USE_INTERNAL_XZ_LIBRARY)
add_subdirectory (xz)
endif()
diff --git a/contrib/yaml-cpp b/contrib/yaml-cpp
new file mode 160000
index 00000000000..0c86adac6d1
--- /dev/null
+++ b/contrib/yaml-cpp
@@ -0,0 +1 @@
+Subproject commit 0c86adac6d117ee2b4afcedb8ade19036ca0327d
diff --git a/contrib/yaml-cpp-cmake/CMakeLists.txt b/contrib/yaml-cpp-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..ed0287de110
--- /dev/null
+++ b/contrib/yaml-cpp-cmake/CMakeLists.txt
@@ -0,0 +1,39 @@
+set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp)
+
+set (SRCS
+ ${LIBRARY_DIR}/src/binary.cpp
+ ${LIBRARY_DIR}/src/emitterutils.cpp
+ ${LIBRARY_DIR}/src/null.cpp
+ ${LIBRARY_DIR}/src/scantoken.cpp
+ ${LIBRARY_DIR}/src/convert.cpp
+ ${LIBRARY_DIR}/src/exceptions.cpp
+ ${LIBRARY_DIR}/src/ostream_wrapper.cpp
+ ${LIBRARY_DIR}/src/simplekey.cpp
+ ${LIBRARY_DIR}/src/depthguard.cpp
+ ${LIBRARY_DIR}/src/exp.cpp
+ ${LIBRARY_DIR}/src/parse.cpp
+ ${LIBRARY_DIR}/src/singledocparser.cpp
+ ${LIBRARY_DIR}/src/directives.cpp
+ ${LIBRARY_DIR}/src/memory.cpp
+ ${LIBRARY_DIR}/src/parser.cpp
+ ${LIBRARY_DIR}/src/stream.cpp
+ ${LIBRARY_DIR}/src/emit.cpp
+ ${LIBRARY_DIR}/src/nodebuilder.cpp
+ ${LIBRARY_DIR}/src/regex_yaml.cpp
+ ${LIBRARY_DIR}/src/tag.cpp
+ ${LIBRARY_DIR}/src/emitfromevents.cpp
+ ${LIBRARY_DIR}/src/node.cpp
+ ${LIBRARY_DIR}/src/scanner.cpp
+ ${LIBRARY_DIR}/src/emitter.cpp
+ ${LIBRARY_DIR}/src/node_data.cpp
+ ${LIBRARY_DIR}/src/scanscalar.cpp
+ ${LIBRARY_DIR}/src/emitterstate.cpp
+ ${LIBRARY_DIR}/src/nodeevents.cpp
+ ${LIBRARY_DIR}/src/scantag.cpp
+)
+
+add_library (yaml-cpp ${SRCS})
+
+
+target_include_directories(yaml-cpp PRIVATE ${LIBRARY_DIR}/include/yaml-cpp)
+target_include_directories(yaml-cpp SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include)
diff --git a/programs/server/config-example.yaml.disabled b/programs/server/config-example.yaml.disabled
new file mode 100644
index 00000000000..a83acf50de0
--- /dev/null
+++ b/programs/server/config-example.yaml.disabled
@@ -0,0 +1,86 @@
+# We can use 3 main node types in YAML: Scalar, Map and Sequence.
+
+
+
+# A Scalar is a simple key-value pair:
+
+scalar: 123
+
+# Here we have a key "scalar" and value "123"
+# If we rewrite this in XML, we will get 123
+
+# We can also represent an empty value with '':
+
+key: ''
+
+
+
+# A Map is a node, which contains other nodes:
+
+map:
+ key1: value1
+ key2: value2
+ small_map:
+ key3: value3
+
+# This map can be converted into:
+#
+
+
+
+# A Sequence is a node, which contains also other nodes.
+# The main difference from Map is that Sequence can also contain simple values.
+
+sequence:
+ - val1
+ - val2
+ - key: 123
+ - map:
+ mkey1: foo
+ mkey2: bar
+
+# We can represent it in XML this way:
+# val1
+# val2
+#
+# 123
+#
+#
+#
+#
+
+
+
+# YAML does not have direct support for structures like XML attributes.
+# We represent them as nodes with @ prefix in key. Note, that @ is reserved by YAML standard,
+# so you will need to write double quotes around the key. Both Map and Sequence can have
+# attributes as children nodes
+
+map:
+ "@attr1": value1
+ "@attr2": value2
+ key: 123
+
+# This gives us:
+#
+
+sequence:
+ - "@attr1": value1
+ - "@attr2": value2
+ - 123
+ - abc
+
+# And this gives us:
+#
+#
diff --git a/src/Common/Config/CMakeLists.txt b/src/Common/Config/CMakeLists.txt
index a7914fb17ec..3da44be2af6 100644
--- a/src/Common/Config/CMakeLists.txt
+++ b/src/Common/Config/CMakeLists.txt
@@ -3,6 +3,7 @@ set (SRCS
ConfigProcessor.cpp
configReadClient.cpp
ConfigReloader.cpp
+ YAMLParser.cpp
)
add_library(clickhouse_common_config ${SRCS})
@@ -15,3 +16,10 @@ target_link_libraries(clickhouse_common_config
PRIVATE
string_utils
)
+
+if (USE_YAML_CPP)
+target_link_libraries(clickhouse_common_config
+ PRIVATE
+ yaml-cpp
+)
+endif()
diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp
index bc2a8a27943..fa9e9b72087 100644
--- a/src/Common/Config/ConfigProcessor.cpp
+++ b/src/Common/Config/ConfigProcessor.cpp
@@ -1,4 +1,8 @@
+#if !defined(ARCADIA_BUILD)
+ #include
+#endif
#include "ConfigProcessor.h"
+#include "YAMLParser.h"
#include
#include
@@ -20,10 +24,8 @@
#include
#include
-
#define PREPROCESSED_SUFFIX "-preprocessed"
-
namespace fs = std::filesystem;
using namespace Poco::XML;
@@ -438,8 +440,10 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
std::string base_name = path.getBaseName();
// Skip non-config and temporary files
- if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, "."))
- files.push_back(file.path());
+ if (file.isFile() && (extension == "xml" || extension == "conf" || extension == "yaml" || extension == "yml") && !startsWith(base_name, "."))
+ {
+ files.push_back(file.path());
+ }
}
}
@@ -453,12 +457,21 @@ XMLDocumentPtr ConfigProcessor::processConfig(
zkutil::ZooKeeperNodeCache * zk_node_cache,
const zkutil::EventPtr & zk_changed_event)
{
- XMLDocumentPtr config;
LOG_DEBUG(log, "Processing configuration file '{}'.", path);
+ XMLDocumentPtr config;
+
if (fs::exists(path))
{
- config = dom_parser.parse(path);
+ fs::path p(path);
+ if (p.extension() == ".xml")
+ {
+ config = dom_parser.parse(path);
+ }
+ else if (p.extension() == ".yaml" || p.extension() == ".yml")
+ {
+ config = YAMLParser::parse(path);
+ }
}
else
{
@@ -493,8 +506,20 @@ XMLDocumentPtr ConfigProcessor::processConfig(
{
LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
- XMLDocumentPtr with = dom_parser.parse(merge_file);
+ XMLDocumentPtr with;
+
+ fs::path p(merge_file);
+ if (p.extension() == ".yaml" || p.extension() == ".yml")
+ {
+ with = YAMLParser::parse(merge_file);
+ }
+ else
+ {
+ with = dom_parser.parse(merge_file);
+ }
+
merge(config, with);
+
contributing_files.push_back(merge_file);
}
catch (Exception & e)
diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h
index 7a4102140d9..5b16bc0cb1b 100644
--- a/src/Common/Config/ConfigProcessor.h
+++ b/src/Common/Config/ConfigProcessor.h
@@ -1,5 +1,9 @@
#pragma once
+#if !defined(ARCADIA_BUILD)
+ #include
+#endif
+
#include
#include
#include
@@ -141,3 +145,4 @@ private:
};
}
+
diff --git a/src/Common/Config/YAMLParser.cpp b/src/Common/Config/YAMLParser.cpp
new file mode 100644
index 00000000000..9eaf1cdc1ad
--- /dev/null
+++ b/src/Common/Config/YAMLParser.cpp
@@ -0,0 +1,166 @@
+#if !defined(ARCADIA_BUILD)
+ #include
+#endif
+
+#if USE_YAML_CPP
+#include "YAMLParser.h"
+
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include // Y_IGNORE
+
+#include
+
+using namespace Poco::XML;
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int CANNOT_OPEN_FILE;
+ extern const int CANNOT_PARSE_YAML;
+}
+
+/// A prefix symbol in yaml key
+/// We add attributes to nodes by using a prefix symbol in the key part.
+/// Currently we use @ as a prefix symbol. Note, that @ is reserved
+/// by YAML standard, so we need to write a key-value pair like this: "@attribute": attr_value
+const char YAML_ATTRIBUTE_PREFIX = '@';
+
+namespace
+{
+
+Poco::AutoPtr createCloneNode(Poco::XML::Element & original_node)
+{
+ Poco::AutoPtr clone_node = original_node.ownerDocument()->createElement(original_node.nodeName());
+ original_node.parentNode()->appendChild(clone_node);
+ return clone_node;
+}
+
+void processNode(const YAML::Node & node, Poco::XML::Element & parent_xml_element)
+{
+ auto * xml_document = parent_xml_element.ownerDocument();
+ switch (node.Type())
+ {
+ case YAML::NodeType::Scalar:
+ {
+ auto value = node.as();
+ Poco::AutoPtr xml_value = xml_document->createTextNode(value);
+ parent_xml_element.appendChild(xml_value);
+ break;
+ }
+
+ /// We process YAML Sequences as a
+ /// list of value tags with same key and different values.
+ /// For example, we translate this sequence
+ /// seq:
+ /// - val1
+ /// - val2
+ ///
+ /// into this:
+ /// val1
+ /// val2
+ case YAML::NodeType::Sequence:
+ {
+ for (const auto & child_node : node)
+ if (parent_xml_element.hasChildNodes())
+ {
+ /// We want to process sequences like that:
+ /// seq:
+ /// - val1
+ /// - k2: val2
+ /// - val3
+ /// - k4: val4
+ /// - val5
+ /// into xml like this:
+ /// val1
+ ///
+ /// val2
+ ///
+ /// val3
+ ///
+ /// val4
+ ///
+ /// val5
+ /// So, we create a new parent node with same tag for each child node
+ processNode(child_node, *createCloneNode(parent_xml_element));
+ }
+ else
+ {
+ processNode(child_node, parent_xml_element);
+ }
+ break;
+ }
+ case YAML::NodeType::Map:
+ {
+ for (const auto & key_value_pair : node)
+ {
+ const auto & key_node = key_value_pair.first;
+ const auto & value_node = key_value_pair.second;
+ auto key = key_node.as();
+ bool is_attribute = (key.starts_with(YAML_ATTRIBUTE_PREFIX) && value_node.IsScalar());
+ if (is_attribute)
+ {
+ /// we use substr(1) here to remove YAML_ATTRIBUTE_PREFIX from key
+ auto attribute_name = key.substr(1);
+ auto value = value_node.as();
+ parent_xml_element.setAttribute(attribute_name, value);
+ }
+ else
+ {
+ Poco::AutoPtr xml_key = xml_document->createElement(key);
+ parent_xml_element.appendChild(xml_key);
+ processNode(value_node, *xml_key);
+ }
+ }
+ break;
+ }
+ case YAML::NodeType::Null: break;
+ case YAML::NodeType::Undefined:
+ {
+ throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "YAMLParser has encountered node with undefined type and cannot continue parsing of the file");
+ }
+ }
+}
+
+}
+
+Poco::AutoPtr YAMLParser::parse(const String& path)
+{
+ YAML::Node node_yml;
+ try
+ {
+ node_yml = YAML::LoadFile(path);
+ }
+ catch (const YAML::ParserException& e)
+ {
+ /// yaml-cpp cannot parse the file because its contents are incorrect
+ throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "Unable to parse YAML configuration file {}", path, e.what());
+ }
+ catch (const YAML::BadFile&)
+ {
+ /// yaml-cpp cannot open the file even though it exists
+ throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Unable to open YAML configuration file {}", path);
+ }
+ Poco::AutoPtr xml = new Document;
+ Poco::AutoPtr root_node = xml->createElement("yandex");
+ xml->appendChild(root_node);
+ processNode(node_yml, *root_node);
+ return xml;
+}
+
+}
+#endif
diff --git a/src/Common/Config/YAMLParser.h b/src/Common/Config/YAMLParser.h
new file mode 100644
index 00000000000..a716f3de9b1
--- /dev/null
+++ b/src/Common/Config/YAMLParser.h
@@ -0,0 +1,55 @@
+#pragma once
+
+#if !defined(ARCADIA_BUILD)
+ #include
+#endif
+
+#include
+
+#include
+#include "Poco/DOM/AutoPtr.h"
+#include
+
+#if USE_YAML_CPP
+
+namespace DB
+{
+
+/// Real YAML parser: loads yaml file into a YAML::Node
+class YAMLParserImpl
+{
+public:
+ static Poco::AutoPtr parse(const String& path);
+};
+
+using YAMLParser = YAMLParserImpl;
+
+}
+
+#else
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int CANNOT_PARSE_YAML;
+}
+
+/// Fake YAML parser: throws an exception if we try to parse YAML configs in a build without yaml-cpp
+class DummyYAMLParser
+{
+public:
+ static Poco::AutoPtr parse(const String& path)
+ {
+ Poco::AutoPtr xml = new Poco::XML::Document;
+ throw Exception(ErrorCodes::CANNOT_PARSE_YAML, "Unable to parse YAML configuration file {} without usage of yaml-cpp library", path);
+ return xml;
+ }
+};
+
+using YAMLParser = DummyYAMLParser;
+
+}
+
+#endif
diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp
index b1efd791bf8..330a193212b 100644
--- a/src/Common/ErrorCodes.cpp
+++ b/src/Common/ErrorCodes.cpp
@@ -552,6 +552,7 @@
M(582, NO_SUCH_PROJECTION_IN_TABLE) \
M(583, ILLEGAL_PROJECTION) \
M(584, PROJECTION_NOT_USED) \
+ M(585, CANNOT_PARSE_YAML) \
\
M(998, POSTGRESQL_CONNECTION_FAILURE) \
M(999, KEEPER_EXCEPTION) \
diff --git a/src/Common/config.h.in b/src/Common/config.h.in
index 28a21ea7764..6844f0fa9e3 100644
--- a/src/Common/config.h.in
+++ b/src/Common/config.h.in
@@ -16,3 +16,4 @@
#cmakedefine01 USE_STATS
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
#cmakedefine01 USE_DATASKETCHES
+#cmakedefine01 USE_YAML_CPP
diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 5bd608ef758..6287064b616 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -126,7 +126,8 @@ class ClickHouseCluster:
"""
def __init__(self, base_path, name=None, base_config_dir=None, server_bin_path=None, client_bin_path=None,
- odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None, custom_dockerd_host=None):
+ odbc_bridge_bin_path=None, library_bridge_bin_path=None, zookeeper_config_path=None,
+ custom_dockerd_host=None):
for param in list(os.environ.keys()):
print("ENV %40s %s" % (param, os.environ[param]))
self.base_dir = p.dirname(base_path)
@@ -219,7 +220,9 @@ class ClickHouseCluster:
with_redis=False, with_minio=False, with_cassandra=False,
hostname=None, env_variables=None, image="yandex/clickhouse-integration-test", tag=None,
stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None,
- zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True):
+ zookeeper_docker_compose_path=None, zookeeper_use_tmpfs=True, minio_certs_dir=None, use_keeper=True,
+ main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True):
+
"""Add an instance to the cluster.
name - the name of the instance directory and the value of the 'instance' macro in ClickHouse.
@@ -280,6 +283,9 @@ class ClickHouseCluster:
ipv4_address=ipv4_address,
ipv6_address=ipv6_address,
with_installed_binary=with_installed_binary,
+ main_config_name=main_config_name,
+ users_config_name=users_config_name,
+ copy_common_configs=copy_common_configs,
tmpfs=tmpfs or [])
docker_compose_yml_dir = get_docker_compose_path()
@@ -944,7 +950,7 @@ class ClickHouseCluster:
subprocess_check_call(self.base_zookeeper_cmd + ["start", n])
-CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/config.xml --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log"
+CLICKHOUSE_START_COMMAND = "clickhouse server --config-file=/etc/clickhouse-server/{main_config_file} --log-file=/var/log/clickhouse-server/clickhouse-server.log --errorlog-file=/var/log/clickhouse-server/clickhouse-server.err.log"
CLICKHOUSE_STAY_ALIVE_COMMAND = 'bash -c "{} --daemon; tail -f /dev/null"'.format(CLICKHOUSE_START_COMMAND)
@@ -1000,6 +1006,8 @@ class ClickHouseInstance:
macros, with_zookeeper, zookeeper_config_path, with_mysql, with_mysql_cluster, with_kafka, with_kerberized_kafka, with_rabbitmq, with_kerberized_hdfs,
with_mongo, with_redis, with_minio,
with_cassandra, server_bin_path, odbc_bridge_bin_path, library_bridge_bin_path, clickhouse_path_dir, with_odbc_drivers,
+ clickhouse_start_command=CLICKHOUSE_START_COMMAND,
+ main_config_name="config.xml", users_config_name="users.xml", copy_common_configs=True,
hostname=None, env_variables=None,
image="yandex/clickhouse-integration-test", tag="latest",
stay_alive=False, ipv4_address=None, ipv6_address=None, with_installed_binary=False, tmpfs=None):
@@ -1036,6 +1044,12 @@ class ClickHouseInstance:
self.with_minio = with_minio
self.with_cassandra = with_cassandra
+ self.main_config_name = main_config_name
+ self.users_config_name = users_config_name
+ self.copy_common_configs = copy_common_configs
+
+ self.clickhouse_start_command = clickhouse_start_command.replace("{main_config_file}", self.main_config_name)
+
self.path = p.join(self.cluster.instances_dir, name)
self.docker_compose_path = p.join(self.path, 'docker-compose.yml')
self.env_variables = env_variables or {}
@@ -1177,7 +1191,7 @@ class ClickHouseInstance:
if not self.stay_alive:
raise Exception("clickhouse can be started again only with stay_alive=True instance")
- self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid()))
+ self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()))
# wait start
from helpers.test_tools import assert_eq_with_retry
assert_eq_with_retry(self, "select 1", "1", retry_count=int(start_wait_sec / 0.5), sleep_time=0.5)
@@ -1263,7 +1277,7 @@ class ClickHouseInstance:
self.exec_in_container(["bash", "-c",
"cp /usr/share/clickhouse-odbc-bridge_fresh /usr/bin/clickhouse-odbc-bridge && chmod 777 /usr/bin/clickhouse"],
user='root')
- self.exec_in_container(["bash", "-c", "{} --daemon".format(CLICKHOUSE_START_COMMAND)], user=str(os.getuid()))
+ self.exec_in_container(["bash", "-c", "{} --daemon".format(self.clickhouse_start_command)], user=str(os.getuid()))
from helpers.test_tools import assert_eq_with_retry
# wait start
assert_eq_with_retry(self, "select 1", "1", retry_count=retries)
@@ -1404,8 +1418,10 @@ class ClickHouseInstance:
os.makedirs(instance_config_dir)
print("Copy common default production configuration from {}".format(self.base_config_dir))
- shutil.copyfile(p.join(self.base_config_dir, 'config.xml'), p.join(instance_config_dir, 'config.xml'))
- shutil.copyfile(p.join(self.base_config_dir, 'users.xml'), p.join(instance_config_dir, 'users.xml'))
+
+ shutil.copyfile(p.join(self.base_config_dir, self.main_config_name), p.join(instance_config_dir, self.main_config_name))
+
+ shutil.copyfile(p.join(self.base_config_dir, self.users_config_name), p.join(instance_config_dir, self.users_config_name))
print("Create directory for configuration generated in this helper")
# used by all utils with any config
@@ -1423,7 +1439,9 @@ class ClickHouseInstance:
print("Copy common configuration from helpers")
# The file is named with 0_ prefix to be processed before other configuration overloads.
- shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir)
+ if self.copy_common_configs:
+ shutil.copy(p.join(HELPERS_DIR, '0_common_instance_config.xml'), self.config_d_dir)
+
shutil.copy(p.join(HELPERS_DIR, '0_common_instance_users.xml'), users_d_dir)
if len(self.custom_dictionaries_paths):
shutil.copy(p.join(HELPERS_DIR, '0_common_enable_dictionaries.xml'), self.config_d_dir)
@@ -1502,11 +1520,11 @@ class ClickHouseInstance:
self._create_odbc_config_file()
odbc_ini_path = '- ' + self.odbc_ini_path
- entrypoint_cmd = CLICKHOUSE_START_COMMAND
+ entrypoint_cmd = self.clickhouse_start_command
if self.stay_alive:
- entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND
-
+ entrypoint_cmd = CLICKHOUSE_STAY_ALIVE_COMMAND.replace("{main_config_file}", self.main_config_name)
+
print("Entrypoint cmd: {}".format(entrypoint_cmd))
networks = app_net = ipv4_address = ipv6_address = net_aliases = net_alias1 = ""
diff --git a/tests/integration/runner b/tests/integration/runner
index ee116c29aa5..16eb31dfd5e 100755
--- a/tests/integration/runner
+++ b/tests/integration/runner
@@ -70,11 +70,11 @@ def check_args_and_update_paths(args):
if not os.path.exists(path):
raise Exception("Path {} doesn't exist".format(path))
- if not os.path.exists(os.path.join(args.base_configs_dir, "config.xml")):
- raise Exception("No configs.xml in {}".format(args.base_configs_dir))
+ if (not os.path.exists(os.path.join(args.base_configs_dir, "config.xml"))) and (not os.path.exists(os.path.join(args.base_configs_dir, "config.yaml"))):
+ raise Exception("No configs.xml or configs.yaml in {}".format(args.base_configs_dir))
- if not os.path.exists(os.path.join(args.base_configs_dir, "users.xml")):
- raise Exception("No users.xml in {}".format(args.base_configs_dir))
+ if (not os.path.exists(os.path.join(args.base_configs_dir, "users.xml"))) and (not os.path.exists(os.path.join(args.base_configs_dir, "users.yaml"))):
+ raise Exception("No users.xml or users.yaml in {}".format(args.base_configs_dir))
def docker_kill_handler_handler(signum, frame):
subprocess.check_call('docker kill $(docker ps -a -q --filter name={name} --format="{{{{.ID}}}}")'.format(name=CONTAINER_NAME), shell=True)
diff --git a/tests/integration/test_config_xml_full/__init__.py b/tests/integration/test_config_xml_full/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_xml_full/configs/config.d/access_control.xml b/tests/integration/test_config_xml_full/configs/config.d/access_control.xml
new file mode 100644
index 00000000000..6567c39f171
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/access_control.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
+ users.xml
+
+
+
+ access/
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml b/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml
new file mode 100644
index 00000000000..b21df47bc85
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/keeper_port.xml
@@ -0,0 +1,23 @@
+
+
+ 9181
+ 1
+
+
+ 10000
+ 30000
+ false
+ 60000
+
+ 1000000000000000
+
+
+
+
+ 1
+ localhost
+ 44444
+
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml b/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml
new file mode 100644
index 00000000000..227c53647f3
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/log_to_console.xml
@@ -0,0 +1,7 @@
+
+
+ true
+
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml
new file mode 100644
index 00000000000..2c34585437b
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/logging_no_rotate.xml
@@ -0,0 +1,8 @@
+
+
+
+ never
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/macros.xml b/tests/integration/test_config_xml_full/configs/config.d/macros.xml
new file mode 100644
index 00000000000..4902b12bc81
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/macros.xml
@@ -0,0 +1,9 @@
+
+
+ Hello, world!
+ s1
+ r1
+ /clickhouse/tables/{database}/{shard}/
+ table_{table}
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml b/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml
new file mode 100644
index 00000000000..0ca9f162416
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/metric_log.xml
@@ -0,0 +1,8 @@
+
+
+ system
+
+ 7500
+ 1000
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml b/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml
new file mode 100644
index 00000000000..aecbf9e0ba7
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/more_clusters.xml
@@ -0,0 +1,49 @@
+
+
+
+
+
+
+
+
+ localhost
+ 9001
+
+
+
+
+
+
+
+ localhost
+ 9001
+
+
+
+
+ localhost
+ 9002
+
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 9001
+
+
+
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/part_log.xml b/tests/integration/test_config_xml_full/configs/config.d/part_log.xml
new file mode 100644
index 00000000000..6c6fc9c6982
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/part_log.xml
@@ -0,0 +1,8 @@
+
+
+ system
+
+
+ 7500
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/path.xml b/tests/integration/test_config_xml_full/configs/config.d/path.xml
new file mode 100644
index 00000000000..466ed0d1663
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/path.xml
@@ -0,0 +1,8 @@
+
+ ./
+ ./tmp/
+ ./user_files/
+ ./format_schemas/
+ ./access/
+ ./top_level_domains/
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml
new file mode 100644
index 00000000000..5a854848f3d
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/query_masking_rules.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+ TOPSECRET.TOPSECRET
+ [hidden]
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml b/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml
new file mode 100644
index 00000000000..19046054c16
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/tcp_with_proxy.xml
@@ -0,0 +1,3 @@
+
+ 9010
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/text_log.xml b/tests/integration/test_config_xml_full/configs/config.d/text_log.xml
new file mode 100644
index 00000000000..3699a23578c
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/text_log.xml
@@ -0,0 +1,7 @@
+
+
+ system
+
+ 7500
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml b/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml
new file mode 100644
index 00000000000..06ed7fcd39f
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.d/zookeeper.xml
@@ -0,0 +1,8 @@
+
+
+
+ localhost
+ 9181
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/config.xml b/tests/integration/test_config_xml_full/configs/config.xml
new file mode 100644
index 00000000000..a7bb9d49e95
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/config.xml
@@ -0,0 +1,1118 @@
+
+
+
+
+
+ trace
+ /var/log/clickhouse-server/clickhouse-server.log
+ /var/log/clickhouse-server/clickhouse-server.err.log
+
+ 1000M
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 8123
+
+
+ 9000
+
+
+ 9004
+
+
+ 9005
+
+
+
+
+
+
+
+
+
+
+
+ 9009
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 4096
+
+
+ 3
+
+
+
+
+ false
+
+
+ /path/to/ssl_cert_file
+ /path/to/ssl_key_file
+
+
+ false
+
+
+ /path/to/ssl_ca_cert_file
+
+
+ deflate
+
+
+ medium
+
+
+ -1
+ -1
+
+
+ false
+
+
+
+
+
+
+ /etc/clickhouse-server/server.crt
+ /etc/clickhouse-server/server.key
+
+ /etc/clickhouse-server/dhparam.pem
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ RejectCertificateHandler
+
+
+
+
+
+
+
+
+ 100
+
+
+ 0
+
+
+
+ 10000
+
+
+ 0.9
+
+
+ 4194304
+
+
+ 0
+
+
+
+
+
+ 8589934592
+
+
+ 5368709120
+
+
+
+ 1000
+
+
+
+ /var/lib/clickhouse/
+
+
+ /var/lib/clickhouse/tmp/
+
+
+
+
+
+ /var/lib/clickhouse/user_files/
+
+
+
+
+
+
+
+
+
+
+
+
+ users.xml
+
+
+
+ /var/lib/clickhouse/access/
+
+
+
+
+
+
+ default
+
+
+
+
+
+
+
+
+
+
+
+ default
+
+
+
+
+
+
+
+
+ true
+
+
+ false
+
+ ' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ clickhouse-jdbc-bridge &
+
+ * [CentOS/RHEL]
+ export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
+ export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ clickhouse-jdbc-bridge &
+
+ Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+ 127.0.0.1
+ 9000
+
+
+
+
+ 127.0.0.2
+ 9000
+
+
+
+
+
+ true
+
+ 127.0.0.1
+ 9000
+
+
+
+ true
+
+ 127.0.0.2
+ 9000
+
+
+
+
+
+
+ localhost
+ 9440
+ 1
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 3600
+
+
+
+ 3600
+
+
+ 60
+
+
+
+
+
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+
+
+
+
+
+ 7500
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+
+
+
+
+ system
+
+ 7500
+ 1000
+
+
+
+
+ system
+
+
+ 60000
+
+
+
+
+
+
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+
+ system
+
+ 7500
+
+
+
+
+
+ system
+
+
+
+ 1000
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ *_dictionary.xml
+
+
+
+
+
+
+
+ /clickhouse/task_queue/ddl
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ click_cost
+ any
+
+ 0
+ 3600
+
+
+ 86400
+ 60
+
+
+
+ max
+
+ 0
+ 60
+
+
+ 3600
+ 300
+
+
+ 86400
+ 3600
+
+
+
+
+
+ /var/lib/clickhouse/format_schemas/
+
+
+
+
+ hide encrypt/decrypt arguments
+ ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)
+
+ \1(???)
+
+
+
+
+
+
+
+
+
+ false
+
+ false
+
+
+ https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277
+
+
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/embedded.xml b/tests/integration/test_config_xml_full/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/embedded.xml
@@ -0,0 +1,40 @@
+
+
+
+
+ trace
+ true
+
+
+ 8123
+ 9000
+ 9004
+
+ ./
+
+ 8589934592
+ 5368709120
+ true
+
+
+
+
+
+
+ ::/0
+
+
+ default
+ default
+ 1
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml
new file mode 100644
index 00000000000..b94e95bc043
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/users.d/allow_introspection_functions.xml
@@ -0,0 +1,8 @@
+
+
+
+
+ 1
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml b/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml
new file mode 100644
index 00000000000..25261072ade
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/users.d/log_queries.xml
@@ -0,0 +1,7 @@
+
+
+
+ 1
+
+
+
diff --git a/tests/integration/test_config_xml_full/configs/users.xml b/tests/integration/test_config_xml_full/configs/users.xml
new file mode 100644
index 00000000000..829748dafb1
--- /dev/null
+++ b/tests/integration/test_config_xml_full/configs/users.xml
@@ -0,0 +1,120 @@
+
+
+
+
+
+
+
+ 10000000000
+ 64999
+
+
+ random
+
+
+
+
+ 1
+
+
+
+
+
+
+
+
+
+
+
+
+ ::/0
+
+
+
+ default
+
+
+ default
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 3600
+
+
+ 0
+ 0
+ 0
+ 0
+ 0
+
+
+
+
diff --git a/tests/integration/test_config_xml_full/test.py b/tests/integration/test_config_xml_full/test.py
new file mode 100644
index 00000000000..a8650a0dc55
--- /dev/null
+++ b/tests/integration/test_config_xml_full/test.py
@@ -0,0 +1,40 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_xml_full_conf():
+ # all configs are in XML
+ cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml')
+
+ all_confd = ['configs/config.d/access_control.xml',
+ 'configs/config.d/keeper_port.xml',
+ 'configs/config.d/logging_no_rotate.xml',
+ 'configs/config.d/log_to_console.xml',
+ 'configs/config.d/macros.xml',
+ 'configs/config.d/metric_log.xml',
+ 'configs/config.d/more_clusters.xml',
+ 'configs/config.d/part_log.xml',
+ 'configs/config.d/path.xml',
+ 'configs/config.d/query_masking_rules.xml',
+ 'configs/config.d/tcp_with_proxy.xml',
+ 'configs/config.d/text_log.xml',
+ 'configs/config.d/zookeeper.xml']
+
+ all_userd = ['configs/users.d/allow_introspection_functions.xml',
+ 'configs/users.d/log_queries.xml']
+
+ node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False)
+
+ try:
+ cluster.start()
+ assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+ assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+ finally:
+ cluster.shutdown()
diff --git a/tests/integration/test_config_xml_main/__init__.py b/tests/integration/test_config_xml_main/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml b/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml
new file mode 100644
index 00000000000..d8ead517c86
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/access_control.yaml
@@ -0,0 +1,7 @@
+user_directories:
+ users_xml:
+ path: users.xml
+ local_directory:
+ path: access/
+ "@replace": replace
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml b/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml
new file mode 100644
index 00000000000..91723bc372f
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/keeper_port.yaml
@@ -0,0 +1,15 @@
+keeper_server:
+ tcp_port: 9181
+ server_id: 1
+ coordination_settings:
+ operation_timeout_ms: 10000
+ session_timeout_ms: 30000
+ force_sync: false
+ startup_timeout: 60000
+ reserved_log_items: 1000000000000000
+ raft_configuration:
+ server:
+ id: 1
+ hostname: localhost
+ port: 44444
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml b/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml
new file mode 100644
index 00000000000..7b59339800b
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/log_to_console.yaml
@@ -0,0 +1,7 @@
+logger:
+ console: true
+ log:
+ "@remove": remove
+ errorlog:
+ "@remove": remove
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml b/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml
new file mode 100644
index 00000000000..513cd3f7dc5
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/logging_no_rotate.yaml
@@ -0,0 +1,2 @@
+logger:
+ size: never
diff --git a/tests/integration/test_config_xml_main/configs/config.d/macros.yaml b/tests/integration/test_config_xml_main/configs/config.d/macros.yaml
new file mode 100644
index 00000000000..a9c61e270ab
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/macros.yaml
@@ -0,0 +1,7 @@
+macros:
+ test: 'Hello, world!'
+ shard: s1
+ replica: r1
+ default_path_test: '/clickhouse/tables/{database}/{shard}/'
+ default_name_test: 'table_{table}'
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml
new file mode 100644
index 00000000000..aafad939c37
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/metric_log.yaml
@@ -0,0 +1,6 @@
+metric_log:
+ database: system
+ table: metric_log
+ flush_interval_milliseconds: 7500
+ collect_interval_milliseconds: 1000
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml b/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml
new file mode 100644
index 00000000000..7da07190894
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/more_clusters.yaml
@@ -0,0 +1,23 @@
+remote_servers:
+ single_remote_shard_at_port_9001:
+ shard:
+ replica:
+ host: localhost
+ port: 9001
+ two_remote_shards_at_port_9001_9002:
+ shard:
+ - replica:
+ host: localhost
+ port: 9001
+ - replica:
+ host: localhost
+ port: 9002
+ two_shards_one_local_one_remote_at_port_9001:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 9001
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml
new file mode 100644
index 00000000000..43f60146dca
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/part_log.yaml
@@ -0,0 +1,5 @@
+part_log:
+ database: system
+ table: part_log
+ flush_interval_milliseconds: 7500
+
diff --git a/tests/integration/test_config_xml_main/configs/config.d/path.yaml b/tests/integration/test_config_xml_main/configs/config.d/path.yaml
new file mode 100644
index 00000000000..3e26e8906ee
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/path.yaml
@@ -0,0 +1,18 @@
+path:
+ - ./
+ - "@replace": replace
+tmp_path:
+ - ./tmp/
+ - "@replace": replace
+user_files_path:
+ - ./user_files/
+ - "@replace": replace
+format_schema_path:
+ - ./format_schemas/
+ - "@replace": replace
+access_control_path:
+ - ./access/
+ - "@replace": replace
+top_level_domains_path:
+ - ./top_level_domains/
+ - "@replace": replace
diff --git a/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml b/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml
new file mode 100644
index 00000000000..38163429d81
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/query_masking_rules.yaml
@@ -0,0 +1,4 @@
+query_masking_rules:
+ rule:
+ regexp: TOPSECRET.TOPSECRET
+ replace: '[hidden]'
diff --git a/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml
new file mode 100644
index 00000000000..b0349f5a9b9
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/tcp_with_proxy.yaml
@@ -0,0 +1 @@
+tcp_with_proxy_port: 9010
diff --git a/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml b/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml
new file mode 100644
index 00000000000..309a7daa81d
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/test_cluster_with_incorrect_pw.yaml
@@ -0,0 +1,11 @@
+remote_servers:
+ test_cluster_with_incorrect_pw:
+ shard:
+ internal_replication: true
+ replica:
+ - host: 127.0.0.1
+ port: 9000
+ password: foo
+ - host: 127.0.0.2
+ port: 9000
+ password: foo
diff --git a/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml b/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml
new file mode 100644
index 00000000000..4d188020e37
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/text_log.yaml
@@ -0,0 +1,4 @@
+text_log:
+ database: system
+ table: text_log
+ flush_interval_milliseconds: 7500
diff --git a/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml b/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml
new file mode 100644
index 00000000000..be02c516798
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.d/zookeeper.yaml
@@ -0,0 +1,5 @@
+zookeeper:
+ node:
+ host: localhost
+ port: 9181
+ "@index": 1
diff --git a/tests/integration/test_config_xml_main/configs/config.xml b/tests/integration/test_config_xml_main/configs/config.xml
new file mode 100644
index 00000000000..3b5dab50ffe
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/config.xml
@@ -0,0 +1,277 @@
+
+
+
+ trace
+ /var/log/clickhouse-server/clickhouse-server.log
+ /var/log/clickhouse-server/clickhouse-server.err.log
+ 1000M
+ 10
+
+ 8123
+ 9000
+ 9004
+ 9005
+ 9009
+ 4096
+ 3
+
+ false
+ /path/to/ssl_cert_file
+ /path/to/ssl_key_file
+ false
+ /path/to/ssl_ca_cert_file
+ deflate
+ medium
+ -1
+ -1
+ false
+
+
+
+ /etc/clickhouse-server/server.crt
+ /etc/clickhouse-server/server.key
+ /etc/clickhouse-server/dhparam.pem
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ true
+ true
+ sslv2,sslv3
+ true
+
+ RejectCertificateHandler
+
+
+
+ 100
+ 0
+ 10000
+
+ 0.9
+
+ 4194304
+
+ 0
+
+ 8589934592
+
+ 5368709120
+
+ 1000
+
+ /var/lib/clickhouse/
+
+ /var/lib/clickhouse/tmp/
+ /var/lib/clickhouse/user_files/
+
+
+
+
+ users.xml
+
+
+ /var/lib/clickhouse/access/
+
+
+ default
+
+ default
+ true
+
+ false
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+ 127.0.0.1
+ 9000
+
+
+
+
+ 127.0.0.2
+ 9000
+
+
+
+
+
+ true
+
+ 127.0.0.1
+ 9000
+
+
+
+ true
+
+ 127.0.0.2
+ 9000
+
+
+
+
+
+
+ localhost
+ 9440
+ 1
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 1
+
+
+
+
+
+ 3600
+
+ 3600
+
+ 60
+
+
+ system
+
+
+ toYYYYMM(event_date)
+
+ 7500
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+ system
+
+ 7500
+ 1000
+
+
+
+ system
+
+
+ 60000
+
+
+
+
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+
+ system
+
+ 7500
+
+
+
+ system
+
+
+
+ 1000
+
+
+
+
+
+ *_dictionary.xml
+
+ /clickhouse/task_queue/ddl
+
+
+
+ click_cost
+ any
+
+ 0
+ 3600
+
+
+ 86400
+ 60
+
+
+
+ max
+
+ 0
+ 60
+
+
+ 3600
+ 300
+
+
+ 86400
+ 3600
+
+
+
+ /var/lib/clickhouse/format_schemas/
+
+
+ hide encrypt/decrypt arguments
+ ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)
+ \1(???)
+
+
+
+ false
+ false
+ https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277
+
+
diff --git a/tests/integration/test_config_xml_main/configs/embedded.xml b/tests/integration/test_config_xml_main/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/embedded.xml
@@ -0,0 +1,40 @@
+
+
+
+
+ trace
+ true
+
+
+ 8123
+ 9000
+ 9004
+
+ ./
+
+ 8589934592
+ 5368709120
+ true
+
+
+
+
+
+
+ ::/0
+
+
+ default
+ default
+ 1
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml b/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml
new file mode 100644
index 00000000000..84612c198c9
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/users.d/allow_introspection_functions.yaml
@@ -0,0 +1,3 @@
+profiles:
+ default:
+ allow_introspection_functions: 1
diff --git a/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml b/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml
new file mode 100644
index 00000000000..88574e8f764
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/users.d/log_queries.yaml
@@ -0,0 +1,3 @@
+profiles:
+ default:
+ log_queries: 1
diff --git a/tests/integration/test_config_xml_main/configs/users.xml b/tests/integration/test_config_xml_main/configs/users.xml
new file mode 100644
index 00000000000..b473413bdfa
--- /dev/null
+++ b/tests/integration/test_config_xml_main/configs/users.xml
@@ -0,0 +1,19 @@
+
+
+
+
+ 10000000000
+ 64999
+
+
+
+
+
+
+
+ ::/0
+
+ default
+
+
+
diff --git a/tests/integration/test_config_xml_main/test.py b/tests/integration/test_config_xml_main/test.py
new file mode 100644
index 00000000000..052f9adb01f
--- /dev/null
+++ b/tests/integration/test_config_xml_main/test.py
@@ -0,0 +1,43 @@
+
+
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_xml_main_conf():
+ # main configs are in XML; config.d and users.d are in YAML
+ cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml')
+
+ all_confd = ['configs/config.d/access_control.yaml',
+ 'configs/config.d/keeper_port.yaml',
+ 'configs/config.d/logging_no_rotate.yaml',
+ 'configs/config.d/log_to_console.yaml',
+ 'configs/config.d/macros.yaml',
+ 'configs/config.d/metric_log.yaml',
+ 'configs/config.d/more_clusters.yaml',
+ 'configs/config.d/part_log.yaml',
+ 'configs/config.d/path.yaml',
+ 'configs/config.d/query_masking_rules.yaml',
+ 'configs/config.d/tcp_with_proxy.yaml',
+ 'configs/config.d/test_cluster_with_incorrect_pw.yaml',
+ 'configs/config.d/text_log.yaml',
+ 'configs/config.d/zookeeper.yaml']
+
+ all_userd = ['configs/users.d/allow_introspection_functions.yaml',
+ 'configs/users.d/log_queries.yaml']
+
+ node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False)
+
+ try:
+ cluster.start()
+ assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+ assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+ finally:
+ cluster.shutdown()
diff --git a/tests/integration/test_config_xml_yaml_mix/__init__.py b/tests/integration/test_config_xml_yaml_mix/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml
new file mode 100644
index 00000000000..62e4ba8c744
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/0_common_instance_config.yaml
@@ -0,0 +1,6 @@
+timezone: Europe/Moscow
+listen_host: 0.0.0.0
+custom_settings_prefixes: custom_
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+users_config: users.yaml
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml
new file mode 100644
index 00000000000..ce2e23839ef
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/access_control.yaml
@@ -0,0 +1,7 @@
+user_directories:
+ users_xml:
+ path: users.yaml
+ local_directory:
+ path: access/
+ "@replace": replace
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml
new file mode 100644
index 00000000000..b21df47bc85
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/keeper_port.xml
@@ -0,0 +1,23 @@
+
+
+ 9181
+ 1
+
+
+ 10000
+ 30000
+ false
+ 60000
+
+ 1000000000000000
+
+
+
+
+ 1
+ localhost
+ 44444
+
+
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml
new file mode 100644
index 00000000000..7b59339800b
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/log_to_console.yaml
@@ -0,0 +1,7 @@
+logger:
+ console: true
+ log:
+ "@remove": remove
+ errorlog:
+ "@remove": remove
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml
new file mode 100644
index 00000000000..2c34585437b
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/logging_no_rotate.xml
@@ -0,0 +1,8 @@
+
+
+
+ never
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml
new file mode 100644
index 00000000000..a9c61e270ab
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/macros.yaml
@@ -0,0 +1,7 @@
+macros:
+ test: 'Hello, world!'
+ shard: s1
+ replica: r1
+ default_path_test: '/clickhouse/tables/{database}/{shard}/'
+ default_name_test: 'table_{table}'
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml
new file mode 100644
index 00000000000..0ca9f162416
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/metric_log.xml
@@ -0,0 +1,8 @@
+
+
+ system
+
+ 7500
+ 1000
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml
new file mode 100644
index 00000000000..7da07190894
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/more_clusters.yaml
@@ -0,0 +1,23 @@
+remote_servers:
+ single_remote_shard_at_port_9001:
+ shard:
+ replica:
+ host: localhost
+ port: 9001
+ two_remote_shards_at_port_9001_9002:
+ shard:
+ - replica:
+ host: localhost
+ port: 9001
+ - replica:
+ host: localhost
+ port: 9002
+ two_shards_one_local_one_remote_at_port_9001:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 9001
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml
new file mode 100644
index 00000000000..6c6fc9c6982
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/part_log.xml
@@ -0,0 +1,8 @@
+
+
+ system
+
+
+ 7500
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml
new file mode 100644
index 00000000000..3e26e8906ee
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/path.yaml
@@ -0,0 +1,18 @@
+path:
+ - ./
+ - "@replace": replace
+tmp_path:
+ - ./tmp/
+ - "@replace": replace
+user_files_path:
+ - ./user_files/
+ - "@replace": replace
+format_schema_path:
+ - ./format_schemas/
+ - "@replace": replace
+access_control_path:
+ - ./access/
+ - "@replace": replace
+top_level_domains_path:
+ - ./top_level_domains/
+ - "@replace": replace
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml
new file mode 100644
index 00000000000..5a854848f3d
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/query_masking_rules.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+ TOPSECRET.TOPSECRET
+ [hidden]
+
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml
new file mode 100644
index 00000000000..b0349f5a9b9
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/tcp_with_proxy.yaml
@@ -0,0 +1 @@
+tcp_with_proxy_port: 9010
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml
new file mode 100644
index 00000000000..109e35afc37
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/test_cluster_with_incorrect_pw.xml
@@ -0,0 +1,21 @@
+
+
+
+
+ true
+
+ 127.0.0.1
+ 9000
+
+ foo
+
+
+ 127.0.0.2
+ 9000
+
+ foo
+
+
+
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml
new file mode 100644
index 00000000000..4d188020e37
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/text_log.yaml
@@ -0,0 +1,4 @@
+text_log:
+ database: system
+ table: text_log
+ flush_interval_milliseconds: 7500
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml
new file mode 100644
index 00000000000..06ed7fcd39f
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.d/zookeeper.xml
@@ -0,0 +1,8 @@
+
+
+
+ localhost
+ 9181
+
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/config.xml b/tests/integration/test_config_xml_yaml_mix/configs/config.xml
new file mode 100644
index 00000000000..e6a2b6d5324
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/config.xml
@@ -0,0 +1,277 @@
+
+
+
+ trace
+ /var/log/clickhouse-server/clickhouse-server.log
+ /var/log/clickhouse-server/clickhouse-server.err.log
+ 1000M
+ 10
+
+ 8123
+ 9000
+ 9004
+ 9005
+ 9009
+ 4096
+ 3
+
+ false
+ /path/to/ssl_cert_file
+ /path/to/ssl_key_file
+ false
+ /path/to/ssl_ca_cert_file
+ deflate
+ medium
+ -1
+ -1
+ false
+
+
+
+ /etc/clickhouse-server/server.crt
+ /etc/clickhouse-server/server.key
+ /etc/clickhouse-server/dhparam.pem
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ true
+ true
+ sslv2,sslv3
+ true
+
+ RejectCertificateHandler
+
+
+
+ 100
+ 0
+ 10000
+
+ 0.9
+
+ 4194304
+
+ 0
+
+ 8589934592
+
+ 5368709120
+
+ 1000
+
+ /var/lib/clickhouse/
+
+ /var/lib/clickhouse/tmp/
+ /var/lib/clickhouse/user_files/
+
+
+
+
+ users.yaml
+
+
+ /var/lib/clickhouse/access/
+
+
+ default
+
+ default
+ true
+
+ false
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+
+
+ 127.0.0.1
+ 9000
+
+
+
+
+ 127.0.0.2
+ 9000
+
+
+
+
+
+ true
+
+ 127.0.0.1
+ 9000
+
+
+
+ true
+
+ 127.0.0.2
+ 9000
+
+
+
+
+
+
+ localhost
+ 9440
+ 1
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 1
+
+
+
+
+
+ 3600
+
+ 3600
+
+ 60
+
+
+ system
+
+
+ toYYYYMM(event_date)
+
+ 7500
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+ system
+
+ 7500
+ 1000
+
+
+
+ system
+
+
+ 60000
+
+
+
+
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+
+ system
+
+ 7500
+
+
+
+ system
+
+
+
+ 1000
+
+
+
+
+
+ *_dictionary.xml
+
+ /clickhouse/task_queue/ddl
+
+
+
+ click_cost
+ any
+
+ 0
+ 3600
+
+
+ 86400
+ 60
+
+
+
+ max
+
+ 0
+ 60
+
+
+ 3600
+ 300
+
+
+ 86400
+ 3600
+
+
+
+ /var/lib/clickhouse/format_schemas/
+
+
+ hide encrypt/decrypt arguments
+ ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)
+ \1(???)
+
+
+
+ false
+ false
+ https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml b/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/embedded.xml
@@ -0,0 +1,40 @@
+
+
+
+
+ trace
+ true
+
+
+ 8123
+ 9000
+ 9004
+
+ ./
+
+ 8589934592
+ 5368709120
+ true
+
+
+
+
+
+
+ ::/0
+
+
+ default
+ default
+ 1
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml
new file mode 100644
index 00000000000..b94e95bc043
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/users.d/allow_introspection_functions.xml
@@ -0,0 +1,8 @@
+
+
+
+
+ 1
+
+
+
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml b/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml
new file mode 100644
index 00000000000..88574e8f764
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/users.d/log_queries.yaml
@@ -0,0 +1,3 @@
+profiles:
+ default:
+ log_queries: 1
diff --git a/tests/integration/test_config_xml_yaml_mix/configs/users.yaml b/tests/integration/test_config_xml_yaml_mix/configs/users.yaml
new file mode 100644
index 00000000000..a87a8c82819
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/configs/users.yaml
@@ -0,0 +1,12 @@
+profiles:
+ default:
+ max_memory_usage: 10000000000
+ max_block_size: 64999
+users:
+ default:
+ password: ''
+ networks:
+ "@replace": replace
+ ip: '::/0'
+ profile: default
+
diff --git a/tests/integration/test_config_xml_yaml_mix/test.py b/tests/integration/test_config_xml_yaml_mix/test.py
new file mode 100644
index 00000000000..90ee8a2dea5
--- /dev/null
+++ b/tests/integration/test_config_xml_yaml_mix/test.py
@@ -0,0 +1,43 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_extra_yaml_mix():
+ # some configs are written in XML, others are written in YAML
+ cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml')
+
+ all_confd = ['configs/config.d/0_common_instance_config.yaml',
+ 'configs/config.d/access_control.yaml',
+ 'configs/config.d/keeper_port.xml',
+ 'configs/config.d/logging_no_rotate.xml',
+ 'configs/config.d/log_to_console.yaml',
+ 'configs/config.d/macros.yaml',
+ 'configs/config.d/metric_log.xml',
+ 'configs/config.d/more_clusters.yaml',
+ 'configs/config.d/part_log.xml',
+ 'configs/config.d/path.yaml',
+ 'configs/config.d/query_masking_rules.xml',
+ 'configs/config.d/tcp_with_proxy.yaml',
+ 'configs/config.d/test_cluster_with_incorrect_pw.xml',
+ 'configs/config.d/text_log.yaml',
+ 'configs/config.d/zookeeper.xml']
+
+ all_userd = ['configs/users.d/allow_introspection_functions.xml',
+ 'configs/users.d/log_queries.yaml']
+
+ node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd, with_zookeeper=False,
+ users_config_name="users.yaml", copy_common_configs=False)
+
+ try:
+ cluster.start()
+ assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+ assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+ finally:
+ cluster.shutdown()
diff --git a/tests/integration/test_config_yaml_full/__init__.py b/tests/integration/test_config_yaml_full/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml
new file mode 100644
index 00000000000..62e4ba8c744
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/0_common_instance_config.yaml
@@ -0,0 +1,6 @@
+timezone: Europe/Moscow
+listen_host: 0.0.0.0
+custom_settings_prefixes: custom_
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+users_config: users.yaml
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml b/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml
new file mode 100644
index 00000000000..ce2e23839ef
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/access_control.yaml
@@ -0,0 +1,7 @@
+user_directories:
+ users_xml:
+ path: users.yaml
+ local_directory:
+ path: access/
+ "@replace": replace
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml b/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml
new file mode 100644
index 00000000000..91723bc372f
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/keeper_port.yaml
@@ -0,0 +1,15 @@
+keeper_server:
+ tcp_port: 9181
+ server_id: 1
+ coordination_settings:
+ operation_timeout_ms: 10000
+ session_timeout_ms: 30000
+ force_sync: false
+ startup_timeout: 60000
+ reserved_log_items: 1000000000000000
+ raft_configuration:
+ server:
+ id: 1
+ hostname: localhost
+ port: 44444
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml b/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml
new file mode 100644
index 00000000000..7b59339800b
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/log_to_console.yaml
@@ -0,0 +1,7 @@
+logger:
+ console: true
+ log:
+ "@remove": remove
+ errorlog:
+ "@remove": remove
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml b/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml
new file mode 100644
index 00000000000..513cd3f7dc5
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/logging_no_rotate.yaml
@@ -0,0 +1,2 @@
+logger:
+ size: never
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml b/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml
new file mode 100644
index 00000000000..a9c61e270ab
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/macros.yaml
@@ -0,0 +1,7 @@
+macros:
+ test: 'Hello, world!'
+ shard: s1
+ replica: r1
+ default_path_test: '/clickhouse/tables/{database}/{shard}/'
+ default_name_test: 'table_{table}'
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml
new file mode 100644
index 00000000000..aafad939c37
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/metric_log.yaml
@@ -0,0 +1,6 @@
+metric_log:
+ database: system
+ table: metric_log
+ flush_interval_milliseconds: 7500
+ collect_interval_milliseconds: 1000
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml b/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml
new file mode 100644
index 00000000000..7da07190894
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/more_clusters.yaml
@@ -0,0 +1,23 @@
+remote_servers:
+ single_remote_shard_at_port_9001:
+ shard:
+ replica:
+ host: localhost
+ port: 9001
+ two_remote_shards_at_port_9001_9002:
+ shard:
+ - replica:
+ host: localhost
+ port: 9001
+ - replica:
+ host: localhost
+ port: 9002
+ two_shards_one_local_one_remote_at_port_9001:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 9001
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml
new file mode 100644
index 00000000000..43f60146dca
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/part_log.yaml
@@ -0,0 +1,5 @@
+part_log:
+ database: system
+ table: part_log
+ flush_interval_milliseconds: 7500
+
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/path.yaml b/tests/integration/test_config_yaml_full/configs/config.d/path.yaml
new file mode 100644
index 00000000000..3e26e8906ee
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/path.yaml
@@ -0,0 +1,18 @@
+path:
+ - ./
+ - "@replace": replace
+tmp_path:
+ - ./tmp/
+ - "@replace": replace
+user_files_path:
+ - ./user_files/
+ - "@replace": replace
+format_schema_path:
+ - ./format_schemas/
+ - "@replace": replace
+access_control_path:
+ - ./access/
+ - "@replace": replace
+top_level_domains_path:
+ - ./top_level_domains/
+ - "@replace": replace
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml b/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml
new file mode 100644
index 00000000000..38163429d81
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/query_masking_rules.yaml
@@ -0,0 +1,4 @@
+query_masking_rules:
+ rule:
+ regexp: TOPSECRET.TOPSECRET
+ replace: '[hidden]'
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml b/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml
new file mode 100644
index 00000000000..b0349f5a9b9
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/tcp_with_proxy.yaml
@@ -0,0 +1 @@
+tcp_with_proxy_port: 9010
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml b/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml
new file mode 100644
index 00000000000..309a7daa81d
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/test_cluster_with_incorrect_pw.yaml
@@ -0,0 +1,11 @@
+remote_servers:
+ test_cluster_with_incorrect_pw:
+ shard:
+ internal_replication: true
+ replica:
+ - host: 127.0.0.1
+ port: 9000
+ password: foo
+ - host: 127.0.0.2
+ port: 9000
+ password: foo
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml b/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml
new file mode 100644
index 00000000000..4d188020e37
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/text_log.yaml
@@ -0,0 +1,4 @@
+text_log:
+ database: system
+ table: text_log
+ flush_interval_milliseconds: 7500
diff --git a/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml b/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml
new file mode 100644
index 00000000000..be02c516798
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.d/zookeeper.yaml
@@ -0,0 +1,5 @@
+zookeeper:
+ node:
+ host: localhost
+ port: 9181
+ "@index": 1
diff --git a/tests/integration/test_config_yaml_full/configs/config.yaml b/tests/integration/test_config_yaml_full/configs/config.yaml
new file mode 100644
index 00000000000..619a3735269
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/config.yaml
@@ -0,0 +1,183 @@
+logger:
+ level: trace
+ log: /var/log/clickhouse-server/clickhouse-server.log
+ errorlog: /var/log/clickhouse-server/clickhouse-server.err.log
+ size: 1000M
+ count: 10
+http_port: 8123
+tcp_port: 9000
+mysql_port: 9004
+postgresql_port: 9005
+interserver_http_port: 9009
+max_connections: 4096
+keep_alive_timeout: 3
+grpc:
+ enable_ssl: false
+ ssl_cert_file: /path/to/ssl_cert_file
+ ssl_key_file: /path/to/ssl_key_file
+ ssl_require_client_auth: false
+ ssl_ca_cert_file: /path/to/ssl_ca_cert_file
+ compression: deflate
+ compression_level: medium
+ max_send_message_size: -1
+ max_receive_message_size: -1
+ verbose_logs: false
+openSSL:
+ server:
+ certificateFile: /etc/clickhouse-server/server.crt
+ privateKeyFile: /etc/clickhouse-server/server.key
+ dhParamsFile: /etc/clickhouse-server/dhparam.pem
+ verificationMode: none
+ loadDefaultCAFile: true
+ cacheSessions: true
+ disableProtocols: 'sslv2,sslv3'
+ preferServerCiphers: true
+ client:
+ loadDefaultCAFile: true
+ cacheSessions: true
+ disableProtocols: 'sslv2,sslv3'
+ preferServerCiphers: true
+ invalidCertificateHandler:
+ name: RejectCertificateHandler
+max_concurrent_queries: 100
+max_server_memory_usage: 0
+max_thread_pool_size: 10000
+max_server_memory_usage_to_ram_ratio: 0.9
+total_memory_profiler_step: 4194304
+total_memory_tracker_sample_probability: 0
+uncompressed_cache_size: 8589934592
+mark_cache_size: 5368709120
+mmap_cache_size: 1000
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+user_files_path: /var/lib/clickhouse/user_files/
+ldap_servers: ''
+user_directories:
+ users_xml:
+ path: users.yaml
+ local_directory:
+ path: /var/lib/clickhouse/access/
+default_profile: default
+custom_settings_prefixes: ''
+default_database: default
+mlock_executable: true
+remap_executable: false
+remote_servers:
+ test_shard_localhost:
+ shard:
+ replica:
+ host: localhost
+ port: 9000
+ test_cluster_two_shards_localhost:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 9000
+ test_cluster_two_shards:
+ shard:
+ - replica:
+ host: 127.0.0.1
+ port: 9000
+ - replica:
+ host: 127.0.0.2
+ port: 9000
+ test_cluster_two_shards_internal_replication:
+ shard:
+ - internal_replication: true
+ replica:
+ host: 127.0.0.1
+ port: 9000
+ - internal_replication: true
+ replica:
+ host: 127.0.0.2
+ port: 9000
+ test_shard_localhost_secure:
+ shard:
+ replica:
+ host: localhost
+ port: 9440
+ secure: 1
+ test_unavailable_shard:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 1
+builtin_dictionaries_reload_interval: 3600
+max_session_timeout: 3600
+default_session_timeout: 60
+query_log:
+ database: system
+ table: query_log
+ partition_by: toYYYYMM(event_date)
+ flush_interval_milliseconds: 7500
+trace_log:
+ database: system
+ table: trace_log
+ partition_by: toYYYYMM(event_date)
+ flush_interval_milliseconds: 7500
+query_thread_log:
+ database: system
+ table: query_thread_log
+ partition_by: toYYYYMM(event_date)
+ flush_interval_milliseconds: 7500
+metric_log:
+ database: system
+ table: metric_log
+ flush_interval_milliseconds: 7500
+ collect_interval_milliseconds: 1000
+asynchronous_metric_log:
+ database: system
+ table: asynchronous_metric_log
+ flush_interval_milliseconds: 60000
+opentelemetry_span_log:
+ engine: |-
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+ database: system
+ table: opentelemetry_span_log
+ flush_interval_milliseconds: 7500
+crash_log:
+ database: system
+ table: crash_log
+ partition_by: ''
+ flush_interval_milliseconds: 1000
+top_level_domains_lists: ''
+dictionaries_config: '*_dictionary.xml'
+distributed_ddl:
+ path: /clickhouse/task_queue/ddl
+graphite_rollup_example:
+ pattern:
+ regexp: click_cost
+ function: any
+ retention:
+ - age: 0
+ precision: 3600
+ - age: 86400
+ precision: 60
+ default:
+ function: max
+ retention:
+ - age: 0
+ precision: 60
+ - age: 3600
+ precision: 300
+ - age: 86400
+ precision: 3600
+format_schema_path: /var/lib/clickhouse/format_schemas/
+query_masking_rules:
+ rule:
+ name: hide encrypt/decrypt arguments
+ regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)'
+ replace: \1(???)
+send_crash_reports:
+ enabled: false
+ anonymize: false
+ endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
+
diff --git a/tests/integration/test_config_yaml_full/configs/embedded.xml b/tests/integration/test_config_yaml_full/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/embedded.xml
@@ -0,0 +1,40 @@
+
+
+
+
+ trace
+ true
+
+
+ 8123
+ 9000
+ 9004
+
+ ./
+
+ 8589934592
+ 5368709120
+ true
+
+
+
+
+
+
+ ::/0
+
+
+ default
+ default
+ 1
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml b/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml
new file mode 100644
index 00000000000..84612c198c9
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/users.d/allow_introspection_functions.yaml
@@ -0,0 +1,3 @@
+profiles:
+ default:
+ allow_introspection_functions: 1
diff --git a/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml b/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml
new file mode 100644
index 00000000000..88574e8f764
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/users.d/log_queries.yaml
@@ -0,0 +1,3 @@
+profiles:
+ default:
+ log_queries: 1
diff --git a/tests/integration/test_config_yaml_full/configs/users.yaml b/tests/integration/test_config_yaml_full/configs/users.yaml
new file mode 100644
index 00000000000..a87a8c82819
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/configs/users.yaml
@@ -0,0 +1,12 @@
+profiles:
+ default:
+ max_memory_usage: 10000000000
+ max_block_size: 64999
+users:
+ default:
+ password: ''
+ networks:
+ "@replace": replace
+ ip: '::/0'
+ profile: default
+
diff --git a/tests/integration/test_config_yaml_full/test.py b/tests/integration/test_config_yaml_full/test.py
new file mode 100644
index 00000000000..bc4fa40384c
--- /dev/null
+++ b/tests/integration/test_config_yaml_full/test.py
@@ -0,0 +1,42 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+def test_yaml_full_conf():
+ # all configs are in YAML
+ cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.yaml')
+
+ all_confd = ['configs/config.d/0_common_instance_config.yaml',
+ 'configs/config.d/access_control.yaml',
+ 'configs/config.d/keeper_port.yaml',
+ 'configs/config.d/logging_no_rotate.yaml',
+ 'configs/config.d/log_to_console.yaml',
+ 'configs/config.d/macros.yaml',
+ 'configs/config.d/metric_log.yaml',
+ 'configs/config.d/more_clusters.yaml',
+ 'configs/config.d/part_log.yaml',
+ 'configs/config.d/path.yaml',
+ 'configs/config.d/query_masking_rules.yaml',
+ 'configs/config.d/tcp_with_proxy.yaml',
+ 'configs/config.d/test_cluster_with_incorrect_pw.yaml',
+ 'configs/config.d/text_log.yaml',
+ 'configs/config.d/zookeeper.yaml']
+
+ all_userd = ['configs/users.d/allow_introspection_functions.yaml',
+ 'configs/users.d/log_queries.yaml']
+
+ node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd,
+ with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False)
+
+ try:
+ cluster.start()
+ assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+ assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+ finally:
+ cluster.shutdown()
diff --git a/tests/integration/test_config_yaml_main/__init__.py b/tests/integration/test_config_yaml_main/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml
new file mode 100644
index 00000000000..62e4ba8c744
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/0_common_instance_config.yaml
@@ -0,0 +1,6 @@
+timezone: Europe/Moscow
+listen_host: 0.0.0.0
+custom_settings_prefixes: custom_
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+users_config: users.yaml
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml b/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml
new file mode 100644
index 00000000000..b61f89bd904
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/access_control.xml
@@ -0,0 +1,13 @@
+
+
+
+
+
+ users.yaml
+
+
+
+ access/
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml b/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml
new file mode 100644
index 00000000000..b21df47bc85
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/keeper_port.xml
@@ -0,0 +1,23 @@
+
+
+ 9181
+ 1
+
+
+ 10000
+ 30000
+ false
+ 60000
+
+ 1000000000000000
+
+
+
+
+ 1
+ localhost
+ 44444
+
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml b/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml
new file mode 100644
index 00000000000..227c53647f3
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/log_to_console.xml
@@ -0,0 +1,7 @@
+
+
+ true
+
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml b/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml
new file mode 100644
index 00000000000..2c34585437b
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/logging_no_rotate.xml
@@ -0,0 +1,8 @@
+
+
+
+ never
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/macros.xml b/tests/integration/test_config_yaml_main/configs/config.d/macros.xml
new file mode 100644
index 00000000000..4902b12bc81
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/macros.xml
@@ -0,0 +1,9 @@
+
+
+ Hello, world!
+ s1
+ r1
+ /clickhouse/tables/{database}/{shard}/
+ table_{table}
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml
new file mode 100644
index 00000000000..0ca9f162416
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/metric_log.xml
@@ -0,0 +1,8 @@
+
+
+ system
+
+ 7500
+ 1000
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml b/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml
new file mode 100644
index 00000000000..aecbf9e0ba7
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/more_clusters.xml
@@ -0,0 +1,49 @@
+
+
+
+
+
+
+
+
+ localhost
+ 9001
+
+
+
+
+
+
+
+ localhost
+ 9001
+
+
+
+
+ localhost
+ 9002
+
+
+
+
+
+
+
+ localhost
+ 9000
+
+
+
+
+ localhost
+ 9001
+
+
+
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml
new file mode 100644
index 00000000000..6c6fc9c6982
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/part_log.xml
@@ -0,0 +1,8 @@
+
+
+ system
+
+
+ 7500
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/path.xml b/tests/integration/test_config_yaml_main/configs/config.d/path.xml
new file mode 100644
index 00000000000..466ed0d1663
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/path.xml
@@ -0,0 +1,8 @@
+
+ ./
+ ./tmp/
+ ./user_files/
+ ./format_schemas/
+ ./access/
+ ./top_level_domains/
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml b/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml
new file mode 100644
index 00000000000..5a854848f3d
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/query_masking_rules.xml
@@ -0,0 +1,10 @@
+
+
+
+
+
+ TOPSECRET.TOPSECRET
+ [hidden]
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml b/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml
new file mode 100644
index 00000000000..19046054c16
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/tcp_with_proxy.xml
@@ -0,0 +1,3 @@
+
+ 9010
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml b/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml
new file mode 100644
index 00000000000..109e35afc37
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/test_cluster_with_incorrect_pw.xml
@@ -0,0 +1,21 @@
+
+
+
+
+ true
+
+ 127.0.0.1
+ 9000
+
+ foo
+
+
+ 127.0.0.2
+ 9000
+
+ foo
+
+
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml b/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml
new file mode 100644
index 00000000000..3699a23578c
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/text_log.xml
@@ -0,0 +1,7 @@
+
+
+ system
+
+ 7500
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml b/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml
new file mode 100644
index 00000000000..06ed7fcd39f
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.d/zookeeper.xml
@@ -0,0 +1,8 @@
+
+
+
+ localhost
+ 9181
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/config.yaml b/tests/integration/test_config_yaml_main/configs/config.yaml
new file mode 100644
index 00000000000..e5a36b1e49b
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/config.yaml
@@ -0,0 +1,183 @@
+logger:
+ level: trace
+ log: /var/log/clickhouse-server/clickhouse-server.log
+ errorlog: /var/log/clickhouse-server/clickhouse-server.err.log
+ size: 1000M
+ count: 10
+http_port: 8123
+tcp_port: 9000
+mysql_port: 9004
+postgresql_port: 9005
+interserver_http_port: 9009
+max_connections: 4096
+keep_alive_timeout: 3
+grpc:
+ enable_ssl: false
+ ssl_cert_file: /path/to/ssl_cert_file
+ ssl_key_file: /path/to/ssl_key_file
+ ssl_require_client_auth: false
+ ssl_ca_cert_file: /path/to/ssl_ca_cert_file
+ compression: deflate
+ compression_level: medium
+ max_send_message_size: -1
+ max_receive_message_size: -1
+ verbose_logs: false
+openSSL:
+ server:
+ certificateFile: /etc/clickhouse-server/server.crt
+ privateKeyFile: /etc/clickhouse-server/server.key
+ dhParamsFile: /etc/clickhouse-server/dhparam.pem
+ verificationMode: none
+ loadDefaultCAFile: true
+ cacheSessions: true
+ disableProtocols: 'sslv2,sslv3'
+ preferServerCiphers: true
+ client:
+ loadDefaultCAFile: true
+ cacheSessions: true
+ disableProtocols: 'sslv2,sslv3'
+ preferServerCiphers: true
+ invalidCertificateHandler:
+ name: RejectCertificateHandler
+max_concurrent_queries: 100
+max_server_memory_usage: 0
+max_thread_pool_size: 10000
+max_server_memory_usage_to_ram_ratio: 0.9
+total_memory_profiler_step: 4194304
+total_memory_tracker_sample_probability: 0
+uncompressed_cache_size: 8589934592
+mark_cache_size: 5368709120
+mmap_cache_size: 1000
+path: /var/lib/clickhouse/
+tmp_path: /var/lib/clickhouse/tmp/
+user_files_path: /var/lib/clickhouse/user_files/
+ldap_servers: ''
+user_directories:
+ users_xml:
+ path: users.yml
+ local_directory:
+ path: /var/lib/clickhouse/access/
+default_profile: default
+custom_settings_prefixes: ''
+default_database: default
+mlock_executable: true
+remap_executable: false
+remote_servers:
+ test_shard_localhost:
+ shard:
+ replica:
+ host: localhost
+ port: 9000
+ test_cluster_two_shards_localhost:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 9000
+ test_cluster_two_shards:
+ shard:
+ - replica:
+ host: 127.0.0.1
+ port: 9000
+ - replica:
+ host: 127.0.0.2
+ port: 9000
+ test_cluster_two_shards_internal_replication:
+ shard:
+ - internal_replication: true
+ replica:
+ host: 127.0.0.1
+ port: 9000
+ - internal_replication: true
+ replica:
+ host: 127.0.0.2
+ port: 9000
+ test_shard_localhost_secure:
+ shard:
+ replica:
+ host: localhost
+ port: 9440
+ secure: 1
+ test_unavailable_shard:
+ shard:
+ - replica:
+ host: localhost
+ port: 9000
+ - replica:
+ host: localhost
+ port: 1
+builtin_dictionaries_reload_interval: 3600
+max_session_timeout: 3600
+default_session_timeout: 60
+query_log:
+ database: system
+ table: query_log
+ partition_by: toYYYYMM(event_date)
+ flush_interval_milliseconds: 7500
+trace_log:
+ database: system
+ table: trace_log
+ partition_by: toYYYYMM(event_date)
+ flush_interval_milliseconds: 7500
+query_thread_log:
+ database: system
+ table: query_thread_log
+ partition_by: toYYYYMM(event_date)
+ flush_interval_milliseconds: 7500
+metric_log:
+ database: system
+ table: metric_log
+ flush_interval_milliseconds: 7500
+ collect_interval_milliseconds: 1000
+asynchronous_metric_log:
+ database: system
+ table: asynchronous_metric_log
+ flush_interval_milliseconds: 60000
+opentelemetry_span_log:
+ engine: |-
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+ database: system
+ table: opentelemetry_span_log
+ flush_interval_milliseconds: 7500
+crash_log:
+ database: system
+ table: crash_log
+ partition_by: ''
+ flush_interval_milliseconds: 1000
+top_level_domains_lists: ''
+dictionaries_config: '*_dictionary.xml'
+distributed_ddl:
+ path: /clickhouse/task_queue/ddl
+graphite_rollup_example:
+ pattern:
+ regexp: click_cost
+ function: any
+ retention:
+ - age: 0
+ precision: 3600
+ - age: 86400
+ precision: 60
+ default:
+ function: max
+ retention:
+ - age: 0
+ precision: 60
+ - age: 3600
+ precision: 300
+ - age: 86400
+ precision: 3600
+format_schema_path: /var/lib/clickhouse/format_schemas/
+query_masking_rules:
+ rule:
+ name: hide encrypt/decrypt arguments
+ regexp: '((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:''(?:\\''|.)+''|.*?)\s*\)'
+ replace: \1(???)
+send_crash_reports:
+ enabled: false
+ anonymize: false
+ endpoint: 'https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277'
+
diff --git a/tests/integration/test_config_yaml_main/configs/embedded.xml b/tests/integration/test_config_yaml_main/configs/embedded.xml
new file mode 100644
index 00000000000..a66f57d1eb7
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/embedded.xml
@@ -0,0 +1,40 @@
+
+
+
+
+ trace
+ true
+
+
+ 8123
+ 9000
+ 9004
+
+ ./
+
+ 8589934592
+ 5368709120
+ true
+
+
+
+
+
+
+ ::/0
+
+
+ default
+ default
+ 1
+
+
+
+
+
+
+
+
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml b/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml
new file mode 100644
index 00000000000..b94e95bc043
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/users.d/allow_introspection_functions.xml
@@ -0,0 +1,8 @@
+
+
+
+
+ 1
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml b/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml
new file mode 100644
index 00000000000..25261072ade
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/users.d/log_queries.xml
@@ -0,0 +1,7 @@
+
+
+
+ 1
+
+
+
diff --git a/tests/integration/test_config_yaml_main/configs/users.yaml b/tests/integration/test_config_yaml_main/configs/users.yaml
new file mode 100644
index 00000000000..a87a8c82819
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/configs/users.yaml
@@ -0,0 +1,12 @@
+profiles:
+ default:
+ max_memory_usage: 10000000000
+ max_block_size: 64999
+users:
+ default:
+ password: ''
+ networks:
+ "@replace": replace
+ ip: '::/0'
+ profile: default
+
diff --git a/tests/integration/test_config_yaml_main/test.py b/tests/integration/test_config_yaml_main/test.py
new file mode 100644
index 00000000000..f4de16c35a2
--- /dev/null
+++ b/tests/integration/test_config_yaml_main/test.py
@@ -0,0 +1,43 @@
+import time
+import threading
+from os import path as p, unlink
+from tempfile import NamedTemporaryFile
+
+import helpers
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+def test_yaml_main_conf():
+ # main configs are in YAML; config.d and users.d are in XML
+ cluster = ClickHouseCluster(__file__, zookeeper_config_path='configs/config.d/zookeeper.xml')
+
+ all_confd = ['configs/config.d/0_common_instance_config.yaml',
+ 'configs/config.d/access_control.xml',
+ 'configs/config.d/keeper_port.xml',
+ 'configs/config.d/logging_no_rotate.xml',
+ 'configs/config.d/log_to_console.xml',
+ 'configs/config.d/macros.xml',
+ 'configs/config.d/metric_log.xml',
+ 'configs/config.d/more_clusters.xml',
+ 'configs/config.d/part_log.xml',
+ 'configs/config.d/path.xml',
+ 'configs/config.d/query_masking_rules.xml',
+ 'configs/config.d/tcp_with_proxy.xml',
+ 'configs/config.d/test_cluster_with_incorrect_pw.xml',
+ 'configs/config.d/text_log.xml',
+ 'configs/config.d/zookeeper.xml']
+
+ all_userd = ['configs/users.d/allow_introspection_functions.xml',
+ 'configs/users.d/log_queries.xml']
+
+ node = cluster.add_instance('node', base_config_dir='configs', main_configs=all_confd, user_configs=all_userd,
+ with_zookeeper=False, main_config_name="config.yaml", users_config_name="users.yaml", copy_common_configs=False)
+
+ try:
+ cluster.start()
+ assert(node.query("select value from system.settings where name = 'max_memory_usage'") == "10000000000\n")
+ assert(node.query("select value from system.settings where name = 'max_block_size'") == "64999\n")
+
+ finally:
+ cluster.shutdown()