mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge branch 'master' into patch-6
This commit is contained in:
mergify[bot]
GitHub
commit
8b9940f295
@ -1 +0,0 @@
|
||||
../../../en/faq/use-cases/time-series.md
|
||||
21
docs/zh/faq/use-cases/time-series.md
Normal file
21
docs/zh/faq/use-cases/time-series.md
Normal file
@ -0,0 +1,21 @@
|
||||
---
|
||||
title: 我能把 ClickHouse 当做时序数据库来使用吗?
|
||||
toc_hidden: true
|
||||
toc_priority: 101
|
||||
---
|
||||
|
||||
# 我能把 ClickHouse 当做时序数据库来使用吗? {#can-i-use-clickhouse-as-a-time-series-database}
|
||||
|
||||
ClickHouse是一个通用的数据存储解决方案[OLAP](../../faq/general/olap.md)的工作负载，而有许多专门的时间序列数据库管理系统。然而，ClickHouse的[专注于查询执行速度](../../faq/general/why-clickhouse-is-so-fast.md)使得它在许多情况下的性能优于专门的系统。关于这个话题有很多独立的基准，所以我们不打算在这里进行论述。相反，让我们将重点放在ClickHouse的重要功能(如果这是你的用例)上。
|
||||
|
||||
|
||||
|
||||
首先，有 **[specialized codecs](../../sql-reference/statements/create/table.md#create-query-specialized-codecs)**，这是典型的时间序列。无论是常见的算法，如“DoubleDelta”和“Gorilla”，或特定的ClickHouse 数据类型如“T64”。
|
||||
|
||||
|
||||
|
||||
其次，时间序列查询通常只访问最近的数据，比如一天或一周以前的数据。使用具有快速nVME/SSD驱动器和高容量HDD驱动器的服务器是有意义的。ClickHouse [TTL](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes)特性允许配置在快速硬盘上保持新鲜的热数据，并随着数据的老化逐渐移动到较慢的硬盘上。如果您的需求需要，也可以汇总或删除更旧的数据。
|
||||
|
||||
|
||||
|
||||
尽管这与ClickHouse存储和处理原始数据的理念相违背，但你可以使用[materialized views](../../sql-reference/statements/create/view.md)来适应更紧迫的延迟或成本需求。
|
||||
@ -330,8 +330,6 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
DB::ServerUUID::load(path + "/uuid", log);
|
||||
|
||||
const Settings & settings = global_context->getSettingsRef();
|
||||
|
||||
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
|
||||
|
||||
GlobalThreadPool::initialize(
|
||||
@ -377,8 +375,8 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
|
||||
{
|
||||
Poco::Net::ServerSocket socket;
|
||||
auto address = socketBindListen(socket, listen_host, port);
|
||||
socket.setReceiveTimeout(settings.receive_timeout);
|
||||
socket.setSendTimeout(settings.send_timeout);
|
||||
socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
|
||||
socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
|
||||
servers->emplace_back(
|
||||
listen_host,
|
||||
port_name,
|
||||
@ -393,8 +391,8 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
|
||||
#if USE_SSL
|
||||
Poco::Net::SecureServerSocket socket;
|
||||
auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
|
||||
socket.setReceiveTimeout(settings.receive_timeout);
|
||||
socket.setSendTimeout(settings.send_timeout);
|
||||
socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
|
||||
socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
|
||||
servers->emplace_back(
|
||||
listen_host,
|
||||
secure_port_name,
|
||||
|
||||
@ -967,6 +967,83 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
},
|
||||
/* already_loaded = */ false); /// Reload it right now (initial loading)
|
||||
|
||||
const auto listen_hosts = getListenHosts(config());
|
||||
const auto listen_try = getListenTry(config());
|
||||
|
||||
if (config().has("keeper_server"))
|
||||
{
|
||||
#if USE_NURAFT
|
||||
//// If we don't have configured connection probably someone trying to use clickhouse-server instead
|
||||
//// of clickhouse-keeper, so start synchronously.
|
||||
bool can_initialize_keeper_async = false;
|
||||
|
||||
if (has_zookeeper) /// We have configured connection to some zookeeper cluster
|
||||
{
|
||||
/// If we cannot connect to some other node from our cluster then we have to wait our Keeper start
|
||||
/// synchronously.
|
||||
can_initialize_keeper_async = global_context->tryCheckClientConnectionToMyKeeperCluster();
|
||||
}
|
||||
/// Initialize keeper RAFT.
|
||||
global_context->initializeKeeperDispatcher(can_initialize_keeper_async);
|
||||
FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher());
|
||||
|
||||
for (const auto & listen_host : listen_hosts)
|
||||
{
|
||||
/// TCP Keeper
|
||||
const char * port_name = "keeper_server.tcp_port";
|
||||
createServer(
|
||||
config(), listen_host, port_name, listen_try, /* start_server: */ false,
|
||||
servers_to_start_before_tables,
|
||||
[&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
Poco::Net::ServerSocket socket;
|
||||
auto address = socketBindListen(socket, listen_host, port);
|
||||
socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
|
||||
socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
|
||||
return ProtocolServerAdapter(
|
||||
listen_host,
|
||||
port_name,
|
||||
"Keeper (tcp): " + address.toString(),
|
||||
std::make_unique<TCPServer>(
|
||||
new KeeperTCPHandlerFactory(*this, false), server_pool, socket));
|
||||
});
|
||||
|
||||
const char * secure_port_name = "keeper_server.tcp_port_secure";
|
||||
createServer(
|
||||
config(), listen_host, secure_port_name, listen_try, /* start_server: */ false,
|
||||
servers_to_start_before_tables,
|
||||
[&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
#if USE_SSL
|
||||
Poco::Net::SecureServerSocket socket;
|
||||
auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
|
||||
socket.setReceiveTimeout(config().getUInt64("keeper_server.socket_receive_timeout_sec", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC));
|
||||
socket.setSendTimeout(config().getUInt64("keeper_server.socket_send_timeout_sec", DBMS_DEFAULT_SEND_TIMEOUT_SEC));
|
||||
return ProtocolServerAdapter(
|
||||
listen_host,
|
||||
secure_port_name,
|
||||
"Keeper with secure protocol (tcp_secure): " + address.toString(),
|
||||
std::make_unique<TCPServer>(
|
||||
new KeeperTCPHandlerFactory(*this, true), server_pool, socket));
|
||||
#else
|
||||
UNUSED(port);
|
||||
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
|
||||
ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
#endif
|
||||
});
|
||||
}
|
||||
#else
|
||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse server built without NuRaft library. Cannot use internal coordination.");
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
for (auto & server : servers_to_start_before_tables)
|
||||
{
|
||||
server.start();
|
||||
LOG_INFO(log, "Listening for {}", server.getDescription());
|
||||
}
|
||||
|
||||
auto & access_control = global_context->getAccessControl();
|
||||
if (config().has("custom_settings_prefixes"))
|
||||
access_control.setCustomSettingsPrefixes(config().getString("custom_settings_prefixes"));
|
||||
@ -1075,83 +1152,6 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
/// try set up encryption. There are some errors in config, error will be printed and server wouldn't start.
|
||||
CompressionCodecEncrypted::Configuration::instance().load(config(), "encryption_codecs");
|
||||
|
||||
const auto listen_hosts = getListenHosts(config());
|
||||
const auto listen_try = getListenTry(config());
|
||||
|
||||
if (config().has("keeper_server"))
|
||||
{
|
||||
#if USE_NURAFT
|
||||
//// If we don't have configured connection probably someone trying to use clickhouse-server instead
|
||||
//// of clickhouse-keeper, so start synchronously.
|
||||
bool can_initialize_keeper_async = false;
|
||||
|
||||
if (has_zookeeper) /// We have configured connection to some zookeeper cluster
|
||||
{
|
||||
/// If we cannot connect to some other node from our cluster then we have to wait our Keeper start
|
||||
/// synchronously.
|
||||
can_initialize_keeper_async = global_context->tryCheckClientConnectionToMyKeeperCluster();
|
||||
}
|
||||
/// Initialize keeper RAFT.
|
||||
global_context->initializeKeeperDispatcher(can_initialize_keeper_async);
|
||||
FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher());
|
||||
|
||||
for (const auto & listen_host : listen_hosts)
|
||||
{
|
||||
/// TCP Keeper
|
||||
const char * port_name = "keeper_server.tcp_port";
|
||||
createServer(
|
||||
config(), listen_host, port_name, listen_try, /* start_server: */ false,
|
||||
servers_to_start_before_tables,
|
||||
[&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
Poco::Net::ServerSocket socket;
|
||||
auto address = socketBindListen(socket, listen_host, port);
|
||||
socket.setReceiveTimeout(settings.receive_timeout);
|
||||
socket.setSendTimeout(settings.send_timeout);
|
||||
return ProtocolServerAdapter(
|
||||
listen_host,
|
||||
port_name,
|
||||
"Keeper (tcp): " + address.toString(),
|
||||
std::make_unique<TCPServer>(
|
||||
new KeeperTCPHandlerFactory(*this, false), server_pool, socket));
|
||||
});
|
||||
|
||||
const char * secure_port_name = "keeper_server.tcp_port_secure";
|
||||
createServer(
|
||||
config(), listen_host, secure_port_name, listen_try, /* start_server: */ false,
|
||||
servers_to_start_before_tables,
|
||||
[&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
#if USE_SSL
|
||||
Poco::Net::SecureServerSocket socket;
|
||||
auto address = socketBindListen(socket, listen_host, port, /* secure = */ true);
|
||||
socket.setReceiveTimeout(settings.receive_timeout);
|
||||
socket.setSendTimeout(settings.send_timeout);
|
||||
return ProtocolServerAdapter(
|
||||
listen_host,
|
||||
secure_port_name,
|
||||
"Keeper with secure protocol (tcp_secure): " + address.toString(),
|
||||
std::make_unique<TCPServer>(
|
||||
new KeeperTCPHandlerFactory(*this, true), server_pool, socket));
|
||||
#else
|
||||
UNUSED(port);
|
||||
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
|
||||
ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
#endif
|
||||
});
|
||||
}
|
||||
#else
|
||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "ClickHouse server built without NuRaft library. Cannot use internal coordination.");
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
for (auto & server : servers_to_start_before_tables)
|
||||
{
|
||||
server.start();
|
||||
LOG_INFO(log, "Listening for {}", server.getDescription());
|
||||
}
|
||||
|
||||
SCOPE_EXIT({
|
||||
/// Stop reloading of the main config. This must be done before `global_context->shutdown()` because
|
||||
/// otherwise the reloading may pass a changed config to some destroyed parts of ContextSharedPart.
|
||||
|
||||
@ -145,14 +145,14 @@ enum class AccessType
|
||||
M(SYSTEM_RELOAD_EMBEDDED_DICTIONARIES, "RELOAD EMBEDDED DICTIONARIES", GLOBAL, SYSTEM_RELOAD) /* implicitly enabled by the grant SYSTEM_RELOAD_DICTIONARY ON *.* */\
|
||||
M(SYSTEM_RELOAD, "", GROUP, SYSTEM) \
|
||||
M(SYSTEM_RESTART_DISK, "SYSTEM RESTART DISK", GLOBAL, SYSTEM) \
|
||||
M(SYSTEM_MERGES, "SYSTEM STOP MERGES, SYSTEM START MERGES, STOP_MERGES, START MERGES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_MERGES, "SYSTEM STOP MERGES, SYSTEM START MERGES, STOP MERGES, START MERGES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_TTL_MERGES, "SYSTEM STOP TTL MERGES, SYSTEM START TTL MERGES, STOP TTL MERGES, START TTL MERGES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_FETCHES, "SYSTEM STOP FETCHES, SYSTEM START FETCHES, STOP FETCHES, START FETCHES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_MOVES, "SYSTEM STOP MOVES, SYSTEM START MOVES, STOP MOVES, START MOVES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_DISTRIBUTED_SENDS, "SYSTEM STOP DISTRIBUTED SENDS, SYSTEM START DISTRIBUTED SENDS, STOP DISTRIBUTED SENDS, START DISTRIBUTED SENDS", TABLE, SYSTEM_SENDS) \
|
||||
M(SYSTEM_REPLICATED_SENDS, "SYSTEM STOP REPLICATED SENDS, SYSTEM START REPLICATED SENDS, STOP_REPLICATED_SENDS, START REPLICATED SENDS", TABLE, SYSTEM_SENDS) \
|
||||
M(SYSTEM_REPLICATED_SENDS, "SYSTEM STOP REPLICATED SENDS, SYSTEM START REPLICATED SENDS, STOP REPLICATED SENDS, START REPLICATED SENDS", TABLE, SYSTEM_SENDS) \
|
||||
M(SYSTEM_SENDS, "SYSTEM STOP SENDS, SYSTEM START SENDS, STOP SENDS, START SENDS", GROUP, SYSTEM) \
|
||||
M(SYSTEM_REPLICATION_QUEUES, "SYSTEM STOP REPLICATION QUEUES, SYSTEM START REPLICATION QUEUES, STOP_REPLICATION_QUEUES, START REPLICATION QUEUES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_REPLICATION_QUEUES, "SYSTEM STOP REPLICATION QUEUES, SYSTEM START REPLICATION QUEUES, STOP REPLICATION QUEUES, START REPLICATION QUEUES", TABLE, SYSTEM) \
|
||||
M(SYSTEM_DROP_REPLICA, "DROP REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_SYNC_REPLICA, "SYNC REPLICA", TABLE, SYSTEM) \
|
||||
M(SYSTEM_RESTART_REPLICA, "RESTART REPLICA", TABLE, SYSTEM) \
|
||||
|
||||
@ -141,12 +141,17 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type,
|
||||
auto manager = getContext()->getActionLocksManager();
|
||||
manager->cleanExpired();
|
||||
|
||||
auto access = getContext()->getAccess();
|
||||
auto required_access_type = getRequiredAccessType(action_type);
|
||||
|
||||
if (volume_ptr && action_type == ActionLocks::PartsMerge)
|
||||
{
|
||||
access->checkAccess(required_access_type);
|
||||
volume_ptr->setAvoidMergesUserOverride(!start);
|
||||
}
|
||||
else if (table_id)
|
||||
{
|
||||
access->checkAccess(required_access_type, table_id.database_name, table_id.table_name);
|
||||
auto table = DatabaseCatalog::instance().tryGetTable(table_id, getContext());
|
||||
if (table)
|
||||
{
|
||||
@ -161,7 +166,6 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type,
|
||||
}
|
||||
else
|
||||
{
|
||||
auto access = getContext()->getAccess();
|
||||
for (auto & elem : DatabaseCatalog::instance().getDatabases())
|
||||
{
|
||||
for (auto iterator = elem.second->getTablesIterator(getContext()); iterator->isValid(); iterator->next())
|
||||
@ -170,14 +174,9 @@ void InterpreterSystemQuery::startStopAction(StorageActionBlockType action_type,
|
||||
if (!table)
|
||||
continue;
|
||||
|
||||
if (!access->isGranted(getRequiredAccessType(action_type), elem.first, iterator->name()))
|
||||
if (!access->isGranted(required_access_type, elem.first, iterator->name()))
|
||||
{
|
||||
LOG_INFO(
|
||||
log,
|
||||
"Access {} denied, skipping {}.{}",
|
||||
toString(getRequiredAccessType(action_type)),
|
||||
elem.first,
|
||||
iterator->name());
|
||||
LOG_INFO(log, "Access {} denied, skipping {}.{}", toString(required_access_type), elem.first, iterator->name());
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -422,8 +421,7 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
restartReplicas(system_context);
|
||||
break;
|
||||
case Type::RESTART_REPLICA:
|
||||
if (!tryRestartReplica(table_id, system_context))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs());
|
||||
restartReplica(table_id, system_context);
|
||||
break;
|
||||
case Type::RESTORE_REPLICA:
|
||||
restoreReplica();
|
||||
@ -483,8 +481,6 @@ void InterpreterSystemQuery::restoreReplica()
|
||||
|
||||
StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica, ContextMutablePtr system_context, bool need_ddl_guard)
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_RESTART_REPLICA, replica);
|
||||
|
||||
auto table_ddl_guard = need_ddl_guard
|
||||
? DatabaseCatalog::instance().getDDLGuard(replica.getDatabaseName(), replica.getTableName())
|
||||
: nullptr;
|
||||
@ -529,15 +525,36 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica,
|
||||
return table;
|
||||
}
|
||||
|
||||
void InterpreterSystemQuery::restartReplica(const StorageID & replica, ContextMutablePtr system_context)
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_RESTART_REPLICA, replica);
|
||||
if (!tryRestartReplica(replica, system_context))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), replica.getNameForLogs());
|
||||
}
|
||||
|
||||
void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context)
|
||||
{
|
||||
std::vector<StorageID> replica_names;
|
||||
auto & catalog = DatabaseCatalog::instance();
|
||||
|
||||
auto access = getContext()->getAccess();
|
||||
bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_RESTART_REPLICA);
|
||||
|
||||
for (auto & elem : catalog.getDatabases())
|
||||
{
|
||||
for (auto it = elem.second->getTablesIterator(getContext()); it->isValid(); it->next())
|
||||
{
|
||||
if (dynamic_cast<const StorageReplicatedMergeTree *>(it->table().get()))
|
||||
{
|
||||
if (!access_is_granted_globally && !access->isGranted(AccessType::SYSTEM_RESTART_REPLICA, elem.first, it->name()))
|
||||
{
|
||||
LOG_INFO(log, "Access {} denied, skipping {}.{}", "SYSTEM RESTART REPLICA", elem.first, it->name());
|
||||
continue;
|
||||
}
|
||||
replica_names.emplace_back(it->databaseName(), it->name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (replica_names.empty())
|
||||
return;
|
||||
@ -583,14 +600,22 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query)
|
||||
}
|
||||
else if (query.is_drop_whole_replica)
|
||||
{
|
||||
getContext()->checkAccess(AccessType::SYSTEM_DROP_REPLICA);
|
||||
auto databases = DatabaseCatalog::instance().getDatabases();
|
||||
auto access = getContext()->getAccess();
|
||||
bool access_is_granted_globally = access->isGranted(AccessType::SYSTEM_DROP_REPLICA);
|
||||
|
||||
for (auto & elem : databases)
|
||||
{
|
||||
DatabasePtr & database = elem.second;
|
||||
for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next())
|
||||
{
|
||||
if (!access_is_granted_globally && !access->isGranted(AccessType::SYSTEM_DROP_REPLICA, elem.first, iterator->name()))
|
||||
{
|
||||
LOG_INFO(log, "Access {} denied, skipping {}.{}", "SYSTEM DROP REPLICA", elem.first, iterator->name());
|
||||
continue;
|
||||
}
|
||||
dropReplicaImpl(query, iterator->table());
|
||||
}
|
||||
LOG_TRACE(log, "Dropped replica {} from database {}", query.replica, backQuoteIfNeed(database->getDatabaseName()));
|
||||
}
|
||||
}
|
||||
|
||||
@ -47,6 +47,7 @@ private:
|
||||
/// Returns pointer to a newly created table if the restart was successful
|
||||
StoragePtr tryRestartReplica(const StorageID & replica, ContextMutablePtr context, bool need_ddl_guard = true);
|
||||
|
||||
void restartReplica(const StorageID & replica, ContextMutablePtr system_context);
|
||||
void restartReplicas(ContextMutablePtr system_context);
|
||||
void syncReplica(ASTSystemQuery & query);
|
||||
|
||||
|
||||
@ -0,0 +1 @@
|
||||
#!/usr/bin/env python3
|
||||
@ -0,0 +1,36 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<clickhouse>
|
||||
<keeper_server>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
|
||||
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>5000</operation_timeout_ms>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
<session_timeout_ms>10000</session_timeout_ms>
|
||||
</coordination_settings>
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<can_become_leader>true</can_become_leader>
|
||||
<hostname>node1</hostname>
|
||||
<id>1</id>
|
||||
<port>2888</port>
|
||||
<priority>1</priority>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
|
||||
<user_directories>
|
||||
<replicated>
|
||||
<zookeeper_path>/clickhouse/access</zookeeper_path>
|
||||
</replicated>
|
||||
</user_directories>
|
||||
|
||||
<zookeeper>
|
||||
<node index="1">
|
||||
<host>node1</host>
|
||||
<port>9181</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
</clickhouse>
|
||||
21
tests/integration/test_keeper_and_access_storage/test.py
Normal file
21
tests/integration/test_keeper_and_access_storage/test.py
Normal file
@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import pytest
|
||||
|
||||
from helpers.cluster import ClickHouseCluster
|
||||
|
||||
cluster = ClickHouseCluster(__file__)
|
||||
|
||||
node1 = cluster.add_instance('node1', main_configs=['configs/keeper.xml'], stay_alive=True)
|
||||
|
||||
# test that server is able to start
|
||||
@pytest.fixture(scope="module")
|
||||
def started_cluster():
|
||||
try:
|
||||
cluster.start()
|
||||
yield cluster
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
def test_create_replicated(started_cluster):
|
||||
assert node1.query("SELECT 1") == "1\n"
|
||||
@ -1,11 +1,14 @@
|
||||
-- Tags: no-parallel
|
||||
-- ^^^^^^^^^^^ otherwise you may hit TOO_DEEP_RECURSION error during querying system.columns
|
||||
|
||||
DROP TABLE IF EXISTS merge1;
|
||||
DROP TABLE IF EXISTS merge2;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS merge1 (x UInt64) ENGINE = Merge(currentDatabase(), '^merge\\d$');
|
||||
CREATE TABLE IF NOT EXISTS merge2 (x UInt64) ENGINE = Merge(currentDatabase(), '^merge\\d$');
|
||||
|
||||
SELECT * FROM merge1; -- { serverError 306 }
|
||||
SELECT * FROM merge2; -- { serverError 306 }
|
||||
SELECT * FROM merge1; -- { serverError TOO_DEEP_RECURSION }
|
||||
SELECT * FROM merge2; -- { serverError TOO_DEEP_RECURSION }
|
||||
|
||||
DROP TABLE merge1;
|
||||
DROP TABLE merge2;
|
||||
|
||||
@ -99,14 +99,14 @@ SYSTEM RELOAD FUNCTION ['SYSTEM RELOAD FUNCTIONS','RELOAD FUNCTION','RELOAD FUNC
|
||||
SYSTEM RELOAD EMBEDDED DICTIONARIES ['RELOAD EMBEDDED DICTIONARIES'] GLOBAL SYSTEM RELOAD
|
||||
SYSTEM RELOAD [] \N SYSTEM
|
||||
SYSTEM RESTART DISK ['SYSTEM RESTART DISK'] GLOBAL SYSTEM
|
||||
SYSTEM MERGES ['SYSTEM STOP MERGES','SYSTEM START MERGES','STOP_MERGES','START MERGES'] TABLE SYSTEM
|
||||
SYSTEM MERGES ['SYSTEM STOP MERGES','SYSTEM START MERGES','STOP MERGES','START MERGES'] TABLE SYSTEM
|
||||
SYSTEM TTL MERGES ['SYSTEM STOP TTL MERGES','SYSTEM START TTL MERGES','STOP TTL MERGES','START TTL MERGES'] TABLE SYSTEM
|
||||
SYSTEM FETCHES ['SYSTEM STOP FETCHES','SYSTEM START FETCHES','STOP FETCHES','START FETCHES'] TABLE SYSTEM
|
||||
SYSTEM MOVES ['SYSTEM STOP MOVES','SYSTEM START MOVES','STOP MOVES','START MOVES'] TABLE SYSTEM
|
||||
SYSTEM DISTRIBUTED SENDS ['SYSTEM STOP DISTRIBUTED SENDS','SYSTEM START DISTRIBUTED SENDS','STOP DISTRIBUTED SENDS','START DISTRIBUTED SENDS'] TABLE SYSTEM SENDS
|
||||
SYSTEM REPLICATED SENDS ['SYSTEM STOP REPLICATED SENDS','SYSTEM START REPLICATED SENDS','STOP_REPLICATED_SENDS','START REPLICATED SENDS'] TABLE SYSTEM SENDS
|
||||
SYSTEM REPLICATED SENDS ['SYSTEM STOP REPLICATED SENDS','SYSTEM START REPLICATED SENDS','STOP REPLICATED SENDS','START REPLICATED SENDS'] TABLE SYSTEM SENDS
|
||||
SYSTEM SENDS ['SYSTEM STOP SENDS','SYSTEM START SENDS','STOP SENDS','START SENDS'] \N SYSTEM
|
||||
SYSTEM REPLICATION QUEUES ['SYSTEM STOP REPLICATION QUEUES','SYSTEM START REPLICATION QUEUES','STOP_REPLICATION_QUEUES','START REPLICATION QUEUES'] TABLE SYSTEM
|
||||
SYSTEM REPLICATION QUEUES ['SYSTEM STOP REPLICATION QUEUES','SYSTEM START REPLICATION QUEUES','STOP REPLICATION QUEUES','START REPLICATION QUEUES'] TABLE SYSTEM
|
||||
SYSTEM DROP REPLICA ['DROP REPLICA'] TABLE SYSTEM
|
||||
SYSTEM SYNC REPLICA ['SYNC REPLICA'] TABLE SYSTEM
|
||||
SYSTEM RESTART REPLICA ['RESTART REPLICA'] TABLE SYSTEM
|
||||
|
||||
@ -27,7 +27,6 @@ issue_17653 = "https://github.com/ClickHouse/ClickHouse/issues/17653"
|
||||
issue_17655 = "https://github.com/ClickHouse/ClickHouse/issues/17655"
|
||||
issue_17766 = "https://github.com/ClickHouse/ClickHouse/issues/17766"
|
||||
issue_18110 = "https://github.com/ClickHouse/ClickHouse/issues/18110"
|
||||
issue_18206 = "https://github.com/ClickHouse/ClickHouse/issues/18206"
|
||||
issue_21083 = "https://github.com/ClickHouse/ClickHouse/issues/21083"
|
||||
issue_21084 = "https://github.com/ClickHouse/ClickHouse/issues/21084"
|
||||
issue_25413 = "https://github.com/ClickHouse/ClickHouse/issues/25413"
|
||||
@ -122,20 +121,6 @@ xfails = {
|
||||
[(Fail, issue_17655)],
|
||||
"privileges/public tables/sensitive tables":
|
||||
[(Fail, issue_18110)],
|
||||
"privileges/system merges/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/system ttl merges/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/system moves/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/system sends/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/system fetches/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/system restart replica/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/system replication queues/:/:/:/:/SYSTEM:":
|
||||
[(Fail, issue_18206)],
|
||||
"privileges/: row policy/nested live:":
|
||||
[(Fail, issue_21083)],
|
||||
"privileges/: row policy/nested mat:":
|
||||
|
||||
288
utils/c++expr
Executable file
288
utils/c++expr
Executable file
@ -0,0 +1,288 @@
|
||||
#!/usr/bin/env bash
|
||||
set -e
|
||||
|
||||
usage() {
|
||||
cat <<EOF >&2
|
||||
USAGE: c++expr [-c CXX | -C | -I] [-i INCLUDE] [-b STEPS] [-t TESTS] [-o FILE] [-O CXX_OPTS...] [-g 'GLOBAL CODE'] 'MAIN CODE'
|
||||
OPTIONS:
|
||||
-c CXX use specified c++ compiler
|
||||
-C use cmake
|
||||
-I integrate into ClickHouse build tree in current directory
|
||||
-i INC add #include <INC>
|
||||
-l LIB link against LIB (only for -I or -C)
|
||||
-b STEPS_NUM make program to benchmark specified code snippet and run tests with STEPS_NUM each
|
||||
-b perf-top run infinite benchmark and show perf top
|
||||
-t TESTS_NUM make program to benchmark specified code snippet and run TESTS_NUM tests
|
||||
-o FILE do not run, just save binary executable file
|
||||
-O CXX_OPTS forward option compiler (e.g. -O "-O3 -std=c++20")
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
SOURCE_FILE=main.cpp
|
||||
GLOBAL=
|
||||
OUTPUT_EXECUTABLE=
|
||||
INCS="vector iostream typeinfo cstdlib cmath sys/time.h"
|
||||
LIBS=""
|
||||
BENCHMARK_STEPS=0
|
||||
RUN_PERFTOP=
|
||||
BENCHMARK_TESTS=5
|
||||
USE_CMAKE=
|
||||
USE_CLICKHOUSE=
|
||||
CXX=g++
|
||||
CXX_OPTS=
|
||||
CMD_PARAMS=
|
||||
|
||||
#
|
||||
# Parse command line
|
||||
#
|
||||
|
||||
if [ "$1" == "--help" ]; then usage; fi
|
||||
while getopts "vc:CIi:l:b:t:o:O:g:" OPT; do
|
||||
case "$OPT" in
|
||||
v) set -x; ;;
|
||||
c) CXX="$OPTARG"; ;;
|
||||
C) USE_CMAKE=y; ;;
|
||||
I) USE_CLICKHOUSE=y; LIBS="$LIBS clickhouse_common_io"; ;;
|
||||
i) INCS="$INCS $OPTARG"; ;;
|
||||
l) LIBS="$LIBS $OPTARG"; ;;
|
||||
b) if [ "$OPTARG" = perf-top ]; then BENCHMARK_STEPS=-1; RUN_PERFTOP=y; else BENCHMARK_STEPS="$OPTARG"; fi; ;;
|
||||
t) BENCHMARK_TESTS="$OPTARG"; ;;
|
||||
o) OUTPUT_EXECUTABLE="$OPTARG"; ;;
|
||||
O) CXX_OPTS="$CXX_OPTS $OPTARG"; ;;
|
||||
g) GLOBAL="$OPTARG"; ;;
|
||||
esac
|
||||
done
|
||||
shift $(( $OPTIND - 1 ))
|
||||
|
||||
#
|
||||
# Positional arguments
|
||||
#
|
||||
|
||||
EXPR=$1
|
||||
shift
|
||||
|
||||
if [ -z "$EXPR" ]; then usage; fi
|
||||
|
||||
#
|
||||
# Arguments forwarded to program should go after main code and before --
|
||||
#
|
||||
|
||||
while [ -n "$1" ] && [ "$1" != "--" ]; do
|
||||
CMD_PARAMS="$CMD_PARAMS $1"
|
||||
shift
|
||||
done
|
||||
if [ "$1" == "--" ]; then shift; fi
|
||||
|
||||
#
|
||||
# Setup workdir
|
||||
#
|
||||
|
||||
find_clickhouse_root () {
|
||||
local DIR="`pwd`"
|
||||
while [ $DIR != "/" ]; do
|
||||
if [ ! -e "$DIR/CMakeLists.txt" ]; then
|
||||
echo "error: $DIR has no CMakeLists.txt"
|
||||
return 1
|
||||
fi
|
||||
if grep "project(ClickHouse)" "$DIR/CMakeLists.txt" >/dev/null 2>&1; then
|
||||
echo $DIR
|
||||
return 0
|
||||
fi
|
||||
DIR="`dirname $DIR`"
|
||||
done
|
||||
echo "error: unable to find Clickhouse root folder"
|
||||
return 1
|
||||
}
|
||||
|
||||
find_clickhouse_build () {
|
||||
local CLICKHOUSE_ROOT="`find_clickhouse_root`"
|
||||
if [ -e "$CLICKHOUSE_ROOT/build/CMakeCache.txt" ]; then
|
||||
echo "$CLICKHOUSE_ROOT/build"
|
||||
return 0
|
||||
fi
|
||||
echo "error: $CLICKHOUSE_ROOT/build/CMakeCache.txt doesn't exist"
|
||||
return 1
|
||||
}
|
||||
|
||||
CALL_DIR=`pwd`
|
||||
EXECUTABLE=cppexpr_$$
|
||||
EXECUTABLE_DIR=.
|
||||
|
||||
if [ -n "$USE_CLICKHOUSE" ]; then
|
||||
SUBDIR=cppexpr_$$
|
||||
WORKDIR=$CALL_DIR/$SUBDIR
|
||||
if [ ! -e $CALL_DIR/CMakeLists.txt ]; then
|
||||
echo "error: $CALL_DIR/CMakeLists.txt is required for integration" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
CLICKHOUSE_ROOT="`find_clickhouse_root`"
|
||||
BUILD_ROOT="`find_clickhouse_build`"
|
||||
CLICKHOUSE_PATH="${WORKDIR/$CLICKHOUSE_ROOT}"
|
||||
EXECUTABLE_DIR="${BUILD_ROOT}${CLICKHOUSE_PATH}"
|
||||
|
||||
if [ -z "$CLICKHOUSE_ROOT" ] || [ -z "$BUILD_ROOT" ] || [ -z "$CLICKHOUSE_PATH" ]; then
|
||||
echo "error: unable to locate ClickHouse" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cp $CALL_DIR/CMakeLists.txt $CALL_DIR/CMakeLists.txt.backup.$$
|
||||
echo "add_subdirectory ($SUBDIR)" >>$CALL_DIR/CMakeLists.txt
|
||||
cleanup() {
|
||||
mv $CALL_DIR/CMakeLists.txt.backup.$$ $CALL_DIR/CMakeLists.txt
|
||||
rm -rf $WORKDIR
|
||||
rm -rf ${BUILD_ROOT}${CLICKHOUSE_PATH}
|
||||
}
|
||||
else
|
||||
WORKDIR=/var/tmp/cppexpr_$$
|
||||
cleanup() {
|
||||
rm -rf $WORKDIR
|
||||
}
|
||||
fi
|
||||
|
||||
mkdir -p $WORKDIR
|
||||
cd $WORKDIR
|
||||
|
||||
#
|
||||
# Generate CMakeLists.txt
|
||||
#
|
||||
if [ -n "$USE_CMAKE" ]; then
|
||||
cat <<EOF >>CMakeLists.txt
|
||||
project(CppExpr)
|
||||
SET(PROJECT_NAME CppExpr)
|
||||
SET(CMAKE_INCLUDE_CURRENT_DIR TRUE)
|
||||
cmake_minimum_required(VERSION 2.8)
|
||||
set(CMAKE_CXX_FLAGS -fPIC)
|
||||
set(CMAKE_C_FLAGS -fPIC)
|
||||
set(CMAKE_BUILD_TYPE Release)
|
||||
set(SOURCES $SOURCE_FILE)
|
||||
add_executable($EXECUTABLE \${SOURCES})
|
||||
EOF
|
||||
fi
|
||||
|
||||
#
|
||||
# Generate CMakeLists.txt for integration
|
||||
#
|
||||
if [ -n "$USE_CLICKHOUSE" ]; then
|
||||
cat <<EOF >>CMakeLists.txt
|
||||
add_executable($EXECUTABLE $SOURCE_FILE)
|
||||
EOF
|
||||
fi
|
||||
|
||||
#
|
||||
# Add libraries to CMakeLists.txt
|
||||
#
|
||||
if [ -n "$LIBS" ]; then
|
||||
cat <<EOF >>CMakeLists.txt
|
||||
target_link_libraries($EXECUTABLE PRIVATE $LIBS)
|
||||
EOF
|
||||
fi
|
||||
|
||||
#
|
||||
# Generate source code
|
||||
#
|
||||
>$SOURCE_FILE
|
||||
for INC in $INCS; do
|
||||
echo "#include <$INC>" >> $SOURCE_FILE
|
||||
done
|
||||
cat <<EOF >>$SOURCE_FILE
|
||||
|
||||
#define OUT(expr) std::cout << #expr << " -> " << (expr) << std::endl;
|
||||
size_t max_tests = $BENCHMARK_TESTS;
|
||||
size_t max_steps = $BENCHMARK_STEPS;
|
||||
$GLOBAL
|
||||
int main(int argc, char** argv) {
|
||||
(void)argc; (void)argv;
|
||||
try {
|
||||
EOF
|
||||
|
||||
if [ $BENCHMARK_STEPS -eq 0 ]; then
|
||||
cat <<EOF >>$SOURCE_FILE
|
||||
$EXPR
|
||||
EOF
|
||||
else
|
||||
cat <<EOF >>$SOURCE_FILE
|
||||
std::cout << "Steps per test: " << max_steps << std::endl;
|
||||
if (max_steps == 0) max_steps = 1;
|
||||
double total = 0.0;
|
||||
for (size_t test = 0; test < max_tests; test++) {
|
||||
timeval beg, end;
|
||||
gettimeofday(&beg, nullptr);
|
||||
for (size_t step = 0; step < max_steps; step++) {
|
||||
asm volatile("" ::: "memory");
|
||||
$EXPR
|
||||
}
|
||||
gettimeofday(&end, nullptr);
|
||||
double interval = (end.tv_sec - beg.tv_sec)*1e6 + (end.tv_usec - beg.tv_usec);
|
||||
std::cout << "Test #" << test << ": " << interval / max_steps << " us\t" << max_steps * 1e6 / interval << " sps" << std::endl;
|
||||
total += interval;
|
||||
}
|
||||
std::cout << "Average: " << total / max_tests / max_steps << " us\t" << max_steps * 1e6 / (total / max_tests) << " sps" << std::endl;
|
||||
EOF
|
||||
fi
|
||||
|
||||
cat <<EOF >>$SOURCE_FILE
|
||||
return 0;
|
||||
} catch (std::exception& e) {
|
||||
std::cerr << "unhandled exception (" << typeid(e).name() << "):" << e.what() << std::endl;
|
||||
} catch (...) {
|
||||
std::cerr << "unknown unhandled exception\n";
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
#ifdef OUT
|
||||
#undef OUT
|
||||
#endif
|
||||
EOF
|
||||
|
||||
#
|
||||
# Compile
|
||||
#
|
||||
if [ -n "$USE_CMAKE" ]; then
|
||||
if ! (cmake . && make); then
|
||||
cat -n $SOURCE_FILE
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
elif [ -n "$USE_CLICKHOUSE" ]; then
|
||||
if ! (cd $BUILD_ROOT && ninja $EXECUTABLE) >stdout.log 2>stderr.log; then
|
||||
cat stdout.log
|
||||
cat stderr.log >&2
|
||||
cat -n $SOURCE_FILE
|
||||
cleanup
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
RET=0
|
||||
$CXX $CXX_OPTS -I$CALL_DIR -o $EXECUTABLE $SOURCE_FILE || RET=$?
|
||||
if [ $RET -ne 0 ]; then
|
||||
cat -n $SOURCE_FILE
|
||||
cleanup
|
||||
exit $RET
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Execute
|
||||
#
|
||||
RET=0
|
||||
if [ -z "$OUTPUT_EXECUTABLE" ]; then
|
||||
if [ -z "$RUN_PERFTOP" ]; then
|
||||
"$@" $EXECUTABLE_DIR/$EXECUTABLE $CMD_PARAMS || RET=$?
|
||||
else
|
||||
"$@" $EXECUTABLE_DIR/$EXECUTABLE $CMD_PARAMS &
|
||||
PID=$!
|
||||
perf top -p $PID
|
||||
kill $PID
|
||||
fi
|
||||
else
|
||||
cp $EXECUTABLE_DIR/$EXECUTABLE $CALL_DIR/$OUTPUT_EXECUTABLE
|
||||
fi
|
||||
|
||||
#
|
||||
# Cleanup
|
||||
#
|
||||
cleanup
|
||||
echo "Exit code: $RET"
|
||||
exit $RET
|
||||
@ -184,7 +184,9 @@ tables_with_database_column=(
|
||||
tests_with_database_column=( $(
|
||||
find $ROOT_PATH/tests/queries -iname '*.sql' -or -iname '*.sh' -or -iname '*.py' -or -iname '*.j2' |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
xargs grep --with-filename $(printf -- "-e %s " "${tables_with_database_column[@]}") | cut -d: -f1 | sort -u
|
||||
xargs grep --with-filename $(printf -- "-e %s " "${tables_with_database_column[@]}") |
|
||||
grep -v -e ':--' -e ':#' |
|
||||
cut -d: -f1 | sort -u
|
||||
) )
|
||||
for test_case in "${tests_with_database_column[@]}"; do
|
||||
grep -qE database.*currentDatabase "$test_case" || {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user