mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Merge pull request #21690 from ClickHouse/test_multiple_nodes
Run three nodes with Replicated database and NuKeeper in CI
This commit is contained in:
tavplubix
GitHub
commit
6c1e2a0d87
@ -3,7 +3,7 @@
|
||||
<mysql_port remove="remove"/>
|
||||
<interserver_http_port remove="remove"/>
|
||||
<tcp_with_proxy_port remove="remove"/>
|
||||
<test_keeper_server remove="remove"/>
|
||||
<keeper_server remove="remove"/>
|
||||
<listen_host>::</listen_host>
|
||||
|
||||
<logger>
|
||||
|
||||
@ -13,6 +13,25 @@ dpkg -i package_folder/clickhouse-test_*.deb
|
||||
|
||||
function start()
|
||||
{
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
# NOTE We run "clickhouse server" instead of "clickhouse-server"
|
||||
# to make "pidof clickhouse-server" return single pid of the main instance.
|
||||
# We wil run main instance using "service clickhouse-server start"
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
|
||||
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
|
||||
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
||||
--mysql_port 19004 \
|
||||
--keeper_server.tcp_port 19181 --keeper_server.server_id 2
|
||||
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
|
||||
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
|
||||
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
|
||||
--mysql_port 29004 \
|
||||
--keeper_server.tcp_port 29181 --keeper_server.server_id 3
|
||||
fi
|
||||
|
||||
counter=0
|
||||
until clickhouse-client --query "SELECT 1"
|
||||
do
|
||||
@ -35,9 +54,8 @@ start
|
||||
/s3downloader --dataset-names $DATASETS
|
||||
chmod 777 -R /var/lib/clickhouse
|
||||
clickhouse-client --query "SHOW DATABASES"
|
||||
clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
|
||||
clickhouse-client --query "CREATE DATABASE test"
|
||||
|
||||
clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
|
||||
service clickhouse-server restart
|
||||
|
||||
# Wait for server to start accepting connections
|
||||
@ -47,15 +65,36 @@ for _ in {1..120}; do
|
||||
done
|
||||
|
||||
clickhouse-client --query "SHOW TABLES FROM datasets"
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
clickhouse-client --query "CREATE DATABASE test ON CLUSTER 'test_cluster_database_replicated'
|
||||
ENGINE=Replicated('/test/clickhouse/db/test', '{shard}', '{replica}')"
|
||||
|
||||
clickhouse-client --query "CREATE TABLE test.hits AS datasets.hits_v1"
|
||||
clickhouse-client --query "CREATE TABLE test.visits AS datasets.visits_v1"
|
||||
|
||||
clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1"
|
||||
clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1"
|
||||
|
||||
clickhouse-client --query "DROP TABLE datasets.hits_v1"
|
||||
clickhouse-client --query "DROP TABLE datasets.visits_v1"
|
||||
|
||||
MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours)
|
||||
MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited)
|
||||
else
|
||||
clickhouse-client --query "CREATE DATABASE test"
|
||||
clickhouse-client --query "SHOW TABLES FROM test"
|
||||
clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
|
||||
clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
|
||||
clickhouse-client --query "SHOW TABLES FROM test"
|
||||
|
||||
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test ; then
|
||||
SKIP_LIST_OPT="--use-skip-list"
|
||||
fi
|
||||
|
||||
clickhouse-client --query "SHOW TABLES FROM test"
|
||||
clickhouse-client --query "SELECT count() FROM test.hits"
|
||||
clickhouse-client --query "SELECT count() FROM test.visits"
|
||||
|
||||
function run_tests()
|
||||
{
|
||||
set -x
|
||||
# We can have several additional options so we path them as array because it's
|
||||
# more idiologically correct.
|
||||
read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
|
||||
@ -64,7 +103,12 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
|
||||
ADDITIONAL_OPTIONS+=('--replicated-database')
|
||||
fi
|
||||
|
||||
clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --print-time "$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" "$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
clickhouse-test --testname --shard --zookeeper --no-stateless --hung-check --use-skip-list --print-time "${ADDITIONAL_OPTIONS[@]}" \
|
||||
"$SKIP_TESTS_OPTION" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee test_output/test_result.txt
|
||||
}
|
||||
|
||||
export -f run_tests
|
||||
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
|
||||
|
||||
./process_functional_tests_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
|
||||
|
||||
@ -73,3 +117,9 @@ mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
|
||||
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
|
||||
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
|
||||
fi
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
|
||||
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
|
||||
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
|
||||
fi
|
||||
|
||||
@ -12,6 +12,8 @@ UNKNOWN_SIGN = "[ UNKNOWN "
|
||||
SKIPPED_SIGN = "[ SKIPPED "
|
||||
HUNG_SIGN = "Found hung queries in processlist"
|
||||
|
||||
NO_TASK_TIMEOUT_SIGN = "All tests have finished"
|
||||
|
||||
def process_test_log(log_path):
|
||||
total = 0
|
||||
skipped = 0
|
||||
@ -19,10 +21,13 @@ def process_test_log(log_path):
|
||||
failed = 0
|
||||
success = 0
|
||||
hung = False
|
||||
task_timeout = True
|
||||
test_results = []
|
||||
with open(log_path, 'r') as test_file:
|
||||
for line in test_file:
|
||||
line = line.strip()
|
||||
if NO_TASK_TIMEOUT_SIGN in line:
|
||||
task_timeout = False
|
||||
if HUNG_SIGN in line:
|
||||
hung = True
|
||||
if any(sign in line for sign in (OK_SIGN, FAIL_SING, UNKNOWN_SIGN, SKIPPED_SIGN)):
|
||||
@ -52,7 +57,7 @@ def process_test_log(log_path):
|
||||
else:
|
||||
success += int(OK_SIGN in line)
|
||||
test_results.append((test_name, "OK", test_time))
|
||||
return total, skipped, unknown, failed, success, hung, test_results
|
||||
return total, skipped, unknown, failed, success, hung, task_timeout, test_results
|
||||
|
||||
def process_result(result_path):
|
||||
test_results = []
|
||||
@ -68,7 +73,7 @@ def process_result(result_path):
|
||||
state = "error"
|
||||
|
||||
if result_path and os.path.exists(result_path):
|
||||
total, skipped, unknown, failed, success, hung, test_results = process_test_log(result_path)
|
||||
total, skipped, unknown, failed, success, hung, task_timeout, test_results = process_test_log(result_path)
|
||||
is_flacky_check = 1 < int(os.environ.get('NUM_TRIES', 1))
|
||||
# If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately)
|
||||
# But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped.
|
||||
@ -78,6 +83,9 @@ def process_result(result_path):
|
||||
if hung:
|
||||
description = "Some queries hung, "
|
||||
state = "failure"
|
||||
elif task_timeout:
|
||||
description = "Timeout, "
|
||||
state = "failure"
|
||||
else:
|
||||
description = ""
|
||||
|
||||
|
||||
@ -34,17 +34,37 @@ if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
|
||||
# simpliest way to forward env variables to server
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon
|
||||
sleep 5
|
||||
else
|
||||
service clickhouse-server start && sleep 5
|
||||
service clickhouse-server start
|
||||
fi
|
||||
|
||||
if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
|
||||
SKIP_LIST_OPT="--use-skip-list"
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
|
||||
-- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
|
||||
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
|
||||
--mysql_port 19004 \
|
||||
--keeper_server.tcp_port 19181 --keeper_server.server_id 2 \
|
||||
--macros.replica r2 # It doesn't work :(
|
||||
|
||||
sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server2/config.xml --daemon \
|
||||
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
|
||||
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
|
||||
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
|
||||
--mysql_port 29004 \
|
||||
--keeper_server.tcp_port 29181 --keeper_server.server_id 3 \
|
||||
--macros.shard s2 # It doesn't work :(
|
||||
|
||||
MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000)) # min(MAX_RUN_TIME, 2.5 hours)
|
||||
MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000)) # set to 2.5 hours if 0 (unlimited)
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
|
||||
function run_tests()
|
||||
{
|
||||
set -x
|
||||
# We can have several additional options so we path them as array because it's
|
||||
# more idiologically correct.
|
||||
read -ra ADDITIONAL_OPTIONS <<< "${ADDITIONAL_OPTIONS:-}"
|
||||
@ -63,8 +83,7 @@ function run_tests()
|
||||
fi
|
||||
|
||||
clickhouse-test --testname --shard --zookeeper --hung-check --print-time \
|
||||
--test-runs "$NUM_TRIES" \
|
||||
"$SKIP_LIST_OPT" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
--use-skip-list --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a test_output/test_result.txt
|
||||
}
|
||||
@ -88,3 +107,10 @@ if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
|
||||
fi
|
||||
tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||:
|
||||
tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.gz ||:
|
||||
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
|
||||
mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
|
||||
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
|
||||
fi
|
||||
|
||||
@ -78,7 +78,10 @@ String Macros::expand(const String & s,
|
||||
|
||||
/// Prefer explicit macros over implicit.
|
||||
if (it != macros.end() && !info.expand_special_macros_only)
|
||||
{
|
||||
res += it->second;
|
||||
info.expanded_other = true;
|
||||
}
|
||||
else if (macro_name == "database" && !info.table_id.database_name.empty())
|
||||
{
|
||||
res += info.table_id.database_name;
|
||||
|
||||
@ -40,6 +40,7 @@ public:
|
||||
bool expanded_database = false;
|
||||
bool expanded_table = false;
|
||||
bool expanded_uuid = false;
|
||||
bool expanded_other = false;
|
||||
bool has_unknown = false;
|
||||
};
|
||||
|
||||
|
||||
@ -23,6 +23,7 @@
|
||||
#include <Parsers/parseQuery.h>
|
||||
#include <Interpreters/InterpreterCreateQuery.h>
|
||||
#include <Parsers/formatAST.h>
|
||||
#include <Common/Macros.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -309,20 +310,66 @@ void DatabaseReplicated::loadStoredObjects(Context & context, bool has_force_res
|
||||
ddl_worker->startup();
|
||||
}
|
||||
|
||||
BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const Context & query_context)
|
||||
void DatabaseReplicated::checkQueryValid(const ASTPtr & query, const Context & query_context) const
|
||||
{
|
||||
if (is_readonly)
|
||||
throw Exception(ErrorCodes::NO_ZOOKEEPER, "Database is in readonly mode, because it cannot connect to ZooKeeper");
|
||||
|
||||
if (query_context.getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY)
|
||||
throw Exception(ErrorCodes::INCORRECT_QUERY, "It's not initial query. ON CLUSTER is not allowed for Replicated database.");
|
||||
|
||||
/// Replicas will set correct name of current database in query context (database name can be different on replicas)
|
||||
if (auto * ddl_query = query->as<ASTQueryWithTableAndOutput>())
|
||||
if (auto * ddl_query = dynamic_cast<ASTQueryWithTableAndOutput *>(query.get()))
|
||||
{
|
||||
if (ddl_query->database != getDatabaseName())
|
||||
throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed");
|
||||
ddl_query->database.clear();
|
||||
|
||||
if (auto * create = query->as<ASTCreateQuery>())
|
||||
{
|
||||
bool replicated_table = create->storage && create->storage->engine && startsWith(create->storage->engine->name, "Replicated");
|
||||
if (!replicated_table || !create->storage->engine->arguments)
|
||||
return;
|
||||
|
||||
ASTs & args = create->storage->engine->arguments->children;
|
||||
if (args.size() < 2)
|
||||
return;
|
||||
|
||||
ASTLiteral * arg1 = args[0]->as<ASTLiteral>();
|
||||
ASTLiteral * arg2 = args[1]->as<ASTLiteral>();
|
||||
if (!arg1 || !arg2 || arg1->value.getType() != Field::Types::String || arg2->value.getType() != Field::Types::String)
|
||||
return;
|
||||
|
||||
String maybe_path = arg1->value.get<String>();
|
||||
String maybe_replica = arg2->value.get<String>();
|
||||
|
||||
/// Looks like it's ReplicatedMergeTree with explicit zookeeper_path and replica_name arguments.
|
||||
/// Let's ensure that some macros are used.
|
||||
/// NOTE: we cannot check here that substituted values will be actually different on shards and replicas.
|
||||
|
||||
Macros::MacroExpansionInfo info;
|
||||
info.table_id = {getDatabaseName(), create->table, create->uuid};
|
||||
query_context.getMacros()->expand(maybe_path, info);
|
||||
bool maybe_shard_macros = info.expanded_other;
|
||||
info.expanded_other = false;
|
||||
query_context.getMacros()->expand(maybe_replica, info);
|
||||
bool maybe_replica_macros = info.expanded_other;
|
||||
bool enable_functional_tests_helper = global_context.getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros");
|
||||
|
||||
if (!enable_functional_tests_helper)
|
||||
LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
|
||||
|
||||
if (maybe_shard_macros && maybe_replica_macros)
|
||||
return;
|
||||
|
||||
if (enable_functional_tests_helper)
|
||||
{
|
||||
if (maybe_path.empty() || maybe_path.back() != '/')
|
||||
maybe_path += '/';
|
||||
arg1->value = maybe_path + "auto_{shard}";
|
||||
arg2->value = maybe_replica + "auto_{replica}";
|
||||
return;
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::INCORRECT_QUERY,
|
||||
"Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. "
|
||||
"If you really want to specify it explicitly, then you should use some macros "
|
||||
"to distinguish different shards and replicas");
|
||||
}
|
||||
}
|
||||
|
||||
if (const auto * query_alter = query->as<ASTAlterQuery>())
|
||||
@ -343,7 +390,17 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const
|
||||
"Use DETACH TABLE PERMANENTLY or SYSTEM RESTART REPLICA or set "
|
||||
"database_replicated_always_detach_permanently to 1");
|
||||
}
|
||||
}
|
||||
|
||||
BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, const Context & query_context)
|
||||
{
|
||||
if (is_readonly)
|
||||
throw Exception(ErrorCodes::NO_ZOOKEEPER, "Database is in readonly mode, because it cannot connect to ZooKeeper");
|
||||
|
||||
if (query_context.getClientInfo().query_kind != ClientInfo::QueryKind::INITIAL_QUERY)
|
||||
throw Exception(ErrorCodes::INCORRECT_QUERY, "It's not initial query. ON CLUSTER is not allowed for Replicated database.");
|
||||
|
||||
checkQueryValid(query, query_context);
|
||||
LOG_DEBUG(log, "Proposing query: {}", queryToString(query));
|
||||
|
||||
DDLLogEntry entry;
|
||||
|
||||
@ -71,6 +71,8 @@ private:
|
||||
bool createDatabaseNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);
|
||||
void createReplicaNodesInZooKeeper(const ZooKeeperPtr & current_zookeeper);
|
||||
|
||||
void checkQueryValid(const ASTPtr & query, const Context & query_context) const;
|
||||
|
||||
void recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 max_log_ptr);
|
||||
std::map<String, String> tryGetConsistentMetadataSnapshot(const ZooKeeperPtr & zookeeper, UInt32 & max_log_ptr);
|
||||
|
||||
|
||||
@ -344,6 +344,17 @@ String DatabaseReplicatedTask::getShardID() const
|
||||
return database->shard_name;
|
||||
}
|
||||
|
||||
void DatabaseReplicatedTask::parseQueryFromEntry(const Context & context)
|
||||
{
|
||||
DDLTaskBase::parseQueryFromEntry(context);
|
||||
if (auto * ddl_query = dynamic_cast<ASTQueryWithTableAndOutput *>(query.get()))
|
||||
{
|
||||
/// Update database name with actual name of local database
|
||||
assert(ddl_query->database.empty());
|
||||
ddl_query->database = database->getDatabaseName();
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<Context> DatabaseReplicatedTask::makeQueryContext(Context & from_context, const ZooKeeperPtr & zookeeper)
|
||||
{
|
||||
auto query_context = DDLTaskBase::makeQueryContext(from_context, zookeeper);
|
||||
|
||||
@ -93,7 +93,7 @@ struct DDLTaskBase
|
||||
DDLTaskBase(const DDLTaskBase &) = delete;
|
||||
virtual ~DDLTaskBase() = default;
|
||||
|
||||
void parseQueryFromEntry(const Context & context);
|
||||
virtual void parseQueryFromEntry(const Context & context);
|
||||
|
||||
virtual String getShardID() const = 0;
|
||||
|
||||
@ -134,6 +134,7 @@ struct DatabaseReplicatedTask : public DDLTaskBase
|
||||
DatabaseReplicatedTask(const String & name, const String & path, DatabaseReplicated * database_);
|
||||
|
||||
String getShardID() const override;
|
||||
void parseQueryFromEntry(const Context & context) override;
|
||||
std::unique_ptr<Context> makeQueryContext(Context & from_context, const ZooKeeperPtr & zookeeper) override;
|
||||
|
||||
DatabaseReplicated * database;
|
||||
|
||||
@ -48,6 +48,7 @@ BlockIO InterpreterAlterQuery::execute()
|
||||
|
||||
context.checkAccess(getRequiredAccess());
|
||||
auto table_id = context.resolveStorageID(alter, Context::ResolveOrdinary);
|
||||
query_ptr->as<ASTAlterQuery &>().database = table_id.database_name;
|
||||
|
||||
DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name);
|
||||
if (typeid_cast<DatabaseReplicated *>(database.get()) && context.getClientInfo().query_kind != ClientInfo::QueryKind::SECONDARY_QUERY)
|
||||
|
||||
@ -82,7 +82,7 @@ void InterpreterDropQuery::waitForTableToBeActuallyDroppedOrDetached(const ASTDr
|
||||
db->waitDetachedTableNotInUse(uuid_to_wait);
|
||||
}
|
||||
|
||||
BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query)
|
||||
BlockIO InterpreterDropQuery::executeToTable(ASTDropQuery & query)
|
||||
{
|
||||
DatabasePtr database;
|
||||
UUID table_to_wait_on = UUIDHelpers::Nil;
|
||||
@ -92,7 +92,7 @@ BlockIO InterpreterDropQuery::executeToTable(const ASTDropQuery & query)
|
||||
return res;
|
||||
}
|
||||
|
||||
BlockIO InterpreterDropQuery::executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait)
|
||||
BlockIO InterpreterDropQuery::executeToTableImpl(ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait)
|
||||
{
|
||||
/// NOTE: it does not contain UUID, we will resolve it with locked DDLGuard
|
||||
auto table_id = StorageID(query);
|
||||
@ -101,7 +101,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ASTDropQuery & query, Dat
|
||||
if (context.tryResolveStorageID(table_id, Context::ResolveExternal))
|
||||
return executeToTemporaryTable(table_id.getTableName(), query.kind);
|
||||
else
|
||||
table_id.database_name = context.getCurrentDatabase();
|
||||
query.database = table_id.database_name = context.getCurrentDatabase();
|
||||
}
|
||||
|
||||
if (query.temporary)
|
||||
@ -212,7 +212,7 @@ BlockIO InterpreterDropQuery::executeToDictionary(
|
||||
String database_name = context.resolveDatabase(database_name_);
|
||||
|
||||
auto ddl_guard = (!no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name) : nullptr);
|
||||
|
||||
query_ptr->as<ASTDropQuery>()->database = database_name;
|
||||
DatabasePtr database = tryGetDatabase(database_name, if_exists);
|
||||
|
||||
bool is_drop_or_detach_database = query_ptr->as<ASTDropQuery>()->table.empty();
|
||||
|
||||
@ -34,8 +34,8 @@ private:
|
||||
BlockIO executeToDatabase(const ASTDropQuery & query);
|
||||
BlockIO executeToDatabaseImpl(const ASTDropQuery & query, DatabasePtr & database, std::vector<UUID> & uuids_to_wait);
|
||||
|
||||
BlockIO executeToTable(const ASTDropQuery & query);
|
||||
BlockIO executeToTableImpl(const ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait);
|
||||
BlockIO executeToTable(ASTDropQuery & query);
|
||||
BlockIO executeToTableImpl(ASTDropQuery & query, DatabasePtr & db, UUID & uuid_to_wait);
|
||||
|
||||
static void waitForTableToBeActuallyDroppedOrDetached(const ASTDropQuery & query, const DatabasePtr & db, const UUID & uuid_to_wait);
|
||||
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
#include <AggregateFunctions/parseAggregateFunctionParameters.h>
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/evaluateConstantExpression.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -410,6 +411,35 @@ static StoragePtr create(const StorageFactory::Arguments & args)
|
||||
throw Exception(msg, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
}
|
||||
|
||||
if (is_extended_storage_def)
|
||||
{
|
||||
/// Allow expressions in engine arguments.
|
||||
/// In new syntax argument can be literal or identifier or array/tuple of identifiers.
|
||||
size_t arg_idx = 0;
|
||||
try
|
||||
{
|
||||
for (; arg_idx < engine_args.size(); ++arg_idx)
|
||||
{
|
||||
auto & arg = engine_args[arg_idx];
|
||||
auto * arg_func = arg->as<ASTFunction>();
|
||||
if (!arg_func)
|
||||
continue;
|
||||
|
||||
/// If we got ASTFunction, let's evaluate it and replace with ASTLiteral.
|
||||
/// Do not try evaluate array or tuple, because it's array or tuple of column identifiers.
|
||||
if (arg_func->name == "array" || arg_func->name == "tuple")
|
||||
continue;
|
||||
Field value = evaluateConstantExpression(arg, args.local_context).first;
|
||||
arg = std::make_shared<ASTLiteral>(value);
|
||||
}
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot evaluate engine argument {}: {} {}",
|
||||
arg_idx, e.message(), getMergeTreeVerboseHelp(is_extended_storage_def));
|
||||
}
|
||||
}
|
||||
|
||||
/// For Replicated.
|
||||
String zookeeper_path;
|
||||
String replica_name;
|
||||
|
||||
@ -108,7 +108,7 @@ def remove_control_characters(s):
|
||||
|
||||
def get_db_engine(args, database_name):
|
||||
if args.replicated_database:
|
||||
return " ENGINE=Replicated('/test/clickhouse/db/{}', 's1', 'r1')".format(database_name)
|
||||
return " ON CLUSTER test_cluster_database_replicated ENGINE=Replicated('/test/clickhouse/db/{}', '{{shard}}', '{{replica}}')".format(database_name)
|
||||
if args.db_engine:
|
||||
return " ENGINE=" + args.db_engine
|
||||
return "" # Will use default engine
|
||||
@ -180,7 +180,10 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std
|
||||
clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
|
||||
seconds_left = max(args.timeout - (datetime.now() - start_time).total_seconds(), 20)
|
||||
try:
|
||||
clickhouse_proc_create.communicate(("DROP DATABASE " + database), timeout=seconds_left)
|
||||
drop_database_query = "DROP DATABASE " + database
|
||||
if args.replicated_database:
|
||||
drop_database_query += " ON CLUSTER test_cluster_database_replicated"
|
||||
clickhouse_proc_create.communicate((drop_database_query), timeout=seconds_left)
|
||||
except TimeoutExpired:
|
||||
# kill test process because it can also hung
|
||||
if proc.returncode is None:
|
||||
@ -201,6 +204,9 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std
|
||||
os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=database, file=stdout_file))
|
||||
if not args.show_db_name:
|
||||
os.system("LC_ALL=C sed -i -e 's/{test_db}/default/g' {file}".format(test_db=database, file=stderr_file))
|
||||
if args.replicated_database:
|
||||
os.system("LC_ALL=C sed -i -e 's|/auto_{{shard}}||g' {file}".format(file=stdout_file))
|
||||
os.system("LC_ALL=C sed -i -e 's|auto_{{replica}}||g' {file}".format(file=stdout_file))
|
||||
|
||||
stdout = open(stdout_file, 'rb').read() if os.path.exists(stdout_file) else b''
|
||||
stdout = str(stdout, errors='replace', encoding='utf-8')
|
||||
@ -216,8 +222,12 @@ def need_retry(stderr):
|
||||
|
||||
def get_processlist(args):
|
||||
try:
|
||||
query = b"SHOW PROCESSLIST FORMAT Vertical"
|
||||
if args.replicated_database:
|
||||
query = b"SELECT materialize((hostName(), tcpPort())) as host, * " \
|
||||
b"FROM clusterAllReplicas('r', system.processes) WHERE query NOT LIKE '%system.processes%' FORMAT Vertical"
|
||||
clickhouse_proc = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE)
|
||||
(stdout, _) = clickhouse_proc.communicate((b"SHOW PROCESSLIST FORMAT Vertical"), timeout=20)
|
||||
(stdout, _) = clickhouse_proc.communicate((query), timeout=20)
|
||||
return False, stdout.decode('utf-8')
|
||||
except Exception as ex:
|
||||
print("Exception", ex)
|
||||
@ -849,6 +859,8 @@ def main(args):
|
||||
if total_tests_run == 0:
|
||||
print("No tests were run.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("All tests have finished.")
|
||||
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
77
tests/config/config.d/database_replicated.xml
Normal file
77
tests/config/config.d/database_replicated.xml
Normal file
@ -0,0 +1,77 @@
|
||||
<yandex>
|
||||
<zookeeper>
|
||||
<node index="1">
|
||||
<host>localhost</host>
|
||||
<port>9181</port>
|
||||
</node>
|
||||
<node index="2">
|
||||
<host>localhost</host>
|
||||
<port>19181</port>
|
||||
</node>
|
||||
<node index="3">
|
||||
<host>localhost</host>
|
||||
<port>29181</port>
|
||||
</node>
|
||||
</zookeeper>
|
||||
|
||||
<keeper_server>
|
||||
<tcp_port>9181</tcp_port>
|
||||
<server_id>1</server_id>
|
||||
|
||||
<coordination_settings>
|
||||
<operation_timeout_ms>5000</operation_timeout_ms>
|
||||
<session_timeout_ms>10000</session_timeout_ms>
|
||||
<raft_logs_level>trace</raft_logs_level>
|
||||
<force_sync>false</force_sync>
|
||||
</coordination_settings>
|
||||
|
||||
<raft_configuration>
|
||||
<server>
|
||||
<id>1</id>
|
||||
<hostname>localhost</hostname>
|
||||
<port>44444</port>
|
||||
<can_become_leader>true</can_become_leader>
|
||||
<priority>3</priority>
|
||||
</server>
|
||||
<server>
|
||||
<id>2</id>
|
||||
<hostname>localhost</hostname>
|
||||
<port>44445</port>
|
||||
<can_become_leader>true</can_become_leader>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
<priority>2</priority>
|
||||
</server>
|
||||
<server>
|
||||
<id>3</id>
|
||||
<hostname>localhost</hostname>
|
||||
<port>44446</port>
|
||||
<can_become_leader>true</can_become_leader>
|
||||
<start_as_follower>true</start_as_follower>
|
||||
<priority>1</priority>
|
||||
</server>
|
||||
</raft_configuration>
|
||||
</keeper_server>
|
||||
|
||||
<remote_servers>
|
||||
<test_cluster_database_replicated>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>localhost</host>
|
||||
<port>9000</port>
|
||||
</replica>
|
||||
<replica>
|
||||
<host>localhost</host>
|
||||
<port>19000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
<shard>
|
||||
<replica>
|
||||
<host>localhost</host>
|
||||
<port>29000</port>
|
||||
</replica>
|
||||
</shard>
|
||||
</test_cluster_database_replicated>
|
||||
</remote_servers>
|
||||
|
||||
<_functional_tests_helper_database_replicated_replace_args_macros>1</_functional_tests_helper_database_replicated_replace_args_macros>
|
||||
</yandex>
|
||||
@ -65,6 +65,31 @@ if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; th
|
||||
fi
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
ln -sf $SRC_PATH/users.d/database_replicated.xml $DEST_SERVER_PATH/users.d/
|
||||
ln -sf $SRC_PATH/config.d/database_replicated.xml $DEST_SERVER_PATH/config.d/
|
||||
rm /etc/clickhouse-server/config.d/zookeeper.xml
|
||||
rm /etc/clickhouse-server/config.d/keeper_port.xml
|
||||
|
||||
# There is a bug in config reloading, so we cannot override macros using --macros.replica r2
|
||||
# And we have to copy configs...
|
||||
mkdir /etc/clickhouse-server1
|
||||
mkdir /etc/clickhouse-server2
|
||||
chown clickhouse /etc/clickhouse-server1
|
||||
chown clickhouse /etc/clickhouse-server2
|
||||
chgrp clickhouse /etc/clickhouse-server1
|
||||
chgrp clickhouse /etc/clickhouse-server2
|
||||
sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server1
|
||||
sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server2
|
||||
rm /etc/clickhouse-server1/config.d/macros.xml
|
||||
rm /etc/clickhouse-server2/config.d/macros.xml
|
||||
sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<replica>r1</replica>|<replica>r2</replica>|" > /etc/clickhouse-server1/config.d/macros.xml
|
||||
sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<shard>s1</shard>|<shard>s2</shard>|" > /etc/clickhouse-server2/config.d/macros.xml
|
||||
|
||||
sudo mkdir /var/lib/clickhouse1
|
||||
sudo mkdir /var/lib/clickhouse2
|
||||
sudo chown clickhouse /var/lib/clickhouse1
|
||||
sudo chown clickhouse /var/lib/clickhouse2
|
||||
sudo chgrp clickhouse /var/lib/clickhouse1
|
||||
sudo chgrp clickhouse /var/lib/clickhouse2
|
||||
fi
|
||||
|
||||
ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml
|
||||
|
||||
@ -35,8 +35,17 @@ def started_cluster():
|
||||
cluster.shutdown()
|
||||
|
||||
def test_create_replicated_table(started_cluster):
|
||||
assert "Explicit zookeeper_path and replica_name are specified" in \
|
||||
main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) "
|
||||
"ENGINE=ReplicatedMergeTree('/test/tmp', 'r') ORDER BY k PARTITION BY toYYYYMM(d);")
|
||||
|
||||
assert "Explicit zookeeper_path and replica_name are specified" in \
|
||||
main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) "
|
||||
"ENGINE=ReplicatedMergeTree('/test/tmp', 'r', d, k, 8192);")
|
||||
|
||||
assert "Old syntax is not allowed" in \
|
||||
main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/test/tmp', 'r', d, k, 8192);")
|
||||
main_node.query_and_get_error("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) "
|
||||
"ENGINE=ReplicatedMergeTree('/test/tmp/{shard}', '{replica}', d, k, 8192);")
|
||||
|
||||
main_node.query("CREATE TABLE testdb.replicated_table (d Date, k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree ORDER BY k PARTITION BY toYYYYMM(d);")
|
||||
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
# Introduction to jepsen.nukeeper
|
||||
# Introduction to jepsen.keeper
|
||||
|
||||
TODO: write [great documentation](http://jacobian.org/writing/what-to-write/)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
(defproject jepsen.nukeeper "0.1.0-SNAPSHOT"
|
||||
(defproject jepsen.keeper "0.1.0-SNAPSHOT"
|
||||
:injections [(.. System (setProperty "zookeeper.request.timeout" "10000"))]
|
||||
:description "A jepsen tests for ClickHouse Keeper"
|
||||
:url "https://clickhouse.tech/"
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
(dorun (map (fn [v] (zk/delete conn v)) (take 10 (zk-range)))))
|
||||
|
||||
(deftest a-test
|
||||
(testing "nukeeper connection"
|
||||
(testing "keeper connection"
|
||||
(.setLevel
|
||||
(LoggerFactory/getLogger "org.apache.zookeeper") Level/OFF)
|
||||
(let [conn (zk/connect "localhost:9181" :timeout-msec 5000)]
|
||||
|
||||
@ -4,13 +4,13 @@ DROP TABLE IF EXISTS part_header_r2;
|
||||
SET replication_alter_partitions_sync = 2;
|
||||
|
||||
CREATE TABLE part_header_r1(x UInt32, y UInt32)
|
||||
ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00814/part_header', '1') ORDER BY x
|
||||
ENGINE ReplicatedMergeTree('/clickhouse/tables/'||currentDatabase()||'/test_00814/part_header/{shard}', '1{replica}') ORDER BY x
|
||||
SETTINGS use_minimalistic_part_header_in_zookeeper = 0,
|
||||
old_parts_lifetime = 1,
|
||||
cleanup_delay_period = 0,
|
||||
cleanup_delay_period_random_add = 0;
|
||||
CREATE TABLE part_header_r2(x UInt32, y UInt32)
|
||||
ENGINE ReplicatedMergeTree('/clickhouse/tables/test_00814/part_header', '2') ORDER BY x
|
||||
ENGINE ReplicatedMergeTree('/clickhouse/tables/'||currentDatabase()||'/test_00814/part_header/{shard}', '2{replica}') ORDER BY x
|
||||
SETTINGS use_minimalistic_part_header_in_zookeeper = 1,
|
||||
old_parts_lifetime = 1,
|
||||
cleanup_delay_period = 0,
|
||||
@ -39,10 +39,10 @@ SELECT sleep(3) FORMAT Null;
|
||||
SELECT '*** Test part removal ***';
|
||||
SELECT '*** replica 1 ***';
|
||||
SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r1';
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/test_00814/part_header/replicas/1/parts';
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/'||currentDatabase()||'/test_00814/part_header/s1/replicas/1r1/parts';
|
||||
SELECT '*** replica 2 ***';
|
||||
SELECT name FROM system.parts WHERE active AND database = currentDatabase() AND table = 'part_header_r2';
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/test_00814/part_header/replicas/1/parts';
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/'||currentDatabase()||'/test_00814/part_header/s1/replicas/1r1/parts';
|
||||
|
||||
SELECT '*** Test ALTER ***';
|
||||
ALTER TABLE part_header_r1 MODIFY COLUMN y String;
|
||||
|
||||
@ -15,7 +15,7 @@ CREATE TABLE elog (
|
||||
engine_id UInt32,
|
||||
referrer String
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog', 'test')
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/{shard}', '{replica}')
|
||||
PARTITION BY date
|
||||
ORDER BY (engine_id)
|
||||
SETTINGS replicated_deduplication_window = 2, cleanup_delay_period=4, cleanup_delay_period_random_add=0;"
|
||||
@ -28,35 +28,37 @@ $CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 3, 'h
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 3 rows
|
||||
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'")
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'")
|
||||
|
||||
while [[ $count != 2 ]]
|
||||
do
|
||||
sleep 1
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'")
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'")
|
||||
done
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 1, 'hello')"
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 4 rows
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'")
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'")
|
||||
|
||||
|
||||
while [[ $count != 2 ]]
|
||||
do
|
||||
sleep 1
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'")
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'")
|
||||
done
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')"
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="SELECT count(*) from elog" # 5 rows
|
||||
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'")
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'")
|
||||
|
||||
while [[ $count != 2 ]]
|
||||
do
|
||||
sleep 1
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/blocks'")
|
||||
count=$($CLICKHOUSE_CLIENT --query="SELECT COUNT(*) FROM system.zookeeper where path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/elog/s1/blocks'")
|
||||
|
||||
done
|
||||
|
||||
$CLICKHOUSE_CLIENT --query="INSERT INTO elog VALUES (toDate('2018-10-01'), 2, 'hello')"
|
||||
|
||||
@ -8,13 +8,13 @@ CREATE TABLE versioned_collapsing_table(
|
||||
sign Int8,
|
||||
version UInt16
|
||||
)
|
||||
ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/versioned_collapsing_table', '1', sign, version)
|
||||
ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/versioned_collapsing_table/{shard}', '{replica}', sign, version)
|
||||
PARTITION BY d
|
||||
ORDER BY (key1, key2);
|
||||
|
||||
INSERT INTO versioned_collapsing_table VALUES (toDate('2019-10-10'), 1, 1, 'Hello', -1, 1);
|
||||
|
||||
SELECT value FROM system.zookeeper WHERE path = '/clickhouse/versioned_collapsing_table' and name = 'metadata';
|
||||
SELECT value FROM system.zookeeper WHERE path = '/clickhouse/versioned_collapsing_table/s1' and name = 'metadata';
|
||||
|
||||
SELECT COUNT() FROM versioned_collapsing_table;
|
||||
|
||||
|
||||
@ -12,8 +12,8 @@ SCALE=5000
|
||||
$CLICKHOUSE_CLIENT -n --query "
|
||||
DROP TABLE IF EXISTS r1;
|
||||
DROP TABLE IF EXISTS r2;
|
||||
CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10;
|
||||
CREATE TABLE r2 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '2') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10;
|
||||
CREATE TABLE r1 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}', '1{replica}') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10;
|
||||
CREATE TABLE r2 (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}', '2{replica}') ORDER BY x SETTINGS old_parts_lifetime = 1, cleanup_delay_period = 0, cleanup_delay_period_random_add = 1, parts_to_throw_insert = 100000, max_replicated_logs_to_keep = 10;
|
||||
DETACH TABLE r2;
|
||||
"
|
||||
|
||||
@ -29,16 +29,16 @@ for _ in {1..60}; do
|
||||
done
|
||||
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r' AND name = 'log'";
|
||||
$CLICKHOUSE_CLIENT --query "SELECT numChildren < $((SCALE / 4)) FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1' AND name = 'log'";
|
||||
echo -e '\n---\n';
|
||||
$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/1' AND name = 'is_lost'";
|
||||
$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/2' AND name = 'is_lost'";
|
||||
$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/1r1' AND name = 'is_lost'";
|
||||
$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/2r1' AND name = 'is_lost'";
|
||||
echo -e '\n---\n';
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "ATTACH TABLE r2"
|
||||
$CLICKHOUSE_CLIENT --receive_timeout 600 --query "SYSTEM SYNC REPLICA r2" # Need to increase timeout, otherwise it timed out in debug build
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r/replicas/2' AND name = 'is_lost'";
|
||||
$CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/2r1' AND name = 'is_lost'";
|
||||
|
||||
$CLICKHOUSE_CLIENT -n --query "
|
||||
DROP TABLE IF EXISTS r1;
|
||||
|
||||
@ -2,4 +2,4 @@
|
||||
10
|
||||
10
|
||||
24
|
||||
CREATE TABLE default.replicated_mutations_empty_partitions\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/01586_replicated_mutations_empty_partitions\', \'1\')\nPARTITION BY key\nORDER BY key\nSETTINGS index_granularity = 8192
|
||||
CREATE TABLE default.replicated_mutations_empty_partitions\n(\n `key` UInt64,\n `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/test/default/01586_replicated_mutations_empty_partitions/{shard}\', \'{replica}\')\nPARTITION BY key\nORDER BY key\nSETTINGS index_granularity = 8192
|
||||
|
||||
@ -5,7 +5,7 @@ CREATE TABLE replicated_mutations_empty_partitions
|
||||
key UInt64,
|
||||
value String
|
||||
)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/test/01586_replicated_mutations_empty_partitions', '1')
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/{shard}', '{replica}')
|
||||
ORDER BY key
|
||||
PARTITION by key;
|
||||
|
||||
@ -13,7 +13,7 @@ INSERT INTO replicated_mutations_empty_partitions SELECT number, toString(number
|
||||
|
||||
SELECT count(distinct value) FROM replicated_mutations_empty_partitions;
|
||||
|
||||
SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/01586_replicated_mutations_empty_partitions/block_numbers';
|
||||
SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/s1/block_numbers';
|
||||
|
||||
ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '3';
|
||||
ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '4';
|
||||
@ -21,7 +21,7 @@ ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '5';
|
||||
ALTER TABLE replicated_mutations_empty_partitions DROP PARTITION '9';
|
||||
|
||||
-- still ten records
|
||||
SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/01586_replicated_mutations_empty_partitions/block_numbers';
|
||||
SELECT count() FROM system.zookeeper WHERE path = '/clickhouse/test/'||currentDatabase()||'/01586_replicated_mutations_empty_partitions/s1/block_numbers';
|
||||
|
||||
ALTER TABLE replicated_mutations_empty_partitions MODIFY COLUMN value UInt64 SETTINGS replication_alter_partitions_sync=2;
|
||||
|
||||
|
||||
@ -1,16 +0,0 @@
|
||||
499999500000
|
||||
499999500000
|
||||
499999500000
|
||||
499999500000
|
||||
499999500000
|
||||
Metadata version on replica 1 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_1\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'1\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 2 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_2\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'2\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 3 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_3\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'3\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 4 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_4\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'4\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 5 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_5\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_default\', \'5\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
499999500000
|
||||
@ -0,0 +1,16 @@
|
||||
499999500000
|
||||
499999500000
|
||||
499999500000
|
||||
499999500000
|
||||
499999500000
|
||||
Metadata version on replica 1 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_1\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_long_default/{shard}\', \'{replica}1\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 2 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_2\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_long_default/{shard}\', \'{replica}2\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 3 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_3\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_long_default/{shard}\', \'{replica}3\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 4 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_4\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_long_default/{shard}\', \'{replica}4\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
Metadata version on replica 5 equal with first replica, OK
|
||||
CREATE TABLE default.concurrent_kill_5\n(\n `key` UInt64,\n `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01593_concurrent_alter_mutations_kill_many_replicas_long_default/{shard}\', \'{replica}5\')\nORDER BY key\nSETTINGS max_replicated_mutations_in_queue = 1000, number_of_free_entries_in_pool_to_execute_mutation = 0, max_replicated_merges_in_queue = 1000, index_granularity = 8192
|
||||
499999500000
|
||||
@ -11,7 +11,10 @@ for i in $(seq $REPLICAS); do
|
||||
done
|
||||
|
||||
for i in $(seq $REPLICAS); do
|
||||
$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_kill_$i (key UInt64, value String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000"
|
||||
$CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_kill_$i (key UInt64, value String) ENGINE =
|
||||
ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{shard}', '{replica}$i') ORDER BY key
|
||||
SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0,max_replicated_merges_in_queue=1000"
|
||||
|
||||
done
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_kill_1 SELECT number, toString(number) FROM numbers(1000000)"
|
||||
@ -77,9 +80,10 @@ while true; do
|
||||
done
|
||||
|
||||
|
||||
metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicas/$i/' and name = 'metadata_version'")
|
||||
metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/r1$i/' and name = 'metadata_version'")
|
||||
for i in $(seq $REPLICAS); do
|
||||
replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/replicas/$i/' and name = 'metadata_version'")
|
||||
replica_metadata_version=$($CLICKHOUSE_CLIENT --query "SELECT value FROM system.zookeeper WHERE path = '/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/s1/replicas/r1$i/' and name = 'metadata_version'")
|
||||
|
||||
if [ "$metadata_version" != "$replica_metadata_version" ]; then
|
||||
echo "Metadata version on replica $i differs from the first replica, FAIL"
|
||||
else
|
||||
@ -1,16 +1,14 @@
|
||||
block_numbers
|
||||
blocks
|
||||
1
|
||||
r1
|
||||
========
|
||||
block_numbers
|
||||
blocks
|
||||
1
|
||||
r1
|
||||
========
|
||||
block_numbers
|
||||
blocks
|
||||
========
|
||||
1
|
||||
failed_parts
|
||||
last_part
|
||||
leader_election-0000000000
|
||||
parallel
|
||||
|
||||
@ -3,17 +3,20 @@ DROP TABLE IF EXISTS sample_table;
|
||||
CREATE TABLE sample_table (
|
||||
key UInt64
|
||||
)
|
||||
ENGINE ReplicatedMergeTree('/clickhouse/01700_system_zookeeper_path_in', '1')
|
||||
ENGINE ReplicatedMergeTree('/clickhouse/01700_system_zookeeper_path_in/{shard}', '{replica}')
|
||||
ORDER BY tuple();
|
||||
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/01700_system_zookeeper_path_in' AND name like 'block%' ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/01700_system_zookeeper_path_in/replicas' ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/01700_system_zookeeper_path_in/s1' AND name like 'block%' ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path = '/clickhouse/01700_system_zookeeper_path_in/s1/replicas' AND name LIKE '%r1%' ORDER BY name;
|
||||
|
||||
SELECT '========';
|
||||
SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in') AND name LIKE 'block%' ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in/replicas') ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in/s1') AND name LIKE 'block%' ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in/s1/replicas') AND name LIKE '%r1%' ORDER BY name;
|
||||
SELECT '========';
|
||||
SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in','/clickhouse/01700_system_zookeeper_path_in/replicas') AND name LIKE 'block%' ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path IN ('/clickhouse/01700_system_zookeeper_path_in/s1',
|
||||
'/clickhouse/01700_system_zookeeper_path_in/s1/replicas') AND name LIKE 'block%' ORDER BY name;
|
||||
SELECT '========';
|
||||
SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/01700_system_zookeeper_path_in/', name) FROM system.zookeeper WHERE (path = '/clickhouse/01700_system_zookeeper_path_in')) ORDER BY name;
|
||||
SELECT name FROM system.zookeeper WHERE path IN (SELECT concat('/clickhouse/01700_system_zookeeper_path_in/s1/', name)
|
||||
FROM system.zookeeper WHERE (name != 'replicas' AND name NOT LIKE 'leader_election%' AND path = '/clickhouse/01700_system_zookeeper_path_in/s1')) ORDER BY name;
|
||||
|
||||
DROP TABLE IF EXISTS sample_table;
|
||||
|
||||
@ -118,6 +118,17 @@
|
||||
"01148_zookeeper_path_macros_unfolding",
|
||||
"01294_system_distributed_on_cluster",
|
||||
"01269_create_with_null",
|
||||
"01451_replicated_detach_drop_and_quorum",
|
||||
"01188_attach_table_from_path",
|
||||
"01149_zookeeper_mutation_stuck_after_replace_partition",
|
||||
/// user_files
|
||||
"01721_engine_file_truncate_on_insert",
|
||||
/// Fails due to additional replicas or shards
|
||||
"quorum",
|
||||
"01650_drop_part_and_deduplication_zookeeper",
|
||||
"01532_execute_merges_on_single_replica",
|
||||
"00652_replicated_mutations_default_database_zookeeper",
|
||||
"00620_optimize_on_nonleader_replica_zookeeper",
|
||||
/// grep -c
|
||||
"01018_ddl_dictionaries_bad_queries",
|
||||
"00908_bloom_filter_index",
|
||||
@ -136,6 +147,7 @@
|
||||
"00626_replace_partition_from_table_zookeeper",
|
||||
"00626_replace_partition_from_table",
|
||||
"00152_insert_different_granularity",
|
||||
"00054_merge_tree_partitions",
|
||||
/// Old syntax is not allowed
|
||||
"01062_alter_on_mutataion_zookeeper",
|
||||
"00925_zookeeper_empty_replicated_merge_tree_optimize_final",
|
||||
@ -151,7 +163,8 @@
|
||||
"00083_create_merge_tree_zookeeper",
|
||||
"00062_replicated_merge_tree_alter_zookeeper",
|
||||
/// Does not support renaming of multiple tables in single query
|
||||
"00634_rename_view"
|
||||
"00634_rename_view",
|
||||
"00140_rename"
|
||||
],
|
||||
"polymorphic-parts": [
|
||||
"01508_partition_pruning_long", /// bug, shoud be fixed
|
||||
|
||||
@ -58,7 +58,7 @@ int main(int argc, char *argv[])
|
||||
Poco::Logger::root().setChannel(channel);
|
||||
Poco::Logger::root().setLevel("trace");
|
||||
}
|
||||
auto * logger = &Poco::Logger::get("nukeeper-dumper");
|
||||
auto * logger = &Poco::Logger::get("keeper-dumper");
|
||||
ResponsesQueue queue;
|
||||
SnapshotsQueue snapshots_queue{1};
|
||||
CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
|
||||
|
||||
Loading…
Reference in New Issue
Block a user