thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Alexander Tokmakov 2021-03-13 13:22:48 +03:00
parent 8e00ff7898
commit e62e256e77
8 changed files with 137 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
docker/test/stateful/run.sh
View File

@ -13,6 +13,22 @@ dpkg -i package_folder/clickhouse-test_*.deb
function start()
{
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server2/config.xml --daemon \
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
--tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
--mysql_port 19004 \
--test_keeper_server.tcp_port 19181 --test_keeper_server.server_id 2
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server3/config.xml --daemon \
-- --path /var/lib/clickhouse3/ --logger.stderr /var/log/clickhouse-server/stderr3.log \
--logger.log /var/log/clickhouse-server/clickhouse-server3.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server3.err.log \
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
--mysql_port 29004 \
--test_keeper_server.tcp_port 29181 --test_keeper_server.server_id 3
fi
counter=0
until clickhouse-client --query "SELECT 1"
do
@ -73,3 +89,9 @@ mv /var/log/clickhouse-server/stderr.log /test_output/ ||:
if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
tar -chf /test_output/clickhouse_coverage.tar.gz /profraw ||:
fi
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
pigz < /var/log/clickhouse-server/clickhouse-server3.log > /test_output/clickhouse-server3.log.gz ||:
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
mv /var/log/clickhouse-server/stderr3.log /test_output/ ||:
fi

22
docker/test/stateless/clusters2.xml Normal file
View File

@ -0,0 +1,22 @@
<yandex>
<remote_servers>
<test_cluster_database_replicated2>
<shard>
<replica>
<host>localhost</host>
<port>9000</port>
</replica>
<replica>
<host>localhost</host>
<port>19000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>29000</port>
</replica>
</shard>
</test_cluster_database_replicated2>
</remote_servers>
</yandex>

38
docker/test/stateless/enable_test_keeper1.xml Normal file
View File

@ -0,0 +1,38 @@
<yandex>
<test_keeper_server>
<tcp_port>9181</tcp_port>
<server_id>1</server_id>
<coordination_settings>
<operation_timeout_ms>5000</operation_timeout_ms>
<session_timeout_ms>10000</session_timeout_ms>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>localhost</hostname>
<port>44444</port>
<can_become_leader>true</can_become_leader>
<priority>3</priority>
</server>
<server>
<id>2</id>
<hostname>localhost</hostname>
<port>44445</port>
<can_become_leader>true</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>2</priority>
</server>
<server>
<id>3</id>
<hostname>localhost</hostname>
<port>44446</port>
<can_become_leader>true</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>1</priority>
</server>
</raft_configuration>
</test_keeper_server>
</yandex>

38
docker/test/stateless/run.sh
View File

@ -11,20 +11,6 @@ dpkg -i package_folder/clickhouse-test_*.deb
# install test configs
/usr/share/clickhouse-test/config/install.sh
#cp /use_test_keeper.xml /etc/clickhouse-server/config.d/zookeeper.xml
#cp /enable_test_keeper1.xml /etc/clickhouse-server/config.d/test_keeper_port.xml
#cp /clusters2.xml /etc/clickhouse-server/config.d/
#mkdir /etc/clickhouse-server2
#chown clickhouse /etc/clickhouse-server2
#chgrp clickhouse /etc/clickhouse-server2
#sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server2
#rm /etc/clickhouse-server2/config.d/macros.xml
#sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<replica>r1</replica>|<replica>r2</replica>|" > /etc/clickhouse-server2/config.d/macros.xml
#cat /usr/bin/clickhouse-test | sed "s| ENGINE=Replicated('/test/clickhouse/db/{}', 's1', 'r1')| ON CLUSTER test_cluster_database_replicated ENGINE=Replicated('/test/clickhouse/db/{}', '{{shard}}', '{{replica}}')|" > /usr/bin/clickhouse-test-tmp
#mv /usr/bin/clickhouse-test-tmp /usr/bin/clickhouse-test
#chmod a+x /usr/bin/clickhouse-test
# For flaky check we also enable thread fuzzer
if [ "$NUM_TRIES" -gt "1" ]; then
@ -53,17 +39,7 @@ else
fi
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
# There is a bug in config reloading, so we cannot override macros using --macros.replica r2
# And we have to copy configs...
mkdir /etc/clickhouse-server2
chown clickhouse /etc/clickhouse-server2
chgrp clickhouse /etc/clickhouse-server2
sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server2
rm /etc/clickhouse-server2/config.d/macros.xml
sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<replica>r1</replica>|<replica>r2</replica>|" > /etc/clickhouse-server2/config.d/macros.xml
sudo mkdir /var/lib/clickhouse2
sudo chmod a=rwx /var/lib/clickhouse2
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server2/config.xml --daemon \
-- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
--logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
@ -71,6 +47,14 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
--mysql_port 19004 \
--test_keeper_server.tcp_port 19181 --test_keeper_server.server_id 2 \
--macros.replica r2 # It doesn't work :(
sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server3/config.xml --daemon \
-- --path /var/lib/clickhouse3/ --logger.stderr /var/log/clickhouse-server/stderr3.log \
--logger.log /var/log/clickhouse-server/clickhouse-server3.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server3.err.log \
--tcp_port 29000 --tcp_port_secure 29440 --http_port 28123 --https_port 28443 --interserver_http_port 29009 --tcp_with_proxy_port 29010 \
--mysql_port 29004 \
--test_keeper_server.tcp_port 29181 --test_keeper_server.server_id 3 \
--macros.shard s2 # It doesn't work :(
fi
sleep 5
@ -104,10 +88,6 @@ function run_tests()
| tee -a test_output/test_result.txt
}
#clickhouse-client --port 9000 -q "SELECT * FROM system.macros"
#clickhouse-client --port 19000 -q "SELECT * FROM system.macros"
#clickhouse-client --port 19000 -q "SELECT 2"
export -f run_tests
timeout "$MAX_RUN_TIME" bash -c run_tests ||:
@ -124,5 +104,7 @@ tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_l
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
pigz < /var/log/clickhouse-server/clickhouse-server3.log > /test_output/clickhouse-server3.log.gz ||:
mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
mv /var/log/clickhouse-server/stderr3.log /test_output/ ||:
fi

16
docker/test/stateless/use_test_keeper.xml Normal file
View File

@ -0,0 +1,16 @@
<yandex>
<zookeeper>
<node index="1">
<host>localhost</host>
<port>9181</port>
</node>
<node index="2">
<host>localhost</host>
<port>19181</port>
</node>
<node index="3">
<host>localhost</host>
<port>29181</port>
</node>
</zookeeper>
</yandex>

2
tests/clickhouse-test
View File

@ -107,7 +107,7 @@ def remove_control_characters(s):
def get_db_engine(args, database_name):
if args.replicated_database:
return " ON CLUSTER test_cluster_database_replicated ENGINE=Replicated('/test/clickhouse/db/{}', 's1', 'r1')".format(database_name)
return " ON CLUSTER test_cluster_database_replicated ENGINE=Replicated('/test/clickhouse/db/{}', '{{shard}}', '{{replica}}')".format(database_name)
if args.db_engine:
return " ENGINE=" + args.db_engine
return "" # Will use default engine

6
tests/config/config.d/database_replicated.xml
View File

@ -63,6 +63,12 @@
<port>19000</port>
</replica>
</shard>
<shard>
<replica>
<host>localhost</host>
<port>29000</port>
</replica>
</shard>
</test_cluster_database_replicated>
</remote_servers>

22
tests/config/install.sh
View File

@ -68,6 +68,28 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
ln -sf $SRC_PATH/config.d/database_replicated.xml $DEST_SERVER_PATH/config.d/
rm /etc/clickhouse-server/config.d/zookeeper.xml
rm /etc/clickhouse-server/config.d/test_keeper_port.xml
# There is a bug in config reloading, so we cannot override macros using --macros.replica r2
# And we have to copy configs...
mkdir /etc/clickhouse-server2
mkdir /etc/clickhouse-server3
chown clickhouse /etc/clickhouse-server2
chown clickhouse /etc/clickhouse-server3
chgrp clickhouse /etc/clickhouse-server2
chgrp clickhouse /etc/clickhouse-server3
sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server2
sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server3
rm /etc/clickhouse-server2/config.d/macros.xml
rm /etc/clickhouse-server3/config.d/macros.xml
sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<replica>r1</replica>|<replica>r2</replica>|" > /etc/clickhouse-server2/config.d/macros.xml
sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<shard>s1</shard>|<shard>s2</shard>|" > /etc/clickhouse-server3/config.d/macros.xml
sudo mkdir /var/lib/clickhouse2
sudo mkdir /var/lib/clickhouse3
sudo chown clickhouse /var/lib/clickhouse2
sudo chown clickhouse /var/lib/clickhouse3
sudo chgrp clickhouse /var/lib/clickhouse2
sudo chgrp clickhouse /var/lib/clickhouse3
fi
ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml