try run two nodes with Replicated db and NuKeeper

docker/test/stateless/run.sh

@@ -11,6 +11,20 @@ dpkg -i package_folder/clickhouse-test_*.deb
 
 # install test configs
 /usr/share/clickhouse-test/config/install.sh
+#cp /use_test_keeper.xml /etc/clickhouse-server/config.d/zookeeper.xml
+#cp /enable_test_keeper1.xml /etc/clickhouse-server/config.d/test_keeper_port.xml
+#cp /clusters2.xml /etc/clickhouse-server/config.d/
+
+#mkdir /etc/clickhouse-server2
+#chown clickhouse /etc/clickhouse-server2
+#chgrp clickhouse /etc/clickhouse-server2
+#sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server2
+#rm /etc/clickhouse-server2/config.d/macros.xml
+#sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<replica>r1</replica>|<replica>r2</replica>|" > /etc/clickhouse-server2/config.d/macros.xml
+
+#cat /usr/bin/clickhouse-test | sed "s| ENGINE=Replicated('/test/clickhouse/db/{}', 's1', 'r1')| ON CLUSTER test_cluster_database_replicated ENGINE=Replicated('/test/clickhouse/db/{}', '{{shard}}', '{{replica}}')|" > /usr/bin/clickhouse-test-tmp
+#mv /usr/bin/clickhouse-test-tmp /usr/bin/clickhouse-test
+#chmod a+x /usr/bin/clickhouse-test
 
 # For flaky check we also enable thread fuzzer
 if [ "$NUM_TRIES" -gt "1" ]; then
@@ -34,11 +48,33 @@ if [ "$NUM_TRIES" -gt "1" ]; then
 
     # simpliest way to forward env variables to server
     sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon
-    sleep 5
 else
-    service clickhouse-server start && sleep 5
+    service clickhouse-server start
 fi
 
+if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    # There is a bug in config reloading, so we cannot override macros using --macros.replica r2
+    # And we have to copy configs...
+    mkdir /etc/clickhouse-server2
+    chown clickhouse /etc/clickhouse-server2
+    chgrp clickhouse /etc/clickhouse-server2
+    sudo -u clickhouse cp -r /etc/clickhouse-server/* /etc/clickhouse-server2
+    rm /etc/clickhouse-server2/config.d/macros.xml
+    sudo -u clickhouse cat /etc/clickhouse-server/config.d/macros.xml | sed "s|<replica>r1</replica>|<replica>r2</replica>|" > /etc/clickhouse-server2/config.d/macros.xml
+
+    sudo mkdir /var/lib/clickhouse2
+    sudo chmod a=rwx /var/lib/clickhouse2
+    sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server2/config.xml --daemon \
+    -- --path /var/lib/clickhouse2/ --logger.stderr /var/log/clickhouse-server/stderr2.log \
+    --logger.log /var/log/clickhouse-server/clickhouse-server2.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server2.err.log \
+    --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
+    --mysql_port 19004 \
+    --test_keeper_server.tcp_port 19181 --test_keeper_server.server_id 2 \
+    --macros.replica r2   # It doesn't work :(
+fi
+
+sleep 5
+
 if grep -q -- "--use-skip-list" /usr/bin/clickhouse-test; then
     SKIP_LIST_OPT="--use-skip-list"
 fi
@@ -68,6 +104,10 @@ function run_tests()
         | tee -a test_output/test_result.txt
 }
 
+#clickhouse-client --port 9000 -q "SELECT * FROM system.macros"
+#clickhouse-client --port 19000 -q "SELECT * FROM system.macros"
+#clickhouse-client --port 19000 -q "SELECT 2"
+
 export -f run_tests
 
 timeout "$MAX_RUN_TIME" bash -c run_tests ||:
@@ -81,3 +121,8 @@ if [[ -n "$WITH_COVERAGE" ]] && [[ "$WITH_COVERAGE" -eq 1 ]]; then
 fi
 tar -chf /test_output/text_log_dump.tar /var/lib/clickhouse/data/system/text_log ||:
 tar -chf /test_output/query_log_dump.tar /var/lib/clickhouse/data/system/query_log ||:
+
+if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
+    pigz < /var/log/clickhouse-server/clickhouse-server2.log > /test_output/clickhouse-server2.log.gz ||:
+    mv /var/log/clickhouse-server/stderr2.log /test_output/ ||:
+fi

tests/clickhouse-test

@@ -107,7 +107,7 @@ def remove_control_characters(s):
 
 def get_db_engine(args, database_name):
     if args.replicated_database:
-        return " ENGINE=Replicated('/test/clickhouse/db/{}', 's1', 'r1')".format(database_name)
+        return " ON CLUSTER test_cluster_database_replicated ENGINE=Replicated('/test/clickhouse/db/{}', 's1', 'r1')".format(database_name)
     if args.db_engine:
         return " ENGINE=" + args.db_engine
     return ""   # Will use default engine
@@ -172,7 +172,10 @@ def run_single_test(args, ext, server_logs_level, client_options, case_file, std
         clickhouse_proc_create = Popen(shlex.split(args.client), stdin=PIPE, stdout=PIPE, stderr=PIPE, universal_newlines=True)
         seconds_left = max(args.timeout - (datetime.now() - start_time).total_seconds(), 10)
         try:
-            clickhouse_proc_create.communicate(("DROP DATABASE " + database), timeout=seconds_left)
+            drop_database_query = "DROP DATABASE " + database
+            if args.replicated_database:
+                drop_database_query += " ON CLUSTER test_cluster_database_replicated"
+            clickhouse_proc_create.communicate((drop_database_query), timeout=seconds_left)
         except TimeoutExpired:
             # kill test process because it can also hung
             if proc.returncode is None:

tests/config/config.d/database_replicated.xml

@@ -0,0 +1,69 @@
+<yandex>
+    <zookeeper>
+        <node index="1">
+            <host>localhost</host>
+            <port>9181</port>
+        </node>
+        <node index="2">
+            <host>localhost</host>
+            <port>19181</port>
+        </node>
+        <node index="3">
+            <host>localhost</host>
+            <port>29181</port>
+        </node>
+    </zookeeper>
+
+    <test_keeper_server>
+        <tcp_port>9181</tcp_port>
+        <server_id>1</server_id>
+
+        <coordination_settings>
+            <operation_timeout_ms>5000</operation_timeout_ms>
+            <session_timeout_ms>10000</session_timeout_ms>
+            <raft_logs_level>trace</raft_logs_level>
+        </coordination_settings>
+
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>localhost</hostname>
+                <port>44444</port>
+                <can_become_leader>true</can_become_leader>
+                <priority>3</priority>
+            </server>
+            <server>
+                <id>2</id>
+                <hostname>localhost</hostname>
+                <port>44445</port>
+                <can_become_leader>true</can_become_leader>
+                <start_as_follower>true</start_as_follower>
+                <priority>2</priority>
+            </server>
+            <server>
+                <id>3</id>
+                <hostname>localhost</hostname>
+                <port>44446</port>
+                <can_become_leader>true</can_become_leader>
+                <start_as_follower>true</start_as_follower>
+                <priority>1</priority>
+            </server>
+        </raft_configuration>
+    </test_keeper_server>
+
+    <remote_servers>
+        <test_cluster_database_replicated>
+            <shard>
+                <replica>
+                    <host>localhost</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>localhost</host>
+                    <port>19000</port>
+                </replica>
+            </shard>
+        </test_cluster_database_replicated>
+    </remote_servers>
+
+</yandex>

tests/config/install.sh

@@ -65,6 +65,9 @@ if [[ -n "$USE_DATABASE_ORDINARY" ]] && [[ "$USE_DATABASE_ORDINARY" -eq 1 ]]; th
 fi
 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     ln -sf $SRC_PATH/users.d/database_replicated.xml $DEST_SERVER_PATH/users.d/
+    ln -sf $SRC_PATH/config.d/database_replicated.xml $DEST_SERVER_PATH/config.d/
+    rm /etc/clickhouse-server/config.d/zookeeper.xml
+    rm /etc/clickhouse-server/config.d/test_keeper_port.xml
 fi
 
 ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml