diff --git a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml index 9ddeb84aa53..d04c8a2f3a6 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml @@ -1,20 +1,20 @@ version: '2.3' services: - postgres_1: + postgres2: image: postgres restart: always environment: POSTGRES_PASSWORD: mysecretpassword ports: - 5421:5432 - postgres_2: + postgres3: image: postgres restart: always environment: POSTGRES_PASSWORD: mysecretpassword ports: - 5441:5432 - postgres_3: + postgres4: image: postgres restart: always environment: diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 9cc64260db4..1e1be363621 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -672,6 +672,8 @@ class ClickHouseCluster: print('Setup MySQL') subprocess_check_call(self.base_mysql_cluster_cmd + common_opts) self.wait_mysql_to_start(120, port=3348) + self.wait_mysql_to_start(120, port=3368) + self.wait_mysql_to_start(120, port=3388) if self.with_postgres and self.base_postgres_cmd: print('Setup Postgres') @@ -681,7 +683,9 @@ class ClickHouseCluster: if self.with_postgres_cluster and self.base_postgres_cluster_cmd: print('Setup Postgres') subprocess_check_call(self.base_postgres_cluster_cmd + common_opts) + self.wait_postgres_to_start(120, port=5421) self.wait_postgres_to_start(120, port=5441) + self.wait_postgres_to_start(120, port=5461) if self.with_kafka and self.base_kafka_cmd: print('Setup Kafka') diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index 9049c05f2db..4829811f239 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -194,6 +194,7 @@ def test_mysql_distributed(started_cluster): conn1 = get_mysql_conn(port=3348) conn2 = get_mysql_conn(port=3388) conn3 = get_mysql_conn(port=3368) + conn4 = get_mysql_conn(port=3308) create_mysql_db(conn1, 'clickhouse') create_mysql_db(conn2, 'clickhouse') @@ -202,20 +203,22 @@ def test_mysql_distributed(started_cluster): create_mysql_table(conn1, table_name) create_mysql_table(conn2, table_name) create_mysql_table(conn3, table_name) + create_mysql_table(conn4, table_name) # Storage with with 3 replicas node2.query(''' CREATE TABLE test_replicas (id UInt32, name String, age UInt32, money UInt32) - ENGINE = MySQL(`mysql_{1|2|3}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') + ENGINE = MySQL(`mysql{2|3|4}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') # Fill remote tables with different data to be able to check - for i in range(1, 4): - node2.query(''' + nodes = [node1, node2, node2, node2] + for i in range(1, 5): + nodes[i-1].query(''' CREATE TABLE test_replica{} (id UInt32, name String, age UInt32, money UInt32) - ENGINE = MySQL(`mysql_{}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse');'''.format(i, i)) - node2.query("INSERT INTO test_replica{} (id, name) SELECT number, 'host{}' from numbers(10) ".format(i, i)) + ENGINE = MySQL(`mysql{}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse');'''.format(i, i)) + nodes[i-1].query("INSERT INTO test_replica{} (id, name) SELECT number, 'host{}' from numbers(10) ".format(i, i)) # check all replicas are traversed query = "SELECT * FROM (" @@ -223,27 +226,36 @@ def test_mysql_distributed(started_cluster): query += "SELECT name FROM test_replicas UNION DISTINCT " query += "SELECT name FROM test_replicas)" - result = node2.query(query.format(t=table_name)) - assert(result == 'host1\nhost2\nhost3\n') + result = node2.query(query) + assert(result == 'host2\nhost3\nhost4\n') + + # test table function (all replicas are invalid except for one) + result = node2.query('''SELECT DISTINCT(name) FROM mysql(`mysql{7|8|9|3|6}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') + assert(result == 'host3\n') # Storage with with two shards, each has 2 replicas node1.query(''' CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('MySQL', `mysql_{1|2}:3306,mysql_{3|4}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') + ENGINE = ExternalDistributed('MySQL', `mysql{1|2|3}:3306,mysql{4|5}:3306`, 'clickhouse', 'test_replicas', 'root', 'clickhouse'); ''') # Check only one replica in each shard is used result = node1.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - assert(result == 'host1\nhost3\n') + assert(result == 'host1\nhost4\n') # check all replicas are traversed query = "SELECT name FROM (" - for i in range (2): + for i in range (3): query += "SELECT name FROM test_shards UNION DISTINCT " query += "SELECT name FROM test_shards) ORDER BY name" - result = node1.query(query.format(t=table_name)) - assert(result == 'host1\nhost2\nhost3\n') + result = node1.query(query) + assert(result == 'host1\nhost2\nhost3\nhost4\n') + + # disconnect mysql1 + started_cluster.pause_container('mysql1') + result = node1.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") + assert(result == 'host2\nhost4\n' or result == 'host3\nhost4\n') if __name__ == '__main__': diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 0956d1ed3c6..ed254b3754e 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -233,16 +233,18 @@ def test_concurrent_queries(started_cluster): def test_postgres_distributed(started_cluster): + conn0 = get_postgres_conn(port=5432, database=True) conn1 = get_postgres_conn(port=5421, database=True) conn2 = get_postgres_conn(port=5441, database=True) conn3 = get_postgres_conn(port=5461, database=True) + cursor0 = conn0.cursor() cursor1 = conn1.cursor() cursor2 = conn2.cursor() cursor3 = conn3.cursor() - cursors = [cursor1, cursor2, cursor3] + cursors = [cursor0, cursor1, cursor2, cursor3] - for i in range(3): + for i in range(4): cursors[i].execute('CREATE TABLE test_replicas (id Integer, name Text)') cursors[i].execute("""INSERT INTO test_replicas select i, 'host{}' from generate_series(0, 99) as t(i);""".format(i + 1)); @@ -250,7 +252,7 @@ def test_postgres_distributed(started_cluster): node2.query(''' CREATE TABLE test_replicas (id UInt32, name String) - ENGINE = PostgreSQL(`postgres_{1|2|3}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword'); ''') + ENGINE = PostgreSQL(`postgres{2|3|4}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword'); ''') # check all replicas are traversed query = "SELECT name FROM (" @@ -259,26 +261,35 @@ def test_postgres_distributed(started_cluster): query += "SELECT name FROM test_replicas) ORDER BY name" result = node2.query(query) - assert(result == 'host1\nhost2\nhost3\n') + assert(result == 'host2\nhost3\nhost4\n') + + # test table function (all replicas are invalid except for one) + result = node2.query('''SELECT DISTINCT(name) FROM postgresql(`postgres{7|8|9|3|6}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword'); ''') + assert(result == 'host3\n') # Storage with with two two shards, each has 2 replicas node2.query(''' CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) - ENGINE = ExternalDistributed('PostgreSQL', `postgres_{1|2}:5432,postgres_{3|4}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword'); ''') + ENGINE = ExternalDistributed('PostgreSQL', `postgres{1|2|3}:5432,postgres{4|5}:5432`, 'clickhouse', 'test_replicas', 'postgres', 'mysecretpassword'); ''') # Check only one replica in each shard is used result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") - assert(result == 'host1\nhost3\n') + assert(result == 'host1\nhost4\n') # check all replicas are traversed query = "SELECT name FROM (" - for i in range (2): + for i in range (3): query += "SELECT name FROM test_shards UNION DISTINCT " query += "SELECT name FROM test_shards) ORDER BY name" result = node2.query(query) - assert(result == 'host1\nhost2\nhost3\n') + assert(result == 'host1\nhost2\nhost3\nhost4\n') + + # disconnect postgres1 + started_cluster.pause_container('postgres1') + result = node2.query("SELECT DISTINCT(name) FROM test_shards ORDER BY name") + assert(result == 'host2\nhost4\n' or result == 'host3\nhost4\n') if __name__ == '__main__':