From cbbeb4001b2bb968debf06a97ce9c9eca8bf43a8 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 11 Jun 2021 15:00:40 +0300 Subject: [PATCH 01/35] more changes --- tests/integration/helpers/cluster.py | 8 +++----- tests/integration/helpers/test_tools.py | 7 ++++--- .../test_backup_with_other_granularity/test.py | 13 +++++-------- .../test_cluster_copier/test_three_nodes.py | 10 +++++----- .../test_cluster_copier/test_two_nodes.py | 4 ++-- 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 2aaa4e91cee..cc5c428ed2e 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -575,7 +575,7 @@ class ClickHouseCluster: self.base_cmd.extend(['--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')]) self.base_hdfs_cmd = ['docker-compose', '--env-file', instance.env_file, '--project-name', self.project_name, '--file', p.join(docker_compose_yml_dir, 'docker_compose_hdfs.yml')] - print("HDFS BASE CMD:{}".format(self.base_hdfs_cmd)) + logging.debug("HDFS BASE CMD:{self.base_hdfs_cmd)}") return self.base_hdfs_cmd def setup_kerberized_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir): @@ -1816,8 +1816,7 @@ class ClickHouseInstance: self.start_clickhouse(stop_start_wait_sec) def exec_in_container(self, cmd, detach=False, nothrow=False, **kwargs): - container_id = self.get_docker_handle().id - return self.cluster.exec_in_container(container_id, cmd, detach, nothrow, **kwargs) + return self.cluster.exec_in_container(self.docker_id, cmd, detach, nothrow, **kwargs) def contains_in_log(self, substring): result = self.exec_in_container( @@ -1857,8 +1856,7 @@ class ClickHouseInstance: ["bash", "-c", "echo $(if [ -e '{}' ]; then echo 'yes'; else echo 'no'; fi)".format(path)]) == 'yes\n' def copy_file_to_container(self, local_path, dest_path): - container_id = self.get_docker_handle().id - return self.cluster.copy_file_to_container(container_id, local_path, dest_path) + return self.cluster.copy_file_to_container(self.docker_id, local_path, dest_path) def get_process_pid(self, process_name): output = self.exec_in_container(["bash", "-c", diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index 93478c4dd49..ef530c4836b 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -1,5 +1,6 @@ import difflib import time +import logging from io import IOBase @@ -56,7 +57,7 @@ def assert_eq_with_retry(instance, query, expectation, retry_count=20, sleep_tim break time.sleep(sleep_time) except Exception as ex: - print(("assert_eq_with_retry retry {} exception {}".format(i + 1, ex))) + logging.exception(f"assert_eq_with_retry retry {i+1} exception {ex}") time.sleep(sleep_time) else: val = TSV(get_result(instance.query(query, user=user, stdin=stdin, timeout=timeout, settings=settings, @@ -76,7 +77,7 @@ def assert_logs_contain_with_retry(instance, substring, retry_count=20, sleep_ti break time.sleep(sleep_time) except Exception as ex: - print("contains_in_log_with_retry retry {} exception {}".format(i + 1, ex)) + logging.exception(f"contains_in_log_with_retry retry {i+1} exception {ex}") time.sleep(sleep_time) else: raise AssertionError("'{}' not found in logs".format(substring)) @@ -89,7 +90,7 @@ def exec_query_with_retry(instance, query, retry_count=40, sleep_time=0.5, setti break except Exception as ex: exception = ex - print("Failed to execute query '", query, "' on instance", instance.name, "will retry") + logging.exception(f"Failed to execute query '{query}' on instance '{instance.name}' will retry") time.sleep(sleep_time) else: raise exception diff --git a/tests/integration/test_backup_with_other_granularity/test.py b/tests/integration/test_backup_with_other_granularity/test.py index 5ed1cb06787..832c1cf35ce 100644 --- a/tests/integration/test_backup_with_other_granularity/test.py +++ b/tests/integration/test_backup_with_other_granularity/test.py @@ -43,8 +43,8 @@ def test_backup_from_old_version(started_cluster): assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" - node1.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node1.exec_in_container(['find', '/var/lib/clickhouse/shadow/1/data/default/source_table']) + node1.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) assert node1.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -81,8 +81,7 @@ def test_backup_from_old_version_setting(started_cluster): assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" - node2.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node2.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) assert node2.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -123,8 +122,7 @@ def test_backup_from_old_version_config(started_cluster): assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" - node3.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/ /var/lib/clickhouse/data/default/dest_table/detached']) + node3.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/default/source_table/all_1_1_0/', '/var/lib/clickhouse/data/default/dest_table/detached']) assert node3.query("SELECT COUNT() FROM dest_table") == "1\n" @@ -156,8 +154,7 @@ def test_backup_and_alter(started_cluster): node4.query("ALTER TABLE test.backup_table DROP PARTITION tuple()") - node4.exec_in_container(['bash', '-c', - 'cp -r /var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/ /var/lib/clickhouse/data/test/backup_table/detached']) + node4.exec_in_container(['cp', '-r', '/var/lib/clickhouse/shadow/1/data/test/backup_table/all_1_1_0/', '/var/lib/clickhouse/data/test/backup_table/detached']) node4.query("ALTER TABLE test.backup_table ATTACH PARTITION tuple()") diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index acdc191154c..e9fcad7247f 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -39,7 +39,7 @@ class Task: for instance_name, _ in cluster.instances.items(): instance = cluster.instances[instance_name] instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, './task_taxi_data.xml'), self.container_task_file) - print("Copied task file to container of '{}' instance. Path {}".format(instance_name, self.container_task_file)) + logging.debug(f"Copied task file to container of '{instance_name}' instance. Path {self.container_task_file}") def start(self): @@ -187,10 +187,10 @@ def execute_task(started_cluster, task, cmd_options): task.start() zk = started_cluster.get_kazoo_client('zoo1') - print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) + logging.debug("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) # Run cluster-copier processes on each node - docker_api = docker.from_env().api + docker_api = started_cluster.docker_client.api copiers_exec_ids = [] cmd = ['/usr/bin/clickhouse', 'copier', @@ -201,9 +201,9 @@ def execute_task(started_cluster, task, cmd_options): '--base-dir', '/var/log/clickhouse-server/copier'] cmd += cmd_options - print(cmd) + logging.debug(f"execute_task cmd: {cmd}") - for instance_name, instance in started_cluster.instances.items(): + for instance_name in started_cluster.instances.keys(): instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs_three_nodes/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") diff --git a/tests/integration/test_cluster_copier/test_two_nodes.py b/tests/integration/test_cluster_copier/test_two_nodes.py index a6b2c82e00f..817c3571833 100644 --- a/tests/integration/test_cluster_copier/test_two_nodes.py +++ b/tests/integration/test_cluster_copier/test_two_nodes.py @@ -430,7 +430,7 @@ def execute_task(started_cluster, task, cmd_options): print("Use ZooKeeper server: {}:{}".format(zk.hosts[0][0], zk.hosts[0][1])) # Run cluster-copier processes on each node - docker_api = docker.from_env().api + docker_api = started_cluster.docker_client.api copiers_exec_ids = [] cmd = ['/usr/bin/clickhouse', 'copier', @@ -443,7 +443,7 @@ def execute_task(started_cluster, task, cmd_options): print(cmd) - for instance_name, instance in started_cluster.instances.items(): + for instance_name in started_cluster.instances.keys(): instance = started_cluster.instances[instance_name] container = instance.get_docker_handle() instance.copy_file_to_container(os.path.join(CURRENT_TEST_DIR, "configs_two_nodes/config-copier.xml"), "/etc/clickhouse-server/config-copier.xml") From e96f23d46b37ad32d49a88b8dd9ea87149372e15 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 1 Jun 2021 16:42:56 -0700 Subject: [PATCH 02/35] H3 - add function h3ToGeo --- src/Functions/h3toGeo.cpp | 102 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 104 insertions(+) create mode 100644 src/Functions/h3toGeo.cpp diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp new file mode 100644 index 00000000000..a4e7e2638cb --- /dev/null +++ b/src/Functions/h3toGeo.cpp @@ -0,0 +1,102 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; + +} + +namespace +{ + +/// Implements the function h3ToGeo which takes a single argument (h3Index) +/// and returns the latitude and longitude that correspond to the provided h3 index +class FunctionH3ToGeo : public IFunction +{ +public: + static constexpr auto name = "H3ToGeo"; + + static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + return std::make_shared( + DataTypes{std::make_shared(), std::make_shared()}, + Strings{"latitude", "longitude"}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_index = arguments[0].column.get(); + + auto dst = ColumnVector::create(); + + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + ColumnPtr res_column; + + + auto latitude = ColumnFloat64::create(input_rows_count); + auto longitude = ColumnFloat64::create(input_rows_count); + + ColumnFloat64::Container & lon_data = longitude->getData(); + ColumnFloat64::Container & lat_data = latitude->getData(); + + for (const auto row : ext::range(0, input_rows_count)) + { + const UInt64 h3Index = col_index->getUInt(row); + GeoCoord coord; + h3ToGeo(h3Index,&coord); + lon_data[row] = coord.lon; + lat_data[row] = coord.lat; + + } + MutableColumns result; + result.emplace_back(std::move(latitude)); + result.emplace_back(std::move(longitude)); + return ColumnTuple::create(std::move(result)); + } +}; + +} + +void registerFunctionH3ToGeo(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 605dd4dcba0..1ab475345a0 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -28,6 +28,7 @@ void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); +void registerFunctionH3ToGeo(FunctionFactory &); void registerFunctionH3EdgeAngle(FunctionFactory &); void registerFunctionH3EdgeLengthM(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); @@ -66,6 +67,7 @@ void registerFunctionsGeo(FunctionFactory & factory) #if USE_H3 registerFunctionGeoToH3(factory); + registerFunctionH3ToGeo(factory); registerFunctionH3EdgeAngle(factory); registerFunctionH3EdgeLengthM(factory); registerFunctionH3GetResolution(factory); From 99ae86305509d70948cc3353140c63c7ea172868 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 00:00:14 -0700 Subject: [PATCH 03/35] h3ToGeo - few fixes --- src/Functions/h3toGeo.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index a4e7e2638cb..e886248c644 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -32,11 +32,11 @@ namespace { /// Implements the function h3ToGeo which takes a single argument (h3Index) -/// and returns the latitude and longitude that correspond to the provided h3 index +/// and returns the longitude and latitude that correspond to the provided h3 index class FunctionH3ToGeo : public IFunction { public: - static constexpr auto name = "H3ToGeo"; + static constexpr auto name = "h3ToGeo"; static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } @@ -54,7 +54,7 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); return std::make_shared( DataTypes{std::make_shared(), std::make_shared()}, - Strings{"latitude", "longitude"}); + Strings{"longitude", "latitude"}); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override @@ -79,13 +79,13 @@ public: const UInt64 h3Index = col_index->getUInt(row); GeoCoord coord; h3ToGeo(h3Index,&coord); - lon_data[row] = coord.lon; - lat_data[row] = coord.lat; + lon_data[row] = radsToDegs(coord.lon); + lat_data[row] = radsToDegs(coord.lat); } MutableColumns result; - result.emplace_back(std::move(latitude)); result.emplace_back(std::move(longitude)); + result.emplace_back(std::move(latitude)); return ColumnTuple::create(std::move(result)); } }; From 4487745dde2278fdf3bb9d250b926aba0656e119 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 00:07:49 -0700 Subject: [PATCH 04/35] h3ToGeo - docs --- docs/en/sql-reference/functions/geo/h3.md | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 20dc7b29902..6c03f55cebe 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -195,6 +195,41 @@ Result: └────────────────────┘ ``` +## h3ToGeo {#h3togeo} + +Returns `(lon, lat)` that corresponds to the provided H3 index. + +**Syntax** + +``` sql +h3ToGeo(h3Index) +``` + +**Arguments** + +- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Returned values** + +- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). +- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). + + +**Example** + +Query: + +``` sql +SELECT h3ToGeo(644325524701193974) coordinates; +``` + +Result: + +``` text +┌─coordinates───────────────────────────┐ +│ (37.79506616830252,55.71290243145668) │ +└───────────────────────────────────────┘ +``` ## h3kRing {#h3kring} Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order. From 1284a04a8b957c97e3158a65e2e9a724c3da9e8a Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 09:46:02 -0700 Subject: [PATCH 05/35] h3ToGeo - add tests --- tests/queries/0_stateless/01889_h3_to_geo.reference | 4 ++++ tests/queries/0_stateless/01889_h3_to_geo.sql | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/01889_h3_to_geo.reference create mode 100644 tests/queries/0_stateless/01889_h3_to_geo.sql diff --git a/tests/queries/0_stateless/01889_h3_to_geo.reference b/tests/queries/0_stateless/01889_h3_to_geo.reference new file mode 100644 index 00000000000..2821b897dd4 --- /dev/null +++ b/tests/queries/0_stateless/01889_h3_to_geo.reference @@ -0,0 +1,4 @@ +(37.79506616830252,55.71290243145668) +(37.79507532320379,55.712909934159995) +(37.79510864783612,55.71292405516229) +(37.79511780275313,55.712931557864124) diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01889_h3_to_geo.sql new file mode 100644 index 00000000000..fc69cec0fb4 --- /dev/null +++ b/tests/queries/0_stateless/01889_h3_to_geo.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS h3_indexes; + +CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; + +INSERT INTO h3_indexes VALUES(644325524701193974); +INSERT INTO h3_indexes VALUES(644325524701193975); +INSERT INTO h3_indexes VALUES(644325524701193976); +INSERT INTO h3_indexes VALUES(644325524701193977); + +select h3ToGeo(h3_index) from h3_indexes; + +DROP TABLE h3_indexes; \ No newline at end of file From cabd4144da938cbbee70b411c7b42e8c3a1536dc Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 2 Jun 2021 13:20:08 -0700 Subject: [PATCH 06/35] h3ToGeo - fixes to code --- src/Functions/h3toGeo.cpp | 14 +++----------- tests/queries/0_stateless/01889_h3_to_geo.sql | 4 ++-- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index e886248c644..be483f8065b 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -24,8 +24,6 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int ILLEGAL_COLUMN; - } namespace @@ -61,13 +59,7 @@ public: { const auto * col_index = arguments[0].column.get(); - auto dst = ColumnVector::create(); - - auto & dst_data = dst->getData(); - dst_data.resize(input_rows_count); ColumnPtr res_column; - - auto latitude = ColumnFloat64::create(input_rows_count); auto longitude = ColumnFloat64::create(input_rows_count); @@ -76,13 +68,13 @@ public: for (const auto row : ext::range(0, input_rows_count)) { - const UInt64 h3Index = col_index->getUInt(row); + const UInt64 h3index = col_index->getUInt(row); GeoCoord coord; - h3ToGeo(h3Index,&coord); + h3ToGeo(h3index,&coord); lon_data[row] = radsToDegs(coord.lon); lat_data[row] = radsToDegs(coord.lat); - } + MutableColumns result; result.emplace_back(std::move(longitude)); result.emplace_back(std::move(latitude)); diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01889_h3_to_geo.sql index fc69cec0fb4..04e83a84331 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.sql +++ b/tests/queries/0_stateless/01889_h3_to_geo.sql @@ -7,6 +7,6 @@ INSERT INTO h3_indexes VALUES(644325524701193975); INSERT INTO h3_indexes VALUES(644325524701193976); INSERT INTO h3_indexes VALUES(644325524701193977); -select h3ToGeo(h3_index) from h3_indexes; +select h3ToGeo(h3_index) from h3_indexes order by h3_index; -DROP TABLE h3_indexes; \ No newline at end of file +DROP TABLE h3_indexes; From b17c335074167c263852396a4cea783ccc77cb45 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sun, 6 Jun 2021 15:15:31 -0700 Subject: [PATCH 07/35] h3 - update submodule --- contrib/h3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/h3 b/contrib/h3 index e209086ae1b..5c44b06c406 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit e209086ae1b5477307f545a0f6111780edc59940 +Subproject commit 5c44b06c406613b7792a60b11d04b871116f6e30 From 03060518b77533a49aeef37c7dd716cbca2e5c41 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sun, 6 Jun 2021 17:06:59 -0700 Subject: [PATCH 08/35] h3ToGeo - better tests --- .../0_stateless/01659_h3_buffer_overflow.sql | 1 + .../0_stateless/01889_h3_to_geo.reference | 20 ++++++++++++---- tests/queries/0_stateless/01889_h3_to_geo.sql | 24 +++++++++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/01659_h3_buffer_overflow.sql b/tests/queries/0_stateless/01659_h3_buffer_overflow.sql index b752059da48..f2d77641ec9 100644 --- a/tests/queries/0_stateless/01659_h3_buffer_overflow.sql +++ b/tests/queries/0_stateless/01659_h3_buffer_overflow.sql @@ -7,3 +7,4 @@ SELECT h3kRing(0xFFFFFFFFFFFFFF, 1000) FORMAT Null; SELECT h3GetBaseCell(0xFFFFFFFFFFFFFF) FORMAT Null; SELECT h3GetResolution(0xFFFFFFFFFFFFFF) FORMAT Null; SELECT h3kRing(0xFFFFFFFFFFFFFF, 10) FORMAT Null; +SELECT h3ToGeo(0xFFFFFFFFFFFFFF) FORMAT Null; diff --git a/tests/queries/0_stateless/01889_h3_to_geo.reference b/tests/queries/0_stateless/01889_h3_to_geo.reference index 2821b897dd4..5a45c5f3c30 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.reference +++ b/tests/queries/0_stateless/01889_h3_to_geo.reference @@ -1,4 +1,16 @@ -(37.79506616830252,55.71290243145668) -(37.79507532320379,55.712909934159995) -(37.79510864783612,55.71292405516229) -(37.79511780275313,55.712931557864124) +(-173.6412167681162,-14.130272474941535) +(59.48137613600854,58.020407687755686) +(172.68095885060296,-83.6576608516349) +(-94.46556851304558,-69.1999982492279) +(-8.188263637093279,-55.856179102736284) +(77.25594891852249,47.39278564360122) +(135.11348004704536,36.60778126579667) +(39.28534828967223,49.07710003066973) +(124.71163478198051,-27.481172161567258) +(-147.4887686066785,76.73237945824442) +(86.63291906118863,-25.52526285188784) +(23.27751790712118,13.126101362212724) +(-70.40163237204142,-63.12562536833242) +(15.642428355535966,40.285813505163574) +(-76.53411447979884,54.5560449693637) +(8.19906334981474,67.69370966550179) diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01889_h3_to_geo.sql index 04e83a84331..f16e03b2bd9 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.sql +++ b/tests/queries/0_stateless/01889_h3_to_geo.sql @@ -2,10 +2,26 @@ DROP TABLE IF EXISTS h3_indexes; CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; -INSERT INTO h3_indexes VALUES(644325524701193974); -INSERT INTO h3_indexes VALUES(644325524701193975); -INSERT INTO h3_indexes VALUES(644325524701193976); -INSERT INTO h3_indexes VALUES(644325524701193977); +-- values are random h3 indexes at various resolutions from 0 to 15. +-- generated using h3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c + +INSERT INTO h3_indexes VALUES (579205133326352383); +INSERT INTO h3_indexes VALUES (581263419093549055); +INSERT INTO h3_indexes VALUES (589753847883235327); +INSERT INTO h3_indexes VALUES (594082350283882495); +INSERT INTO h3_indexes VALUES (598372386957426687); +INSERT INTO h3_indexes VALUES (599542359671177215); +INSERT INTO h3_indexes VALUES (604296355086598143); +INSERT INTO h3_indexes VALUES (608785214872748031); +INSERT INTO h3_indexes VALUES (615732192485572607); +INSERT INTO h3_indexes VALUES (617056794467368959); +INSERT INTO h3_indexes VALUES (624586477873168383); +INSERT INTO h3_indexes VALUES (627882919484481535); +INSERT INTO h3_indexes VALUES (634600058503392255); +INSERT INTO h3_indexes VALUES (635544851677385791); +INSERT INTO h3_indexes VALUES (639763125756281263); +INSERT INTO h3_indexes VALUES (644178757620501158); + select h3ToGeo(h3_index) from h3_indexes order by h3_index; From 78b61237907d99df209ab174cf29fd88ab99b07e Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sun, 13 Jun 2021 09:14:29 -0700 Subject: [PATCH 09/35] h3ToGeo - rename test files --- .../{01889_h3_to_geo.reference => 01906_h3_to_geo.reference} | 0 .../0_stateless/{01889_h3_to_geo.sql => 01906_h3_to_geo.sql} | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/queries/0_stateless/{01889_h3_to_geo.reference => 01906_h3_to_geo.reference} (100%) rename tests/queries/0_stateless/{01889_h3_to_geo.sql => 01906_h3_to_geo.sql} (75%) diff --git a/tests/queries/0_stateless/01889_h3_to_geo.reference b/tests/queries/0_stateless/01906_h3_to_geo.reference similarity index 100% rename from tests/queries/0_stateless/01889_h3_to_geo.reference rename to tests/queries/0_stateless/01906_h3_to_geo.reference diff --git a/tests/queries/0_stateless/01889_h3_to_geo.sql b/tests/queries/0_stateless/01906_h3_to_geo.sql similarity index 75% rename from tests/queries/0_stateless/01889_h3_to_geo.sql rename to tests/queries/0_stateless/01906_h3_to_geo.sql index f16e03b2bd9..c78270d2808 100644 --- a/tests/queries/0_stateless/01889_h3_to_geo.sql +++ b/tests/queries/0_stateless/01906_h3_to_geo.sql @@ -2,8 +2,8 @@ DROP TABLE IF EXISTS h3_indexes; CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; --- values are random h3 indexes at various resolutions from 0 to 15. --- generated using h3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c +-- Random geo coordinates were generated using the H3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c at various resolutions from 0 to 15. +-- Corresponding H3 index values were in turn generated with those geo coordinates using `geoToH3(lon, lat, res)` ClickHouse function for the following test. INSERT INTO h3_indexes VALUES (579205133326352383); INSERT INTO h3_indexes VALUES (581263419093549055); From d073161455b9b11ef7c2b933c975a0a941ae71ba Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 14 Jun 2021 11:36:41 -0700 Subject: [PATCH 10/35] h3ToGeo - try fix msan fuzzer errors --- src/Functions/h3toGeo.cpp | 74 ++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp index be483f8065b..8ce0b15dc8c 100644 --- a/src/Functions/h3toGeo.cpp +++ b/src/Functions/h3toGeo.cpp @@ -23,7 +23,7 @@ namespace DB { namespace ErrorCodes { - extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_TYPE_OF_ARGUMENT; } namespace @@ -36,50 +36,52 @@ class FunctionH3ToGeo : public IFunction public: static constexpr auto name = "h3ToGeo"; - static FunctionPtr create(ContextConstPtr) { return std::make_shared(); } + static FunctionPtr create(ContextPtr) { return std::make_shared(); } std::string getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } bool useDefaultImplementationForConstants() const override { return true; } - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared( + DataTypes{std::make_shared(), std::make_shared()}, + Strings{"longitude", "latitude"}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_index = arguments[0].column.get(); + + auto latitude = ColumnFloat64::create(input_rows_count); + auto longitude = ColumnFloat64::create(input_rows_count); + + ColumnFloat64::Container & lon_data = longitude->getData(); + ColumnFloat64::Container & lat_data = latitude->getData(); + + + for (size_t row = 0; row < input_rows_count; ++row) { - const auto * arg = arguments[0].get(); - if (!WhichDataType(arg).isUInt64()) - throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return std::make_shared( - DataTypes{std::make_shared(), std::make_shared()}, - Strings{"longitude", "latitude"}); + H3Index h3index = col_index->getUInt(row); + GeoCoord coord{}; + + h3ToGeo(h3index,&coord); + lon_data[row] = radsToDegs(coord.lon); + lat_data[row] = radsToDegs(coord.lat); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - const auto * col_index = arguments[0].column.get(); - - ColumnPtr res_column; - auto latitude = ColumnFloat64::create(input_rows_count); - auto longitude = ColumnFloat64::create(input_rows_count); - - ColumnFloat64::Container & lon_data = longitude->getData(); - ColumnFloat64::Container & lat_data = latitude->getData(); - - for (const auto row : ext::range(0, input_rows_count)) - { - const UInt64 h3index = col_index->getUInt(row); - GeoCoord coord; - h3ToGeo(h3index,&coord); - lon_data[row] = radsToDegs(coord.lon); - lat_data[row] = radsToDegs(coord.lat); - } - - MutableColumns result; - result.emplace_back(std::move(longitude)); - result.emplace_back(std::move(latitude)); - return ColumnTuple::create(std::move(result)); - } + MutableColumns columns; + columns.emplace_back(std::move(longitude)); + columns.emplace_back(std::move(latitude)); + return ColumnTuple::create(std::move(columns)); + } }; } From c1e57efda3b99fa7e2ef36985e4f8c9963ffaa52 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 15 Jun 2021 08:53:30 -0700 Subject: [PATCH 11/35] h3ToGeo - test h3ToGeo(geoToH3(lat, lon, res)) --- .../0_stateless/01906_h3_to_geo.reference | 16 +++++++++ tests/queries/0_stateless/01906_h3_to_geo.sql | 35 ++++++++++++++++++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01906_h3_to_geo.reference b/tests/queries/0_stateless/01906_h3_to_geo.reference index 5a45c5f3c30..93e8600576c 100644 --- a/tests/queries/0_stateless/01906_h3_to_geo.reference +++ b/tests/queries/0_stateless/01906_h3_to_geo.reference @@ -14,3 +14,19 @@ (15.642428355535966,40.285813505163574) (-76.53411447979884,54.5560449693637) (8.19906334981474,67.69370966550179) +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok diff --git a/tests/queries/0_stateless/01906_h3_to_geo.sql b/tests/queries/0_stateless/01906_h3_to_geo.sql index c78270d2808..aa6ecca1754 100644 --- a/tests/queries/0_stateless/01906_h3_to_geo.sql +++ b/tests/queries/0_stateless/01906_h3_to_geo.sql @@ -23,6 +23,39 @@ INSERT INTO h3_indexes VALUES (639763125756281263); INSERT INTO h3_indexes VALUES (644178757620501158); -select h3ToGeo(h3_index) from h3_indexes order by h3_index; +SELECT h3ToGeo(h3_index) FROM h3_indexes ORDER BY h3_index; DROP TABLE h3_indexes; + +DROP TABLE IF EXISTS h3_geo; + +-- compare if the results of h3ToGeo and geoToH3 are the same + +CREATE TABLE h3_geo(lat Float64, lon Float64, res UInt8) ENGINE = Memory; + +INSERT INTO h3_geo VALUES (-173.6412167681162, -14.130272474941535, 0); +INSERT INTO h3_geo VALUES (59.48137613600854, 58.020407687755686, 1); +INSERT INTO h3_geo VALUES (172.68095885060296, -83.6576608516349, 2); +INSERT INTO h3_geo VALUES (-94.46556851304558, -69.1999982492279, 3); +INSERT INTO h3_geo VALUES (-8.188263637093279, -55.856179102736284, 4); +INSERT INTO h3_geo VALUES (77.25594891852249, 47.39278564360122, 5); +INSERT INTO h3_geo VALUES (135.11348004704536, 36.60778126579667, 6); +INSERT INTO h3_geo VALUES (39.28534828967223, 49.07710003066973, 7); +INSERT INTO h3_geo VALUES (124.71163478198051, -27.481172161567258, 8); +INSERT INTO h3_geo VALUES (-147.4887686066785, 76.73237945824442, 9); +INSERT INTO h3_geo VALUES (86.63291906118863, -25.52526285188784, 10); +INSERT INTO h3_geo VALUES (23.27751790712118, 13.126101362212724, 11); +INSERT INTO h3_geo VALUES (-70.40163237204142, -63.12562536833242, 12); +INSERT INTO h3_geo VALUES (15.642428355535966, 40.285813505163574, 13); +INSERT INTO h3_geo VALUES (-76.53411447979884, 54.5560449693637, 14); +INSERT INTO h3_geo VALUES (8.19906334981474, 67.69370966550179, 15); + +SELECT result FROM ( + SELECT + (lat, lon) AS input_geo, + h3ToGeo(geoToH3(lat, lon, res)) AS output_geo, + if(input_geo = output_geo, 'ok', 'fail') AS result + FROM h3_geo +); + +DROP TABLE h3_geo; From f4d587a17a7b189b798f1b0b0858b34a667d8f49 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 16 Jun 2021 15:31:19 +0300 Subject: [PATCH 12/35] hardening --- tests/integration/helpers/cluster.py | 15 ++++++------- .../test_cluster_copier/test_three_nodes.py | 22 +++++++++---------- .../test.py | 2 +- .../integration/test_distributed_ddl/test.py | 7 +++--- .../integration/test_zookeeper_config/test.py | 2 ++ .../test_zookeeper_config/test_password.py | 2 ++ 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index cc5c428ed2e..d40505bcb89 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -1421,9 +1421,9 @@ class ClickHouseCluster: instance.docker_client = self.docker_client instance.ip_address = self.get_instance_ip(instance.name) - logging.debug("Waiting for ClickHouse start...") + logging.debug("Waiting for ClickHouse start in {instance}, ip: {instance.ip_address}...") instance.wait_for_start(start_timeout) - logging.debug("ClickHouse started") + logging.debug("ClickHouse {instance} started") instance.client = Client(instance.ip_address, command=self.client_bin_path) @@ -1911,6 +1911,7 @@ class ClickHouseInstance: self.get_docker_handle().start() def wait_for_start(self, start_timeout=None, connection_timeout=None): + handle = self.get_docker_handle() if start_timeout is None or start_timeout <= 0: raise Exception("Invalid timeout: {}".format(start_timeout)) @@ -1933,11 +1934,10 @@ class ClickHouseInstance: return False while True: - handle = self.get_docker_handle() + handle.reload() status = handle.status if status == 'exited': - raise Exception("Instance `{}' failed to start. Container status: {}, logs: {}" - .format(self.name, status, handle.logs().decode('utf-8'))) + raise Exception(f"Instance `{self.name}' failed to start. Container status: {status}, logs: {handle.logs().decode('utf-8')}") deadline = start_time + start_timeout # It is possible that server starts slowly. @@ -1947,9 +1947,8 @@ class ClickHouseInstance: current_time = time.time() if current_time >= deadline: - raise Exception("Timed out while waiting for instance `{}' with ip address {} to start. " - "Container status: {}, logs: {}".format(self.name, self.ip_address, status, - handle.logs().decode('utf-8'))) + raise Exception(f"Timed out while waiting for instance `{self.name}' with ip address {self.ip_address} to start. " \ + f"Container status: {status}, logs: {handle.logs().decode('utf-8')}") socket_timeout = min(start_timeout, deadline - current_time) diff --git a/tests/integration/test_cluster_copier/test_three_nodes.py b/tests/integration/test_cluster_copier/test_three_nodes.py index e9fcad7247f..63b0bcc6679 100644 --- a/tests/integration/test_cluster_copier/test_three_nodes.py +++ b/tests/integration/test_cluster_copier/test_three_nodes.py @@ -48,11 +48,11 @@ class Task: node.query("DROP DATABASE IF EXISTS dailyhistory SYNC;") node.query("DROP DATABASE IF EXISTS monthlyhistory SYNC;") - instance = cluster.instances['first'] + first = cluster.instances['first'] # daily partition database - instance.query("CREATE DATABASE IF NOT EXISTS dailyhistory on cluster events;") - instance.query("""CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events + first.query("CREATE DATABASE IF NOT EXISTS dailyhistory on cluster events;") + first.query("""CREATE TABLE dailyhistory.yellow_tripdata_staging ON CLUSTER events ( id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -84,12 +84,12 @@ class Task: ORDER BY (tpep_pickup_datetime, id) PARTITION BY (toYYYYMMDD(tpep_pickup_datetime))""") - instance.query("""CREATE TABLE dailyhistory.yellow_tripdata + first.query("""CREATE TABLE dailyhistory.yellow_tripdata ON CLUSTER events AS dailyhistory.yellow_tripdata_staging ENGINE = Distributed('events', 'dailyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""") - instance.query("""INSERT INTO dailyhistory.yellow_tripdata + first.query("""INSERT INTO dailyhistory.yellow_tripdata SELECT * FROM generateRandom( 'id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -119,8 +119,8 @@ class Task: 1, 10, 2) LIMIT 50;""") # monthly partition database - instance.query("create database IF NOT EXISTS monthlyhistory on cluster events;") - instance.query("""CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events + first.query("create database IF NOT EXISTS monthlyhistory on cluster events;") + first.query("""CREATE TABLE monthlyhistory.yellow_tripdata_staging ON CLUSTER events ( id UUID DEFAULT generateUUIDv4(), vendor_id String, @@ -153,16 +153,16 @@ class Task: ORDER BY (tpep_pickup_datetime, id) PARTITION BY (pickup_location_id, toYYYYMM(tpep_pickup_datetime))""") - instance.query("""CREATE TABLE monthlyhistory.yellow_tripdata + first.query("""CREATE TABLE monthlyhistory.yellow_tripdata ON CLUSTER events AS monthlyhistory.yellow_tripdata_staging ENGINE = Distributed('events', 'monthlyhistory', yellow_tripdata_staging, sipHash64(id) % 3);""") def check(self): - instance = cluster.instances["first"] - a = TSV(instance.query("SELECT count() from dailyhistory.yellow_tripdata")) - b = TSV(instance.query("SELECT count() from monthlyhistory.yellow_tripdata")) + first = cluster.instances["first"] + a = TSV(first.query("SELECT count() from dailyhistory.yellow_tripdata")) + b = TSV(first.query("SELECT count() from monthlyhistory.yellow_tripdata")) assert a == b, "Distributed tables" for instance_name, instance in cluster.instances.items(): diff --git a/tests/integration/test_dictionaries_update_and_reload/test.py b/tests/integration/test_dictionaries_update_and_reload/test.py index f1e6e9bb5a9..8e375b7b327 100644 --- a/tests/integration/test_dictionaries_update_and_reload/test.py +++ b/tests/integration/test_dictionaries_update_and_reload/test.py @@ -150,7 +150,7 @@ def test_reload_after_loading(started_cluster): time.sleep(1) # see the comment above replace_in_file_in_container('/etc/clickhouse-server/dictionaries/executable.xml', '82', '83') replace_in_file_in_container('/etc/clickhouse-server/dictionaries/file.txt', '102', '103') - time.sleep(7) + time.sleep(10) assert query("SELECT dictGetInt32('file', 'a', toUInt64(9))") == "103\n" assert query("SELECT dictGetInt32('executable', 'a', toUInt64(7))") == "83\n" diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 87e793a7acb..ae0739770ba 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -53,6 +53,7 @@ def test_default_database(test_cluster): def test_create_view(test_cluster): instance = test_cluster.instances['ch3'] + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.super_simple_view ON CLUSTER 'cluster'") test_cluster.ddl_check_query(instance, "CREATE VIEW test.super_simple_view ON CLUSTER 'cluster' AS SELECT * FROM system.numbers FORMAT TSV") test_cluster.ddl_check_query(instance, @@ -76,7 +77,7 @@ def test_on_server_fail(test_cluster): kill_instance.get_docker_handle().stop() request = instance.get_query_request("CREATE TABLE test.test_server_fail ON CLUSTER 'cluster' (i Int8) ENGINE=Null", - timeout=30) + timeout=180) kill_instance.get_docker_handle().start() test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'") @@ -106,11 +107,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout): def test_on_connection_loss(test_cluster): - _test_on_connection_losses(test_cluster, 5) # connection loss will occur only (3 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 10) # connection loss will occur only (10 sec ZK timeout in config) def test_on_session_expired(test_cluster): - _test_on_connection_losses(test_cluster, 15) # session should be expired (3 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 30) # session should be expired (10 sec ZK timeout in config) def test_simple_alters(test_cluster): diff --git a/tests/integration/test_zookeeper_config/test.py b/tests/integration/test_zookeeper_config/test.py index 732816ea224..95d9db27a7d 100644 --- a/tests/integration/test_zookeeper_config/test.py +++ b/tests/integration/test_zookeeper_config/test.py @@ -30,6 +30,7 @@ def started_cluster(): def test_chroot_with_same_root(started_cluster): for i, node in enumerate([node1, node2]): + node.query('DROP TABLE IF EXISTS simple SYNC') node.query(''' CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); @@ -44,6 +45,7 @@ def test_chroot_with_same_root(started_cluster): def test_chroot_with_different_root(started_cluster): for i, node in [(1, node1), (3, node3)]: + node.query('DROP TABLE IF EXISTS simple_different SYNC') node.query(''' CREATE TABLE simple_different (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple_different', '{replica}', date, id, 8192); diff --git a/tests/integration/test_zookeeper_config/test_password.py b/tests/integration/test_zookeeper_config/test_password.py index c0ed4375978..09c15cfd0cf 100644 --- a/tests/integration/test_zookeeper_config/test_password.py +++ b/tests/integration/test_zookeeper_config/test_password.py @@ -22,6 +22,8 @@ def started_cluster(): cluster.shutdown() def test_identity(started_cluster): + node1.query('DROP TABLE IF EXISTS simple SYNC') + node1.query(''' CREATE TABLE simple (date Date, id UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/0/simple', '{replica}', date, id, 8192); From 1d6253b9dcafc27b0d0f51045d4b9fe2d2bd471b Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 16 Jun 2021 18:37:51 +0300 Subject: [PATCH 13/35] hardening --- tests/integration/test_distributed_ddl/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index ae0739770ba..28ebc69bc05 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -99,7 +99,7 @@ def _test_on_connection_losses(test_cluster, zk_timeout): with PartitionManager() as pm: pm.drop_instance_zk_connections(kill_instance) - request = instance.get_query_request("DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'", timeout=20) + request = instance.get_query_request("DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'", timeout=40) time.sleep(zk_timeout) pm.restore_instance_zk_connections(kill_instance) @@ -107,11 +107,11 @@ def _test_on_connection_losses(test_cluster, zk_timeout): def test_on_connection_loss(test_cluster): - _test_on_connection_losses(test_cluster, 10) # connection loss will occur only (10 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 15) # connection loss will occur only (10 sec ZK timeout in config) def test_on_session_expired(test_cluster): - _test_on_connection_losses(test_cluster, 30) # session should be expired (10 sec ZK timeout in config) + _test_on_connection_losses(test_cluster, 60) # session should be expired (10 sec ZK timeout in config) def test_simple_alters(test_cluster): From dc893f9644bafd2e9a710c6b9351f06561fafb24 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 17 Jun 2021 19:37:08 +0300 Subject: [PATCH 14/35] Update NuRaft buffer to 64 bit size --- contrib/NuRaft | 2 +- src/Coordination/Changelog.h | 3 ++- src/Coordination/KeeperSnapshotManager.cpp | 2 +- src/Coordination/KeeperSnapshotManager.h | 5 ++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contrib/NuRaft b/contrib/NuRaft index 2a1bf7d87b4..976874b7aa7 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit 2a1bf7d87b4a03561fc66fbb49cee8a288983c5d +Subproject commit 976874b7aa7f422bf4ea595bb7d1166c617b1c26 diff --git a/src/Coordination/Changelog.h b/src/Coordination/Changelog.h index a50b734c5b4..893fe16abdf 100644 --- a/src/Coordination/Changelog.h +++ b/src/Coordination/Changelog.h @@ -23,9 +23,10 @@ using IndexToLogEntry = std::unordered_map; enum class ChangelogVersion : uint8_t { V0 = 0, + V1 = 1, /// with 64 bit buffer header }; -static constexpr auto CURRENT_CHANGELOG_VERSION = ChangelogVersion::V0; +static constexpr auto CURRENT_CHANGELOG_VERSION = ChangelogVersion::V1; struct ChangelogRecordHeader { diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index 7520f9b3ba2..3575966410c 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -204,7 +204,7 @@ SnapshotMetadataPtr KeeperStorageSnapshot::deserialize(KeeperStorage & storage, uint8_t version; readBinary(version, in); SnapshotVersion current_version = static_cast(version); - if (current_version > SnapshotVersion::V1) + if (current_version > CURRENT_SNAPSHOT_VERSION) throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unsupported snapshot version {}", version); SnapshotMetadataPtr result = deserializeSnapshotMetadata(in); diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h index 3dbd7c9328e..57347c37b9f 100644 --- a/src/Coordination/KeeperSnapshotManager.h +++ b/src/Coordination/KeeperSnapshotManager.h @@ -14,8 +14,11 @@ enum SnapshotVersion : uint8_t { V0 = 0, V1 = 1, /// with ACL map + V2 = 2, /// with 64 bit buffer header }; +static constexpr auto CURRENT_SNAPSHOT_VERSION = SnapshotVersion::V2; + struct KeeperStorageSnapshot { public: @@ -30,7 +33,7 @@ public: KeeperStorage * storage; - SnapshotVersion version = SnapshotVersion::V1; + SnapshotVersion version = CURRENT_SNAPSHOT_VERSION; SnapshotMetadataPtr snapshot_meta; int64_t session_id; size_t snapshot_container_size; From fabff86788de636fe2d5960693f48180e43fc0bd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 20 Jun 2021 22:34:18 +0300 Subject: [PATCH 15/35] Fix NOT parsing --- src/Parsers/ExpressionListParsers.cpp | 5 +++-- src/Parsers/ExpressionListParsers.h | 6 +++--- tests/queries/0_stateless/01921_not_chain.reference | 9 +++++++++ tests/queries/0_stateless/01921_not_chain.sql | 5 +++++ 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/01921_not_chain.reference create mode 100644 tests/queries/0_stateless/01921_not_chain.sql diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index e6af11399de..5047ce39d47 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -22,9 +22,10 @@ const char * ParserMultiplicativeExpression::operators[] = nullptr }; -const char * ParserUnaryMinusExpression::operators[] = +const char * ParserUnaryExpression::operators[] = { "-", "negate", + "NOT", "not", nullptr }; @@ -539,7 +540,7 @@ bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, ASTPtr & node, Ex } -bool ParserUnaryMinusExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +bool ParserUnaryExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { /// As an exception, negative numbers should be parsed as literals, and not as an application of the operator. diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index 3a65141d751..bd4763297d4 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -245,14 +245,14 @@ protected: }; -class ParserUnaryMinusExpression : public IParserBase +class ParserUnaryExpression : public IParserBase { private: static const char * operators[]; ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique()}; protected: - const char * getName() const override { return "unary minus expression"; } + const char * getName() const override { return "unary expression"; } bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; @@ -262,7 +262,7 @@ class ParserMultiplicativeExpression : public IParserBase { private: static const char * operators[]; - ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; + ParserLeftAssociativeBinaryOperatorList operator_parser {operators, std::make_unique()}; protected: const char * getName() const override { return "multiplicative expression"; } diff --git a/tests/queries/0_stateless/01921_not_chain.reference b/tests/queries/0_stateless/01921_not_chain.reference new file mode 100644 index 00000000000..c29c66f1274 --- /dev/null +++ b/tests/queries/0_stateless/01921_not_chain.reference @@ -0,0 +1,9 @@ +-- { echo } +SELECT 1 != (NOT 1); +1 +SELECT 1 != NOT 1; +1 +EXPLAIN SYNTAX SELECT 1 != (NOT 1); +SELECT 1 != NOT 1 +EXPLAIN SYNTAX SELECT 1 != NOT 1; +SELECT 1 != NOT 1 diff --git a/tests/queries/0_stateless/01921_not_chain.sql b/tests/queries/0_stateless/01921_not_chain.sql new file mode 100644 index 00000000000..52549e8ce51 --- /dev/null +++ b/tests/queries/0_stateless/01921_not_chain.sql @@ -0,0 +1,5 @@ +-- { echo } +SELECT 1 != (NOT 1); +SELECT 1 != NOT 1; +EXPLAIN SYNTAX SELECT 1 != (NOT 1); +EXPLAIN SYNTAX SELECT 1 != NOT 1; From c7781e24d51ba9f32bc7543c50bcffd23153a349 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 11:02:27 +0300 Subject: [PATCH 16/35] Fix some flaky tests --- tests/integration/helpers/cluster.py | 12 +++++++++--- tests/integration/pytest.ini | 2 +- tests/integration/test_merge_tree_s3/test.py | 4 ++-- .../test_reload_max_table_size_to_drop/test.py | 5 +++-- tests/integration/test_storage_s3/test.py | 5 +++-- tests/integration/test_system_flush_logs/test.py | 5 ++--- 6 files changed, 20 insertions(+), 13 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 7888e07587c..eb7ed55c7a2 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -29,7 +29,6 @@ from dict2xml import dict2xml from kazoo.client import KazooClient from kazoo.exceptions import KazooException from minio import Minio -from minio.deleteobjects import DeleteObject from helpers.test_tools import assert_eq_with_retry import docker @@ -172,6 +171,13 @@ def enable_consistent_hash_plugin(rabbitmq_id): p.communicate() return p.returncode == 0 +def get_instances_dir(): + if 'INTEGRATION_TESTS_RUN_ID' in os.environ: + return '_instances_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) + else: + return '_instances' + + class ClickHouseCluster: """ClickHouse cluster with several instances and (possibly) ZooKeeper. @@ -1232,8 +1238,8 @@ class ClickHouseCluster: for bucket in buckets: if minio_client.bucket_exists(bucket): delete_object_list = map( - lambda x: DeleteObject(x.object_name), - minio_client.list_objects(bucket, recursive=True), + lambda x: x.object_name, + minio_client.list_objects_v2(bucket, recursive=True), ) errors = minio_client.remove_objects(bucket, delete_object_list) for error in errors: diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini index 737a37ee9d0..6d451adf7eb 100644 --- a/tests/integration/pytest.ini +++ b/tests/integration/pytest.ini @@ -1,6 +1,6 @@ [pytest] python_files = test*.py -norecursedirs = _instances +norecursedirs = _instances* timeout = 1800 junit_duration_report = call junit_suite_name = integration diff --git a/tests/integration/test_merge_tree_s3/test.py b/tests/integration/test_merge_tree_s3/test.py index 4b30287ca1e..31df6dff374 100644 --- a/tests/integration/test_merge_tree_s3/test.py +++ b/tests/integration/test_merge_tree_s3/test.py @@ -6,7 +6,7 @@ import threading import os import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, get_instances_dir # By default the exceptions that was throwed in threads will be ignored @@ -30,7 +30,7 @@ class SafeThread(threading.Thread): SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node/configs/config.d/storage_conf.xml') +CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/storage_conf.xml'.format(get_instances_dir())) def replace_config(old, new): diff --git a/tests/integration/test_reload_max_table_size_to_drop/test.py b/tests/integration/test_reload_max_table_size_to_drop/test.py index 5f2083d742e..7e7219088b8 100644 --- a/tests/integration/test_reload_max_table_size_to_drop/test.py +++ b/tests/integration/test_reload_max_table_size_to_drop/test.py @@ -2,13 +2,14 @@ import os import time import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, get_instances_dir cluster = ClickHouseCluster(__file__) node = cluster.add_instance('node', main_configs=["configs/max_table_size_to_drop.xml"]) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node/configs/config.d/max_table_size_to_drop.xml') + +CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node/configs/config.d/max_table_size_to_drop.xml'.format(get_instances_dir())) @pytest.fixture(scope="module") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 545ca4256f3..52b021a07c5 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -9,12 +9,13 @@ import time import helpers.client import pytest -from helpers.cluster import ClickHouseCluster, ClickHouseInstance +from helpers.cluster import ClickHouseCluster, ClickHouseInstance, get_instances_dir MINIO_INTERNAL_PORT = 9001 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/dummy/configs/config.d/defaultS3.xml') + +CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/dummy/configs/config.d/defaultS3.xml'.format(get_instances_dir())) # Creates S3 bucket for tests and allows anonymous read-write access to it. diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py index 6dc843e101a..a1a21f24099 100644 --- a/tests/integration/test_system_flush_logs/test.py +++ b/tests/integration/test_system_flush_logs/test.py @@ -20,9 +20,8 @@ system_logs = [ ('system.metric_log', 1), ] -# Default timeout for flush is 60 -# decrease timeout for the test to show possible issues. -timeout = pytest.mark.timeout(30) +# 60 -- Default timeout for flush +timeout = pytest.mark.timeout(60) @pytest.fixture(scope='module', autouse=True) From a63da94ae40857793230440078b5428d17b456b3 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 11:14:26 +0300 Subject: [PATCH 17/35] Revert "Merge pull request #25526 from ClickHouse/revert-25459-add_run_id_option" This reverts commit 7c7e96f26c0b9e221a9ea54706394bba8e912e7a, reversing changes made to a54cbef5b7f02f68ae11c82297124af0251bacf7. --- tests/integration/ci-runner.py | 4 ++-- tests/integration/conftest.py | 8 +++++++- tests/integration/helpers/cluster.py | 19 +++++++++++++++++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 8b8462a7125..0af76fe2648 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -377,8 +377,8 @@ class ClickhouseIntegrationTestsRunner: test_cmd = ' '.join([test for test in sorted(test_names)]) parallel_cmd = " --parallel {} ".format(num_workers) if num_workers > 0 else "" - cmd = "cd {}/tests/integration && ./runner --tmpfs {} -t {} {} '-ss -rfEp --color=no --durations=0 {}' | tee {}".format( - repo_path, image_cmd, test_cmd, parallel_cmd, _get_deselect_option(self.should_skip_tests()), output_path) + cmd = "cd {}/tests/integration && ./runner --tmpfs {} -t {} {} '-ss -rfEp --run-id={} --color=no --durations=0 {}' | tee {}".format( + repo_path, image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), output_path) with open(log_path, 'w') as log: logging.info("Executing cmd: %s", cmd) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index fa14e2b06d6..993e7a6e973 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -28,4 +28,10 @@ def cleanup_environment(): logging.exception(f"cleanup_environment:{str(e)}") pass - yield \ No newline at end of file + yield + +def pytest_addoption(parser): + parser.addoption("--run-id", default="", help="run-id is used as postfix in _instances_{} directory") + +def pytest_configure(config): + os.environ['INTEGRATION_TESTS_RUN_ID'] = config.option.run_id diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 4eeb07d62b1..eb7ed55c7a2 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -209,7 +209,14 @@ class ClickHouseCluster: project_name = pwd.getpwuid(os.getuid()).pw_name + p.basename(self.base_dir) + self.name # docker-compose removes everything non-alphanumeric from project names so we do it too. self.project_name = re.sub(r'[^a-z0-9]', '', project_name.lower()) - self.instances_dir = p.join(self.base_dir, '_instances' + ('' if not self.name else '_' + self.name)) + instances_dir_name = '_instances' + if self.name: + instances_dir_name += '_' + self.name + + if 'INTEGRATION_TESTS_RUN_ID' in os.environ: + instances_dir_name += '_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) + + self.instances_dir = p.join(self.base_dir, instances_dir_name) self.docker_logs_path = p.join(self.instances_dir, 'docker.log') self.env_file = p.join(self.instances_dir, DEFAULT_ENV_NAME) self.env_variables = {} @@ -427,7 +434,15 @@ class ClickHouseCluster: pass def get_docker_handle(self, docker_id): - return self.docker_client.containers.get(docker_id) + exception = None + for i in range(5): + try: + return self.docker_client.containers.get(docker_id) + except Exception as ex: + print("Got exception getting docker handle", str(ex)) + time.sleep(i * 2) + exception = ex + raise exception def get_client_cmd(self): cmd = self.client_bin_path From c0732ddc12cca6d7a0f1a20a1278434befe3a753 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 08:18:38 +0000 Subject: [PATCH 18/35] Fix datetime with timezone --- programs/odbc-bridge/ODBCBlockInputStream.cpp | 2 +- src/DataStreams/PostgreSQLBlockInputStream.cpp | 6 +++--- src/Formats/MySQLBlockInputStream.cpp | 2 +- tests/integration/test_storage_postgresql/test.py | 15 +++++++++++++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/programs/odbc-bridge/ODBCBlockInputStream.cpp b/programs/odbc-bridge/ODBCBlockInputStream.cpp index b23d09e0481..25c953c0b71 100644 --- a/programs/odbc-bridge/ODBCBlockInputStream.cpp +++ b/programs/odbc-bridge/ODBCBlockInputStream.cpp @@ -132,7 +132,7 @@ void ODBCBlockInputStream::insertValue( auto value = row.get(idx); ReadBufferFromString in(value); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); if (time < 0) time = 0; assert_cast(column).insertValue(time); diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index a41280847a5..0df974b2405 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -170,7 +170,7 @@ void PostgreSQLBlockInputStream::insertValue(IColumn & column, std::string_view { ReadBufferFromString in(value); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); if (time < 0) time = 0; assert_cast(column).insertValue(time); @@ -272,11 +272,11 @@ void PostgreSQLBlockInputStream::prepareArrayInfo(size_t column_idx, const DataT else if (which.isDate()) parser = [](std::string & field) -> Field { return UInt16{LocalDate{field}.getDayNum()}; }; else if (which.isDateTime()) - parser = [](std::string & field) -> Field + parser = [data_type](std::string & field) -> Field { ReadBufferFromString in(field); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); return time; }; else if (which.isDecimal32()) diff --git a/src/Formats/MySQLBlockInputStream.cpp b/src/Formats/MySQLBlockInputStream.cpp index 3ea86c82fa3..91e51de4603 100644 --- a/src/Formats/MySQLBlockInputStream.cpp +++ b/src/Formats/MySQLBlockInputStream.cpp @@ -169,7 +169,7 @@ namespace { ReadBufferFromString in(value); time_t time = 0; - readDateTimeText(time, in); + readDateTimeText(time, in, assert_cast(data_type).getTimeZone()); if (time < 0) time = 0; assert_cast(column).insertValue(time); diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index f81033822c8..716f16c6211 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -308,6 +308,21 @@ def test_postgres_distributed(started_cluster): assert(result == 'host2\nhost4\n' or result == 'host3\nhost4\n') +def test_datetime_with_timezone(started_cluster): + conn = get_postgres_conn(started_cluster, started_cluster.postgres_ip, True) + cursor = conn.cursor() + cursor.execute("CREATE TABLE test_timezone (ts timestamp without time zone, ts_z timestamp with time zone)") + cursor.execute("insert into test_timezone select '2014-04-04 20:00:00', '2014-04-04 20:00:00'::timestamptz at time zone 'America/New_York';") + cursor.execute("select * from test_timezone") + result = cursor.fetchall()[0] + print(result[0], str(result[1])[:-6]) + node1.query("create table test_timezone ( ts DateTime, ts_z DateTime('America/New_York')) ENGINE PostgreSQL('postgres1:5432', 'clickhouse', 'test_timezone', 'postgres', 'mysecretpassword');") + assert(node1.query("select ts from test_timezone").strip() == str(result[0])) + # [:-6] because 2014-04-04 16:00:00+00:00 -> 2014-04-04 16:00:00 + assert(node1.query("select ts_z from test_timezone").strip() == str(result[1])[:-6]) + assert(node1.query("select * from test_timezone") == "2014-04-04 20:00:00\t2014-04-04 16:00:00\n") + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From 8cf5f8e748a2a4f936e10abb75e71e58e266e36a Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 13:30:39 +0300 Subject: [PATCH 19/35] Followup fix --- tests/integration/helpers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index eb7ed55c7a2..59e598ce6ba 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -172,7 +172,7 @@ def enable_consistent_hash_plugin(rabbitmq_id): return p.returncode == 0 def get_instances_dir(): - if 'INTEGRATION_TESTS_RUN_ID' in os.environ: + if 'INTEGRATION_TESTS_RUN_ID' in os.environ and os.environ['INTEGRATION_TESTS_RUN_ID']: return '_instances_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) else: return '_instances' From fb9c92cb6c702b02ce20577b03394f2fa9cc6498 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 10:32:40 +0000 Subject: [PATCH 20/35] Fix --- src/DataStreams/PostgreSQLBlockInputStream.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/DataStreams/PostgreSQLBlockInputStream.cpp b/src/DataStreams/PostgreSQLBlockInputStream.cpp index 0df974b2405..a486df83025 100644 --- a/src/DataStreams/PostgreSQLBlockInputStream.cpp +++ b/src/DataStreams/PostgreSQLBlockInputStream.cpp @@ -272,11 +272,11 @@ void PostgreSQLBlockInputStream::prepareArrayInfo(size_t column_idx, const DataT else if (which.isDate()) parser = [](std::string & field) -> Field { return UInt16{LocalDate{field}.getDayNum()}; }; else if (which.isDateTime()) - parser = [data_type](std::string & field) -> Field + parser = [nested](std::string & field) -> Field { ReadBufferFromString in(field); time_t time = 0; - readDateTimeText(time, in, assert_cast(data_type.get())->getTimeZone()); + readDateTimeText(time, in, assert_cast(nested.get())->getTimeZone()); return time; }; else if (which.isDecimal32()) From 7d3dee172c747447d11272ad522a038212892c04 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 21 Jun 2021 12:19:25 +0000 Subject: [PATCH 21/35] FIx progress --- src/Common/ProgressIndication.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 56362015245..044d8cb1a89 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -39,7 +39,7 @@ public: void setFileProgressCallback(ContextMutablePtr context, bool write_progress_on_update = false); /// How much seconds passed since query execution start. - UInt64 elapsedSeconds() const { return watch.elapsedSeconds(); } + double elapsedSeconds() const { return watch.elapsedSeconds(); } private: /// This flag controls whether to show the progress bar. We start showing it after From ffa56bde24cf21234db2bda2023103ab6eb59580 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 21 Jun 2021 15:34:05 +0300 Subject: [PATCH 22/35] fix usage of index with array columns and ARRAY JOIN --- src/Storages/MergeTree/KeyCondition.cpp | 13 +++++++++++++ src/Storages/MergeTree/KeyCondition.h | 2 ++ .../01922_array_join_with_index.reference | 1 + .../0_stateless/01922_array_join_with_index.sql | 10 ++++++++++ 4 files changed, 26 insertions(+) create mode 100644 tests/queries/0_stateless/01922_array_join_with_index.reference create mode 100644 tests/queries/0_stateless/01922_array_join_with_index.sql diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 246d00b831b..476032e66aa 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -423,6 +423,9 @@ KeyCondition::KeyCondition( */ Block block_with_constants = getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context); + for (const auto & [name, _] : query_info.syntax_analyzer_result->array_join_result_to_source) + array_joined_columns.insert(name); + const ASTSelectQuery & select = query_info.query->as(); if (select.where() || select.prewhere()) { @@ -610,6 +613,10 @@ bool KeyCondition::canConstantBeWrappedByMonotonicFunctions( DataTypePtr & out_type) { String expr_name = node->getColumnNameWithoutAlias(); + + if (array_joined_columns.count(expr_name)) + return false; + if (key_subexpr_names.count(expr_name) == 0) return false; @@ -714,6 +721,9 @@ bool KeyCondition::canConstantBeWrappedByFunctions( { String expr_name = ast->getColumnNameWithoutAlias(); + if (array_joined_columns.count(expr_name)) + return false; + if (key_subexpr_names.count(expr_name) == 0) { /// Let's check another one case. @@ -1075,6 +1085,9 @@ bool KeyCondition::isKeyPossiblyWrappedByMonotonicFunctionsImpl( // Key columns should use canonical names for index analysis String name = node->getColumnNameWithoutAlias(); + if (array_joined_columns.count(name)) + return false; + auto it = key_columns.find(name); if (key_columns.end() != it) { diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h index 7e7b767b53b..c957c65fc40 100644 --- a/src/Storages/MergeTree/KeyCondition.h +++ b/src/Storages/MergeTree/KeyCondition.h @@ -459,6 +459,8 @@ private: const ExpressionActionsPtr key_expr; /// All intermediate columns are used to calculate key_expr. const NameSet key_subexpr_names; + + NameSet array_joined_columns; PreparedSets prepared_sets; // If true, always allow key_expr to be wrapped by function diff --git a/tests/queries/0_stateless/01922_array_join_with_index.reference b/tests/queries/0_stateless/01922_array_join_with_index.reference new file mode 100644 index 00000000000..b7539310ef3 --- /dev/null +++ b/tests/queries/0_stateless/01922_array_join_with_index.reference @@ -0,0 +1 @@ +a c diff --git a/tests/queries/0_stateless/01922_array_join_with_index.sql b/tests/queries/0_stateless/01922_array_join_with_index.sql new file mode 100644 index 00000000000..1444c639692 --- /dev/null +++ b/tests/queries/0_stateless/01922_array_join_with_index.sql @@ -0,0 +1,10 @@ +DROP TABLE IF EXISTS t_array_index; + +CREATE TABLE t_array_index (n Nested(key String, value String)) +ENGINE = MergeTree ORDER BY n.key; + +INSERT INTO t_array_index VALUES (['a', 'b'], ['c', 'd']); + +SELECT * FROM t_array_index ARRAY JOIN n WHERE n.key = 'a'; + +DROP TABLE IF EXISTS t_array_index; From eaf3f9c6e6182a42781324ac033f12d84d344f5f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 21 Jun 2021 16:59:19 +0300 Subject: [PATCH 23/35] Remove Testflows, because they are unstable --- tests/testflows/regression.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index c2e143a4b1c..661bc03509b 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -22,15 +22,15 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): tasks = [] with Pool(8) as pool: try: - run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) #run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) + #run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) finally: join(tasks) From e04957ce821306f6774bf7f262d9a06cb7eea422 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 21 Jun 2021 18:42:40 +0300 Subject: [PATCH 24/35] more --- tests/integration/helpers/client.py | 2 + .../config.d/zookeeper_session_timeout.xml | 2 +- .../config.d/zookeeper_session_timeout.xml | 2 +- .../integration/test_distributed_ddl/test.py | 41 +++++-------- .../test_replicated_alter.py | 12 ++-- tests/integration/test_storage_mysql/test.py | 59 ++++++++++++++----- 6 files changed, 69 insertions(+), 49 deletions(-) diff --git a/tests/integration/helpers/client.py b/tests/integration/helpers/client.py index 95f8a58dbf1..ceebf3c23bf 100644 --- a/tests/integration/helpers/client.py +++ b/tests/integration/helpers/client.py @@ -1,6 +1,7 @@ import os import subprocess as sp import tempfile +import logging from threading import Timer @@ -105,6 +106,7 @@ class CommandRequest: stderr = self.stderr_file.read().decode('utf-8', errors='replace') if self.timer is not None and not self.process_finished_before_timeout and not self.ignore_error: + logging.debug(f"Timed out. Last stdout:{stdout}, stderr:{stderr}") raise QueryTimeoutExceedException('Client timed out!') if (self.process.returncode != 0 or stderr) and not self.ignore_error: diff --git a/tests/integration/test_distributed_ddl/configs/config.d/zookeeper_session_timeout.xml b/tests/integration/test_distributed_ddl/configs/config.d/zookeeper_session_timeout.xml index 3b878bbf4de..caa0ff11137 100644 --- a/tests/integration/test_distributed_ddl/configs/config.d/zookeeper_session_timeout.xml +++ b/tests/integration/test_distributed_ddl/configs/config.d/zookeeper_session_timeout.xml @@ -1,6 +1,6 @@ - 10000 + 15000 diff --git a/tests/integration/test_distributed_ddl/configs_secure/config.d/zookeeper_session_timeout.xml b/tests/integration/test_distributed_ddl/configs_secure/config.d/zookeeper_session_timeout.xml index 3b878bbf4de..caa0ff11137 100644 --- a/tests/integration/test_distributed_ddl/configs_secure/config.d/zookeeper_session_timeout.xml +++ b/tests/integration/test_distributed_ddl/configs_secure/config.d/zookeeper_session_timeout.xml @@ -1,6 +1,6 @@ - 10000 + 15000 diff --git a/tests/integration/test_distributed_ddl/test.py b/tests/integration/test_distributed_ddl/test.py index 28ebc69bc05..18e091de1ec 100755 --- a/tests/integration/test_distributed_ddl/test.py +++ b/tests/integration/test_distributed_ddl/test.py @@ -93,27 +93,6 @@ def test_on_server_fail(test_cluster): test_cluster.ddl_check_query(instance, "DROP TABLE test.test_server_fail ON CLUSTER 'cluster'") -def _test_on_connection_losses(test_cluster, zk_timeout): - instance = test_cluster.instances['ch1'] - kill_instance = test_cluster.instances['ch2'] - - with PartitionManager() as pm: - pm.drop_instance_zk_connections(kill_instance) - request = instance.get_query_request("DROP TABLE IF EXISTS test.__nope__ ON CLUSTER 'cluster'", timeout=40) - time.sleep(zk_timeout) - pm.restore_instance_zk_connections(kill_instance) - - test_cluster.check_all_hosts_successfully_executed(request.get_answer()) - - -def test_on_connection_loss(test_cluster): - _test_on_connection_losses(test_cluster, 15) # connection loss will occur only (10 sec ZK timeout in config) - - -def test_on_session_expired(test_cluster): - _test_on_connection_losses(test_cluster, 60) # session should be expired (10 sec ZK timeout in config) - - def test_simple_alters(test_cluster): instance = test_cluster.instances['ch2'] @@ -191,7 +170,7 @@ def test_implicit_macros(test_cluster): instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}'") + test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test_db ON CLUSTER '{cluster}' SYNC") test_cluster.ddl_check_query(instance, "CREATE DATABASE IF NOT EXISTS test_db ON CLUSTER '{cluster}'") test_cluster.ddl_check_query(instance, """ @@ -271,6 +250,15 @@ def test_create_reserved(test_cluster): def test_rename(test_cluster): instance = test_cluster.instances['ch1'] rules = test_cluster.pm_random_drops.pop_rules() + test_cluster.ddl_check_query(instance, + "DROP TABLE IF EXISTS rename_shard ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, + "DROP TABLE IF EXISTS rename_new ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, + "DROP TABLE IF EXISTS rename_old ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, + "DROP TABLE IF EXISTS rename ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, "CREATE TABLE rename_shard ON CLUSTER cluster (id Int64, sid String DEFAULT concat('old', toString(id))) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{shard}/staging/test_shard', '{replica}') ORDER BY (id)") test_cluster.ddl_check_query(instance, @@ -327,12 +315,15 @@ def test_socket_timeout(test_cluster): def test_replicated_without_arguments(test_cluster): rules = test_cluster.pm_random_drops.pop_rules() instance = test_cluster.instances['ch1'] + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS test_atomic.rmt ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, "DROP DATABASE IF EXISTS test_atomic ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, "CREATE DATABASE test_atomic ON CLUSTER cluster ENGINE=Atomic") assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ instance.query_and_get_error("CREATE TABLE test_atomic.rmt (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree() ORDER BY n") - test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE test_atomic.rmt ON CLUSTER cluster SYNC") test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rmt UUID '12345678-0000-4000-8000-000000000001' ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree ORDER BY n") assert instance.query("SHOW CREATE test_atomic.rmt FORMAT TSVRaw") == \ @@ -350,7 +341,7 @@ def test_replicated_without_arguments(test_cluster): "CREATE TABLE test_atomic.rsmt ON CLUSTER cluster (n UInt64, m UInt64, k UInt64) ENGINE=ReplicatedSummingMergeTree((m, k)) ORDER BY n") test_cluster.ddl_check_query(instance, "CREATE TABLE test_atomic.rvcmt ON CLUSTER cluster (n UInt64, m Int8, k UInt64) ENGINE=ReplicatedVersionedCollapsingMergeTree(m, k) ORDER BY n") - test_cluster.ddl_check_query(instance, "DROP DATABASE test_atomic ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP DATABASE test_atomic ON CLUSTER cluster SYNC") test_cluster.ddl_check_query(instance, "CREATE DATABASE test_ordinary ON CLUSTER cluster ENGINE=Ordinary") assert "are supported only for ON CLUSTER queries with Atomic database engine" in \ @@ -360,7 +351,7 @@ def test_replicated_without_arguments(test_cluster): test_cluster.ddl_check_query(instance, "CREATE TABLE test_ordinary.rmt ON CLUSTER cluster (n UInt64, s String) ENGINE=ReplicatedMergeTree('/{shard}/{table}/', '{replica}') ORDER BY n") assert instance.query("SHOW CREATE test_ordinary.rmt FORMAT TSVRaw") == \ "CREATE TABLE test_ordinary.rmt\n(\n `n` UInt64,\n `s` String\n)\nENGINE = ReplicatedMergeTree('/{shard}/rmt/', '{replica}')\nORDER BY n\nSETTINGS index_granularity = 8192\n" - test_cluster.ddl_check_query(instance, "DROP DATABASE test_ordinary ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP DATABASE test_ordinary ON CLUSTER cluster SYNC") test_cluster.pm_random_drops.push_rules(rules) diff --git a/tests/integration/test_distributed_ddl/test_replicated_alter.py b/tests/integration/test_distributed_ddl/test_replicated_alter.py index 148ad5fca5e..5e7989cb256 100644 --- a/tests/integration/test_distributed_ddl/test_replicated_alter.py +++ b/tests/integration/test_distributed_ddl/test_replicated_alter.py @@ -38,9 +38,9 @@ def test_cluster(request): def test_replicated_alters(test_cluster): instance = test_cluster.instances['ch2'] - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS merge_for_alter ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_32 ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, "DROP TABLE IF EXISTS all_merge_64 ON CLUSTER cluster SYNC") # Temporarily disable random ZK packet drops, they might broke creation if ReplicatedMergeTree replicas firewall_drops_rules = test_cluster.pm_random_drops.pop_rules() @@ -90,10 +90,10 @@ ENGINE = Distributed(cluster, default, merge_for_alter, i) assert TSV(instance.query("SELECT i, s FROM all_merge_64 ORDER BY i")) == TSV( ''.join(['{}\t{}\n'.format(x, x) for x in range(4)])) - test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE merge_for_alter ON CLUSTER cluster SYNC") # Enable random ZK packet drops test_cluster.pm_random_drops.push_rules(firewall_drops_rules) - test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster") - test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster") + test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_32 ON CLUSTER cluster SYNC") + test_cluster.ddl_check_query(instance, "DROP TABLE all_merge_64 ON CLUSTER cluster SYNC") diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py index 2c993e3d696..a044528cacf 100644 --- a/tests/integration/test_storage_mysql/test.py +++ b/tests/integration/test_storage_mysql/test.py @@ -21,16 +21,27 @@ create_table_sql_template = """ PRIMARY KEY (`id`)) ENGINE=InnoDB; """ -def create_mysql_db(conn, name): - with conn.cursor() as cursor: - cursor.execute( - "CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) +drop_table_sql_template = """ + DROP TABLE IF EXISTS `clickhouse`.`{}`; + """ +def get_mysql_conn(started_cluster, host): + conn = pymysql.connect(user='root', password='clickhouse', host=host, port=started_cluster.mysql_port) + return conn def create_mysql_table(conn, tableName): with conn.cursor() as cursor: cursor.execute(create_table_sql_template.format(tableName)) +def drop_mysql_table(conn, tableName): + with conn.cursor() as cursor: + cursor.execute(drop_table_sql_template.format(tableName)) + +def create_mysql_db(conn, name): + with conn.cursor() as cursor: + cursor.execute("DROP DATABASE IF EXISTS {}".format(name)) + cursor.execute("CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) + @pytest.fixture(scope="module") def started_cluster(): @@ -51,7 +62,10 @@ def started_cluster(): def test_many_connections(started_cluster): table_name = 'test_many_connections' + node1.query(f'DROP TABLE IF EXISTS {table_name}') + conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) node1.query(''' @@ -66,14 +80,18 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL query += "SELECT id FROM {t})" assert node1.query(query.format(t=table_name)) == '250\n' + drop_mysql_table(conn, table_name) conn.close() def test_insert_select(started_cluster): table_name = 'test_insert_select' + node1.query(f'DROP TABLE IF EXISTS {table_name}') conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) + node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); '''.format(table_name, table_name)) @@ -87,7 +105,9 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL def test_replace_select(started_cluster): table_name = 'test_replace_select' + node1.query(f'DROP TABLE IF EXISTS {table_name}') conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) node1.query(''' @@ -106,7 +126,9 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL def test_insert_on_duplicate_select(started_cluster): table_name = 'test_insert_on_duplicate_select' + node1.query(f'DROP TABLE IF EXISTS {table_name}') conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) node1.query(''' @@ -125,7 +147,10 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL def test_where(started_cluster): table_name = 'test_where' + node1.query(f'DROP TABLE IF EXISTS {table_name}') + conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); @@ -146,6 +171,7 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL def test_table_function(started_cluster): conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, 'table_function') create_mysql_table(conn, 'table_function') table_function = "mysql('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('table_function') assert node1.query("SELECT count() FROM {}".format(table_function)).rstrip() == '0' @@ -168,6 +194,8 @@ def test_table_function(started_cluster): def test_binary_type(started_cluster): conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, 'binary_type') + with conn.cursor() as cursor: cursor.execute("CREATE TABLE clickhouse.binary_type (id INT PRIMARY KEY, data BINARY(16) NOT NULL)") table_function = "mysql('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse')".format('binary_type') @@ -177,7 +205,10 @@ def test_binary_type(started_cluster): def test_enum_type(started_cluster): table_name = 'test_enum_type' + node1.query(f'DROP TABLE IF EXISTS {table_name}') + conn = get_mysql_conn(started_cluster, cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) node1.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, source Enum8('IP' = 1, 'URL' = 2)) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse', 1); @@ -186,20 +217,8 @@ CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32, source Enum8(' assert node1.query("SELECT source FROM {} LIMIT 1".format(table_name)).rstrip() == 'URL' conn.close() -def get_mysql_conn(started_cluster, host): - conn = pymysql.connect(user='root', password='clickhouse', host=host, port=started_cluster.mysql_port) - return conn -def create_mysql_db(conn, name): - with conn.cursor() as cursor: - cursor.execute("DROP DATABASE IF EXISTS {}".format(name)) - cursor.execute("CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) - -def create_mysql_table(conn, tableName): - with conn.cursor() as cursor: - cursor.execute(create_table_sql_template.format(tableName)) - def test_mysql_distributed(started_cluster): table_name = 'test_replicas' @@ -218,6 +237,8 @@ def test_mysql_distributed(started_cluster): create_mysql_table(conn3, table_name) create_mysql_table(conn4, table_name) + node2.query('DROP TABLE IF EXISTS test_replicas') + # Storage with with 3 replicas node2.query(''' CREATE TABLE test_replicas @@ -227,6 +248,7 @@ def test_mysql_distributed(started_cluster): # Fill remote tables with different data to be able to check nodes = [node1, node2, node2, node2] for i in range(1, 5): + nodes[i-1].query('DROP TABLE IF EXISTS test_replica{}'.format(i)) nodes[i-1].query(''' CREATE TABLE test_replica{} (id UInt32, name String, age UInt32, money UInt32) @@ -249,6 +271,8 @@ def test_mysql_distributed(started_cluster): assert(result == 'host2\nhost3\nhost4\n') # Storage with with two shards, each has 2 replicas + node2.query('DROP TABLE IF EXISTS test_shards') + node2.query(''' CREATE TABLE test_shards (id UInt32, name String, age UInt32, money UInt32) @@ -275,9 +299,12 @@ def test_mysql_distributed(started_cluster): def test_external_settings(started_cluster): table_name = 'test_external_settings' + node1.query(f'DROP TABLE IF EXISTS {table_name}') conn = get_mysql_conn(started_cluster, started_cluster.mysql_ip) + drop_mysql_table(conn, table_name) create_mysql_table(conn, table_name) + node3.query(f'DROP TABLE IF EXISTS {table_name}') node3.query(''' CREATE TABLE {}(id UInt32, name String, age UInt32, money UInt32) ENGINE = MySQL('mysql57:3306', 'clickhouse', '{}', 'root', 'clickhouse'); '''.format(table_name, table_name)) From 426710f8e399e74c2c5b94faadc22f9aab9c0765 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 18:53:47 +0300 Subject: [PATCH 25/35] Followup fix --- tests/integration/helpers/cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 59e598ce6ba..54e129fed11 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -213,7 +213,7 @@ class ClickHouseCluster: if self.name: instances_dir_name += '_' + self.name - if 'INTEGRATION_TESTS_RUN_ID' in os.environ: + if 'INTEGRATION_TESTS_RUN_ID' in os.environ and os.environ['INTEGRATION_TESTS_RUN_ID']: instances_dir_name += '_' + shlex.quote(os.environ['INTEGRATION_TESTS_RUN_ID']) self.instances_dir = p.join(self.base_dir, instances_dir_name) From 2487063be3d9079e8ae86d32dbf7ad0be2ee2e36 Mon Sep 17 00:00:00 2001 From: Evgeniy Gatov Date: Mon, 21 Jun 2021 22:23:13 +0300 Subject: [PATCH 26/35] METR-41529 --- base/mysqlxx/Query.cpp | 2 +- base/mysqlxx/ya.make | 39 +++++++++++++++++++++++++++++++++++++++ base/mysqlxx/ya.make.in | 28 ++++++++++++++++++++++++++++ base/ya.make | 1 + 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 base/mysqlxx/ya.make create mode 100644 base/mysqlxx/ya.make.in diff --git a/base/mysqlxx/Query.cpp b/base/mysqlxx/Query.cpp index d4514c3e988..c0d5c20fdfd 100644 --- a/base/mysqlxx/Query.cpp +++ b/base/mysqlxx/Query.cpp @@ -2,7 +2,7 @@ #include #include #else -#include +#include //Y_IGNORE #include #endif diff --git a/base/mysqlxx/ya.make b/base/mysqlxx/ya.make new file mode 100644 index 00000000000..aabc9922e72 --- /dev/null +++ b/base/mysqlxx/ya.make @@ -0,0 +1,39 @@ +# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it. +LIBRARY() + +OWNER(g:clickhouse) + +CFLAGS(-g0) + +PEERDIR( + contrib/restricted/boost/libs + contrib/libs/libmysql_r + contrib/libs/poco/Foundation + contrib/libs/poco/Util +) + +ADDINCL( + GLOBAL clickhouse/base + clickhouse/base + contrib/libs/libmysql_r +) + +NO_COMPILER_WARNINGS() + +NO_UTIL() + +SRCS( + Connection.cpp + Exception.cpp + Pool.cpp + PoolFactory.cpp + PoolWithFailover.cpp + Query.cpp + ResultBase.cpp + Row.cpp + UseQueryResult.cpp + Value.cpp + +) + +END() diff --git a/base/mysqlxx/ya.make.in b/base/mysqlxx/ya.make.in new file mode 100644 index 00000000000..10755078e20 --- /dev/null +++ b/base/mysqlxx/ya.make.in @@ -0,0 +1,28 @@ +LIBRARY() + +OWNER(g:clickhouse) + +CFLAGS(-g0) + +PEERDIR( + contrib/restricted/boost/libs + contrib/libs/libmysql_r + contrib/libs/poco/Foundation + contrib/libs/poco/Util +) + +ADDINCL( + GLOBAL clickhouse/base + clickhouse/base + contrib/libs/libmysql_r +) + +NO_COMPILER_WARNINGS() + +NO_UTIL() + +SRCS( + +) + +END() diff --git a/base/ya.make b/base/ya.make index 9f4cf0fd4a7..19a16044280 100644 --- a/base/ya.make +++ b/base/ya.make @@ -4,6 +4,7 @@ RECURSE( common daemon loggers + mysqlxx pcg-random widechar_width readpassphrase From c9c22996d41444d59c3bd57f473e36bde5188187 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 22:52:44 +0300 Subject: [PATCH 27/35] One more fixed test --- tests/integration/test_merge_tree_s3_restore/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py index b7543388791..809fff6695c 100644 --- a/tests/integration/test_merge_tree_s3_restore/test.py +++ b/tests/integration/test_merge_tree_s3_restore/test.py @@ -5,10 +5,10 @@ import string import time import pytest -from helpers.cluster import ClickHouseCluster +from helpers.cluster import ClickHouseCluster, get_instances_dir SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -NOT_RESTORABLE_CONFIG_PATH = os.path.join(SCRIPT_DIR, './_instances/node_not_restorable/configs/config.d/storage_conf_not_restorable.xml') +NOT_RESTORABLE_CONFIG_PATH = os.path.join(SCRIPT_DIR, './{}/node_not_restorable/configs/config.d/storage_conf_not_restorable.xml'.format(get_instances_dir())) COMMON_CONFIGS = ["configs/config.d/bg_processing_pool_conf.xml", "configs/config.d/log_conf.xml", "configs/config.d/clusters.xml"] From 8bbe65b4f450cf28f8b76576d4cd62467fb9969d Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jun 2021 23:02:00 +0300 Subject: [PATCH 28/35] Remove timeouts --- tests/integration/test_system_flush_logs/test.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py index a1a21f24099..b69105710fb 100644 --- a/tests/integration/test_system_flush_logs/test.py +++ b/tests/integration/test_system_flush_logs/test.py @@ -20,9 +20,6 @@ system_logs = [ ('system.metric_log', 1), ] -# 60 -- Default timeout for flush -timeout = pytest.mark.timeout(60) - @pytest.fixture(scope='module', autouse=True) def start_cluster(): @@ -38,7 +35,6 @@ def flush_logs(): node.query('SYSTEM FLUSH LOGS') -@timeout @pytest.mark.parametrize('table,exists', system_logs) def test_system_logs(flush_logs, table, exists): q = 'SELECT * FROM {}'.format(table) @@ -50,7 +46,6 @@ def test_system_logs(flush_logs, table, exists): # Logic is tricky, let's check that there is no hang in case of message queue # is not empty (this is another code path in the code). -@timeout def test_system_logs_non_empty_queue(): node.query('SELECT 1', settings={ # right now defaults are the same, From cbc7e61140948fc591d748437208a3001d003161 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 22 Jun 2021 02:23:53 +0300 Subject: [PATCH 29/35] Update arcadia_skip_list.txt --- tests/queries/0_stateless/arcadia_skip_list.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 6e827c3c240..f146913a2e8 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -246,3 +246,4 @@ 01901_test_attach_partition_from 01910_view_dictionary 01824_prefer_global_in_and_join +01576_alias_column_rewrite From eafd96bc83290b9742a86991e68b814a68f15c0f Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Jun 2021 16:26:14 +0300 Subject: [PATCH 30/35] Update adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index fa257a84173..e00b7c04509 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -148,5 +148,6 @@ toc_title: Adopters | kakaocorp | Internet company | — | — | — | [if(kakao)2020 conference](https://if.kakao.com/session/117) | | ООО «МПЗ Богородский» | Agriculture | — | — | — | [Article in Russian, November 2020](https://cloud.yandex.ru/cases/okraina) | | Tesla | Electric vehicle and clean energy company | — | — | — | [Vacancy description, March 2021](https://news.ycombinator.com/item?id=26306170) | +| KGK Global | Vehicle monitoring | — | — | — | [Press release, June 2021](https://zoom.cnews.ru/news/item/530921) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) From 50ebf9d1b2a97901aedb51c1bd53c5105915cb9c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Jun 2021 16:29:11 +0300 Subject: [PATCH 31/35] Update adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index e00b7c04509..02c1e5aebe9 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -149,5 +149,6 @@ toc_title: Adopters | ООО «МПЗ Богородский» | Agriculture | — | — | — | [Article in Russian, November 2020](https://cloud.yandex.ru/cases/okraina) | | Tesla | Electric vehicle and clean energy company | — | — | — | [Vacancy description, March 2021](https://news.ycombinator.com/item?id=26306170) | | KGK Global | Vehicle monitoring | — | — | — | [Press release, June 2021](https://zoom.cnews.ru/news/item/530921) | +| BiliBili | Video sharing | — | — | — | [Blog post, June 2021](https://chowdera.com/2021/06/20210622012241476b.html) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) From 2201993479d83850f1cf2c03913163d6ef6dbd3d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Jun 2021 16:33:12 +0300 Subject: [PATCH 32/35] Update adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 02c1e5aebe9..aa17a041137 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -150,5 +150,6 @@ toc_title: Adopters | Tesla | Electric vehicle and clean energy company | — | — | — | [Vacancy description, March 2021](https://news.ycombinator.com/item?id=26306170) | | KGK Global | Vehicle monitoring | — | — | — | [Press release, June 2021](https://zoom.cnews.ru/news/item/530921) | | BiliBili | Video sharing | — | — | — | [Blog post, June 2021](https://chowdera.com/2021/06/20210622012241476b.html) | +| Gigapipe | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) From bd4fb30735f9093f145ee7ca4c558b497a819804 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Jun 2021 16:35:15 +0300 Subject: [PATCH 33/35] Update adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index aa17a041137..3aae0b09b19 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -151,5 +151,6 @@ toc_title: Adopters | KGK Global | Vehicle monitoring | — | — | — | [Press release, June 2021](https://zoom.cnews.ru/news/item/530921) | | BiliBili | Video sharing | — | — | — | [Blog post, June 2021](https://chowdera.com/2021/06/20210622012241476b.html) | | Gigapipe | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) | +| Hydrolix | Cloud data platform | Main product | — | — | [Documentation](https://docs.hydrolix.io/guide/query) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) From 2987d11268e6a2ef8553a6cd1086da54d7da1975 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 22 Jun 2021 16:36:29 +0300 Subject: [PATCH 34/35] Update adopters.md --- docs/en/introduction/adopters.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 3aae0b09b19..c324a8995de 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -152,5 +152,6 @@ toc_title: Adopters | BiliBili | Video sharing | — | — | — | [Blog post, June 2021](https://chowdera.com/2021/06/20210622012241476b.html) | | Gigapipe | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) | | Hydrolix | Cloud data platform | Main product | — | — | [Documentation](https://docs.hydrolix.io/guide/query) | +| Argedor | ClickHouse support | — | — | — | [Official website](https://www.argedor.com/en/clickhouse/) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) From 668846012f1146ad08bf17b6108160474244aab8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 22 Jun 2021 16:44:59 +0300 Subject: [PATCH 35/35] Revert "Remove Testflows, because they are unstable" This reverts commit eaf3f9c6e6182a42781324ac033f12d84d344f5f. --- tests/testflows/regression.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 661bc03509b..c2e143a4b1c 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -22,15 +22,15 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): tasks = [] with Pool(8) as pool: try: - #run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) #run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) finally: join(tasks)