From 90a18326a5f5edde8038082e82b48be729edfa1d Mon Sep 17 00:00:00 2001 From: kothiga Date: Tue, 6 Feb 2024 13:04:40 -0800 Subject: [PATCH 01/12] Fix KQL distributed tables queries. --- src/Client/HedgedConnections.cpp | 4 ++++ src/Client/MultiplexedConnections.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 7ea13a7dffc..794292bb4b6 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -177,6 +177,10 @@ void HedgedConnections::sendQuery( { Settings modified_settings = settings; + // Kusto queries to replicas are transformed to ClickHouse-SQL. Change the setting before sending. + if (modified_settings.dialect == Dialect::kusto) + modified_settings.dialect = Dialect::clickhouse; + if (disable_two_level_aggregation) { /// Disable two-level aggregation due to version incompatibility. diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index c7d7d0143c8..55708f37051 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -126,6 +126,10 @@ void MultiplexedConnections::sendQuery( Settings modified_settings = settings; + // Kusto queries to replicas are transformed to ClickHouse-SQL. Change the setting before sending. + if (modified_settings.dialect == Dialect::kusto) + modified_settings.dialect = Dialect::clickhouse; + for (auto & replica : replica_states) { if (!replica.connection) From 70248639e852e724ecd778bb69a856d2ab7bc091 Mon Sep 17 00:00:00 2001 From: kothiga Date: Wed, 7 Feb 2024 10:54:52 -0800 Subject: [PATCH 02/12] Include PRQL. Added functional test that tests both KQL and PRQL. --- src/Client/HedgedConnections.cpp | 5 ++-- src/Client/MultiplexedConnections.cpp | 5 ++-- ...dialects_with_distributed_tables.reference | 9 +++++++ ...02985_dialects_with_distributed_tables.sql | 27 +++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference create mode 100644 tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 794292bb4b6..5e08549e5be 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -177,9 +177,8 @@ void HedgedConnections::sendQuery( { Settings modified_settings = settings; - // Kusto queries to replicas are transformed to ClickHouse-SQL. Change the setting before sending. - if (modified_settings.dialect == Dialect::kusto) - modified_settings.dialect = Dialect::clickhouse; + // Kusto and PRQL queries to replicas are transformed to ClickHouse-SQL. Ensure the setting before sending. + modified_settings.dialect = Dialect::clickhouse; if (disable_two_level_aggregation) { diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 55708f37051..0ccb0c6ac55 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -126,9 +126,8 @@ void MultiplexedConnections::sendQuery( Settings modified_settings = settings; - // Kusto queries to replicas are transformed to ClickHouse-SQL. Change the setting before sending. - if (modified_settings.dialect == Dialect::kusto) - modified_settings.dialect = Dialect::clickhouse; + // Kusto and PRQL queries to replicas are transformed to ClickHouse-SQL. Ensure the setting before sending. + modified_settings.dialect = Dialect::clickhouse; for (auto & replica : replica_states) { diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference new file mode 100644 index 00000000000..f22e294ce86 --- /dev/null +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference @@ -0,0 +1,9 @@ +123 +234 +315 +123 +234 +315 +123 +234 +315 diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql new file mode 100644 index 00000000000..d393090ad13 --- /dev/null +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -0,0 +1,27 @@ +-- Tags: distributed, replica, shard, zookeeper + +DROP TABLE IF EXISTS shared_test_table; +DROP TABLE IF EXISTS distributed_test_table; + +CREATE TABLE shared_test_table +ON CLUSTER test_cluster_two_shard_three_replicas_localhost (id Int16) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_02985_{database}/{shard}/shared_test_table', '{replica}') +ORDER BY (id); + +CREATE TABLE distributed_test_table +ON CLUSTER test_cluster_two_shard_three_replicas_localhost (id Int16) +ENGINE = Distributed(test_cluster_two_shard_three_replicas_localhost, currentDatabase(), shared_test_table); + +INSERT INTO shared_test_table VALUES (123), (651), (446), (315), (234), (764); + +SELECT id FROM distributed_test_table LIMIT 3; + +SET dialect = 'kusto'; + +distributed_test_table | take 3; + +SET dialect = 'prql'; + +from distributed_test_table +select {id} +take 1..3 From 0dfcf954a10dd3dc6a656fbbfe39609bfab29cdb Mon Sep 17 00:00:00 2001 From: kothiga Date: Wed, 7 Feb 2024 12:39:37 -0800 Subject: [PATCH 03/12] Specify no-fasttest. --- .../0_stateless/02985_dialects_with_distributed_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index d393090ad13..38776397a03 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -1,4 +1,4 @@ --- Tags: distributed, replica, shard, zookeeper +-- Tags: no-fasttest, distributed, replica, shard, zookeeper DROP TABLE IF EXISTS shared_test_table; DROP TABLE IF EXISTS distributed_test_table; From f5ba7fde9f6ffc39d5834dc303d841b396c0d5ea Mon Sep 17 00:00:00 2001 From: Austin Kothig Date: Thu, 8 Feb 2024 10:09:02 -0500 Subject: [PATCH 04/12] Only set dialect if KQL or PRQL. --- src/Client/HedgedConnections.cpp | 3 ++- src/Client/MultiplexedConnections.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 5e08549e5be..0795a6740c3 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -178,7 +178,8 @@ void HedgedConnections::sendQuery( Settings modified_settings = settings; // Kusto and PRQL queries to replicas are transformed to ClickHouse-SQL. Ensure the setting before sending. - modified_settings.dialect = Dialect::clickhouse; + if (modified_settings.dialect == Dialect::kusto || modified_settings.dialect == Dialect::prql) + modified_settings.dialect = Dialect::clickhouse; if (disable_two_level_aggregation) { diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 0ccb0c6ac55..100a635938d 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -127,7 +127,8 @@ void MultiplexedConnections::sendQuery( Settings modified_settings = settings; // Kusto and PRQL queries to replicas are transformed to ClickHouse-SQL. Ensure the setting before sending. - modified_settings.dialect = Dialect::clickhouse; + if (modified_settings.dialect == Dialect::kusto || modified_settings.dialect == Dialect::prql) + modified_settings.dialect = Dialect::clickhouse; for (auto & replica : replica_states) { From 0458f470b948bded04c2bf729d923c4fae951d8e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 11 Feb 2024 07:55:04 +0100 Subject: [PATCH 05/12] Update 02985_dialects_with_distributed_tables.sql --- .../02985_dialects_with_distributed_tables.sql | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index 38776397a03..7d36b9fd263 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -1,15 +1,13 @@ --- Tags: no-fasttest, distributed, replica, shard, zookeeper +-- Tags: no-fasttest, distributed DROP TABLE IF EXISTS shared_test_table; DROP TABLE IF EXISTS distributed_test_table; CREATE TABLE shared_test_table -ON CLUSTER test_cluster_two_shard_three_replicas_localhost (id Int16) -ENGINE = ReplicatedMergeTree('/clickhouse/tables/test_02985_{database}/{shard}/shared_test_table', '{replica}') +ENGINE = MergeTree ORDER BY (id); CREATE TABLE distributed_test_table -ON CLUSTER test_cluster_two_shard_three_replicas_localhost (id Int16) ENGINE = Distributed(test_cluster_two_shard_three_replicas_localhost, currentDatabase(), shared_test_table); INSERT INTO shared_test_table VALUES (123), (651), (446), (315), (234), (764); @@ -25,3 +23,8 @@ SET dialect = 'prql'; from distributed_test_table select {id} take 1..3 + +SET dialect = 'clickhouse'; + +DROP TABLE distributed_test_table; +DROP TABLE shared_test_table; From 9130c1704ce0d37cbc4945f1802fae2a0180ceb9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 13 Feb 2024 02:04:36 +0100 Subject: [PATCH 06/12] Update 02985_dialects_with_distributed_tables.sql --- .../0_stateless/02985_dialects_with_distributed_tables.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index 7d36b9fd263..0b5a54b752e 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -3,11 +3,11 @@ DROP TABLE IF EXISTS shared_test_table; DROP TABLE IF EXISTS distributed_test_table; -CREATE TABLE shared_test_table +CREATE TABLE shared_test_table (id UInt64) ENGINE = MergeTree ORDER BY (id); -CREATE TABLE distributed_test_table +CREATE TABLE distributed_test_table ENGINE = Distributed(test_cluster_two_shard_three_replicas_localhost, currentDatabase(), shared_test_table); INSERT INTO shared_test_table VALUES (123), (651), (446), (315), (234), (764); From 61640e3da0eb157c30ed16ceb95d9504fdf132b8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 15 Feb 2024 02:51:55 +0100 Subject: [PATCH 07/12] Temporarily turn off Kusto --- .../02985_dialects_with_distributed_tables.reference | 6 ------ .../0_stateless/02985_dialects_with_distributed_tables.sql | 2 ++ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference index f22e294ce86..c69bc512c96 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference @@ -1,9 +1,3 @@ 123 234 315 -123 -234 -315 -123 -234 -315 diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index 0b5a54b752e..695188c52fc 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -14,6 +14,7 @@ INSERT INTO shared_test_table VALUES (123), (651), (446), (315), (234), (764); SELECT id FROM distributed_test_table LIMIT 3; +/* Temporarily disabled with Kusto. SET dialect = 'kusto'; distributed_test_table | take 3; @@ -23,6 +24,7 @@ SET dialect = 'prql'; from distributed_test_table select {id} take 1..3 +*/ SET dialect = 'clickhouse'; From 00f80b171bcd73a8aa9deb6d3bb221c64b68f328 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 16 Feb 2024 03:25:47 +0100 Subject: [PATCH 08/12] Update test --- .../02985_dialects_with_distributed_tables.reference | 3 +++ .../0_stateless/02985_dialects_with_distributed_tables.sql | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference index c69bc512c96..4b9fa8021a4 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference @@ -1,3 +1,6 @@ 123 234 315 +123 +234 +315 diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index 695188c52fc..f03c09000e3 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -18,13 +18,13 @@ SELECT id FROM distributed_test_table LIMIT 3; SET dialect = 'kusto'; distributed_test_table | take 3; +*/ SET dialect = 'prql'; from distributed_test_table select {id} take 1..3 -*/ SET dialect = 'clickhouse'; From 092ed2b16a53aec8a1dc9acc9ae88000aa953171 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 19 Feb 2024 12:17:37 +0100 Subject: [PATCH 09/12] Update 02985_dialects_with_distributed_tables.sql --- .../0_stateless/02985_dialects_with_distributed_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index f03c09000e3..cd8b7dc6b3d 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -24,7 +24,7 @@ SET dialect = 'prql'; from distributed_test_table select {id} -take 1..3 +take 1..3; SET dialect = 'clickhouse'; From 3613464d468cb89116935390ffb49e39676ac706 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 19 Feb 2024 12:19:25 +0100 Subject: [PATCH 10/12] Fix the code --- src/Client/HedgedConnections.cpp | 5 ++--- src/Client/MultiplexedConnections.cpp | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 0795a6740c3..59899fde3fa 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -177,9 +177,8 @@ void HedgedConnections::sendQuery( { Settings modified_settings = settings; - // Kusto and PRQL queries to replicas are transformed to ClickHouse-SQL. Ensure the setting before sending. - if (modified_settings.dialect == Dialect::kusto || modified_settings.dialect == Dialect::prql) - modified_settings.dialect = Dialect::clickhouse; + /// Queries in foreign languages are transformed to ClickHouse-SQL. Ensure the setting before sending. + modified_settings.dialect = Dialect::clickhouse; if (disable_two_level_aggregation) { diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 100a635938d..0997343ef6b 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -126,9 +126,8 @@ void MultiplexedConnections::sendQuery( Settings modified_settings = settings; - // Kusto and PRQL queries to replicas are transformed to ClickHouse-SQL. Ensure the setting before sending. - if (modified_settings.dialect == Dialect::kusto || modified_settings.dialect == Dialect::prql) - modified_settings.dialect = Dialect::clickhouse; + /// Queries in foreign languages are transformed to ClickHouse-SQL. Ensure the setting before sending. + modified_settings.dialect = Dialect::clickhouse; for (auto & replica : replica_states) { From 6c6376de0a4fc0a914e7a3743086549329d8a1ab Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 21 Feb 2024 08:56:55 +0100 Subject: [PATCH 11/12] Fix compatibility --- src/Client/HedgedConnections.cpp | 2 +- src/Client/MultiplexedConnections.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 59899fde3fa..6f5332f1ea0 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -178,7 +178,7 @@ void HedgedConnections::sendQuery( Settings modified_settings = settings; /// Queries in foreign languages are transformed to ClickHouse-SQL. Ensure the setting before sending. - modified_settings.dialect = Dialect::clickhouse; + modified_settings.dialect.resetValueToDefault(); if (disable_two_level_aggregation) { diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 0997343ef6b..59949865ca0 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -127,7 +127,7 @@ void MultiplexedConnections::sendQuery( Settings modified_settings = settings; /// Queries in foreign languages are transformed to ClickHouse-SQL. Ensure the setting before sending. - modified_settings.dialect = Dialect::clickhouse; + modified_settings.dialect.resetValueToDefault(); for (auto & replica : replica_states) { From e23dd5ba5a4ed47310202375ed78bb2d9369c462 Mon Sep 17 00:00:00 2001 From: Austin Kothig Date: Mon, 26 Feb 2024 12:56:14 -0500 Subject: [PATCH 12/12] Enable Kusto part of test. --- .../02985_dialects_with_distributed_tables.reference | 3 +++ .../0_stateless/02985_dialects_with_distributed_tables.sql | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference index 4b9fa8021a4..f22e294ce86 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.reference @@ -4,3 +4,6 @@ 123 234 315 +123 +234 +315 diff --git a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql index cd8b7dc6b3d..6ac36cf5835 100644 --- a/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql +++ b/tests/queries/0_stateless/02985_dialects_with_distributed_tables.sql @@ -14,11 +14,9 @@ INSERT INTO shared_test_table VALUES (123), (651), (446), (315), (234), (764); SELECT id FROM distributed_test_table LIMIT 3; -/* Temporarily disabled with Kusto. SET dialect = 'kusto'; distributed_test_table | take 3; -*/ SET dialect = 'prql';