From a900d08a5f91e2b3a1253c847f9211bf9dc92762 Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 12 Aug 2023 08:16:44 +0000 Subject: [PATCH 1/6] Forbid use_structure_from_insertion_table_in_table_functions when execute scalar --- .../ExecuteScalarSubqueriesVisitor.cpp | 4 ++++ .../02843_insertion_table_schema_infer.reference | 3 +++ .../02843_insertion_table_schema_infer.sh | 16 ++++++++++++++++ tests/queries/0_stateless/data_tsv/mock_data.tsv | 5 +++++ 4 files changed, 28 insertions(+) create mode 100644 tests/queries/0_stateless/02843_insertion_table_schema_infer.reference create mode 100755 tests/queries/0_stateless/02843_insertion_table_schema_infer.sh create mode 100644 tests/queries/0_stateless/data_tsv/mock_data.tsv diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 3d821b60e81..87ec71c1ac6 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -77,6 +77,10 @@ static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqu subquery_settings.max_result_rows = 1; subquery_settings.extremes = false; subquery_context->setSettings(subquery_settings); + + /// When execute `INSERT INTO t WITH ... SELECT ...`, it may lead to `Unknown columns` + /// exception with this settings enabled(https://github.com/ClickHouse/ClickHouse/issues/52494). + subquery_context->getQueryContext()->setSetting("use_structure_from_insertion_table_in_table_functions", false); if (!data.only_analyze && subquery_context->hasQueryContext()) { /// Save current cached scalars in the context before analyzing the query diff --git a/tests/queries/0_stateless/02843_insertion_table_schema_infer.reference b/tests/queries/0_stateless/02843_insertion_table_schema_infer.reference new file mode 100644 index 00000000000..aec86406a98 --- /dev/null +++ b/tests/queries/0_stateless/02843_insertion_table_schema_infer.reference @@ -0,0 +1,3 @@ +user127 1 +user405 1 +user902 1 diff --git a/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh b/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh new file mode 100755 index 00000000000..d806b678456 --- /dev/null +++ b/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-parallel, no-fasttest + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DATA_DIR=$CUR_DIR/data_tsv + +$CLICKHOUSE_LOCAL --multiquery \ +"CREATE VIEW users AS SELECT * FROM file('$DATA_DIR/mock_data.tsv', TSVWithNamesAndTypes); + CREATE TABLE users_output (name String, tag UInt64)ENGINE = Memory; + INSERT INTO users_output WITH (SELECT groupUniqArrayArray(mapKeys(Tags)) FROM users) AS unique_tags SELECT UserName AS name, length(unique_tags) AS tag FROM users; + SELECT * FROM users_output;" diff --git a/tests/queries/0_stateless/data_tsv/mock_data.tsv b/tests/queries/0_stateless/data_tsv/mock_data.tsv new file mode 100644 index 00000000000..fcf2b300b15 --- /dev/null +++ b/tests/queries/0_stateless/data_tsv/mock_data.tsv @@ -0,0 +1,5 @@ +UserName Age Tags +String Int8 Map(String, UInt64) +user127 20 {'test': 123} +user405 43 {'test': 123} +user902 43 {'test': 123} From 102526a21e0ce639f0e473e26dfce6be392e1bbc Mon Sep 17 00:00:00 2001 From: flynn Date: Sat, 12 Aug 2023 08:58:43 +0000 Subject: [PATCH 2/6] fix for analyzer --- src/Analyzer/Passes/QueryAnalysisPass.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index 0b3d19f1861..b0dc5396045 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -1918,6 +1918,9 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden subquery_settings.max_result_rows = 1; subquery_settings.extremes = false; subquery_context->setSettings(subquery_settings); + /// When execute `INSERT INTO t WITH ... SELECT ...`, it may lead to `Unknown columns` + /// exception with this settings enabled(https://github.com/ClickHouse/ClickHouse/issues/52494). + subquery_context->setSetting("use_structure_from_insertion_table_in_table_functions", false); auto options = SelectQueryOptions(QueryProcessingStage::Complete, scope.subquery_depth, true /*is_subquery*/); auto interpreter = std::make_unique(node->toAST(), subquery_context, options); From 9d11f4bf93ee56480c6ed73e7426b88c300df345 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 16 Aug 2023 17:46:50 +0800 Subject: [PATCH 3/6] Bug fix JSON_QUERY while the json path is all number --- .../JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 5 +++++ tests/queries/0_stateless/01889_sql_json_functions.reference | 2 ++ tests/queries/0_stateless/01889_sql_json_functions.sql | 1 + 3 files changed, 8 insertions(+) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index 709ef89dd3c..ed7a0ef54f1 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -51,6 +51,7 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & } const auto * last_begin = *pos->begin == '.' ? pos->begin + 1 : pos->begin; const auto * last_end = pos->end; + const String numberic_member_name = String(last_begin, last_end); ++pos; if (pos.isValid() && pos->type == TokenType::BareWord && pos->begin == last_end) @@ -58,6 +59,10 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & member_name = std::make_shared(String(last_begin, pos->end)); ++pos; } + else if (!pos.isValid() && pos->type == TokenType::EndOfStream) + { + member_name = std::make_shared(numberic_member_name); + } else { return false; diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index cb8e19ea2a0..3a1cafed0bd 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -77,6 +77,8 @@ SELECT JSON_QUERY('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$. [0, 1, 4, 0, -1, -4] SELECT JSON_QUERY('{"1key":1}', '$.1key'); [1] +SELECT JSON_QUERY('{"123":1}', '$.123'); +[1] SELECT JSON_QUERY('{"hello":1}', '$[hello]'); [1] SELECT JSON_QUERY('{"hello":1}', '$["hello"]'); diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 947b0171ec6..511a25d0c12 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -43,6 +43,7 @@ SELECT JSON_QUERY( '{hello:{"world":"!"}}}', '$.hello'); -- invalid json => defa SELECT JSON_QUERY('', '$.hello'); SELECT JSON_QUERY('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$.array[*][0 to 2, 4]'); SELECT JSON_QUERY('{"1key":1}', '$.1key'); +SELECT JSON_QUERY('{"123":1}', '$.123'); SELECT JSON_QUERY('{"hello":1}', '$[hello]'); SELECT JSON_QUERY('{"hello":1}', '$["hello"]'); SELECT JSON_QUERY('{"hello":1}', '$[\'hello\']'); From f75b5c91b70bd3cd4b64e8ba79b6c9de330e2661 Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Wed, 16 Aug 2023 18:09:21 +0800 Subject: [PATCH 4/6] review fix --- src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index ed7a0ef54f1..c506379a34c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -51,7 +51,7 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & } const auto * last_begin = *pos->begin == '.' ? pos->begin + 1 : pos->begin; const auto * last_end = pos->end; - const String numberic_member_name = String(last_begin, last_end); + const String numberic_member_name(last_begin, last_end); ++pos; if (pos.isValid() && pos->type == TokenType::BareWord && pos->begin == last_end) From d65dba10c6ffc67fdb2f49238daa8026c138e41c Mon Sep 17 00:00:00 2001 From: kevinyhzou Date: Thu, 17 Aug 2023 17:45:57 +0800 Subject: [PATCH 5/6] review --- src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp | 3 +-- tests/queries/0_stateless/01889_sql_json_functions.reference | 2 ++ tests/queries/0_stateless/01889_sql_json_functions.sql | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp index c506379a34c..141f25bfe4c 100644 --- a/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp +++ b/src/Functions/JSONPath/Parsers/ParserJSONPathMemberAccess.cpp @@ -51,7 +51,6 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & } const auto * last_begin = *pos->begin == '.' ? pos->begin + 1 : pos->begin; const auto * last_end = pos->end; - const String numberic_member_name(last_begin, last_end); ++pos; if (pos.isValid() && pos->type == TokenType::BareWord && pos->begin == last_end) @@ -61,7 +60,7 @@ bool ParserJSONPathMemberAccess::parseImpl(Pos & pos, ASTPtr & node, Expected & } else if (!pos.isValid() && pos->type == TokenType::EndOfStream) { - member_name = std::make_shared(numberic_member_name); + member_name = std::make_shared(String(last_begin, last_end)); } else { diff --git a/tests/queries/0_stateless/01889_sql_json_functions.reference b/tests/queries/0_stateless/01889_sql_json_functions.reference index 3a1cafed0bd..407b126c4af 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.reference +++ b/tests/queries/0_stateless/01889_sql_json_functions.reference @@ -79,6 +79,8 @@ SELECT JSON_QUERY('{"1key":1}', '$.1key'); [1] SELECT JSON_QUERY('{"123":1}', '$.123'); [1] +SELECT JSON_QUERY('{"123":1}', '$[123]'); + SELECT JSON_QUERY('{"hello":1}', '$[hello]'); [1] SELECT JSON_QUERY('{"hello":1}', '$["hello"]'); diff --git a/tests/queries/0_stateless/01889_sql_json_functions.sql b/tests/queries/0_stateless/01889_sql_json_functions.sql index 511a25d0c12..0683203fcea 100644 --- a/tests/queries/0_stateless/01889_sql_json_functions.sql +++ b/tests/queries/0_stateless/01889_sql_json_functions.sql @@ -44,6 +44,7 @@ SELECT JSON_QUERY('', '$.hello'); SELECT JSON_QUERY('{"array":[[0, 1, 2, 3, 4, 5], [0, -1, -2, -3, -4, -5]]}', '$.array[*][0 to 2, 4]'); SELECT JSON_QUERY('{"1key":1}', '$.1key'); SELECT JSON_QUERY('{"123":1}', '$.123'); +SELECT JSON_QUERY('{"123":1}', '$[123]'); SELECT JSON_QUERY('{"hello":1}', '$[hello]'); SELECT JSON_QUERY('{"hello":1}', '$["hello"]'); SELECT JSON_QUERY('{"hello":1}', '$[\'hello\']'); From 240d29b1887c79b92d8a5f299bd7a64c543f8649 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 23 Aug 2023 09:46:17 +0000 Subject: [PATCH 6/6] Fix broken test --- .../0_stateless/02862_sorted_distinct_sparse_fix.reference | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.reference b/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.reference index 121351b29cd..3a17878aea7 100644 --- a/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.reference +++ b/tests/queries/0_stateless/02862_sorted_distinct_sparse_fix.reference @@ -10,3 +10,4 @@ set max_threads=1; select trimLeft(explain) from (explain pipeline SELECT DISTINCT id, v FROM t_sparse_distinct) where explain ilike '%DistinctSortedChunkTransform%'; DistinctSortedChunkTransform SELECT DISTINCT id, v FROM t_sparse_distinct format Null; +DROP TABLE t_sparse_distinct;