Merge pull request #29206 from azat/strict-external-queries

Add ability to disable converting expressions to local filter for external queries

src/Core/Settings.h

@@ -386,6 +386,7 @@ class IColumn;
     M(Bool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
     M(Bool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
     M(Bool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql', 'postgresql' and 'odbc' table functions.", 0) \
+    M(Bool, external_table_strict_query, false, "If it is set to true, transforming expression to local filter is forbidden for queries to external tables.", 0) \
     \
     M(Bool, allow_hyperscan, true, "Allow functions that use Hyperscan library. Disable to avoid potentially long compilation times and excessive resource usage.", 0) \
     M(UInt64, max_hyperscan_regexp_length, 0, "Max length of regexp than can be used in hyperscan multi-match functions. Zero means unlimited.", 0) \

src/Storages/tests/gtest_transform_query_for_external_database.cpp

@@ -219,3 +219,33 @@ TEST(TransformQueryForExternalDatabase, ForeignColumnInWhere)
           "WHERE column > 2 AND (apply_id = 1 OR table2.num = 1) AND table2.attr != ''",
           R"(SELECT "column", "apply_id" FROM "test"."table" WHERE ("column" > 2) AND ("apply_id" = 1))");
 }
+
+TEST(TransformQueryForExternalDatabase, NoStrict)
+{
+    const State & state = State::instance();
+
+    check(state, 1,
+          "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)",
+          R"(SELECT "field" FROM "test"."table")");
+}
+
+TEST(TransformQueryForExternalDatabase, Strict)
+{
+    const State & state = State::instance();
+    state.context->setSetting("external_table_strict_query", true);
+
+    check(state, 1,
+          "SELECT field FROM table WHERE field = '1'",
+          R"(SELECT "field" FROM "test"."table" WHERE "field" = '1')");
+    check(state, 1,
+          "SELECT field FROM table WHERE field IN ('1', '2')",
+          R"(SELECT "field" FROM "test"."table" WHERE "field" IN ('1', '2'))");
+    check(state, 1,
+          "SELECT field FROM table WHERE field LIKE '%test%'",
+          R"(SELECT "field" FROM "test"."table" WHERE "field" LIKE '%test%')");
+
+    /// removeUnknownSubexpressionsFromWhere() takes place
+    EXPECT_THROW(check(state, 1, "SELECT field FROM table WHERE field IN (SELECT attr FROM table2)", ""), Exception);
+    /// !isCompatible() takes place
+    EXPECT_THROW(check(state, 1, "SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", ""), Exception);
+}

src/Storages/transformQueryForExternalDatabase.cpp

@@ -9,6 +9,7 @@
 #include <Parsers/ASTExpressionList.h>
 #include <Interpreters/TreeRewriter.h>
 #include <Interpreters/InDepthNodeVisitor.h>
+#include <Interpreters/Context.h>
 #include <IO/WriteBufferFromString.h>
 #include <Storages/transformQueryForExternalDatabase.h>
 #include <Storages/MergeTree/KeyCondition.h>
@@ -20,6 +21,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
+    extern const int INCORRECT_QUERY;
 }
 
 namespace
@@ -248,6 +250,7 @@ String transformQueryForExternalDatabase(
 {
     auto clone_query = query_info.query->clone();
     const Names used_columns = query_info.syntax_analyzer_result->requiredSourceColumns();
+    bool strict = context->getSettingsRef().external_table_strict_query;
 
     auto select = std::make_shared<ASTSelectQuery>();
 
@@ -275,6 +278,10 @@ String transformQueryForExternalDatabase(
         {
             select->setExpression(ASTSelectQuery::Expression::WHERE, std::move(original_where));
         }
+        else if (strict)
+        {
+            throw Exception("Query contains non-compatible expressions (and external_table_strict_query=true)", ErrorCodes::INCORRECT_QUERY);
+        }
         else if (const auto * function = original_where->as<ASTFunction>())
         {
             if (function->name == "and")
@@ -292,6 +299,10 @@ String transformQueryForExternalDatabase(
             }
         }
     }
+    else if (strict && original_where)
+    {
+        throw Exception("Query contains non-compatible expressions (and external_table_strict_query=true)", ErrorCodes::INCORRECT_QUERY);
+    }
 
     ASTPtr select_ptr = select;
     dropAliases(select_ptr);

src/Storages/transformQueryForExternalDatabase.h

@@ -22,6 +22,9 @@ class IAST;
   * that contain only compatible expressions.
   *
   * Compatible expressions are comparisons of identifiers, constants, and logical operations on them.
+  *
+  * Throws INCORRECT_QUERY if external_table_strict_query (from context settings)
+  * is set and some expression from WHERE is not compatible.
   */
 String transformQueryForExternalDatabase(
     const SelectQueryInfo & query_info,