Merge pull request #11686 from ClickHouse/proper-query-formatting-in-log

Don't break multiline queries when they are printed in server log

programs/client/Client.cpp

@@ -986,7 +986,10 @@ private:
     /// Process the query that doesn't require transferring data blocks to the server.
     void processOrdinaryQuery()
     {
-        /// We will always rewrite query (even if there are no query_parameters) because it will help to find errors in query formatter.
+        /// Rewrite query only when we have query parameters.
+        /// Note that if query is rewritten, comments in query are lost.
+        /// But the user often wants to see comments in server logs, query log, processlist, etc.
+        if (!query_parameters.empty())
         {
             /// Replace ASTQueryParameter with ASTLiteral for prepared statements.
             ReplaceQueryParameterVisitor visitor(query_parameters);

src/Interpreters/executeQuery.cpp

@@ -22,6 +22,8 @@
 #include <Parsers/ParserQuery.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/queryToString.h>
+#include <Parsers/ASTWatchQuery.h>
+#include <Parsers/Lexer.h>
 
 #include <Storages/StorageInput.h>
 
@@ -41,7 +43,6 @@
 #include <Processors/Transforms/LimitsCheckingTransform.h>
 #include <Processors/Transforms/MaterializingTransform.h>
 #include <Processors/Formats/IOutputFormat.h>
-#include <Parsers/ASTWatchQuery.h>
 
 
 namespace ProfileEvents
@@ -70,11 +71,35 @@ static void checkASTSizeLimits(const IAST & ast, const Settings & settings)
         ast.checkSize(settings.max_ast_elements);
 }
 
-/// NOTE This is wrong in case of single-line comments and in case of multiline string literals.
+
 static String joinLines(const String & query)
 {
-    String res = query;
+    /// Care should be taken. We don't join lines inside non-whitespace tokens (e.g. multiline string literals)
-    std::replace(res.begin(), res.end(), '\n', ' ');
+    ///  and we don't join line after comment (because it can be single-line comment).
+    /// All other whitespaces replaced to a single whitespace.
+
+    String res;
+    const char * begin = query.data();
+    const char * end = begin + query.size();
+
+    Lexer lexer(begin, end);
+    Token token = lexer.nextToken();
+    for (; !token.isEnd(); token = lexer.nextToken())
+    {
+        if (token.type == TokenType::Whitespace)
+        {
+            res += ' ';
+        }
+        else if (token.type == TokenType::Comment)
+        {
+            res.append(token.begin, token.end);
+            if (token.end < end && *token.end == '\n')
+                res += '\n';
+        }
+        else
+            res.append(token.begin, token.end);
+    }
+
     return res;
 }
 

tests/queries/0_stateless/01070_exception_code_in_query_log_table.sql

@@ -3,5 +3,5 @@ SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serve
 CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory();
 SELECT * FROM test_table_for_01070_exception_code_in_query_log_table;
 SYSTEM FLUSH LOGS;
-SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code;
+SELECT exception_code FROM system.query_log WHERE lower(query) LIKE lower('SELECT * FROM test_table_for_01070_exception_code_in_query_log_table%') AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code;
 DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table;

tests/queries/0_stateless/01091_num_threads.sql

@@ -4,7 +4,7 @@ set log_query_threads=1;
 SELECT 1;
 SYSTEM FLUSH LOGS;
 
-WITH 
+WITH
     (
         SELECT query_id
         FROM system.query_log
@@ -19,11 +19,11 @@ WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master
 select sum(number) from numbers(1000000);
 SYSTEM FLUSH LOGS;
 
-WITH 
+WITH
     (
         SELECT query_id
         FROM system.query_log
-        WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1))
+        WHERE (query LIKE 'select sum(number) from numbers(1000000);%') AND (event_date >= (today() - 1))
         ORDER BY event_time DESC
         LIMIT 1
     ) AS id
@@ -34,11 +34,11 @@ WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master
 select sum(number) from numbers_mt(1000000);
 SYSTEM FLUSH LOGS;
 
-WITH 
+WITH
     (
         SELECT query_id
         FROM system.query_log
-        WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1))
+        WHERE (query LIKE 'select sum(number) from numbers_mt(1000000);%') AND (event_date >= (today() - 1))
         ORDER BY event_time DESC
         LIMIT 1
     ) AS id

tests/queries/0_stateless/01319_query_formatting_in_server_log.reference

@@ -0,0 +1,4 @@
+ab\ncd	1
+SeLeCt 'ab
+cd' /* hello */ -- world
+ , 1;

tests/queries/0_stateless/01319_query_formatting_in_server_log.sql

@@ -0,0 +1,6 @@
+SeLeCt 'ab
+cd' /* hello */ -- world
+, 1;
+
+SYSTEM FLUSH LOGS;
+SELECT extract(message, 'SeL.+?;') FROM system.text_log WHERE event_date >= yesterday() AND message LIKE '%SeLeCt \'ab\n%' ORDER BY event_time DESC LIMIT 1 FORMAT TSVRaw;