Merge pull request #11686 from ClickHouse/proper-query-formatting-in-log

Don't break multiline queries when they are printed in server log
This commit is contained in:
alexey-milovidov 2020-06-16 22:17:30 +03:00 committed by GitHub
commit ad53ef2c14
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 49 additions and 11 deletions

View File

@ -986,7 +986,10 @@ private:
/// Process the query that doesn't require transferring data blocks to the server. /// Process the query that doesn't require transferring data blocks to the server.
void processOrdinaryQuery() void processOrdinaryQuery()
{ {
/// We will always rewrite query (even if there are no query_parameters) because it will help to find errors in query formatter. /// Rewrite query only when we have query parameters.
/// Note that if query is rewritten, comments in query are lost.
/// But the user often wants to see comments in server logs, query log, processlist, etc.
if (!query_parameters.empty())
{ {
/// Replace ASTQueryParameter with ASTLiteral for prepared statements. /// Replace ASTQueryParameter with ASTLiteral for prepared statements.
ReplaceQueryParameterVisitor visitor(query_parameters); ReplaceQueryParameterVisitor visitor(query_parameters);

View File

@ -22,6 +22,8 @@
#include <Parsers/ParserQuery.h> #include <Parsers/ParserQuery.h>
#include <Parsers/parseQuery.h> #include <Parsers/parseQuery.h>
#include <Parsers/queryToString.h> #include <Parsers/queryToString.h>
#include <Parsers/ASTWatchQuery.h>
#include <Parsers/Lexer.h>
#include <Storages/StorageInput.h> #include <Storages/StorageInput.h>
@ -41,7 +43,6 @@
#include <Processors/Transforms/LimitsCheckingTransform.h> #include <Processors/Transforms/LimitsCheckingTransform.h>
#include <Processors/Transforms/MaterializingTransform.h> #include <Processors/Transforms/MaterializingTransform.h>
#include <Processors/Formats/IOutputFormat.h> #include <Processors/Formats/IOutputFormat.h>
#include <Parsers/ASTWatchQuery.h>
namespace ProfileEvents namespace ProfileEvents
@ -70,11 +71,35 @@ static void checkASTSizeLimits(const IAST & ast, const Settings & settings)
ast.checkSize(settings.max_ast_elements); ast.checkSize(settings.max_ast_elements);
} }
/// NOTE This is wrong in case of single-line comments and in case of multiline string literals.
static String joinLines(const String & query) static String joinLines(const String & query)
{ {
String res = query; /// Care should be taken. We don't join lines inside non-whitespace tokens (e.g. multiline string literals)
std::replace(res.begin(), res.end(), '\n', ' '); /// and we don't join line after comment (because it can be single-line comment).
/// All other whitespaces replaced to a single whitespace.
String res;
const char * begin = query.data();
const char * end = begin + query.size();
Lexer lexer(begin, end);
Token token = lexer.nextToken();
for (; !token.isEnd(); token = lexer.nextToken())
{
if (token.type == TokenType::Whitespace)
{
res += ' ';
}
else if (token.type == TokenType::Comment)
{
res.append(token.begin, token.end);
if (token.end < end && *token.end == '\n')
res += '\n';
}
else
res.append(token.begin, token.end);
}
return res; return res;
} }

View File

@ -3,5 +3,5 @@ SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; -- { serve
CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory(); CREATE TABLE test_table_for_01070_exception_code_in_query_log_table (value UInt64) ENGINE=Memory();
SELECT * FROM test_table_for_01070_exception_code_in_query_log_table; SELECT * FROM test_table_for_01070_exception_code_in_query_log_table;
SYSTEM FLUSH LOGS; SYSTEM FLUSH LOGS;
SELECT exception_code FROM system.query_log WHERE query = 'SELECT * FROM test_table_for_01070_exception_code_in_query_log_table' AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code; SELECT exception_code FROM system.query_log WHERE lower(query) LIKE lower('SELECT * FROM test_table_for_01070_exception_code_in_query_log_table%') AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY exception_code;
DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table; DROP TABLE IF EXISTS test_table_for_01070_exception_code_in_query_log_table;

View File

@ -4,7 +4,7 @@ set log_query_threads=1;
SELECT 1; SELECT 1;
SYSTEM FLUSH LOGS; SYSTEM FLUSH LOGS;
WITH WITH
( (
SELECT query_id SELECT query_id
FROM system.query_log FROM system.query_log
@ -19,11 +19,11 @@ WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master
select sum(number) from numbers(1000000); select sum(number) from numbers(1000000);
SYSTEM FLUSH LOGS; SYSTEM FLUSH LOGS;
WITH WITH
( (
SELECT query_id SELECT query_id
FROM system.query_log FROM system.query_log
WHERE (query = 'SELECT sum(number) FROM numbers(1000000)') AND (event_date >= (today() - 1)) WHERE (query LIKE 'select sum(number) from numbers(1000000);%') AND (event_date >= (today() - 1))
ORDER BY event_time DESC ORDER BY event_time DESC
LIMIT 1 LIMIT 1
) AS id ) AS id
@ -34,11 +34,11 @@ WHERE (event_date >= (today() - 1)) AND (query_id = id) AND (thread_id != master
select sum(number) from numbers_mt(1000000); select sum(number) from numbers_mt(1000000);
SYSTEM FLUSH LOGS; SYSTEM FLUSH LOGS;
WITH WITH
( (
SELECT query_id SELECT query_id
FROM system.query_log FROM system.query_log
WHERE (query = 'SELECT sum(number) FROM numbers_mt(1000000)') AND (event_date >= (today() - 1)) WHERE (query LIKE 'select sum(number) from numbers_mt(1000000);%') AND (event_date >= (today() - 1))
ORDER BY event_time DESC ORDER BY event_time DESC
LIMIT 1 LIMIT 1
) AS id ) AS id

View File

@ -0,0 +1,4 @@
ab\ncd 1
SeLeCt 'ab
cd' /* hello */ -- world
, 1;

View File

@ -0,0 +1,6 @@
SeLeCt 'ab
cd' /* hello */ -- world
, 1;
SYSTEM FLUSH LOGS;
SELECT extract(message, 'SeL.+?;') FROM system.text_log WHERE event_date >= yesterday() AND message LIKE '%SeLeCt \'ab\n%' ORDER BY event_time DESC LIMIT 1 FORMAT TSVRaw;