thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 23:52:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #62512 from ClickHouse/revert-62508-revert-62123-full-syntax-highlight

Browse Source 
Revert "Revert "Rich syntax highlighting in the client""
This commit is contained in:
Alexey Milovidov 2024-04-12 13:18:51 +02:00 committed by GitHub
commit 5af5be989e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
20 changed files with 311 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Client/ClientBase.cpp
View File

@ -2064,7 +2064,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
return MultiQueryProcessingStage::QUERIES_END;
// Remove leading empty newlines and other whitespace, because they
// are annoying to filter in query log. This is mostly relevant for
// are annoying to filter in the query log. This is mostly relevant for
// the tests.
while (this_query_begin < all_queries_end && isWhitespaceASCII(*this_query_begin))
++this_query_begin;
@ -2098,7 +2098,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
is_interactive,
ignore_error);
}
catch (Exception & e)
catch (const Exception & e)
{
current_exception.reset(e.clone());
return MultiQueryProcessingStage::PARSING_EXCEPTION;
@ -2123,9 +2123,9 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
// INSERT queries may have the inserted data in the query text
// that follow the query itself, e.g. "insert into t format CSV 1;2".
// They need special handling. First of all, here we find where the
// inserted data ends. In multy-query mode, it is delimited by a
// inserted data ends. In multi-query mode, it is delimited by a
// newline.
// The VALUES format needs even more handling -- we also allow the
// The VALUES format needs even more handling - we also allow the
// data to be delimited by semicolon. This case is handled later by
// the format parser itself.
// We can't do multiline INSERTs with inline data, because most

161
src/Client/ClientBaseHelpers.cpp
View File

@ -1,11 +1,14 @@
#include "ClientBaseHelpers.h"
#include <Common/DateLUT.h>
#include <Common/LocalDate.h>
#include <Parsers/Lexer.h>
#include <Parsers/ParserQuery.h>
#include <Parsers/parseQuery.h>
#include <Common/UTF8Helpers.h>
#include <iostream>
namespace DB
{
@ -96,77 +99,103 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
{
using namespace replxx;
static const std::unordered_map<TokenType, Replxx::Color> token_to_color
= {{TokenType::Whitespace, Replxx::Color::DEFAULT},
{TokenType::Comment, Replxx::Color::GRAY},
{TokenType::BareWord, Replxx::Color::DEFAULT},
{TokenType::Number, Replxx::Color::GREEN},
{TokenType::StringLiteral, Replxx::Color::CYAN},
{TokenType::QuotedIdentifier, Replxx::Color::MAGENTA},
{TokenType::OpeningRoundBracket, Replxx::Color::BROWN},
{TokenType::ClosingRoundBracket, Replxx::Color::BROWN},
{TokenType::OpeningSquareBracket, Replxx::Color::BROWN},
{TokenType::ClosingSquareBracket, Replxx::Color::BROWN},
{TokenType::DoubleColon, Replxx::Color::BROWN},
{TokenType::OpeningCurlyBrace, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::ClosingCurlyBrace, replxx::color::bold(Replxx::Color::DEFAULT)},
/// The `colors` array maps to a Unicode code point position in a string into a color.
/// A color is set for every position individually (not for a range).
{TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::HereDoc, Replxx::Color::CYAN},
{TokenType::Plus, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Minus, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Slash, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Percent, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Arrow, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::QuestionMark, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Colon, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Equals, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::NotEquals, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Less, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Greater, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::LessOrEquals, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::GreaterOrEquals, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Spaceship, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Concatenation, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::At, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::DoubleAt, Replxx::Color::MAGENTA},
/// Empty input.
if (colors.empty())
return;
{TokenType::EndOfStream, Replxx::Color::DEFAULT},
/// The colors should be legible (and look gorgeous) in both dark and light themes.
/// When modifying this, check it in both themes.
{TokenType::Error, Replxx::Color::RED},
{TokenType::ErrorMultilineCommentIsNotClosed, Replxx::Color::RED},
{TokenType::ErrorSingleQuoteIsNotClosed, Replxx::Color::RED},
{TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED},
{TokenType::ErrorSinglePipeMark, Replxx::Color::RED},
{TokenType::ErrorWrongNumber, Replxx::Color::RED},
{TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED}};
const Replxx::Color unknown_token_color = Replxx::Color::RED;
Lexer lexer(query.data(), query.data() + query.size());
size_t pos = 0;
for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
static const std::unordered_map<Highlight, Replxx::Color> type_to_color =
{
if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter)
ReplxxLineReader::setLastIsDelimiter(true);
else if (token.type != TokenType::Whitespace)
ReplxxLineReader::setLastIsDelimiter(false);
{Highlight::keyword, replxx::color::bold(Replxx::Color::DEFAULT)},
{Highlight::identifier, Replxx::Color::CYAN},
{Highlight::function, Replxx::Color::BROWN},
{Highlight::alias, replxx::color::rgb666(0, 4, 4)},
{Highlight::substitution, Replxx::Color::MAGENTA},
{Highlight::number, replxx::color::rgb666(0, 4, 0)},
{Highlight::string, Replxx::Color::GREEN},
};
size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
/// We set reasonably small limits for size/depth, because we don't want the CLI to be slow.
/// While syntax highlighting is unneeded for long queries, which the user couldn't read anyway.
const char * begin = query.data();
const char * end = begin + query.size();
Tokens tokens(begin, end, 1000, true);
IParser::Pos token_iterator(tokens, static_cast<uint32_t>(1000), static_cast<uint32_t>(10000));
Expected expected;
expected.enable_highlighting = true;
/// We don't do highlighting for foreign dialects, such as PRQL and Kusto.
/// Only normal ClickHouse SQL queries are highlighted.
/// Currently we highlight only the first query in the multi-query mode.
ParserQuery parser(end);
ASTPtr ast;
bool parse_res = false;
try
{
parse_res = parser.parse(token_iterator, ast, expected);
}
catch (...)
{
/// Skip highlighting in the case of exceptions during parsing.
/// It is ok to ignore unknown exceptions here.
return;
}
size_t pos = 0;
const char * prev = begin;
for (const auto & range : expected.highlights)
{
auto it = type_to_color.find(range.highlight);
if (it != type_to_color.end())
{
if (token_to_color.find(token.type) != token_to_color.end())
colors[pos + code_point_index] = token_to_color.at(token.type);
else
colors[pos + code_point_index] = unknown_token_color;
}
/// We have to map from byte positions to Unicode positions.
pos += UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(prev), range.begin - prev);
size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(range.begin), range.end - range.begin);
pos += utf8_len;
for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
colors[pos + code_point_index] = it->second;
pos += utf8_len;
prev = range.end;
}
}
Token last_token = token_iterator.max();
/// Raw data in INSERT queries, which is not necessarily tokenized.
const char * insert_data = ast ? getInsertData(ast) : nullptr;
/// Highlight the last error in red. If the parser failed or the lexer found an invalid token,
/// or if it didn't parse all the data (except, the data for INSERT query, which is legitimately unparsed)
if ((!parse_res || last_token.isError() || (!token_iterator->isEnd() && token_iterator->type != TokenType::Semicolon))
&& !(insert_data && expected.max_parsed_pos >= insert_data)
&& expected.max_parsed_pos >= prev)
{
pos += UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(prev), expected.max_parsed_pos - prev);
if (pos >= colors.size())
pos = colors.size() - 1;
colors[pos] = Replxx::Color::BRIGHTRED;
}
/// This is a callback for the client/local app to better find query end. Note: this is a kludge, remove it.
if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter
|| query.ends_with(';') || query.ends_with("\\G")) /// This is for raw data in INSERT queries, which is not necessarily tokenized.
{
ReplxxLineReader::setLastIsDelimiter(true);
}
else if (last_token.type != TokenType::Whitespace)
{
ReplxxLineReader::setLastIsDelimiter(false);
}
}
#endif

1
src/Parsers/ASTOrderByElement.cpp
View File

@ -1,4 +1,3 @@
#include <Columns/Collator.h>
#include <Parsers/ASTOrderByElement.h>
#include <Common/SipHash.h>
#include <IO/Operators.h>

2
src/Parsers/CommonParsers.h
View File

@ -601,6 +601,8 @@ public:
constexpr const char * getName() const override { return s.data(); }
Highlight highlight() const override { return Highlight::keyword; }
protected:
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
};

4
src/Parsers/ExpressionElementParsers.cpp
View File

@ -278,7 +278,7 @@ bool ParserTableAsStringLiteralIdentifier::parseImpl(Pos & pos, ASTPtr & node, E
bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
ASTPtr id_list;
if (!ParserList(std::make_unique<ParserIdentifier>(allow_query_parameter), std::make_unique<ParserToken>(TokenType::Dot), false)
if (!ParserList(std::make_unique<ParserIdentifier>(allow_query_parameter, highlight_type), std::make_unique<ParserToken>(TokenType::Dot), false)
.parse(pos, id_list, expected))
return false;
@ -1491,7 +1491,7 @@ const char * ParserAlias::restricted_keywords[] =
bool ParserAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
ParserKeyword s_as(Keyword::AS);
ParserIdentifier id_p;
ParserIdentifier id_p(false, Highlight::alias);
bool has_as_word = s_as.ignore(pos, expected);
if (!allow_alias_without_as_keyword && !has_as_word)

13
src/Parsers/ExpressionElementParsers.h
View File

@ -25,12 +25,15 @@ protected:
class ParserIdentifier : public IParserBase
{
public:
explicit ParserIdentifier(bool allow_query_parameter_ = false) : allow_query_parameter(allow_query_parameter_) {}
explicit ParserIdentifier(bool allow_query_parameter_ = false, Highlight highlight_type_ = Highlight::identifier)
: allow_query_parameter(allow_query_parameter_), highlight_type(highlight_type_) {}
Highlight highlight() const override { return highlight_type; }
protected:
const char * getName() const override { return "identifier"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
bool allow_query_parameter;
Highlight highlight_type;
};
@ -53,8 +56,8 @@ protected:
class ParserCompoundIdentifier : public IParserBase
{
public:
explicit ParserCompoundIdentifier(bool table_name_with_optional_uuid_ = false, bool allow_query_parameter_ = false)
: table_name_with_optional_uuid(table_name_with_optional_uuid_), allow_query_parameter(allow_query_parameter_)
explicit ParserCompoundIdentifier(bool table_name_with_optional_uuid_ = false, bool allow_query_parameter_ = false, Highlight highlight_type_ = Highlight::identifier)
: table_name_with_optional_uuid(table_name_with_optional_uuid_), allow_query_parameter(allow_query_parameter_), highlight_type(highlight_type_)
{
}
@ -63,6 +66,7 @@ protected:
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
bool table_name_with_optional_uuid;
bool allow_query_parameter;
Highlight highlight_type;
};
/** *, t.*, db.table.*, COLUMNS('<regular expression>') APPLY(...) or EXCEPT(...) or REPLACE(...)
@ -253,6 +257,7 @@ class ParserNumber : public IParserBase
protected:
const char * getName() const override { return "number"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
Highlight highlight() const override { return Highlight::number; }
};
/** Unsigned integer, used in right hand side of tuple access operator (x.1).
@ -273,6 +278,7 @@ class ParserStringLiteral : public IParserBase
protected:
const char * getName() const override { return "string literal"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
Highlight highlight() const override { return Highlight::string; }
};
@ -385,6 +391,7 @@ class ParserSubstitution : public IParserBase
protected:
const char * getName() const override { return "substitution"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
Highlight highlight() const override { return Highlight::substitution; }
};

91
src/Parsers/ExpressionListParsers.cpp
View File

@ -441,6 +441,21 @@ bool ParserKeyValuePairsList::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
return parser.parse(pos, node, expected);
}
namespace
{
/// This wrapper is needed to highlight function names differently.
class ParserFunctionName : public IParserBase
{
protected:
const char * getName() const override { return "function name"; }
bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
{
ParserCompoundIdentifier parser(false, true, Highlight::function);
return parser.parse(pos, node, expected);
}
};
}
enum class Action
{
@ -809,6 +824,7 @@ struct ParserExpressionImpl
static const Operator finish_between_operator;
ParserFunctionName function_name_parser;
ParserCompoundIdentifier identifier_parser{false, true};
ParserNumber number_parser;
ParserAsterisk asterisk_parser;
@ -2359,7 +2375,7 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
ASTPtr identifier;
if (ParserCompoundIdentifier(false,true).parse(pos, identifier, expected)
if (ParserFunctionName().parse(pos, identifier, expected)
&& ParserToken(TokenType::OpeningRoundBracket).ignore(pos, expected))
{
auto start = getFunctionLayer(identifier, is_table_function, allow_function_parameters);
@ -2497,7 +2513,7 @@ Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos
{
if (typeid_cast<ViewLayer *>(layers.back().get()) || typeid_cast<KustoLayer *>(layers.back().get()))
{
if (identifier_parser.parse(pos, tmp, expected)
if (function_name_parser.parse(pos, tmp, expected)
&& ParserToken(TokenType::OpeningRoundBracket).ignore(pos, expected))
{
layers.push_back(getFunctionLayer(tmp, layers.front()->is_table_function));
@ -2629,50 +2645,53 @@ Action ParserExpressionImpl::tryParseOperand(Layers & layers, IParser::Pos & pos
{
layers.back()->pushOperand(std::move(tmp));
}
else if (identifier_parser.parse(pos, tmp, expected))
else
{
if (pos->type == TokenType::OpeningRoundBracket)
old_pos = pos;
if (function_name_parser.parse(pos, tmp, expected) && pos->type == TokenType::OpeningRoundBracket)
{
++pos;
layers.push_back(getFunctionLayer(tmp, layers.front()->is_table_function));
return Action::OPERAND;
}
pos = old_pos;
if (identifier_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
}
else if (substitution_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
}
else if (pos->type == TokenType::OpeningRoundBracket)
{
if (subquery_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
return Action::OPERATOR;
}
++pos;
layers.push_back(std::make_unique<RoundBracketsLayer>());
return Action::OPERAND;
}
else if (pos->type == TokenType::OpeningSquareBracket)
{
++pos;
layers.push_back(std::make_unique<ArrayLayer>());
return Action::OPERAND;
}
else if (mysql_global_variable_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
}
else
{
layers.back()->pushOperand(std::move(tmp));
return Action::NONE;
}
}
else if (substitution_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
}
else if (pos->type == TokenType::OpeningRoundBracket)
{
if (subquery_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
return Action::OPERATOR;
}
++pos;
layers.push_back(std::make_unique<RoundBracketsLayer>());
return Action::OPERAND;
}
else if (pos->type == TokenType::OpeningSquareBracket)
{
++pos;
layers.push_back(std::make_unique<ArrayLayer>());
return Action::OPERAND;
}
else if (mysql_global_variable_parser.parse(pos, tmp, expected))
{
layers.back()->pushOperand(std::move(tmp));
}
else
{
return Action::NONE;
}
return Action::OPERATOR;
}

26
src/Parsers/IParser.cpp
View File

@ -9,6 +9,7 @@ namespace ErrorCodes
extern const int TOO_SLOW_PARSING;
}
IParser::Pos & IParser::Pos::operator=(const IParser::Pos & rhs)
{
depth = rhs.depth;
@ -32,4 +33,29 @@ IParser::Pos & IParser::Pos::operator=(const IParser::Pos & rhs)
return *this;
}
template <typename T>
static bool intersects(T a_begin, T a_end, T b_begin, T b_end)
{
return (a_begin <= b_begin && b_begin < a_end)
|| (b_begin <= a_begin && a_begin < b_end);
}
void Expected::highlight(HighlightedRange range)
{
if (!enable_highlighting)
return;
auto it = highlights.lower_bound(range);
while (it != highlights.end() && range.begin < it->end)
{
if (intersects(range.begin, range.end, it->begin, it->end))
it = highlights.erase(it);
else
++it;
}
highlights.insert(range);
}
}

40
src/Parsers/IParser.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <absl/container/inlined_vector.h>
#include <set>
#include <algorithm>
#include <memory>
@ -21,14 +22,43 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
enum class Highlight
{
none = 0,
keyword,
identifier,
function,
alias,
substitution,
number,
string,
};
struct HighlightedRange
{
const char * begin;
const char * end;
Highlight highlight;
auto operator<=>(const HighlightedRange & other) const
{
return begin <=> other.begin;
}
};
/** Collects variants, how parser could proceed further at rightmost position.
* Also collects a mapping of parsed ranges for highlighting,
* which is accumulated through the parsing.
*/
struct Expected
{
absl::InlinedVector<const char *, 7> variants;
const char * max_parsed_pos = nullptr;
bool enable_highlighting = false;
std::set<HighlightedRange> highlights;
/// 'description' should be statically allocated string.
ALWAYS_INLINE void add(const char * current_pos, const char * description)
{
@ -48,6 +78,8 @@ struct Expected
{
add(it->begin, description);
}
void highlight(HighlightedRange range);
};
@ -158,6 +190,14 @@ public:
return parse(pos, node, expected);
}
/** If the parsed fragment should be highlighted in the query editor,
* which type of highlighting to use?
*/
virtual Highlight highlight() const
{
return Highlight::none;
}
virtual ~IParser() = default;
};

19
src/Parsers/IParserBase.cpp
View File

@ -10,8 +10,25 @@ bool IParserBase::parse(Pos & pos, ASTPtr & node, Expected & expected)
return wrapParseImpl(pos, IncreaseDepthTag{}, [&]
{
const char * begin = pos->begin;
bool res = parseImpl(pos, node, expected);
if (!res)
if (res)
{
Highlight type = highlight();
if (pos->begin > begin && type != Highlight::none)
{
Pos prev_token = pos;
--prev_token;
HighlightedRange range;
range.begin = begin;
range.end = prev_token->end;
range.highlight = type;
expected.highlight(range);
}
}
else
node = nullptr;
return res;
});

4
src/Parsers/ParserInsertQuery.cpp
View File

@ -40,7 +40,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
ParserKeyword s_with(Keyword::WITH);
ParserToken s_lparen(TokenType::OpeningRoundBracket);
ParserToken s_rparen(TokenType::ClosingRoundBracket);
ParserToken s_semicolon(TokenType::Semicolon);
ParserIdentifier name_p(true);
ParserList columns_p(std::make_unique<ParserInsertElement>(), std::make_unique<ParserToken>(TokenType::Comma), false);
ParserFunction table_function_p{false};
@ -147,8 +146,9 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
/// If VALUES is defined in query, everything except setting will be parsed as data,
/// and if values followed by semicolon, the data should be null.
if (!s_semicolon.checkWithoutMoving(pos, expected))
if (pos->type != TokenType::Semicolon)
data = pos->begin;
format_str = "Values";
}
else if (s_format.ignore(pos, expected))

15
src/Parsers/parseDatabaseAndTableName.cpp
View File

@ -60,21 +60,6 @@ bool parseDatabaseAndTableAsAST(IParser::Pos & pos, Expected & expected, ASTPtr
}
bool parseDatabase(IParser::Pos & pos, Expected & expected, String & database_str)
{
ParserToken s_dot(TokenType::Dot);
ParserIdentifier identifier_parser;
ASTPtr database;
database_str = "";
if (!identifier_parser.parse(pos, database, expected))
return false;
tryGetIdentifierNameInto(database, database_str);
return true;
}
bool parseDatabaseAsAST(IParser::Pos & pos, Expected & expected, ASTPtr & database)
{
ParserIdentifier identifier_parser(/* allow_query_parameter */true);

58
src/Parsers/parseQuery.cpp
View File

@ -226,6 +226,32 @@ std::string getUnmatchedParenthesesErrorMessage(
}
static ASTInsertQuery * getInsertAST(const ASTPtr & ast)
{
/// Either it is INSERT or EXPLAIN INSERT.
if (auto * explain = ast->as<ASTExplainQuery>())
{
if (auto explained_query = explain->getExplainedQuery())
{
return explained_query->as<ASTInsertQuery>();
}
}
else
{
return ast->as<ASTInsertQuery>();
}
return nullptr;
}
const char * getInsertData(const ASTPtr & ast)
{
if (const ASTInsertQuery * insert = getInsertAST(ast))
return insert->data;
return nullptr;
}
ASTPtr tryParseQuery(
IParser & parser,
const char * & _out_query_end, /* also query begin as input parameter */
@ -270,29 +296,11 @@ ASTPtr tryParseQuery(
if (res && max_parser_depth)
res->checkDepth(max_parser_depth);
ASTInsertQuery * insert = nullptr;
if (parse_res)
{
if (auto * explain = res->as<ASTExplainQuery>())
{
if (auto explained_query = explain->getExplainedQuery())
{
insert = explained_query->as<ASTInsertQuery>();
}
}
else
{
insert = res->as<ASTInsertQuery>();
}
}
// If parsed query ends at data for insertion. Data for insertion could be
// in any format and not necessary be lexical correct, so we can't perform
// most of the checks.
if (insert && insert->data)
{
/// If parsed query ends at data for insertion. Data for insertion could be
/// in any format and not necessary be lexical correct, so we can't perform
/// most of the checks.
if (res && getInsertData(res))
return res;
}
// More granular checks for queries other than INSERT w/inline data.
/// Lexical error
@ -434,11 +442,9 @@ std::pair<const char *, bool> splitMultipartQuery(
ast = parseQueryAndMovePosition(parser, pos, end, "", true, max_query_size, max_parser_depth, max_parser_backtracks);
auto * insert = ast->as<ASTInsertQuery>();
if (insert && insert->data)
if (ASTInsertQuery * insert = getInsertAST(ast))
{
/// Data for INSERT is broken on new line
/// Data for INSERT is broken on the new line
pos = insert->data;
while (*pos && *pos != '\n')
++pos;

5
src/Parsers/parseQuery.h
View File

@ -71,4 +71,9 @@ std::pair<const char *, bool> splitMultipartQuery(
size_t max_parser_backtracks,
bool allow_settings_after_format_in_insert);
/** If the query contains raw data part, such as INSERT ... FORMAT ..., return a pointer to it.
* The SQL parser stops at the raw data part, which is parsed by a separate parser.
*/
const char * getInsertData(const ASTPtr & ast);
}

2
tests/queries/0_stateless/01370_client_autocomplete_word_break_characters.expect
View File

@ -20,7 +20,7 @@ expect_after {
-i $any_spawn_id timeout { exit 1 }
}
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file"
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file --highlight=0"
expect ":) "
# Make a query

19
tests/queries/0_stateless/01565_query_loop_after_client_error.expect
View File

@ -24,30 +24,21 @@ expect_after {
-i $any_spawn_id timeout { exit 1 }
}
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=$history_file"
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion -mn --history_file=$history_file --highlight 0"
expect "\n:) "
send -- "DROP TABLE IF EXISTS t01565;\n"
# NOTE: this is important for -mn mode, you should send "\r" only after reading echoed command
expect "\r\n"
send -- "\r"
send -- "DROP TABLE IF EXISTS t01565;\r"
expect "\nOk."
expect "\n:)"
send -- "CREATE TABLE t01565 (c0 String, c1 Int32) ENGINE = Memory() ;\n"
expect "\r\n"
send -- "\r"
send -- "CREATE TABLE t01565 (c0 String, c1 Int32) ENGINE = Memory() ;\r"
expect "\nOk."
expect "\n:) "
send -- "INSERT INTO t01565(c0, c1) VALUES (\"1\",1) ;\n"
expect "\r\n"
send -- "\r"
send -- "INSERT INTO t01565(c0, c1) VALUES (\"1\",1) ;\r"
expect "\n:) "
send -- "INSERT INTO t01565(c0, c1) VALUES ('1', 1) ;\n"
expect "\r\n"
send -- "\r"
send -- "INSERT INTO t01565(c0, c1) VALUES ('1', 1) ;\r"
expect "\nOk."
expect "\n:) "

2
tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
View File

@ -128,6 +128,7 @@ if __name__ == "__main__":
clickhouse_client = os.environ["CLICKHOUSE_CLIENT"]
args = shlex.split(clickhouse_client)
args.append("--wait_for_suggestions_to_load")
args.append("--highlight=0")
[
run_with_timeout(
test_completion, [args[0], args, comp_word], COMPLETION_TIMEOUT_SECONDS
@ -139,6 +140,7 @@ if __name__ == "__main__":
clickhouse_local = os.environ["CLICKHOUSE_LOCAL"]
args = shlex.split(clickhouse_local)
args.append("--wait_for_suggestions_to_load")
args.append("--highlight=0")
[
run_with_timeout(
test_completion, [args[0], args, comp_word], COMPLETION_TIMEOUT_SECONDS

2
tests/queries/0_stateless/01676_clickhouse_client_autocomplete.sh
View File

@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
python3 "$CURDIR"/01676_clickhouse_client_autocomplete.python
python3 "$CURDIR"/01676_clickhouse_client_autocomplete.python

20
tests/queries/0_stateless/01702_system_query_log.reference
View File

@ -43,16 +43,16 @@ Alter ALTER TABLE sqllt.table UPDATE i = i + 1 WHERE 1;
Alter ALTER TABLE sqllt.table DELETE WHERE i > 65535;
Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\';
System SYSTEM FLUSH LOGS;
System SYSTEM STOP MERGES sqllt.table
System SYSTEM START MERGES sqllt.table
System SYSTEM STOP TTL MERGES sqllt.table
System SYSTEM START TTL MERGES sqllt.table
System SYSTEM STOP MOVES sqllt.table
System SYSTEM START MOVES sqllt.table
System SYSTEM STOP FETCHES sqllt.table
System SYSTEM START FETCHES sqllt.table
System SYSTEM STOP REPLICATED SENDS sqllt.table
System SYSTEM START REPLICATED SENDS sqllt.table
System SYSTEM STOP MERGES sqllt.table;
System SYSTEM START MERGES sqllt.table;
System SYSTEM STOP TTL MERGES sqllt.table;
System SYSTEM START TTL MERGES sqllt.table;
System SYSTEM STOP MOVES sqllt.table;
System SYSTEM START MOVES sqllt.table;
System SYSTEM STOP FETCHES sqllt.table;
System SYSTEM START FETCHES sqllt.table;
System SYSTEM STOP REPLICATED SENDS sqllt.table;
System SYSTEM START REPLICATED SENDS sqllt.table;
Select -- SYSTEM RELOAD DICTIONARY sqllt.dictionary; -- temporary out of order: Code: 210, Connection refused (localhost:9001) (version 21.3.1.1)\n-- DROP REPLICA\n-- haha, no\n-- SYSTEM KILL;\n-- SYSTEM SHUTDOWN;\n\n-- Since we don\'t really care about the actual output, suppress it with `FORMAT Null`.\nSELECT \'SHOW queries\';
Show SHOW CREATE TABLE sqllt.table FORMAT Null;
Show SHOW CREATE DICTIONARY sqllt.dictionary FORMAT Null;

2
tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect
View File

@ -21,7 +21,7 @@ expect_after {
-i $any_spawn_id timeout { exit 1 }
}
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file"
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --history_file=$history_file --highlight=0"
expect ":) "
# Make a query