mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Merge pull request #26671 from kitaisreal/mysql-dictionaries-support-for-custom-query
Lexer introduce heredoc
This commit is contained in:
commit
f6f8bea689
@ -425,6 +425,7 @@ private:
|
|||||||
{TokenType::Semicolon, Replxx::Color::INTENSE},
|
{TokenType::Semicolon, Replxx::Color::INTENSE},
|
||||||
{TokenType::Dot, Replxx::Color::INTENSE},
|
{TokenType::Dot, Replxx::Color::INTENSE},
|
||||||
{TokenType::Asterisk, Replxx::Color::INTENSE},
|
{TokenType::Asterisk, Replxx::Color::INTENSE},
|
||||||
|
{TokenType::HereDoc, Replxx::Color::CYAN},
|
||||||
{TokenType::Plus, Replxx::Color::INTENSE},
|
{TokenType::Plus, Replxx::Color::INTENSE},
|
||||||
{TokenType::Minus, Replxx::Color::INTENSE},
|
{TokenType::Minus, Replxx::Color::INTENSE},
|
||||||
{TokenType::Slash, Replxx::Color::INTENSE},
|
{TokenType::Slash, Replxx::Color::INTENSE},
|
||||||
@ -450,8 +451,7 @@ private:
|
|||||||
{TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED},
|
{TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED},
|
||||||
{TokenType::ErrorSinglePipeMark, Replxx::Color::RED},
|
{TokenType::ErrorSinglePipeMark, Replxx::Color::RED},
|
||||||
{TokenType::ErrorWrongNumber, Replxx::Color::RED},
|
{TokenType::ErrorWrongNumber, Replxx::Color::RED},
|
||||||
{ TokenType::ErrorMaxQuerySizeExceeded,
|
{TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED }};
|
||||||
Replxx::Color::RED }};
|
|
||||||
|
|
||||||
const Replxx::Color unknown_token_color = Replxx::Color::RED;
|
const Replxx::Color unknown_token_color = Replxx::Color::RED;
|
||||||
|
|
||||||
|
@ -1555,26 +1555,37 @@ bool ParserUnsignedInteger::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
|
|||||||
|
|
||||||
bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||||
{
|
{
|
||||||
if (pos->type != TokenType::StringLiteral)
|
if (pos->type != TokenType::StringLiteral && pos->type != TokenType::HereDoc)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
String s;
|
String s;
|
||||||
ReadBufferFromMemory in(pos->begin, pos->size());
|
|
||||||
|
|
||||||
try
|
if (pos->type == TokenType::StringLiteral)
|
||||||
{
|
{
|
||||||
readQuotedStringWithSQLStyle(s, in);
|
ReadBufferFromMemory in(pos->begin, pos->size());
|
||||||
}
|
|
||||||
catch (const Exception &)
|
|
||||||
{
|
|
||||||
expected.add(pos, "string literal");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (in.count() != pos->size())
|
try
|
||||||
|
{
|
||||||
|
readQuotedStringWithSQLStyle(s, in);
|
||||||
|
}
|
||||||
|
catch (const Exception &)
|
||||||
|
{
|
||||||
|
expected.add(pos, "string literal");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (in.count() != pos->size())
|
||||||
|
{
|
||||||
|
expected.add(pos, "string literal");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (pos->type == TokenType::HereDoc)
|
||||||
{
|
{
|
||||||
expected.add(pos, "string literal");
|
std::string_view here_doc(pos->begin, pos->size());
|
||||||
return false;
|
size_t heredoc_size = here_doc.find('$', 1) + 1;
|
||||||
|
assert(heredoc_size != std::string_view::npos);
|
||||||
|
s = String(pos->begin + heredoc_size, pos->size() - heredoc_size * 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
auto literal = std::make_shared<ASTLiteral>(s);
|
auto literal = std::make_shared<ASTLiteral>(s);
|
||||||
|
@ -308,6 +308,7 @@ protected:
|
|||||||
|
|
||||||
|
|
||||||
/** String in single quotes.
|
/** String in single quotes.
|
||||||
|
* String in heredoc $here$txt$here$ equivalent to 'txt'.
|
||||||
*/
|
*/
|
||||||
class ParserStringLiteral : public IParserBase
|
class ParserStringLiteral : public IParserBase
|
||||||
{
|
{
|
||||||
|
@ -2,7 +2,6 @@
|
|||||||
#include <Common/StringUtils/StringUtils.h>
|
#include <Common/StringUtils/StringUtils.h>
|
||||||
#include <common/find_symbols.h>
|
#include <common/find_symbols.h>
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -338,10 +337,33 @@ Token Lexer::nextTokenImpl()
|
|||||||
}
|
}
|
||||||
|
|
||||||
default:
|
default:
|
||||||
if (*pos == '$' && ((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end))
|
if (*pos == '$')
|
||||||
{
|
{
|
||||||
/// Capture standalone dollar sign
|
/// Try to capture dollar sign as start of here doc
|
||||||
return Token(TokenType::DollarSign, token_begin, ++pos);
|
|
||||||
|
std::string_view token_stream(pos, end - pos);
|
||||||
|
auto heredoc_name_end_position = token_stream.find('$', 1);
|
||||||
|
if (heredoc_name_end_position != std::string::npos)
|
||||||
|
{
|
||||||
|
size_t heredoc_size = heredoc_name_end_position + 1;
|
||||||
|
std::string_view heredoc = {token_stream.data(), heredoc_size};
|
||||||
|
|
||||||
|
size_t heredoc_end_position = token_stream.find(heredoc, heredoc_size);
|
||||||
|
if (heredoc_end_position != std::string::npos)
|
||||||
|
{
|
||||||
|
|
||||||
|
pos += heredoc_end_position;
|
||||||
|
pos += heredoc_size;
|
||||||
|
|
||||||
|
return Token(TokenType::HereDoc, token_begin, pos);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end))
|
||||||
|
{
|
||||||
|
/// Capture standalone dollar sign
|
||||||
|
return Token(TokenType::DollarSign, token_begin, ++pos);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (isWordCharASCII(*pos) || *pos == '$')
|
if (isWordCharASCII(*pos) || *pos == '$')
|
||||||
{
|
{
|
||||||
|
@ -33,6 +33,8 @@ namespace DB
|
|||||||
\
|
\
|
||||||
M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \
|
M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \
|
||||||
\
|
\
|
||||||
|
M(HereDoc) \
|
||||||
|
\
|
||||||
M(DollarSign) \
|
M(DollarSign) \
|
||||||
M(Plus) \
|
M(Plus) \
|
||||||
M(Minus) \
|
M(Minus) \
|
||||||
|
11
tests/queries/0_stateless/01948_heredoc.reference
Normal file
11
tests/queries/0_stateless/01948_heredoc.reference
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
|
||||||
|
VALUE
|
||||||
|
|
||||||
|
VALUE
|
||||||
|
\'VALUE\'
|
||||||
|
$do$ $ doc$ $doc $ $doco$
|
||||||
|
$do$ $ doc$ $doc $ $doco$ $do$ $ doc$ $doc $ $doco$
|
||||||
|
ТЕСТ
|
||||||
|
该类型的引擎
|
||||||
|
VALUE
|
||||||
|
VALUE
|
13
tests/queries/0_stateless/01948_heredoc.sql
Normal file
13
tests/queries/0_stateless/01948_heredoc.sql
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
SELECT $$$$;
|
||||||
|
SELECT $$VALUE$$;
|
||||||
|
SELECT $doc$$doc$;
|
||||||
|
SELECT $doc$VALUE$doc$;
|
||||||
|
SELECT $doc$'VALUE'$doc$;
|
||||||
|
SELECT $doc$$do$ $ doc$ $doc $ $doco$$doc$;
|
||||||
|
SELECT $doc$$do$ $ doc$ $doc $ $doco$$doc$, $doc$$do$ $ doc$ $doc $ $doco$$doc$;
|
||||||
|
|
||||||
|
SELECT $doc$ТЕСТ$doc$;
|
||||||
|
SELECT $doc$该类型的引擎$doc$;
|
||||||
|
|
||||||
|
SELECT $РАЗДЕЛИТЕЛЬ$VALUE$РАЗДЕЛИТЕЛЬ$;
|
||||||
|
SELECT $该类型的引擎$VALUE$该类型的引擎$;
|
@ -0,0 +1,6 @@
|
|||||||
|
doc
|
||||||
|
abc
|
||||||
|
abc
|
||||||
|
1
|
||||||
|
1
|
||||||
|
1
|
12
tests/queries/0_stateless/01949_heredoc_unfinished.sh
Executable file
12
tests/queries/0_stateless/01949_heredoc_unfinished.sh
Executable file
@ -0,0 +1,12 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||||
|
# shellcheck source=../shell_config.sh
|
||||||
|
. "$CURDIR"/../shell_config.sh
|
||||||
|
|
||||||
|
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24doc%24%24VALUE%24%24doc%24%24";
|
||||||
|
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24abc%24%24";
|
||||||
|
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24abc%24%24d";
|
||||||
|
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24ab" | grep -c "DB::Exception";
|
||||||
|
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24" | grep -c "DB::Exception";
|
||||||
|
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24" | grep -c "DB::Exception";
|
Loading…
Reference in New Issue
Block a user