thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #26671 from kitaisreal/mysql-dictionaries-support-for-custom-query

Browse Source 
Lexer introduce heredoc
This commit is contained in:
Maksim Kita 2021-07-23 16:20:54 +03:00 committed by GitHub
commit f6f8bea689
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 97 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
programs/client/Client.cpp
View File

@ -425,6 +425,7 @@ private:
{TokenType::Semicolon, Replxx::Color::INTENSE},
{TokenType::Dot, Replxx::Color::INTENSE},
{TokenType::Asterisk, Replxx::Color::INTENSE},
{TokenType::HereDoc, Replxx::Color::CYAN},
{TokenType::Plus, Replxx::Color::INTENSE},
{TokenType::Minus, Replxx::Color::INTENSE},
{TokenType::Slash, Replxx::Color::INTENSE},
@ -450,8 +451,7 @@ private:
{TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED},
{TokenType::ErrorSinglePipeMark, Replxx::Color::RED},
{TokenType::ErrorWrongNumber, Replxx::Color::RED},
{ TokenType::ErrorMaxQuerySizeExceeded,
Replxx::Color::RED }};
{TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED }};
const Replxx::Color unknown_token_color = Replxx::Color::RED;

37
src/Parsers/ExpressionElementParsers.cpp
View File

@ -1555,26 +1555,37 @@ bool ParserUnsignedInteger::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
bool ParserStringLiteral::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
{
if (pos->type != TokenType::StringLiteral)
if (pos->type != TokenType::StringLiteral && pos->type != TokenType::HereDoc)
return false;
String s;
ReadBufferFromMemory in(pos->begin, pos->size());
try
if (pos->type == TokenType::StringLiteral)
{
readQuotedStringWithSQLStyle(s, in);
}
catch (const Exception &)
{
expected.add(pos, "string literal");
return false;
}
ReadBufferFromMemory in(pos->begin, pos->size());
if (in.count() != pos->size())
try
{
readQuotedStringWithSQLStyle(s, in);
}
catch (const Exception &)
{
expected.add(pos, "string literal");
return false;
}
if (in.count() != pos->size())
{
expected.add(pos, "string literal");
return false;
}
}
else if (pos->type == TokenType::HereDoc)
{
expected.add(pos, "string literal");
return false;
std::string_view here_doc(pos->begin, pos->size());
size_t heredoc_size = here_doc.find('$', 1) + 1;
assert(heredoc_size != std::string_view::npos);
s = String(pos->begin + heredoc_size, pos->size() - heredoc_size * 2);
}
auto literal = std::make_shared<ASTLiteral>(s);

1
src/Parsers/ExpressionElementParsers.h
View File

@ -308,6 +308,7 @@ protected:
/** String in single quotes.
* String in heredoc $here$txt$here$ equivalent to 'txt'.
*/
class ParserStringLiteral : public IParserBase
{

30
src/Parsers/Lexer.cpp
View File

@ -2,7 +2,6 @@
#include <Common/StringUtils/StringUtils.h>
#include <common/find_symbols.h>
namespace DB
{
@ -338,10 +337,33 @@ Token Lexer::nextTokenImpl()
}
default:
if (*pos == '$' && ((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end))
if (*pos == '$')
{
/// Capture standalone dollar sign
return Token(TokenType::DollarSign, token_begin, ++pos);
/// Try to capture dollar sign as start of here doc
std::string_view token_stream(pos, end - pos);
auto heredoc_name_end_position = token_stream.find('$', 1);
if (heredoc_name_end_position != std::string::npos)
{
size_t heredoc_size = heredoc_name_end_position + 1;
std::string_view heredoc = {token_stream.data(), heredoc_size};
size_t heredoc_end_position = token_stream.find(heredoc, heredoc_size);
if (heredoc_end_position != std::string::npos)
{
pos += heredoc_end_position;
pos += heredoc_size;
return Token(TokenType::HereDoc, token_begin, pos);
}
}
if (((pos + 1 < end && !isWordCharASCII(pos[1])) || pos + 1 == end))
{
/// Capture standalone dollar sign
return Token(TokenType::DollarSign, token_begin, ++pos);
}
}
if (isWordCharASCII(*pos) || *pos == '$')
{

2
src/Parsers/Lexer.h
View File

@ -33,6 +33,8 @@ namespace DB
\
M(Asterisk) /** Could be used as multiplication operator or on it's own: "SELECT *" */ \
\
M(HereDoc) \
\
M(DollarSign) \
M(Plus) \
M(Minus) \

11
tests/queries/0_stateless/01948_heredoc.reference Normal file
View File

@ -0,0 +1,11 @@
VALUE
VALUE
\'VALUE\'
$do$ $ doc$ $doc $ $doco$
$do$ $ doc$ $doc $ $doco$ $do$ $ doc$ $doc $ $doco$
ТЕСТ
该类型的引擎
VALUE
VALUE

13
tests/queries/0_stateless/01948_heredoc.sql Normal file
View File

@ -0,0 +1,13 @@
SELECT $$$$;
SELECT $$VALUE$$;
SELECT $doc$$doc$;
SELECT $doc$VALUE$doc$;
SELECT $doc$'VALUE'$doc$;
SELECT $doc$$do$ $ doc$ $doc $ $doco$$doc$;
SELECT $doc$$do$ $ doc$ $doc $ $doco$$doc$, $doc$$do$ $ doc$ $doc $ $doco$$doc$;
SELECT $doc$ТЕСТ$doc$;
SELECT $doc$该类型的引擎$doc$;
SELECT $РАЗДЕЛИТЕЛЬ$VALUE$РАЗДЕЛИТЕЛЬ$;
SELECT $该类型的引擎$VALUE$该类型的引擎$;

6
tests/queries/0_stateless/01949_heredoc_unfinished.reference Normal file
View File

@ -0,0 +1,6 @@
doc
abc
abc
1
1
1

12
tests/queries/0_stateless/01949_heredoc_unfinished.sh Executable file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24doc%24%24VALUE%24%24doc%24%24";
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24abc%24%24";
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24abc%24%24d";
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24ab" | grep -c "DB::Exception";
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24%24" | grep -c "DB::Exception";
curl -sS "${CLICKHOUSE_URL}&query=SELECT%20%24" | grep -c "DB::Exception";