Make serializeAST() more regular

This commit is contained in:
Robert Schulze 2023-07-19 18:02:09 +00:00
parent 25ddcc256b
commit bd761c365a
No known key found for this signature in database
GPG Key ID: 26703B55FB13728A
11 changed files with 28 additions and 21 deletions

View File

@ -32,7 +32,7 @@ namespace
/// We need a unique name for a created custom disk, but it needs to be the same /// We need a unique name for a created custom disk, but it needs to be the same
/// after table is reattached or server is restarted, so take a hash of the disk /// after table is reattached or server is restarted, so take a hash of the disk
/// configuration serialized ast as a disk name suffix. /// configuration serialized ast as a disk name suffix.
auto disk_setting_string = serializeAST(function, true); auto disk_setting_string = serializeAST(function);
disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX
+ toString(sipHash128(disk_setting_string.data(), disk_setting_string.size())); + toString(sipHash128(disk_setting_string.data(), disk_setting_string.size()));
} }

View File

@ -60,7 +60,7 @@ public:
/// The SELECT query as plain string, displayed in SYSTEM.QUERY_CACHE. Stored explicitly, i.e. not constructed from the AST, for the /// The SELECT query as plain string, displayed in SYSTEM.QUERY_CACHE. Stored explicitly, i.e. not constructed from the AST, for the
/// sole reason that QueryCache-related SETTINGS are pruned from the AST (see removeQueryCacheSettings()) which will look ugly in /// sole reason that QueryCache-related SETTINGS are pruned from the AST (see removeQueryCacheSettings()) which will look ugly in
/// the SYSTEM.QUERY_CACHE. /// SYSTEM.QUERY_CACHE.
const String query_string; const String query_string;
/// Ctor to construct a Key for writing into query cache. /// Ctor to construct a Key for writing into query cache.

View File

@ -518,7 +518,7 @@ void ThreadStatus::logToQueryThreadLog(QueryThreadLog & thread_log, const String
static String getCleanQueryAst(const ASTPtr q, ContextPtr context) static String getCleanQueryAst(const ASTPtr q, ContextPtr context)
{ {
String res = serializeAST(*q, true); String res = serializeAST(*q);
if (auto * masker = SensitiveDataMasker::getInstance()) if (auto * masker = SensitiveDataMasker::getInstance())
masker->wipeSensitiveData(res); masker->wipeSensitiveData(res);

View File

@ -11,10 +11,10 @@ void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite, bool one_line,
ast.format(settings); ast.format(settings);
} }
String serializeAST(const IAST & ast, bool one_line) String serializeAST(const IAST & ast)
{ {
WriteBufferFromOwnString buf; WriteBufferFromOwnString buf;
formatAST(ast, buf, false, one_line); formatAST(ast, buf, false, true);
return buf.str(); return buf.str();
} }

View File

@ -8,12 +8,13 @@ namespace DB
class WriteBuffer; class WriteBuffer;
/** Takes a syntax tree and turns it back into text. /// Takes a syntax tree and turns it into text.
* In case of INSERT query, the data will be missing. /// Intended for pretty-printing (multi-line + hiliting).
*/ /// In case of INSERT query, the data will be missing.
void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false, bool show_secrets = true); void formatAST(const IAST & ast, WriteBuffer & buf, bool hilite = true, bool one_line = false, bool show_secrets = true);
String serializeAST(const IAST & ast, bool one_line = true); /// Like formatAST() but intended for serialization w/o pretty-printing (single-line, no hiliting).
String serializeAST(const IAST & ast);
inline WriteBuffer & operator<<(WriteBuffer & buf, const IAST & ast) inline WriteBuffer & operator<<(WriteBuffer & buf, const IAST & ast)
{ {

View File

@ -64,7 +64,10 @@ TEST_P(ParserTest, parseQuery)
if (std::string("CREATE USER or ALTER USER query") != parser->getName() if (std::string("CREATE USER or ALTER USER query") != parser->getName()
&& std::string("ATTACH access entity query") != parser->getName()) && std::string("ATTACH access entity query") != parser->getName())
{ {
EXPECT_EQ(expected_ast, serializeAST(*ast->clone(), false)); WriteBufferFromOwnString buf;
formatAST(*ast->clone(), buf, false, false);
String formatted_ast = buf.str();
EXPECT_EQ(expected_ast, formatted_ast);
} }
else else
{ {
@ -75,7 +78,10 @@ TEST_P(ParserTest, parseQuery)
} }
else else
{ {
EXPECT_TRUE(std::regex_match(serializeAST(*ast->clone(), false), std::regex(expected_ast))); WriteBufferFromOwnString buf;
formatAST(*ast->clone(), buf, false, false);
String formatted_ast = buf.str();
EXPECT_TRUE(std::regex_match(formatted_ast, std::regex(expected_ast)));
} }
} }
} }

View File

@ -155,7 +155,7 @@ TEST(ParserDictionaryDDL, AttributesWithMultipleProperties)
EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr); EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr);
EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr); EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr);
EXPECT_EQ(serializeAST(*attributes_children[2]->as<ASTDictionaryAttributeDeclaration>()->expression, true), "(rand() % 100) * 77"); EXPECT_EQ(serializeAST(*attributes_children[2]->as<ASTDictionaryAttributeDeclaration>()->expression), "(rand() % 100) * 77");
EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, false); EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, false);
EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, true); EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, true);
@ -201,7 +201,7 @@ TEST(ParserDictionaryDDL, CustomAttributePropertiesOrder)
EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr); EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr);
EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr); EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->expression, nullptr);
EXPECT_EQ(serializeAST(*attributes_children[2]->as<ASTDictionaryAttributeDeclaration>()->expression, true), "(rand() % 100) * 77"); EXPECT_EQ(serializeAST(*attributes_children[2]->as<ASTDictionaryAttributeDeclaration>()->expression), "(rand() % 100) * 77");
EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, false); EXPECT_EQ(attributes_children[0]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, false);
EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, true); EXPECT_EQ(attributes_children[1]->as<ASTDictionaryAttributeDeclaration>()->hierarchical, true);
@ -288,7 +288,7 @@ TEST(ParserDictionaryDDL, Formatting)
ParserCreateDictionaryQuery parser; ParserCreateDictionaryQuery parser;
ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0); ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0);
ASTCreateQuery * create = ast->as<ASTCreateQuery>(); ASTCreateQuery * create = ast->as<ASTCreateQuery>();
auto str = serializeAST(*create, true); auto str = serializeAST(*create);
EXPECT_EQ(str, "CREATE DICTIONARY test.dict5 (`key_column1` UInt64 DEFAULT 1 HIERARCHICAL INJECTIVE, `key_column2` String DEFAULT '', `second_column` UInt8 EXPRESSION intDiv(50, rand() % 1000), `third_column` UInt8) PRIMARY KEY key_column1, key_column2 SOURCE(MYSQL(HOST 'localhost' PORT 9000 USER 'default' REPLICA (HOST '127.0.0.1' PRIORITY 1) PASSWORD '')) LIFETIME(MIN 1 MAX 10) LAYOUT(CACHE(SIZE_IN_CELLS 50)) RANGE(MIN second_column MAX third_column)"); EXPECT_EQ(str, "CREATE DICTIONARY test.dict5 (`key_column1` UInt64 DEFAULT 1 HIERARCHICAL INJECTIVE, `key_column2` String DEFAULT '', `second_column` UInt8 EXPRESSION intDiv(50, rand() % 1000), `third_column` UInt8) PRIMARY KEY key_column1, key_column2 SOURCE(MYSQL(HOST 'localhost' PORT 9000 USER 'default' REPLICA (HOST '127.0.0.1' PRIORITY 1) PASSWORD '')) LIFETIME(MIN 1 MAX 10) LAYOUT(CACHE(SIZE_IN_CELLS 50)) RANGE(MIN second_column MAX third_column)");
} }
@ -303,7 +303,7 @@ TEST(ParserDictionaryDDL, ParseDropQuery)
EXPECT_TRUE(drop1->is_dictionary); EXPECT_TRUE(drop1->is_dictionary);
EXPECT_EQ(drop1->getDatabase(), "test"); EXPECT_EQ(drop1->getDatabase(), "test");
EXPECT_EQ(drop1->getTable(), "dict1"); EXPECT_EQ(drop1->getTable(), "dict1");
auto str1 = serializeAST(*drop1, true); auto str1 = serializeAST(*drop1);
EXPECT_EQ(input1, str1); EXPECT_EQ(input1, str1);
String input2 = "DROP DICTIONARY IF EXISTS dict2"; String input2 = "DROP DICTIONARY IF EXISTS dict2";
@ -314,7 +314,7 @@ TEST(ParserDictionaryDDL, ParseDropQuery)
EXPECT_TRUE(drop2->is_dictionary); EXPECT_TRUE(drop2->is_dictionary);
EXPECT_EQ(drop2->getDatabase(), ""); EXPECT_EQ(drop2->getDatabase(), "");
EXPECT_EQ(drop2->getTable(), "dict2"); EXPECT_EQ(drop2->getTable(), "dict2");
auto str2 = serializeAST(*drop2, true); auto str2 = serializeAST(*drop2);
EXPECT_EQ(input2, str2); EXPECT_EQ(input2, str2);
} }

View File

@ -73,7 +73,7 @@ void CheckConstraintsTransform::onConsume(Chunk chunk)
"Constraint expression returns nullable column that contains null value", "Constraint expression returns nullable column that contains null value",
backQuote(constraint_ptr->name), backQuote(constraint_ptr->name),
table_id.getNameForLogs(), table_id.getNameForLogs(),
serializeAST(*(constraint_ptr->expr), true)); serializeAST(*(constraint_ptr->expr)));
result_column = nested_column; result_column = nested_column;
} }
@ -116,7 +116,7 @@ void CheckConstraintsTransform::onConsume(Chunk chunk)
backQuote(constraint_ptr->name), backQuote(constraint_ptr->name),
table_id.getNameForLogs(), table_id.getNameForLogs(),
rows_written + row_idx + 1, rows_written + row_idx + 1,
serializeAST(*(constraint_ptr->expr), true), serializeAST(*(constraint_ptr->expr)),
column_values_msg); column_values_msg);
} }
} }

View File

@ -35,7 +35,7 @@ String ConstraintsDescription::toString() const
for (const auto & constraint : constraints) for (const auto & constraint : constraints)
list.children.push_back(constraint); list.children.push_back(constraint);
return serializeAST(list, true); return serializeAST(list);
} }
ConstraintsDescription ConstraintsDescription::parse(const String & str) ConstraintsDescription ConstraintsDescription::parse(const String & str)

View File

@ -151,7 +151,7 @@ String IndicesDescription::toString() const
for (const auto & index : *this) for (const auto & index : *this)
list.children.push_back(index.definition_ast); list.children.push_back(index.definition_ast);
return serializeAST(list, true); return serializeAST(list);
} }

View File

@ -324,7 +324,7 @@ String ProjectionsDescription::toString() const
for (const auto & projection : projections) for (const auto & projection : projections)
list.children.push_back(projection.definition_ast); list.children.push_back(projection.definition_ast);
return serializeAST(list, true); return serializeAST(list);
} }
ProjectionsDescription ProjectionsDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr query_context) ProjectionsDescription ProjectionsDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr query_context)