2020-06-26 05:44:42 +00:00
|
|
|
#include "QueryFuzzer.h"
|
2022-08-11 02:34:10 +00:00
|
|
|
#include <DataTypes/DataTypeArray.h>
|
|
|
|
#include <DataTypes/DataTypeFactory.h>
|
|
|
|
#include <DataTypes/DataTypeFixedString.h>
|
|
|
|
#include <DataTypes/DataTypeLowCardinality.h>
|
|
|
|
#include <DataTypes/DataTypeMap.h>
|
|
|
|
#include <DataTypes/DataTypeNullable.h>
|
|
|
|
#include <DataTypes/DataTypeTuple.h>
|
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypesDecimal.h>
|
|
|
|
#include <DataTypes/IDataType.h>
|
|
|
|
#include <IO/ReadBufferFromString.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
|
|
|
#include <Parsers/ASTColumnDeclaration.h>
|
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/IAST_fwd.h>
|
|
|
|
#include <Parsers/ParserDataType.h>
|
|
|
|
#include <Parsers/ParserInsertQuery.h>
|
2022-08-12 00:32:21 +00:00
|
|
|
#include <Parsers/ASTDropQuery.h>
|
2020-06-26 05:44:42 +00:00
|
|
|
|
|
|
|
#include <unordered_set>
|
|
|
|
|
|
|
|
#include <Core/Types.h>
|
|
|
|
#include <IO/Operators.h>
|
|
|
|
#include <IO/UseSSL.h>
|
2020-11-09 16:05:40 +00:00
|
|
|
#include <IO/WriteBufferFromOStream.h>
|
2020-06-26 05:44:42 +00:00
|
|
|
#include <Parsers/ASTExpressionList.h>
|
|
|
|
#include <Parsers/ASTFunction.h>
|
|
|
|
#include <Parsers/ASTIdentifier.h>
|
|
|
|
#include <Parsers/ASTInsertQuery.h>
|
|
|
|
#include <Parsers/ASTLiteral.h>
|
2020-12-18 00:21:23 +00:00
|
|
|
#include <Parsers/ASTOrderByElement.h>
|
2020-06-26 05:44:42 +00:00
|
|
|
#include <Parsers/ASTQueryWithOutput.h>
|
2023-03-21 02:41:51 +00:00
|
|
|
#include <Parsers/ASTSelectIntersectExceptQuery.h>
|
2020-06-26 05:44:42 +00:00
|
|
|
#include <Parsers/ASTSelectQuery.h>
|
|
|
|
#include <Parsers/ASTSelectWithUnionQuery.h>
|
|
|
|
#include <Parsers/ASTSetQuery.h>
|
|
|
|
#include <Parsers/ASTSubquery.h>
|
|
|
|
#include <Parsers/ASTTablesInSelectQuery.h>
|
|
|
|
#include <Parsers/ASTUseQuery.h>
|
2021-01-13 19:29:52 +00:00
|
|
|
#include <Parsers/ASTWindowDefinition.h>
|
2020-06-26 05:44:42 +00:00
|
|
|
#include <Parsers/ParserQuery.h>
|
|
|
|
#include <Parsers/formatAST.h>
|
|
|
|
#include <Parsers/parseQuery.h>
|
2023-03-21 02:41:51 +00:00
|
|
|
#include <pcg_random.hpp>
|
|
|
|
#include <Common/assert_cast.h>
|
|
|
|
#include <Common/typeid_cast.h>
|
2020-06-26 05:44:42 +00:00
|
|
|
|
2021-05-02 11:19:47 +00:00
|
|
|
|
2020-06-26 05:44:42 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2020-12-22 04:16:56 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int TOO_DEEP_RECURSION;
|
|
|
|
}
|
|
|
|
|
2020-06-26 05:44:42 +00:00
|
|
|
Field QueryFuzzer::getRandomField(int type)
|
|
|
|
{
|
2021-04-06 11:43:27 +00:00
|
|
|
static constexpr Int64 bad_int64_values[]
|
|
|
|
= {-2, -1, 0, 1, 2, 3, 7, 10, 100, 255, 256, 257, 1023, 1024,
|
|
|
|
1025, 65535, 65536, 65537, 1024 * 1024 - 1, 1024 * 1024,
|
|
|
|
1024 * 1024 + 1, INT_MIN - 1ll, INT_MIN, INT_MIN + 1,
|
|
|
|
INT_MAX - 1, INT_MAX, INT_MAX + 1ll, INT64_MIN, INT64_MIN + 1,
|
|
|
|
INT64_MAX - 1, INT64_MAX};
|
2020-07-07 12:35:53 +00:00
|
|
|
switch (type)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
case 0:
|
|
|
|
{
|
2021-04-06 11:43:27 +00:00
|
|
|
return bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values)
|
|
|
|
/ sizeof(*bad_int64_values))];
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
case 1:
|
|
|
|
{
|
2022-09-10 02:07:51 +00:00
|
|
|
static constexpr double values[]
|
2021-04-06 11:43:27 +00:00
|
|
|
= {NAN, INFINITY, -INFINITY, 0., -0., 0.0001, 0.5, 0.9999,
|
|
|
|
1., 1.0001, 2., 10.0001, 100.0001, 1000.0001, 1e10, 1e20,
|
|
|
|
FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
case 2:
|
|
|
|
{
|
|
|
|
static constexpr UInt64 scales[] = {0, 1, 2, 10};
|
|
|
|
return DecimalField<Decimal64>(
|
2022-10-07 10:46:45 +00:00
|
|
|
bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values) / sizeof(*bad_int64_values))],
|
|
|
|
static_cast<UInt32>(scales[fuzz_rand() % (sizeof(scales) / sizeof(*scales))])
|
|
|
|
);
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
default:
|
|
|
|
assert(false);
|
2020-07-03 12:52:16 +00:00
|
|
|
return Null{};
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Field QueryFuzzer::fuzzField(Field field)
|
|
|
|
{
|
|
|
|
const auto type = field.getType();
|
|
|
|
|
|
|
|
int type_index = -1;
|
|
|
|
|
|
|
|
if (type == Field::Types::Int64
|
|
|
|
|| type == Field::Types::UInt64)
|
|
|
|
{
|
|
|
|
type_index = 0;
|
|
|
|
}
|
|
|
|
else if (type == Field::Types::Float64)
|
|
|
|
{
|
|
|
|
type_index = 1;
|
|
|
|
}
|
|
|
|
else if (type == Field::Types::Decimal32
|
|
|
|
|| type == Field::Types::Decimal64
|
|
|
|
|| type == Field::Types::Decimal128)
|
|
|
|
{
|
|
|
|
type_index = 2;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fuzz_rand() % 20 == 0)
|
|
|
|
{
|
|
|
|
return Null{};
|
|
|
|
}
|
|
|
|
|
|
|
|
if (type_index >= 0)
|
|
|
|
{
|
|
|
|
if (fuzz_rand() % 20 == 0)
|
|
|
|
{
|
|
|
|
// Change type sometimes, but not often, because it mostly leads to
|
|
|
|
// boring errors.
|
|
|
|
type_index = fuzz_rand() % 3;
|
|
|
|
}
|
|
|
|
return getRandomField(type_index);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (type == Field::Types::String)
|
|
|
|
{
|
|
|
|
auto & str = field.get<std::string>();
|
|
|
|
UInt64 action = fuzz_rand() % 10;
|
|
|
|
switch (action)
|
|
|
|
{
|
|
|
|
case 0:
|
|
|
|
str = "";
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
str = str + str;
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
str = str + str + str + str;
|
|
|
|
break;
|
|
|
|
case 4:
|
2020-07-08 10:17:06 +00:00
|
|
|
if (!str.empty())
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
str[fuzz_rand() % str.size()] = '\0';
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
// Do nothing
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2022-09-18 03:16:08 +00:00
|
|
|
else if (type == Field::Types::Array)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
2022-09-18 03:16:08 +00:00
|
|
|
auto & arr = field.get<Array>();
|
|
|
|
|
|
|
|
if (fuzz_rand() % 5 == 0 && !arr.empty())
|
|
|
|
{
|
|
|
|
size_t pos = fuzz_rand() % arr.size();
|
|
|
|
arr.erase(arr.begin() + pos);
|
|
|
|
std::cerr << "erased\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
if (fuzz_rand() % 5 == 0)
|
|
|
|
{
|
|
|
|
if (!arr.empty())
|
|
|
|
{
|
|
|
|
size_t pos = fuzz_rand() % arr.size();
|
|
|
|
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
|
|
|
|
std::cerr << fmt::format("inserted (pos {})\n", pos);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
arr.insert(arr.begin(), getRandomField(0));
|
|
|
|
std::cerr << "inserted (0)\n";
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for (auto & element : arr)
|
|
|
|
{
|
|
|
|
element = fuzzField(element);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (type == Field::Types::Tuple)
|
|
|
|
{
|
|
|
|
auto & arr = field.get<Tuple>();
|
2020-06-26 05:44:42 +00:00
|
|
|
|
2020-07-08 10:17:06 +00:00
|
|
|
if (fuzz_rand() % 5 == 0 && !arr.empty())
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
size_t pos = fuzz_rand() % arr.size();
|
|
|
|
arr.erase(arr.begin() + pos);
|
2021-10-10 01:21:07 +00:00
|
|
|
std::cerr << "erased\n";
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (fuzz_rand() % 5 == 0)
|
|
|
|
{
|
|
|
|
if (!arr.empty())
|
|
|
|
{
|
|
|
|
size_t pos = fuzz_rand() % arr.size();
|
|
|
|
arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
|
2021-10-10 01:21:07 +00:00
|
|
|
std::cerr << fmt::format("inserted (pos {})\n", pos);
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
arr.insert(arr.begin(), getRandomField(0));
|
2021-10-10 01:21:07 +00:00
|
|
|
std::cerr << "inserted (0)\n";
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
for (auto & element : arr)
|
|
|
|
{
|
|
|
|
element = fuzzField(element);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return field;
|
|
|
|
}
|
|
|
|
|
|
|
|
ASTPtr QueryFuzzer::getRandomColumnLike()
|
|
|
|
{
|
|
|
|
if (column_like.empty())
|
|
|
|
{
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
ASTPtr new_ast = column_like[fuzz_rand() % column_like.size()]->clone();
|
|
|
|
new_ast->setAlias("");
|
|
|
|
|
|
|
|
return new_ast;
|
|
|
|
}
|
|
|
|
|
2022-06-28 11:09:27 +00:00
|
|
|
ASTPtr QueryFuzzer::getRandomExpressionList()
|
|
|
|
{
|
|
|
|
if (column_like.empty())
|
|
|
|
{
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
ASTPtr new_ast = std::make_shared<ASTExpressionList>();
|
|
|
|
for (size_t i = 0; i < fuzz_rand() % 5 + 1; ++i)
|
|
|
|
{
|
|
|
|
new_ast->children.push_back(getRandomColumnLike());
|
|
|
|
}
|
|
|
|
return new_ast;
|
|
|
|
}
|
|
|
|
|
2020-06-26 05:44:42 +00:00
|
|
|
void QueryFuzzer::replaceWithColumnLike(ASTPtr & ast)
|
|
|
|
{
|
|
|
|
if (column_like.empty())
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
std::string old_alias = ast->tryGetAlias();
|
|
|
|
ast = getRandomColumnLike();
|
|
|
|
ast->setAlias(old_alias);
|
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::replaceWithTableLike(ASTPtr & ast)
|
|
|
|
{
|
|
|
|
if (table_like.empty())
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
ASTPtr new_ast = table_like[fuzz_rand() % table_like.size()]->clone();
|
|
|
|
|
|
|
|
std::string old_alias = ast->tryGetAlias();
|
|
|
|
new_ast->setAlias(old_alias);
|
|
|
|
|
|
|
|
ast = new_ast;
|
|
|
|
}
|
|
|
|
|
2020-12-18 00:21:23 +00:00
|
|
|
void QueryFuzzer::fuzzOrderByElement(ASTOrderByElement * elem)
|
|
|
|
{
|
|
|
|
switch (fuzz_rand() % 10)
|
|
|
|
{
|
|
|
|
case 0:
|
|
|
|
elem->direction = -1;
|
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
elem->direction = 1;
|
|
|
|
break;
|
|
|
|
case 2:
|
|
|
|
elem->nulls_direction = -1;
|
|
|
|
elem->nulls_direction_was_explicitly_specified = true;
|
|
|
|
break;
|
|
|
|
case 3:
|
|
|
|
elem->nulls_direction = 1;
|
|
|
|
elem->nulls_direction_was_explicitly_specified = true;
|
|
|
|
break;
|
|
|
|
case 4:
|
|
|
|
elem->nulls_direction = elem->direction;
|
|
|
|
elem->nulls_direction_was_explicitly_specified = false;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
// do nothing
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::fuzzOrderByList(IAST * ast)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
if (!ast)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-12-18 00:21:23 +00:00
|
|
|
auto * list = assert_cast<ASTExpressionList *>(ast);
|
|
|
|
|
|
|
|
// Remove element
|
|
|
|
if (fuzz_rand() % 50 == 0 && list->children.size() > 1)
|
|
|
|
{
|
|
|
|
// Don't remove last element -- this leads to questionable
|
|
|
|
// constructs such as empty select.
|
|
|
|
list->children.erase(list->children.begin()
|
|
|
|
+ fuzz_rand() % list->children.size());
|
|
|
|
}
|
|
|
|
|
|
|
|
// Add element
|
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
2022-10-18 09:40:12 +00:00
|
|
|
auto * pos = list->children.empty() ? list->children.begin() : list->children.begin() + fuzz_rand() % list->children.size();
|
2020-12-18 00:21:23 +00:00
|
|
|
auto col = getRandomColumnLike();
|
|
|
|
if (col)
|
|
|
|
{
|
|
|
|
auto elem = std::make_shared<ASTOrderByElement>();
|
|
|
|
elem->children.push_back(col);
|
|
|
|
elem->direction = 1;
|
|
|
|
elem->nulls_direction = 1;
|
|
|
|
elem->nulls_direction_was_explicitly_specified = false;
|
|
|
|
elem->with_fill = false;
|
|
|
|
|
|
|
|
list->children.insert(pos, elem);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2021-10-10 01:21:07 +00:00
|
|
|
std::cerr << "No random column.\n";
|
2020-12-18 00:21:23 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// We don't have to recurse here to fuzz the children, this is handled by
|
|
|
|
// the generic recursion into IAST.children.
|
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::fuzzColumnLikeExpressionList(IAST * ast)
|
|
|
|
{
|
|
|
|
if (!ast)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
auto * impl = assert_cast<ASTExpressionList *>(ast);
|
2020-08-05 15:45:44 +00:00
|
|
|
|
|
|
|
// Remove element
|
2020-06-26 05:44:42 +00:00
|
|
|
if (fuzz_rand() % 50 == 0 && impl->children.size() > 1)
|
|
|
|
{
|
|
|
|
// Don't remove last element -- this leads to questionable
|
|
|
|
// constructs such as empty select.
|
|
|
|
impl->children.erase(impl->children.begin()
|
|
|
|
+ fuzz_rand() % impl->children.size());
|
|
|
|
}
|
2020-08-05 15:45:44 +00:00
|
|
|
|
|
|
|
// Add element
|
2020-06-26 05:44:42 +00:00
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
2022-10-18 09:40:12 +00:00
|
|
|
auto * pos = impl->children.empty() ? impl->children.begin() : impl->children.begin() + fuzz_rand() % impl->children.size();
|
2020-06-26 05:44:42 +00:00
|
|
|
auto col = getRandomColumnLike();
|
|
|
|
if (col)
|
|
|
|
impl->children.insert(pos, col);
|
|
|
|
else
|
2021-10-10 01:21:07 +00:00
|
|
|
std::cerr << "No random column.\n";
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
2020-08-05 15:45:44 +00:00
|
|
|
|
|
|
|
// We don't have to recurse here to fuzz the children, this is handled by
|
|
|
|
// the generic recursion into IAST.children.
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
|
2021-05-29 13:12:18 +00:00
|
|
|
void QueryFuzzer::fuzzWindowFrame(ASTWindowDefinition & def)
|
2021-02-03 06:42:54 +00:00
|
|
|
{
|
|
|
|
switch (fuzz_rand() % 40)
|
|
|
|
{
|
|
|
|
case 0:
|
|
|
|
{
|
|
|
|
const auto r = fuzz_rand() % 3;
|
2022-07-25 20:53:53 +00:00
|
|
|
def.frame_type = r == 0 ? WindowFrame::FrameType::ROWS
|
|
|
|
: r == 1 ? WindowFrame::FrameType::RANGE
|
|
|
|
: WindowFrame::FrameType::GROUPS;
|
2021-02-03 06:42:54 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 1:
|
|
|
|
{
|
|
|
|
const auto r = fuzz_rand() % 3;
|
2021-05-29 13:12:18 +00:00
|
|
|
def.frame_begin_type = r == 0 ? WindowFrame::BoundaryType::Unbounded
|
2021-02-03 06:42:54 +00:00
|
|
|
: r == 1 ? WindowFrame::BoundaryType::Current
|
|
|
|
: WindowFrame::BoundaryType::Offset;
|
2021-05-29 13:12:18 +00:00
|
|
|
|
|
|
|
if (def.frame_begin_type == WindowFrame::BoundaryType::Offset)
|
|
|
|
{
|
|
|
|
// The offsets are fuzzed normally through 'children'.
|
|
|
|
def.frame_begin_offset
|
|
|
|
= std::make_shared<ASTLiteral>(getRandomField(0));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
def.frame_begin_offset = nullptr;
|
|
|
|
}
|
2021-02-03 06:42:54 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 2:
|
|
|
|
{
|
|
|
|
const auto r = fuzz_rand() % 3;
|
2021-05-29 13:12:18 +00:00
|
|
|
def.frame_end_type = r == 0 ? WindowFrame::BoundaryType::Unbounded
|
2021-02-03 06:42:54 +00:00
|
|
|
: r == 1 ? WindowFrame::BoundaryType::Current
|
|
|
|
: WindowFrame::BoundaryType::Offset;
|
2021-05-29 13:12:18 +00:00
|
|
|
|
|
|
|
if (def.frame_end_type == WindowFrame::BoundaryType::Offset)
|
|
|
|
{
|
|
|
|
def.frame_end_offset
|
|
|
|
= std::make_shared<ASTLiteral>(getRandomField(0));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
def.frame_end_offset = nullptr;
|
|
|
|
}
|
2021-02-03 06:42:54 +00:00
|
|
|
break;
|
|
|
|
}
|
2021-02-05 09:13:19 +00:00
|
|
|
case 5:
|
|
|
|
{
|
2021-05-29 13:12:18 +00:00
|
|
|
def.frame_begin_preceding = fuzz_rand() % 2;
|
2021-02-05 09:13:19 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case 6:
|
|
|
|
{
|
2021-05-29 13:12:18 +00:00
|
|
|
def.frame_end_preceding = fuzz_rand() % 2;
|
2021-02-05 09:13:19 +00:00
|
|
|
break;
|
|
|
|
}
|
2021-02-03 06:42:54 +00:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
2021-02-03 12:50:25 +00:00
|
|
|
|
2022-07-25 20:53:53 +00:00
|
|
|
if (def.frame_type == WindowFrame::FrameType::RANGE
|
2021-05-29 13:12:18 +00:00
|
|
|
&& def.frame_begin_type == WindowFrame::BoundaryType::Unbounded
|
2021-06-02 12:18:03 +00:00
|
|
|
&& def.frame_begin_preceding
|
2021-05-29 13:12:18 +00:00
|
|
|
&& def.frame_end_type == WindowFrame::BoundaryType::Current)
|
|
|
|
{
|
2021-06-02 12:18:03 +00:00
|
|
|
def.frame_is_default = true; /* NOLINT clang-tidy could you just shut up please */
|
2021-05-29 13:12:18 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
def.frame_is_default = false;
|
|
|
|
}
|
2021-02-03 06:42:54 +00:00
|
|
|
}
|
|
|
|
|
2022-09-28 14:52:10 +00:00
|
|
|
bool QueryFuzzer::isSuitableForFuzzing(const ASTCreateQuery & create)
|
|
|
|
{
|
|
|
|
return create.columns_list && create.columns_list->columns;
|
|
|
|
}
|
|
|
|
|
2022-10-17 21:08:27 +00:00
|
|
|
static String getOriginalTableName(const String & full_name)
|
|
|
|
{
|
|
|
|
return full_name.substr(0, full_name.find("__fuzz_"));
|
|
|
|
}
|
|
|
|
|
|
|
|
static String getFuzzedTableName(const String & original_name, size_t index)
|
|
|
|
{
|
|
|
|
return original_name + "__fuzz_" + toString(index);
|
|
|
|
}
|
|
|
|
|
2022-08-11 02:34:10 +00:00
|
|
|
void QueryFuzzer::fuzzCreateQuery(ASTCreateQuery & create)
|
|
|
|
{
|
|
|
|
if (create.columns_list && create.columns_list->columns)
|
|
|
|
{
|
|
|
|
for (auto & ast : create.columns_list->columns->children)
|
|
|
|
{
|
|
|
|
if (auto * column = ast->as<ASTColumnDeclaration>())
|
|
|
|
{
|
|
|
|
fuzzColumnDeclaration(*column);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (create.storage && create.storage->engine)
|
|
|
|
{
|
2022-09-28 15:55:39 +00:00
|
|
|
/// Replace ReplicatedMergeTree to ordinary MergeTree
|
|
|
|
/// to avoid inconsistency of metadata in zookeeper.
|
2022-08-11 02:34:10 +00:00
|
|
|
auto & engine_name = create.storage->engine->name;
|
|
|
|
if (startsWith(engine_name, "Replicated"))
|
2022-09-28 15:55:39 +00:00
|
|
|
{
|
2022-08-11 02:34:10 +00:00
|
|
|
engine_name = engine_name.substr(strlen("Replicated"));
|
2022-09-28 15:55:39 +00:00
|
|
|
if (auto & arguments = create.storage->engine->arguments)
|
|
|
|
{
|
|
|
|
auto & children = arguments->children;
|
|
|
|
if (children.size() <= 2)
|
|
|
|
arguments.reset();
|
|
|
|
else
|
|
|
|
children.erase(children.begin(), children.begin() + 2);
|
|
|
|
}
|
|
|
|
}
|
2022-08-11 02:34:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
auto full_name = create.getTable();
|
2022-10-17 21:08:27 +00:00
|
|
|
auto original_name = getOriginalTableName(full_name);
|
2022-08-11 02:34:10 +00:00
|
|
|
size_t index = index_of_fuzzed_table[original_name]++;
|
2022-10-17 21:08:27 +00:00
|
|
|
auto new_name = getFuzzedTableName(original_name, index);
|
2022-08-11 02:34:10 +00:00
|
|
|
|
|
|
|
create.setTable(new_name);
|
|
|
|
|
|
|
|
SipHash sip_hash;
|
|
|
|
sip_hash.update(original_name);
|
|
|
|
if (create.columns_list)
|
|
|
|
create.columns_list->updateTreeHash(sip_hash);
|
|
|
|
if (create.storage)
|
2022-08-11 12:24:15 +00:00
|
|
|
create.storage->updateTreeHash(sip_hash);
|
2022-08-11 02:34:10 +00:00
|
|
|
|
|
|
|
IAST::Hash hash;
|
|
|
|
sip_hash.get128(hash);
|
2022-09-28 14:52:10 +00:00
|
|
|
|
|
|
|
/// Save only tables with unique definition.
|
2022-08-11 02:34:10 +00:00
|
|
|
if (created_tables_hashes.insert(hash).second)
|
2022-09-28 15:55:39 +00:00
|
|
|
original_table_name_to_fuzzed[original_name].insert(new_name);
|
2022-08-11 02:34:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::fuzzColumnDeclaration(ASTColumnDeclaration & column)
|
|
|
|
{
|
|
|
|
if (column.type)
|
|
|
|
{
|
|
|
|
auto data_type = fuzzDataType(DataTypeFactory::instance().get(column.type));
|
|
|
|
|
|
|
|
ParserDataType parser;
|
|
|
|
column.type = parseQuery(parser, data_type->getName(), DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
DataTypePtr QueryFuzzer::fuzzDataType(DataTypePtr type)
|
|
|
|
{
|
2022-09-28 14:52:10 +00:00
|
|
|
/// Do not replace Array/Tuple/etc. with not Array/Tuple too often.
|
2022-08-11 02:34:10 +00:00
|
|
|
const auto * type_array = typeid_cast<const DataTypeArray *>(type.get());
|
2022-09-28 21:10:41 +00:00
|
|
|
if (type_array && fuzz_rand() % 4 != 0)
|
2022-08-11 02:34:10 +00:00
|
|
|
return std::make_shared<DataTypeArray>(fuzzDataType(type_array->getNestedType()));
|
|
|
|
|
|
|
|
const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get());
|
2022-09-28 21:10:41 +00:00
|
|
|
if (type_tuple && fuzz_rand() % 4 != 0)
|
2022-08-11 02:34:10 +00:00
|
|
|
{
|
|
|
|
DataTypes elements;
|
|
|
|
for (const auto & element : type_tuple->getElements())
|
|
|
|
elements.push_back(fuzzDataType(element));
|
|
|
|
|
|
|
|
return type_tuple->haveExplicitNames()
|
|
|
|
? std::make_shared<DataTypeTuple>(elements, type_tuple->getElementNames())
|
|
|
|
: std::make_shared<DataTypeTuple>(elements);
|
|
|
|
}
|
|
|
|
|
|
|
|
const auto * type_map = typeid_cast<const DataTypeMap *>(type.get());
|
2022-09-28 21:10:41 +00:00
|
|
|
if (type_map && fuzz_rand() % 4 != 0)
|
2022-08-11 02:34:10 +00:00
|
|
|
{
|
|
|
|
auto key_type = fuzzDataType(type_map->getKeyType());
|
|
|
|
auto value_type = fuzzDataType(type_map->getValueType());
|
|
|
|
if (!DataTypeMap::checkKeyType(key_type))
|
|
|
|
key_type = type_map->getKeyType();
|
|
|
|
|
|
|
|
return std::make_shared<DataTypeMap>(key_type, value_type);
|
|
|
|
}
|
|
|
|
|
|
|
|
const auto * type_nullable = typeid_cast<const DataTypeNullable *>(type.get());
|
|
|
|
if (type_nullable)
|
|
|
|
{
|
|
|
|
size_t tmp = fuzz_rand() % 3;
|
|
|
|
if (tmp == 0)
|
2022-10-04 11:40:59 +00:00
|
|
|
return fuzzDataType(type_nullable->getNestedType());
|
2022-08-11 02:34:10 +00:00
|
|
|
|
|
|
|
if (tmp == 1)
|
|
|
|
{
|
|
|
|
auto nested_type = fuzzDataType(type_nullable->getNestedType());
|
|
|
|
if (nested_type->canBeInsideNullable())
|
|
|
|
return std::make_shared<DataTypeNullable>(nested_type);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
const auto * type_low_cardinality = typeid_cast<const DataTypeLowCardinality *>(type.get());
|
|
|
|
if (type_low_cardinality)
|
|
|
|
{
|
|
|
|
size_t tmp = fuzz_rand() % 3;
|
|
|
|
if (tmp == 0)
|
2022-10-04 11:40:59 +00:00
|
|
|
return fuzzDataType(type_low_cardinality->getDictionaryType());
|
2022-08-11 02:34:10 +00:00
|
|
|
|
|
|
|
if (tmp == 1)
|
|
|
|
{
|
|
|
|
auto nested_type = fuzzDataType(type_low_cardinality->getDictionaryType());
|
|
|
|
if (nested_type->canBeInsideLowCardinality())
|
|
|
|
return std::make_shared<DataTypeLowCardinality>(nested_type);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-09-28 14:52:10 +00:00
|
|
|
size_t tmp = fuzz_rand() % 8;
|
|
|
|
if (tmp == 0)
|
|
|
|
return std::make_shared<DataTypeArray>(type);
|
|
|
|
|
2022-08-11 02:34:10 +00:00
|
|
|
if (tmp <= 1 && type->canBeInsideNullable())
|
|
|
|
return std::make_shared<DataTypeNullable>(type);
|
|
|
|
|
2022-09-28 14:52:10 +00:00
|
|
|
if (tmp <= 2 && type->canBeInsideLowCardinality())
|
2022-08-11 02:34:10 +00:00
|
|
|
return std::make_shared<DataTypeLowCardinality>(type);
|
|
|
|
|
2022-09-28 14:52:10 +00:00
|
|
|
if (tmp <= 3)
|
2022-08-11 02:34:10 +00:00
|
|
|
return getRandomType();
|
|
|
|
|
|
|
|
return type;
|
|
|
|
}
|
|
|
|
|
|
|
|
DataTypePtr QueryFuzzer::getRandomType()
|
|
|
|
{
|
|
|
|
auto type_id = static_cast<TypeIndex>(fuzz_rand() % static_cast<size_t>(TypeIndex::Tuple) + 1);
|
|
|
|
|
|
|
|
if (type_id == TypeIndex::Tuple)
|
|
|
|
{
|
|
|
|
size_t tuple_size = fuzz_rand() % 6 + 1;
|
|
|
|
DataTypes elements;
|
|
|
|
for (size_t i = 0; i < tuple_size; ++i)
|
|
|
|
elements.push_back(getRandomType());
|
|
|
|
return std::make_shared<DataTypeTuple>(elements);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (type_id == TypeIndex::Array)
|
|
|
|
return std::make_shared<DataTypeArray>(getRandomType());
|
|
|
|
|
2022-09-30 13:54:05 +00:00
|
|
|
/// NOLINTBEGIN(bugprone-macro-parentheses)
|
2022-08-11 02:34:10 +00:00
|
|
|
#define DISPATCH(DECIMAL) \
|
|
|
|
if (type_id == TypeIndex::DECIMAL) \
|
2022-08-11 13:39:15 +00:00
|
|
|
return std::make_shared<DataTypeDecimal<DECIMAL>>( \
|
|
|
|
DataTypeDecimal<DECIMAL>::maxPrecision(), \
|
2022-09-28 14:52:10 +00:00
|
|
|
(fuzz_rand() % DataTypeDecimal<DECIMAL>::maxPrecision()) + 1);
|
2022-08-11 02:34:10 +00:00
|
|
|
|
|
|
|
DISPATCH(Decimal32)
|
|
|
|
DISPATCH(Decimal64)
|
|
|
|
DISPATCH(Decimal128)
|
|
|
|
DISPATCH(Decimal256)
|
|
|
|
#undef DISPATCH
|
2022-09-30 13:54:05 +00:00
|
|
|
/// NOLINTEND(bugprone-macro-parentheses)
|
2022-08-11 02:34:10 +00:00
|
|
|
|
|
|
|
if (type_id == TypeIndex::FixedString)
|
|
|
|
return std::make_shared<DataTypeFixedString>(fuzz_rand() % 20);
|
|
|
|
|
|
|
|
if (type_id == TypeIndex::Enum8)
|
|
|
|
return std::make_shared<DataTypeUInt8>();
|
|
|
|
|
|
|
|
if (type_id == TypeIndex::Enum16)
|
|
|
|
return std::make_shared<DataTypeUInt16>();
|
|
|
|
|
|
|
|
return DataTypeFactory::instance().get(String(magic_enum::enum_name(type_id)));
|
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::fuzzTableName(ASTTableExpression & table)
|
|
|
|
{
|
|
|
|
if (!table.database_and_table_name || fuzz_rand() % 3 == 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
const auto * identifier = table.database_and_table_name->as<ASTTableIdentifier>();
|
|
|
|
if (!identifier)
|
|
|
|
return;
|
|
|
|
|
|
|
|
auto table_id = identifier->getTableId();
|
|
|
|
if (table_id.empty())
|
|
|
|
return;
|
|
|
|
|
2022-10-17 21:08:27 +00:00
|
|
|
auto original_name = getOriginalTableName(table_id.getTableName());
|
|
|
|
auto it = original_table_name_to_fuzzed.find(original_name);
|
2022-08-11 02:34:10 +00:00
|
|
|
if (it != original_table_name_to_fuzzed.end() && !it->second.empty())
|
|
|
|
{
|
2022-09-28 15:55:39 +00:00
|
|
|
auto new_table_name = it->second.begin();
|
|
|
|
std::advance(new_table_name, fuzz_rand() % it->second.size());
|
|
|
|
StorageID new_table_id(table_id.database_name, *new_table_name);
|
2022-08-11 02:34:10 +00:00
|
|
|
table.database_and_table_name = std::make_shared<ASTTableIdentifier>(new_table_id);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-21 02:41:51 +00:00
|
|
|
void QueryFuzzer::fuzzExplainQuery(ASTExplainQuery & explain)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
explain.setExplainKind(fuzzExplainKind(explain.getKind()));
|
|
|
|
|
|
|
|
bool settings_have_fuzzed = false;
|
|
|
|
for (auto & child : explain.children)
|
|
|
|
{
|
|
|
|
if (auto * settings_ast = typeid_cast<ASTSetQuery *>(child.get()))
|
|
|
|
{
|
|
|
|
fuzzExplainSettings(*settings_ast, explain.getKind());
|
|
|
|
settings_have_fuzzed = true;
|
|
|
|
}
|
|
|
|
/// Fuzzing other child like Explain Query
|
|
|
|
else
|
|
|
|
{
|
|
|
|
fuzz(child);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!settings_have_fuzzed)
|
|
|
|
{
|
|
|
|
auto settings_ast = std::make_shared<ASTSetQuery>();
|
|
|
|
fuzzExplainSettings(*settings_ast, explain.getKind());
|
|
|
|
explain.setSettings(settings_ast);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
ASTExplainQuery::ExplainKind QueryFuzzer::fuzzExplainKind(ASTExplainQuery::ExplainKind kind)
|
|
|
|
{
|
2023-03-21 02:41:51 +00:00
|
|
|
if (fuzz_rand() % 20 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return kind;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::ParsedAST;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::AnalyzedSyntax;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::QueryTree;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::QueryPlan;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::QueryPipeline;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::QueryEstimates;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::TableOverride;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 11 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
return ASTExplainQuery::ExplainKind::CurrentTransaction;
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
2023-03-21 14:26:39 +00:00
|
|
|
return kind;
|
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::fuzzExplainSettings(ASTSetQuery & settings_ast, ASTExplainQuery::ExplainKind kind)
|
|
|
|
{
|
|
|
|
auto & changes = settings_ast.changes;
|
2023-03-21 02:41:51 +00:00
|
|
|
|
|
|
|
static const std::unordered_map<ASTExplainQuery::ExplainKind, std::vector<String>> settings_by_kind
|
|
|
|
= {{ASTExplainQuery::ExplainKind::ParsedAST, {"graph", "optimize"}},
|
|
|
|
{ASTExplainQuery::ExplainKind::AnalyzedSyntax, {}},
|
|
|
|
{ASTExplainQuery::QueryTree, {"run_passes", "dump_passes", "dump_ast", "passes"}},
|
|
|
|
{ASTExplainQuery::ExplainKind::QueryPlan, {"header, description", "actions", "indexes", "optimize", "json", "sorting"}},
|
|
|
|
{ASTExplainQuery::ExplainKind::QueryPipeline, {"header", "graph=1", "compact"}},
|
|
|
|
{ASTExplainQuery::ExplainKind::QueryEstimates, {}},
|
|
|
|
{ASTExplainQuery::ExplainKind::TableOverride, {}},
|
|
|
|
{ASTExplainQuery::ExplainKind::CurrentTransaction, {}}};
|
|
|
|
|
2023-03-21 14:26:39 +00:00
|
|
|
const auto & settings = settings_by_kind.at(kind);
|
2023-03-21 02:41:51 +00:00
|
|
|
if (fuzz_rand() % 50 == 0 && !changes.empty())
|
|
|
|
{
|
|
|
|
changes.erase(changes.begin() + fuzz_rand() % changes.size());
|
|
|
|
}
|
|
|
|
|
2023-03-21 14:26:39 +00:00
|
|
|
for (const auto & setting : settings)
|
2023-03-21 02:41:51 +00:00
|
|
|
{
|
|
|
|
if (fuzz_rand() % 5 == 0)
|
|
|
|
{
|
2023-03-21 14:26:39 +00:00
|
|
|
changes.emplace_back(setting, true);
|
2023-03-21 02:41:51 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-08-11 02:34:10 +00:00
|
|
|
static ASTPtr tryParseInsertQuery(const String & full_query)
|
|
|
|
{
|
|
|
|
const char * pos = full_query.data();
|
|
|
|
const char * end = full_query.data() + full_query.size();
|
|
|
|
|
|
|
|
ParserInsertQuery parser(end, false);
|
|
|
|
String message;
|
|
|
|
|
|
|
|
return tryParseQuery(parser, pos, end, message, false, "", false, DBMS_DEFAULT_MAX_QUERY_SIZE, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
|
|
|
}
|
|
|
|
|
|
|
|
ASTs QueryFuzzer::getInsertQueriesForFuzzedTables(const String & full_query)
|
|
|
|
{
|
|
|
|
auto parsed_query = tryParseInsertQuery(full_query);
|
|
|
|
if (!parsed_query)
|
|
|
|
return {};
|
|
|
|
|
|
|
|
const auto & insert = *parsed_query->as<ASTInsertQuery>();
|
|
|
|
if (!insert.table)
|
|
|
|
return {};
|
|
|
|
|
|
|
|
auto table_name = insert.getTable();
|
|
|
|
auto it = original_table_name_to_fuzzed.find(table_name);
|
|
|
|
if (it == original_table_name_to_fuzzed.end())
|
|
|
|
return {};
|
|
|
|
|
|
|
|
ASTs queries;
|
|
|
|
for (const auto & fuzzed_name : it->second)
|
|
|
|
{
|
2022-08-12 00:32:21 +00:00
|
|
|
/// Parse query from scratch for each table instead of clone,
|
|
|
|
/// to store proper pointers to inlined data,
|
2022-09-28 14:52:10 +00:00
|
|
|
/// which are not copied during clone.
|
2022-08-11 02:34:10 +00:00
|
|
|
auto & query = queries.emplace_back(tryParseInsertQuery(full_query));
|
|
|
|
query->as<ASTInsertQuery>()->setTable(fuzzed_name);
|
|
|
|
}
|
|
|
|
|
|
|
|
return queries;
|
|
|
|
}
|
|
|
|
|
2022-08-12 00:32:21 +00:00
|
|
|
ASTs QueryFuzzer::getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query)
|
|
|
|
{
|
|
|
|
if (drop_query.kind != ASTDropQuery::Drop)
|
|
|
|
return {};
|
|
|
|
|
|
|
|
auto table_name = drop_query.getTable();
|
2022-09-28 14:52:10 +00:00
|
|
|
auto it = index_of_fuzzed_table.find(table_name);
|
|
|
|
if (it == index_of_fuzzed_table.end())
|
2022-08-12 00:32:21 +00:00
|
|
|
return {};
|
|
|
|
|
|
|
|
ASTs queries;
|
2022-09-28 14:52:10 +00:00
|
|
|
/// Drop all created tables, not only unique ones.
|
|
|
|
for (size_t i = 0; i < it->second; ++i)
|
2022-08-12 00:32:21 +00:00
|
|
|
{
|
2022-10-17 21:08:27 +00:00
|
|
|
auto fuzzed_name = getFuzzedTableName(table_name, i);
|
2022-08-12 00:32:21 +00:00
|
|
|
auto & query = queries.emplace_back(drop_query.clone());
|
|
|
|
query->as<ASTDropQuery>()->setTable(fuzzed_name);
|
2022-09-28 14:52:10 +00:00
|
|
|
/// Just in case add IF EXISTS to avoid exceptions.
|
|
|
|
query->as<ASTDropQuery>()->if_exists = true;
|
2022-08-12 00:32:21 +00:00
|
|
|
}
|
|
|
|
|
2022-09-28 21:06:50 +00:00
|
|
|
index_of_fuzzed_table.erase(it);
|
|
|
|
original_table_name_to_fuzzed.erase(table_name);
|
|
|
|
|
2022-08-12 00:32:21 +00:00
|
|
|
return queries;
|
|
|
|
}
|
|
|
|
|
2022-09-28 15:55:39 +00:00
|
|
|
void QueryFuzzer::notifyQueryFailed(ASTPtr ast)
|
|
|
|
{
|
|
|
|
auto remove_fuzzed_table = [this](const auto & table_name)
|
|
|
|
{
|
|
|
|
auto pos = table_name.find("__fuzz_");
|
|
|
|
if (pos != std::string::npos)
|
|
|
|
{
|
|
|
|
auto original_name = table_name.substr(0, pos);
|
2022-10-17 21:08:27 +00:00
|
|
|
auto it = original_table_name_to_fuzzed.find(original_name);
|
|
|
|
if (it != original_table_name_to_fuzzed.end())
|
|
|
|
it->second.erase(table_name);
|
2022-09-28 15:55:39 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
if (const auto * create = ast->as<ASTCreateQuery>())
|
|
|
|
remove_fuzzed_table(create->getTable());
|
|
|
|
|
|
|
|
if (const auto * insert = ast->as<ASTInsertQuery>())
|
|
|
|
remove_fuzzed_table(insert->getTable());
|
|
|
|
}
|
|
|
|
|
2020-06-26 05:44:42 +00:00
|
|
|
void QueryFuzzer::fuzz(ASTs & asts)
|
|
|
|
{
|
|
|
|
for (auto & ast : asts)
|
|
|
|
{
|
|
|
|
fuzz(ast);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-22 04:02:39 +00:00
|
|
|
struct ScopedIncrement
|
|
|
|
{
|
|
|
|
size_t & counter;
|
|
|
|
|
2020-12-22 07:25:45 +00:00
|
|
|
explicit ScopedIncrement(size_t & counter_) : counter(counter_) { ++counter; }
|
2020-12-22 04:02:39 +00:00
|
|
|
~ScopedIncrement() { --counter; }
|
|
|
|
};
|
|
|
|
|
2020-06-26 05:44:42 +00:00
|
|
|
void QueryFuzzer::fuzz(ASTPtr & ast)
|
|
|
|
{
|
|
|
|
if (!ast)
|
|
|
|
return;
|
|
|
|
|
2020-12-22 04:02:39 +00:00
|
|
|
// Check for exceeding max depth.
|
|
|
|
ScopedIncrement depth_increment(current_ast_depth);
|
2020-12-22 04:16:56 +00:00
|
|
|
if (current_ast_depth > 500)
|
2020-12-22 04:02:39 +00:00
|
|
|
{
|
2020-12-22 04:16:56 +00:00
|
|
|
// The AST is too deep (see the comment for current_ast_depth). Throw
|
|
|
|
// an exception to fail fast and not use this query as an etalon, or we'll
|
|
|
|
// end up in a very slow and useless loop. It also makes sense to set it
|
|
|
|
// lower than the default max parse depth on the server (1000), so that
|
|
|
|
// we don't get the useless error about parse depth from the server either.
|
|
|
|
throw Exception(ErrorCodes::TOO_DEEP_RECURSION,
|
|
|
|
"AST depth exceeded while fuzzing ({})", current_ast_depth);
|
2020-12-22 04:02:39 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Check for loops.
|
|
|
|
auto [_, inserted] = debug_visited_nodes.insert(ast.get());
|
|
|
|
if (!inserted)
|
|
|
|
{
|
|
|
|
fmt::print(stderr, "The AST node '{}' was already visited before."
|
|
|
|
" Depth {}, {} visited nodes, current top AST:\n{}\n",
|
|
|
|
static_cast<void *>(ast.get()), current_ast_depth,
|
|
|
|
debug_visited_nodes.size(), (*debug_top_ast)->dumpTree());
|
|
|
|
assert(false);
|
|
|
|
}
|
|
|
|
|
|
|
|
// The fuzzing.
|
2020-06-26 05:44:42 +00:00
|
|
|
if (auto * with_union = typeid_cast<ASTSelectWithUnionQuery *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzz(with_union->list_of_selects);
|
2023-03-21 14:26:39 +00:00
|
|
|
/// Fuzzing SELECT query to EXPLAIN query randomly
|
|
|
|
if (fuzz_rand() % 20 == 0)
|
|
|
|
{
|
2023-03-21 15:09:10 +00:00
|
|
|
auto explain = std::make_shared<ASTExplainQuery>(fuzzExplainKind());
|
2023-03-21 14:26:39 +00:00
|
|
|
|
|
|
|
auto settings_ast = std::make_shared<ASTSetQuery>();
|
|
|
|
fuzzExplainSettings(*settings_ast, explain->getKind());
|
|
|
|
explain->setSettings(settings_ast);
|
|
|
|
|
|
|
|
explain->setExplainedQuery(ast);
|
|
|
|
ast = explain;
|
|
|
|
}
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
2021-11-10 09:33:48 +00:00
|
|
|
else if (auto * with_intersect_except = typeid_cast<ASTSelectIntersectExceptQuery *>(ast.get()))
|
2021-11-01 13:19:31 +00:00
|
|
|
{
|
2021-11-09 08:16:18 +00:00
|
|
|
auto selects = with_intersect_except->getListOfSelects();
|
|
|
|
fuzz(selects);
|
2021-11-01 13:19:31 +00:00
|
|
|
}
|
2020-06-26 05:44:42 +00:00
|
|
|
else if (auto * tables = typeid_cast<ASTTablesInSelectQuery *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzz(tables->children);
|
|
|
|
}
|
|
|
|
else if (auto * tables_element = typeid_cast<ASTTablesInSelectQueryElement *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzz(tables_element->table_join);
|
|
|
|
fuzz(tables_element->table_expression);
|
|
|
|
fuzz(tables_element->array_join);
|
|
|
|
}
|
|
|
|
else if (auto * table_expr = typeid_cast<ASTTableExpression *>(ast.get()))
|
|
|
|
{
|
2022-08-11 02:34:10 +00:00
|
|
|
fuzzTableName(*table_expr);
|
2020-10-26 15:49:00 +00:00
|
|
|
fuzz(table_expr->children);
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
else if (auto * expr_list = typeid_cast<ASTExpressionList *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzz(expr_list->children);
|
|
|
|
}
|
2020-12-18 00:21:23 +00:00
|
|
|
else if (auto * order_by_element = typeid_cast<ASTOrderByElement *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzzOrderByElement(order_by_element);
|
|
|
|
}
|
2020-06-26 05:44:42 +00:00
|
|
|
else if (auto * fn = typeid_cast<ASTFunction *>(ast.get()))
|
|
|
|
{
|
2020-12-18 00:21:23 +00:00
|
|
|
fuzzColumnLikeExpressionList(fn->arguments.get());
|
|
|
|
fuzzColumnLikeExpressionList(fn->parameters.get());
|
2020-06-26 05:44:42 +00:00
|
|
|
|
2021-01-13 19:29:52 +00:00
|
|
|
if (fn->is_window_function && fn->window_definition)
|
2020-12-18 00:21:23 +00:00
|
|
|
{
|
2021-01-13 19:29:52 +00:00
|
|
|
auto & def = fn->window_definition->as<ASTWindowDefinition &>();
|
|
|
|
fuzzColumnLikeExpressionList(def.partition_by.get());
|
|
|
|
fuzzOrderByList(def.order_by.get());
|
2021-05-29 13:12:18 +00:00
|
|
|
fuzzWindowFrame(def);
|
2020-12-18 00:21:23 +00:00
|
|
|
}
|
2020-06-26 05:44:42 +00:00
|
|
|
|
|
|
|
fuzz(fn->children);
|
|
|
|
}
|
|
|
|
else if (auto * select = typeid_cast<ASTSelectQuery *>(ast.get()))
|
|
|
|
{
|
2020-12-18 00:21:23 +00:00
|
|
|
fuzzColumnLikeExpressionList(select->select().get());
|
2022-06-28 11:09:27 +00:00
|
|
|
|
2022-10-07 12:43:20 +00:00
|
|
|
if (select->groupBy().get())
|
2022-06-28 11:09:27 +00:00
|
|
|
{
|
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->groupBy()->children.clear();
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::GROUP_BY, {});
|
2022-10-07 11:39:47 +00:00
|
|
|
select->group_by_with_grouping_sets = false;
|
|
|
|
select->group_by_with_rollup = false;
|
|
|
|
select->group_by_with_cube = false;
|
|
|
|
select->group_by_with_totals = true;
|
|
|
|
}
|
2022-10-07 12:43:20 +00:00
|
|
|
else if (fuzz_rand() % 100 == 0)
|
2022-10-07 11:39:47 +00:00
|
|
|
{
|
|
|
|
select->group_by_with_grouping_sets = !select->group_by_with_grouping_sets;
|
|
|
|
}
|
2022-10-07 12:43:20 +00:00
|
|
|
else if (fuzz_rand() % 100 == 0)
|
2022-10-07 11:39:47 +00:00
|
|
|
{
|
|
|
|
select->group_by_with_rollup = !select->group_by_with_rollup;
|
|
|
|
}
|
2022-10-07 12:43:20 +00:00
|
|
|
else if (fuzz_rand() % 100 == 0)
|
2022-10-07 11:39:47 +00:00
|
|
|
{
|
|
|
|
select->group_by_with_cube = !select->group_by_with_cube;
|
|
|
|
}
|
2022-10-07 12:43:20 +00:00
|
|
|
else if (fuzz_rand() % 100 == 0)
|
2022-10-07 11:39:47 +00:00
|
|
|
{
|
|
|
|
select->group_by_with_totals = !select->group_by_with_totals;
|
2022-06-28 11:09:27 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::GROUP_BY, getRandomExpressionList());
|
|
|
|
}
|
|
|
|
|
2022-10-07 12:43:20 +00:00
|
|
|
if (select->where().get())
|
2022-06-28 11:09:27 +00:00
|
|
|
{
|
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->where()->children.clear();
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::WHERE, {});
|
|
|
|
}
|
2023-01-12 12:14:09 +00:00
|
|
|
else if (!select->prewhere().get())
|
|
|
|
{
|
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::PREWHERE, select->where()->clone());
|
|
|
|
|
|
|
|
if (fuzz_rand() % 2 == 0)
|
|
|
|
{
|
|
|
|
select->where()->children.clear();
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::WHERE, {});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-06-28 11:09:27 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::WHERE, getRandomColumnLike());
|
|
|
|
}
|
2023-01-12 12:14:09 +00:00
|
|
|
|
|
|
|
if (select->prewhere().get())
|
|
|
|
{
|
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->prewhere()->children.clear();
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::PREWHERE, {});
|
|
|
|
}
|
2023-01-12 13:11:30 +00:00
|
|
|
else if (!select->where().get())
|
|
|
|
{
|
|
|
|
if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::WHERE, select->prewhere()->clone());
|
|
|
|
|
|
|
|
if (fuzz_rand() % 2 == 0)
|
|
|
|
{
|
|
|
|
select->prewhere()->children.clear();
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::PREWHERE, {});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-01-12 12:14:09 +00:00
|
|
|
}
|
|
|
|
else if (fuzz_rand() % 50 == 0)
|
|
|
|
{
|
|
|
|
select->setExpression(ASTSelectQuery::Expression::PREWHERE, getRandomColumnLike());
|
|
|
|
}
|
|
|
|
|
2020-12-18 00:21:23 +00:00
|
|
|
fuzzOrderByList(select->orderBy().get());
|
2020-06-26 05:44:42 +00:00
|
|
|
|
|
|
|
fuzz(select->children);
|
|
|
|
}
|
2021-02-03 12:50:25 +00:00
|
|
|
/*
|
|
|
|
* The time to fuzz the settings has not yet come.
|
|
|
|
* Apparently we don't have any infractructure to validate the values of
|
|
|
|
* the settings, and the first query with max_block_size = -1 breaks
|
|
|
|
* because of overflows here and there.
|
|
|
|
*//*
|
|
|
|
* else if (auto * set = typeid_cast<ASTSetQuery *>(ast.get()))
|
|
|
|
* {
|
|
|
|
* for (auto & c : set->changes)
|
|
|
|
* {
|
|
|
|
* if (fuzz_rand() % 50 == 0)
|
|
|
|
* {
|
|
|
|
* c.value = fuzzField(c.value);
|
|
|
|
* }
|
|
|
|
* }
|
|
|
|
* }
|
|
|
|
*/
|
2020-06-26 05:44:42 +00:00
|
|
|
else if (auto * literal = typeid_cast<ASTLiteral *>(ast.get()))
|
|
|
|
{
|
2020-08-05 15:45:44 +00:00
|
|
|
// There is a caveat with fuzzing the children: many ASTs also keep the
|
|
|
|
// links to particular children in own fields. This means that replacing
|
|
|
|
// the child with another object might lead to error. Many of these fields
|
|
|
|
// are ASTPtr -- this is redundant ownership, but hides the error if the
|
|
|
|
// child field is replaced. Others can be ASTLiteral * or the like, which
|
|
|
|
// leads to segfault if the pointed-to AST is replaced.
|
|
|
|
// Replacing children is safe in case of ASTExpressionList. In a more
|
|
|
|
// general case, we can change the value of ASTLiteral, which is what we
|
|
|
|
// do here.
|
|
|
|
if (fuzz_rand() % 11 == 0)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
literal->value = fuzzField(literal->value);
|
|
|
|
}
|
|
|
|
}
|
2022-08-11 02:34:10 +00:00
|
|
|
else if (auto * create_query = typeid_cast<ASTCreateQuery *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzzCreateQuery(*create_query);
|
|
|
|
}
|
2023-03-21 02:41:51 +00:00
|
|
|
else if (auto * explain_query = typeid_cast<ASTExplainQuery *>(ast.get()))
|
|
|
|
{
|
|
|
|
fuzzExplainQuery(*explain_query);
|
|
|
|
}
|
2020-06-26 05:44:42 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
fuzz(ast->children);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-07 16:31:58 +00:00
|
|
|
/*
|
|
|
|
* This functions collects various parts of query that we can then substitute
|
|
|
|
* to a query being fuzzed.
|
|
|
|
*
|
|
|
|
* TODO: we just stop remembering new parts after our corpus reaches certain size.
|
|
|
|
* This is boring, should implement a random replacement of existing parst with
|
|
|
|
* small probability. Do this after we add this fuzzer to CI and fix all the
|
|
|
|
* problems it can routinely find even in this boring version.
|
|
|
|
*/
|
2022-03-11 18:16:49 +00:00
|
|
|
void QueryFuzzer::collectFuzzInfoMain(ASTPtr ast)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
collectFuzzInfoRecurse(ast);
|
|
|
|
|
|
|
|
aliases.clear();
|
|
|
|
for (const auto & alias : aliases_set)
|
|
|
|
{
|
|
|
|
aliases.push_back(alias);
|
|
|
|
}
|
|
|
|
|
|
|
|
column_like.clear();
|
|
|
|
for (const auto & [name, value] : column_like_map)
|
|
|
|
{
|
|
|
|
column_like.push_back(value);
|
|
|
|
}
|
|
|
|
|
|
|
|
table_like.clear();
|
|
|
|
for (const auto & [name, value] : table_like_map)
|
|
|
|
{
|
|
|
|
table_like.push_back(value);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-11 18:16:49 +00:00
|
|
|
void QueryFuzzer::addTableLike(ASTPtr ast)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
if (table_like_map.size() > 1000)
|
|
|
|
{
|
2021-01-22 17:15:31 +00:00
|
|
|
table_like_map.clear();
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const auto name = ast->formatForErrorMessage();
|
|
|
|
if (name.size() < 200)
|
|
|
|
{
|
|
|
|
table_like_map.insert({name, ast});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-11 18:16:49 +00:00
|
|
|
void QueryFuzzer::addColumnLike(ASTPtr ast)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
if (column_like_map.size() > 1000)
|
|
|
|
{
|
2021-01-22 17:15:31 +00:00
|
|
|
column_like_map.clear();
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
const auto name = ast->formatForErrorMessage();
|
2021-03-15 19:45:45 +00:00
|
|
|
if (name == "Null")
|
|
|
|
{
|
|
|
|
// The `Null` identifier from FORMAT Null clause. We don't quote it
|
|
|
|
// properly when formatting the AST, and while the resulting query
|
|
|
|
// technically works, it has non-standard case for Null (the standard
|
|
|
|
// is NULL), so it breaks the query formatting idempotence check.
|
|
|
|
// Just plug this particular case for now.
|
|
|
|
return;
|
|
|
|
}
|
2020-06-26 05:44:42 +00:00
|
|
|
if (name.size() < 200)
|
|
|
|
{
|
|
|
|
column_like_map.insert({name, ast});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-03-11 18:16:49 +00:00
|
|
|
void QueryFuzzer::collectFuzzInfoRecurse(ASTPtr ast)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
|
|
|
if (auto * impl = dynamic_cast<ASTWithAlias *>(ast.get()))
|
|
|
|
{
|
2021-01-22 17:15:31 +00:00
|
|
|
if (aliases_set.size() > 1000)
|
2020-06-26 05:44:42 +00:00
|
|
|
{
|
2021-01-22 17:15:31 +00:00
|
|
|
aliases_set.clear();
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
2021-01-22 17:15:31 +00:00
|
|
|
|
|
|
|
aliases_set.insert(impl->alias);
|
2020-06-26 05:44:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (typeid_cast<ASTLiteral *>(ast.get()))
|
|
|
|
{
|
|
|
|
addColumnLike(ast);
|
|
|
|
}
|
|
|
|
else if (typeid_cast<ASTIdentifier *>(ast.get()))
|
|
|
|
{
|
|
|
|
addColumnLike(ast);
|
|
|
|
}
|
|
|
|
else if (typeid_cast<ASTFunction *>(ast.get()))
|
|
|
|
{
|
|
|
|
addColumnLike(ast);
|
|
|
|
}
|
|
|
|
else if (typeid_cast<ASTTableExpression *>(ast.get()))
|
|
|
|
{
|
|
|
|
addTableLike(ast);
|
|
|
|
}
|
|
|
|
else if (typeid_cast<ASTSubquery *>(ast.get()))
|
|
|
|
{
|
|
|
|
addTableLike(ast);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (const auto & child : ast->children)
|
|
|
|
{
|
|
|
|
collectFuzzInfoRecurse(child);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void QueryFuzzer::fuzzMain(ASTPtr & ast)
|
|
|
|
{
|
2020-12-22 04:02:39 +00:00
|
|
|
current_ast_depth = 0;
|
|
|
|
debug_visited_nodes.clear();
|
|
|
|
debug_top_ast = *
|
|
|
|
|
2020-06-26 05:44:42 +00:00
|
|
|
collectFuzzInfoMain(ast);
|
|
|
|
fuzz(ast);
|
|
|
|
|
|
|
|
std::cout << std::endl;
|
2020-11-09 16:05:40 +00:00
|
|
|
WriteBufferFromOStream ast_buf(std::cout, 4096);
|
|
|
|
formatAST(*ast, ast_buf, false /*highlight*/);
|
2020-11-11 13:27:54 +00:00
|
|
|
ast_buf.next();
|
2020-06-26 05:44:42 +00:00
|
|
|
std::cout << std::endl << std::endl;
|
|
|
|
}
|
|
|
|
|
2020-07-07 12:35:53 +00:00
|
|
|
}
|