Added draft version of random table function.

Currently unimplemented data generators
This commit is contained in:
Yatsishin Ilya 2020-01-30 23:12:00 +03:00
parent 072043156f
commit 6597176388
29 changed files with 176 additions and 0 deletions

View File

@ -304,6 +304,10 @@ MutableColumnPtr DataTypeAggregateFunction::createColumn() const
return ColumnAggregateFunction::create(function); return ColumnAggregateFunction::create(function);
} }
MutableColumnPtr DataTypeAggregateFunction::createColumnWithRandomData(size_t) const
{
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
/// Create empty state /// Create empty state
Field DataTypeAggregateFunction::getDefault() const Field DataTypeAggregateFunction::getDefault() const

View File

@ -63,6 +63,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;

View File

@ -487,6 +487,13 @@ MutableColumnPtr DataTypeArray::createColumn() const
} }
MutableColumnPtr DataTypeArray::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
Field DataTypeArray::getDefault() const Field DataTypeArray::getDefault() const
{ {
return Array(); return Array();

View File

@ -94,6 +94,7 @@ public:
bool & row_added) const override; bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;

View File

@ -41,6 +41,13 @@ MutableColumnPtr DataTypeDecimalBase<T>::createColumn() const
return ColumnType::create(0, scale); return ColumnType::create(0, scale);
} }
template <typename T>
MutableColumnPtr DataTypeDecimalBase<T>::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
template <typename T> template <typename T>
void DataTypeDecimalBase<T>::serializeBinary(const Field & field, WriteBuffer & ostr) const void DataTypeDecimalBase<T>::serializeBinary(const Field & field, WriteBuffer & ostr) const
{ {

View File

@ -83,6 +83,7 @@ public:
Field getDefault() const override; Field getDefault() const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
bool isParametric() const override { return true; } bool isParametric() const override { return true; }
bool haveSubtypes() const override { return false; } bool haveSubtypes() const override { return false; }

View File

@ -111,6 +111,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override { return ColumnType::create(); } MutableColumnPtr createColumn() const override { return ColumnType::create(); }
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;
void insertDefaultInto(IColumn & column) const override; void insertDefaultInto(IColumn & column) const override;

View File

@ -268,6 +268,12 @@ MutableColumnPtr DataTypeFixedString::createColumn() const
return ColumnFixedString::create(n); return ColumnFixedString::create(n);
} }
MutableColumnPtr DataTypeFixedString::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
Field DataTypeFixedString::getDefault() const Field DataTypeFixedString::getDefault() const
{ {
return String(); return String();

View File

@ -70,6 +70,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;

View File

@ -934,6 +934,12 @@ MutableColumnPtr DataTypeLowCardinality::createColumn() const
return ColumnLowCardinality::create(std::move(dictionary), std::move(indexes)); return ColumnLowCardinality::create(std::move(dictionary), std::move(indexes));
} }
MutableColumnPtr DataTypeLowCardinality::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
Field DataTypeLowCardinality::getDefault() const Field DataTypeLowCardinality::getDefault() const
{ {
return dictionary_type->getDefault(); return dictionary_type->getDefault();

View File

@ -68,6 +68,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;

View File

@ -14,6 +14,14 @@ MutableColumnPtr DataTypeNothing::createColumn() const
return ColumnNothing::create(0); return ColumnNothing::create(0);
} }
MutableColumnPtr DataTypeNothing::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void DataTypeNothing::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const void DataTypeNothing::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const
{ {
size_t size = column.size(); size_t size = column.size();

View File

@ -19,6 +19,7 @@ public:
TypeIndex getTypeId() const override { return TypeIndex::Nothing; } TypeIndex getTypeId() const override { return TypeIndex::Nothing; }
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
/// These methods read and write zero bytes just to allow to figure out size of column. /// These methods read and write zero bytes just to allow to figure out size of column.
void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override;

View File

@ -488,6 +488,11 @@ MutableColumnPtr DataTypeNullable::createColumn() const
return ColumnNullable::create(nested_data_type->createColumn(), ColumnUInt8::create()); return ColumnNullable::create(nested_data_type->createColumn(), ColumnUInt8::create());
} }
MutableColumnPtr DataTypeNullable::createColumnWithRandomData(size_t limit) const
{
return ColumnNullable::create(nested_data_type->createColumnWithRandomData(limit), DataTypeUInt8().createColumnWithRandomData(limit));
}
Field DataTypeNullable::getDefault() const Field DataTypeNullable::getDefault() const
{ {
return Null(); return Null();

View File

@ -76,6 +76,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;

View File

@ -239,6 +239,13 @@ MutableColumnPtr DataTypeNumberBase<T>::createColumn() const
return ColumnVector<T>::create(); return ColumnVector<T>::create();
} }
template <typename T>
MutableColumnPtr DataTypeNumberBase<T>::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
template <typename T> template <typename T>
bool DataTypeNumberBase<T>::isValueRepresentedByInteger() const bool DataTypeNumberBase<T>::isValueRepresentedByInteger() const
{ {

View File

@ -45,6 +45,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
bool isParametric() const override { return false; } bool isParametric() const override { return false; }
bool haveSubtypes() const override { return false; } bool haveSubtypes() const override { return false; }

View File

@ -21,6 +21,7 @@ public:
// Used for expressions analysis. // Used for expressions analysis.
MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); }
MutableColumnPtr createColumnWithRandomData(size_t) const override;
// Used only for debugging, making it DUMPABLE // Used only for debugging, making it DUMPABLE
Field getDefault() const override { return Tuple(); } Field getDefault() const override { return Tuple(); }

View File

@ -360,6 +360,11 @@ MutableColumnPtr DataTypeString::createColumn() const
return ColumnString::create(); return ColumnString::create();
} }
MutableColumnPtr DataTypeString::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
bool DataTypeString::equals(const IDataType & rhs) const bool DataTypeString::equals(const IDataType & rhs) const
{ {

View File

@ -54,6 +54,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;

View File

@ -454,6 +454,14 @@ MutableColumnPtr DataTypeTuple::createColumn() const
return ColumnTuple::create(std::move(tuple_columns)); return ColumnTuple::create(std::move(tuple_columns));
} }
MutableColumnPtr DataTypeTuple::createColumnWithRandomData(size_t limit) const
{
(void)limit;
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
Field DataTypeTuple::getDefault() const Field DataTypeTuple::getDefault() const
{ {
return Tuple(ext::map<Tuple>(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); })); return Tuple(ext::map<Tuple>(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); }));

View File

@ -81,6 +81,7 @@ public:
void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override; void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override;
MutableColumnPtr createColumn() const override; MutableColumnPtr createColumn() const override;
MutableColumnPtr createColumnWithRandomData(size_t) const override;
Field getDefault() const override; Field getDefault() const override;
void insertDefaultInto(IColumn & column) const override; void insertDefaultInto(IColumn & column) const override;

View File

@ -287,6 +287,10 @@ public:
*/ */
virtual MutableColumnPtr createColumn() const = 0; virtual MutableColumnPtr createColumn() const = 0;
/** Create column for corresponding type and fill with random values.
*/
virtual MutableColumnPtr createColumnWithRandomData(size_t size) const = 0;
/** Create ColumnConst for corresponding type, with specified size and value. /** Create ColumnConst for corresponding type, with specified size and value.
*/ */
ColumnPtr createColumnConst(size_t size, const Field & field) const; ColumnPtr createColumnConst(size_t size, const Field & field) const;

View File

@ -42,6 +42,11 @@ public:
throw Exception("Method createColumn() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); throw Exception("Method createColumn() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
} }
MutableColumnPtr createColumnWithRandomData(size_t) const override
{
throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
Field getDefault() const override Field getDefault() const override
{ {
throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED);

View File

@ -0,0 +1,69 @@
#include <Common/typeid_cast.h>
#include <Common/Exception.h>
#include <Core/Block.h>
#include <Storages/StorageValues.h>
#include <DataTypes/DataTypeTuple.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <TableFunctions/ITableFunction.h>
#include <TableFunctions/TableFunctionRandom.h>
#include <TableFunctions/TableFunctionFactory.h>
#include "registerTableFunctions.h"
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int BAD_ARGUMENTS;
}
StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const
{
ASTs & args_func = ast_function->children;
if (args_func.size() != 1)
throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR);
ASTs & args = args_func.at(0)->children;
if (args.size() > 2)
throw Exception("Table function '" + getName() + "' requires one or two arguments: structure (and limit).",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
/// Parsing first argument as table structure and creating a sample block
std::string structure = args[0]->as<ASTLiteral &>().value.safeGet<String>();
UInt64 limit = 1;
/// Parsing second argument if present
if (args.size() == 2)
limit = args[1]->as<ASTLiteral &>().value.safeGet<Uint64>();
if (!limit)
throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS);
ColumnsDescription columns = parseColumnsListFromString(structure, context);
Block res_block;
for (const auto & name_type : columns.getOrdinary())
Column c = name_type.type->createColumnWithRandomData(limit) ;
res_block.insert({ c, name_type.type, name_type.name });
auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block);
res->startup();
return res;
}
void registerTableFunctionRandom(TableFunctionFactory & factory)
{
factory.registerFunction<TableFunctionRandom>(TableFunctionFactory::CaseInsensitive);
}
}

View File

@ -0,0 +1,20 @@
#pragma once
#include <TableFunctions/ITableFunction.h>
namespace DB
{
/* random(structure, limit) - creates a temporary storage filling columns with random data
* random is case-insensitive table function
*/
class TableFunctionRandom : public ITableFunction
{
public:
static constexpr auto name = "generate";
std::string getName() const override { return name; }
private:
StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override;
};
}

View File

@ -15,6 +15,7 @@ void registerTableFunctions()
registerTableFunctionURL(factory); registerTableFunctionURL(factory);
registerTableFunctionValues(factory); registerTableFunctionValues(factory);
registerTableFunctionInput(factory); registerTableFunctionInput(factory);
registerTableFunctionRandom(factory);
#if USE_AWS_S3 #if USE_AWS_S3
registerTableFunctionS3(factory); registerTableFunctionS3(factory);

View File

@ -12,6 +12,7 @@ void registerTableFunctionFile(TableFunctionFactory & factory);
void registerTableFunctionURL(TableFunctionFactory & factory); void registerTableFunctionURL(TableFunctionFactory & factory);
void registerTableFunctionValues(TableFunctionFactory & factory); void registerTableFunctionValues(TableFunctionFactory & factory);
void registerTableFunctionInput(TableFunctionFactory & factory); void registerTableFunctionInput(TableFunctionFactory & factory);
void registerTableFunctionRandom(TableFunctionFactory & factory);
#if USE_AWS_S3 #if USE_AWS_S3
void registerTableFunctionS3(TableFunctionFactory & factory); void registerTableFunctionS3(TableFunctionFactory & factory);

View File

@ -0,0 +1 @@
SELECT * FROM random(3)