thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 18:42:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Buildable code

Browse Source
This commit is contained in:
alesapin 2020-09-21 14:24:10 +03:00
parent 387b97aba5
commit a3e999784b
11 changed files with 27 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Compression/CompressionCodecDelta.cpp
View File

@ -136,7 +136,7 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_
namespace
{
UInt8 getDeltaBytesSize(DataTypePtr column_type)
UInt8 getDeltaBytesSize(const IDataType * column_type)
{
if (!column_type->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Delta is not applicable for {} because the data type is not of fixed size",
@ -155,7 +155,7 @@ UInt8 getDeltaBytesSize(DataTypePtr column_type)
void registerCodecDelta(CompressionCodecFactory & factory)
{
UInt8 method_code = UInt8(CompressionMethodByte::Delta);
factory.registerCompressionCodecWithType("Delta", method_code, [&](const ASTPtr & arguments, DataTypePtr column_type) -> CompressionCodecPtr
factory.registerCompressionCodecWithType("Delta", method_code, [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
{
UInt8 delta_bytes_size = 0;

4
src/Compression/CompressionCodecDoubleDelta.cpp
View File

@ -307,7 +307,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
}
}
UInt8 getDataBytesSize(DataTypePtr column_type)
UInt8 getDataBytesSize(const IDataType * column_type)
{
if (!column_type->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec DoubleDelta is not applicable for {} because the data type is not of fixed size",
@ -413,7 +413,7 @@ void registerCodecDoubleDelta(CompressionCodecFactory & factory)
{
UInt8 method_code = UInt8(CompressionMethodByte::DoubleDelta);
factory.registerCompressionCodecWithType("DoubleDelta", method_code,
[&](const ASTPtr & arguments, DataTypePtr column_type) -> CompressionCodecPtr
[&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
{
if (arguments)
throw Exception("Codec DoubleDelta does not accept any arguments", ErrorCodes::BAD_ARGUMENTS);

4
src/Compression/CompressionCodecGorilla.cpp
View File

@ -222,7 +222,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest)
}
}
UInt8 getDataBytesSize(DataTypePtr column_type)
UInt8 getDataBytesSize(const IDataType * column_type)
{
if (!column_type->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Gorilla is not applicable for {} because the data type is not of fixed size",
@ -329,7 +329,7 @@ void registerCodecGorilla(CompressionCodecFactory & factory)
{
UInt8 method_code = UInt8(CompressionMethodByte::Gorilla);
factory.registerCompressionCodecWithType("Gorilla", method_code,
[&](const ASTPtr & arguments, DataTypePtr column_type) -> CompressionCodecPtr
[&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr
{
if (arguments)
throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS);

4
src/Compression/CompressionCodecT64.cpp
View File

@ -136,7 +136,7 @@ TypeIndex baseType(TypeIndex type_idx)
return TypeIndex::Nothing;
}
TypeIndex typeIdx(const DataTypePtr & data_type)
TypeIndex typeIdx(const IDataType * data_type)
{
if (!data_type)
return TypeIndex::Nothing;
@ -656,7 +656,7 @@ void CompressionCodecT64::updateHash(SipHash & hash) const
void registerCodecT64(CompressionCodecFactory & factory)
{
auto reg_func = [&](const ASTPtr & arguments, DataTypePtr type) -> CompressionCodecPtr
auto reg_func = [&](const ASTPtr & arguments, const IDataType * type) -> CompressionCodecPtr
{
Variant variant = Variant::Byte;

13
src/Compression/CompressionFactory.cpp
View File

@ -57,7 +57,7 @@ void CompressionCodecFactory::validateCodec(const String & family_name, std::opt
}
}
ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, DataTypePtr column_type, bool sanity_check) const
ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check) const
{
if (const auto * func = ast->as<ASTFunction>())
{
@ -140,16 +140,13 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
}
std::shared_ptr<ASTFunction> result = std::make_shared<ASTFunction>();
result->name = "CODEC";
result->arguments = codecs_descriptions;
return result;
return ast;
}
throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC);
}
CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, DataTypePtr column_type, CompressionCodecPtr current_default) const
CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default) const
{
if (current_default == nullptr)
current_default = default_codec;
@ -203,7 +200,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const uint8_t byte_code) const
}
CompressionCodecPtr CompressionCodecFactory::getImpl(const String & family_name, const ASTPtr & arguments, DataTypePtr column_type) const
CompressionCodecPtr CompressionCodecFactory::getImpl(const String & family_name, const ASTPtr & arguments, const IDataType * column_type) const
{
if (family_name == "Multiple")
throw Exception("Codec Multiple cannot be specified directly", ErrorCodes::UNKNOWN_CODEC);
@ -235,7 +232,7 @@ void CompressionCodecFactory::registerCompressionCodecWithType(
void CompressionCodecFactory::registerCompressionCodec(const String & family_name, std::optional<uint8_t> byte_code, Creator creator)
{
registerCompressionCodecWithType(family_name, byte_code, [family_name, creator](const ASTPtr & ast, DataTypePtr /* data_type */)
registerCompressionCodecWithType(family_name, byte_code, [family_name, creator](const ASTPtr & ast, const IDataType * /* data_type */)
{
return creator(ast);
});

12
src/Compression/CompressionFactory.h
View File

@ -26,7 +26,7 @@ class CompressionCodecFactory final : private boost::noncopyable
{
protected:
using Creator = std::function<CompressionCodecPtr(const ASTPtr & parameters)>;
using CreatorWithType = std::function<CompressionCodecPtr(const ASTPtr & parameters, DataTypePtr column_type)>;
using CreatorWithType = std::function<CompressionCodecPtr(const ASTPtr & parameters, const IDataType * column_type)>;
using SimpleCreator = std::function<CompressionCodecPtr()>;
using CompressionCodecsDictionary = std::unordered_map<String, CreatorWithType>;
using CompressionCodecsCodeDictionary = std::unordered_map<uint8_t, CreatorWithType>;
@ -39,6 +39,10 @@ public:
/// Validate codecs AST specified by user and parses codecs description (substitute default parameters)
ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check) const;
ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check) const
{
return validateCodecAndGetPreprocessedAST(ast, column_type.get(), sanity_check);
}
/// Validate codecs AST specified by user
void validateCodec(const String & family_name, std::optional<int> level, bool sanity_check) const;
@ -49,6 +53,10 @@ public:
/// codec, which can be alias to current default codec, which can be changed
/// in runtime.
CompressionCodecPtr get(const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default = nullptr) const;
CompressionCodecPtr get(const ASTPtr & ast, const DataTypePtr & column_type, CompressionCodecPtr current_default = nullptr) const
{
return get(ast, column_type.get(), current_default);
}
/// Get codec by method byte (no params available)
CompressionCodecPtr get(const uint8_t byte_code) const;
@ -65,7 +73,7 @@ public:
void registerSimpleCompressionCodec(const String & family_name, std::optional<uint8_t> byte_code, SimpleCreator creator);
protected:
CompressionCodecPtr getImpl(const String & family_name, const ASTPtr & arguments, DataTypePtr column_type) const;
CompressionCodecPtr getImpl(const String & family_name, const ASTPtr & arguments, const IDataType * column_type) const;
private:
CompressionCodecsDictionary family_name_with_codec;

1
src/Compression/ICompressionCodec.h
View File

@ -17,7 +17,6 @@ using CompressionCodecPtr = std::shared_ptr<ICompressionCodec>;
using Codecs = std::vector<CompressionCodecPtr>;
class IDataType;
using DataTypePtr = std::shared_ptr<const IDataType>;
/**
* Represents interface for compression codecs like LZ4, ZSTD, etc.

2
src/Storages/ColumnsDescription.cpp
View File

@ -426,7 +426,7 @@ CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_
return getCodecOrDefault(column_name, CompressionCodecFactory::instance().getDefaultCodec());
}
ASTPtr ColumnsDescription::getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec)
ASTPtr ColumnsDescription::getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) const
{
const auto it = columns.get<1>().find(column_name);

2
src/Storages/ColumnsDescription.h
View File

@ -115,7 +115,7 @@ public:
bool hasCompressionCodec(const String & column_name) const;
CompressionCodecPtr getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const;
CompressionCodecPtr getCodecOrDefault(const String & column_name) const;
ASTPtr getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec);
ASTPtr getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) const;
String toString() const;
static ColumnsDescription parse(const String & str);

5
src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp
View File

@ -29,14 +29,13 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide(
{
const auto & columns = metadata_snapshot->getColumns();
for (const auto & it : columns_list)
addStreams(it.name, *it.type, columns.getCodecOrDefaultDesc(it.name, default_codec), default_codec, settings.estimated_size);
addStreams(it.name, *it.type, columns.getCodecDescOrDefault(it.name, default_codec), settings.estimated_size);
}
void MergeTreeDataPartWriterWide::addStreams(
const String & name,
const IDataType & type,
const ASTPtr & effective_codec_desc,
const CompressionCodecPtr & default_codec,
size_t estimated_size)
{
IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path, const IDataType & substream_type)
@ -53,7 +52,7 @@ void MergeTreeDataPartWriterWide::addStreams(
data_part->volume->getDisk(),
part_path + stream_name, DATA_FILE_EXTENSION,
part_path + stream_name, marks_file_extension,
effective_codec,
compression_codec,
settings.max_compress_block_size,
estimated_size,
settings.aio_threshold);

1
src/Storages/MergeTree/MergeTreeDataPartWriterWide.h
View File

@ -67,7 +67,6 @@ private:
const String & name,
const IDataType & type,
const ASTPtr & effective_codec_desc,
const CompressionCodecPtr & default_codec,
size_t estimated_size);
SerializationStates serialization_states;