thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove copy-paste

Browse Source
This commit is contained in:
Alexey Milovidov 2021-05-23 03:55:16 +03:00
parent 1a091c7a17
commit 77414b50e6
12 changed files with 152 additions and 362 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Compression/CompressionCodecDensity.cpp
View File

@ -2,7 +2,6 @@
#include <Compression/CompressionInfo.h>
#include <Compression/CompressionCodecDensity.h>
#include <Parsers/ASTLiteral.h>
#include <Common/ErrorCodes.h>
namespace DB

129
src/Compression/CompressionCodecLZSSE.cpp Normal file
View File

@ -0,0 +1,129 @@
#include <Compression/CompressionCodecLZSSE.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressionInfo.h>
#include <Parsers/ASTLiteral.h>
#include <lzsse2/lzsse2.h>
#include <lzsse4/lzsse4.h>
#include <lzsse8/lzsse8.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_COMPRESS;
extern const int CANNOT_DECOMPRESS;
extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
extern const int ILLEGAL_CODEC_PARAMETER;
extern const int LOGICAL_ERROR;
}
CompressionCodecLZSSE::CompressionCodecLZSSE(UInt32 type_, UInt32 level_) : type(type_), level(level_)
{
if (type != 2 && type != 4 && type != 8)
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no LZSSE{} codec", type);
setCodecDescription(fmt::format("LZSSE{}", type), {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
}
uint8_t CompressionCodecLZSSE::getMethodByte() const
{
switch (type)
{
case 2: return static_cast<uint8_t>(CompressionMethodByte::LZSSE2);
case 4: return static_cast<uint8_t>(CompressionMethodByte::LZSSE4);
case 8: return static_cast<uint8_t>(CompressionMethodByte::LZSSE8);
default:
throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no LZSSE{} codec", type);
}
}
void CompressionCodecLZSSE::updateHash(SipHash & hash) const
{
getCodecDesc()->updateTreeHash(hash);
}
UInt32 CompressionCodecLZSSE::getMaxCompressedDataSize(UInt32 uncompressed_size) const
{
return uncompressed_size;
}
UInt32 CompressionCodecLZSSE::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
UInt32 res = 0;
switch (type)
{
case 2:
{
LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size);
res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level);
LZSSE2_FreeOptimalParseState(state);
break;
}
case 4:
{
LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size);
res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level);
LZSSE2_FreeOptimalParseState(state);
break;
}
case 8:
{
LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size);
res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level);
LZSSE2_FreeOptimalParseState(state);
break;
}
default:
break;
}
if (res == 0)
throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress block with LZSSE{}", type);
return res;
}
void CompressionCodecLZSSE::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
UInt32 res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size);
if (res < uncompressed_size)
throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress block with LZSSE{}", type);
}
void registerCodecsLZSSE(CompressionCodecFactory & factory)
{
for (auto [type, method_byte] : std::initializer_list<std::tuple<int, CompressionMethodByte>>
{
{2, CompressionMethodByte::LZSSE2},
{4, CompressionMethodByte::LZSSE4},
{8, CompressionMethodByte::LZSSE8}
})
{
factory.registerCompressionCodec(
fmt::format("LZSSE{}", type),
uint8_t(method_byte),
[type = type](const ASTPtr & arguments) -> CompressionCodecPtr
{
int level = 1;
if (arguments && !arguments->children.empty())
{
if (arguments->children.size() != 1)
throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE,
"LZSSE{} codec must have 1 parameter, {} given", type, arguments->children.size());
const auto children = arguments->children;
const auto * level_literal = children[0]->as<ASTLiteral>();
if (!level_literal)
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER,
"LZSSE{} first codec argument must be integer", type);
level = level_literal->value.safeGet<UInt64>();
}
return std::make_shared<CompressionCodecLZSSE>(type, level);
});
}
}
}

14
src/Compression/CompressionCodecLZSSE8.h → src/Compression/CompressionCodecLZSSE.h
View File

@ -5,29 +5,25 @@
namespace DB
{
class CompressionCodecLZSSE8 : public ICompressionCodec
class CompressionCodecLZSSE : public ICompressionCodec
{
public:
CompressionCodecLZSSE8(int level_);
CompressionCodecLZSSE(UInt32 type_, UInt32 level_);
uint8_t getMethodByte() const override;
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
void updateHash(SipHash & hash) const override;
protected:
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
bool isCompression() const override { return true; }
bool isGenericCompression() const override { return true; }
private:
const int level;
const UInt32 type;
const UInt32 level;
};
}

91
src/Compression/CompressionCodecLZSSE2.cpp
View File

@ -1,91 +0,0 @@
#include <Compression/CompressionCodecLZSSE2.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressionInfo.h>
#include <Parsers/ASTLiteral.h>
#include <lzsse2/lzsse2.h>
#include <Common/ErrorCodes.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_COMPRESS;
extern const int CANNOT_DECOMPRESS;
extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
extern const int ILLEGAL_CODEC_PARAMETER;
}
CompressionCodecLZSSE2::CompressionCodecLZSSE2(int level_) : level(level_)
{
setCodecDescription("LZSSE2", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
}
uint8_t CompressionCodecLZSSE2::getMethodByte() const
{
return static_cast<uint8_t>(CompressionMethodByte::LZSSE2);
}
void CompressionCodecLZSSE2::updateHash(SipHash & hash) const
{
getCodecDesc()->updateTreeHash(hash);
}
UInt32 CompressionCodecLZSSE2::getMaxCompressedDataSize(UInt32 uncompressed_size) const
{
return uncompressed_size;
}
UInt32 CompressionCodecLZSSE2::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
UInt32 res;
LZSSE2_OptimalParseState * state = LZSSE2_MakeOptimalParseState(source_size);
res = LZSSE2_CompressOptimalParse(state, source, source_size, dest, source_size, level);
LZSSE2_FreeOptimalParseState(state);
if (res == 0)
throw Exception("Cannot compress block with LZSSE2", ErrorCodes::CANNOT_COMPRESS);
return res;
}
void CompressionCodecLZSSE2::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
UInt32 res;
res = LZSSE2_Decompress(source, source_size, dest, uncompressed_size);
if (res < uncompressed_size)
throw Exception("Cannot decompress block with LZSSE2", ErrorCodes::CANNOT_DECOMPRESS);
}
void registerCodecLZSSE2(CompressionCodecFactory & factory)
{
UInt8 method_code = UInt8(CompressionMethodByte::LZSSE2);
factory.registerCompressionCodec(
"LZSSE2",
method_code,
[&](const ASTPtr & arguments) -> CompressionCodecPtr
{
int level = 1;
if (arguments && !arguments->children.empty())
{
if (arguments->children.size() != 1)
throw Exception(
"LZSSE2 codec must have 1 parameter, given " + std::to_string(arguments->children.size()),
ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE);
const auto children = arguments->children;
const auto * level_literal = children[0]->as<ASTLiteral>();
if (!level_literal)
throw Exception("LZSSE2 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER);
level = level_literal->value.safeGet<UInt64>();
}
return std::make_shared<CompressionCodecLZSSE2>(level);
});
}
}

33
src/Compression/CompressionCodecLZSSE2.h
View File

@ -1,33 +0,0 @@
#pragma once
#include <Compression/ICompressionCodec.h>
namespace DB
{
class CompressionCodecLZSSE2 : public ICompressionCodec
{
public:
CompressionCodecLZSSE2(int level_);
uint8_t getMethodByte() const override;
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
void updateHash(SipHash & hash) const override;
protected:
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
bool isCompression() const override { return true; }
bool isGenericCompression() const override { return true; }
private:
const int level;
};
}

90
src/Compression/CompressionCodecLZSSE4.cpp
View File

@ -1,90 +0,0 @@
#include <Compression/CompressionCodecLZSSE4.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressionInfo.h>
#include <Parsers/ASTLiteral.h>
#include <lzsse4/lzsse4.h>
#include <Common/ErrorCodes.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_COMPRESS;
extern const int CANNOT_DECOMPRESS;
extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
extern const int ILLEGAL_CODEC_PARAMETER;
}
CompressionCodecLZSSE4::CompressionCodecLZSSE4(int level_) : level(level_)
{
setCodecDescription("LZSSE4", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
}
uint8_t CompressionCodecLZSSE4::getMethodByte() const
{
return static_cast<uint8_t>(CompressionMethodByte::LZSSE4);
}
void CompressionCodecLZSSE4::updateHash(SipHash & hash) const
{
getCodecDesc()->updateTreeHash(hash);
}
UInt32 CompressionCodecLZSSE4::getMaxCompressedDataSize(UInt32 uncompressed_size) const
{
return uncompressed_size;
}
UInt32 CompressionCodecLZSSE4::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
UInt32 res;
LZSSE4_OptimalParseState * state = LZSSE4_MakeOptimalParseState(source_size);
res = LZSSE4_CompressOptimalParse(state, source, source_size, dest, source_size, level);
LZSSE4_FreeOptimalParseState(state);
if (res == 0)
throw Exception("Cannot compress block with LZSSE4", ErrorCodes::CANNOT_COMPRESS);
return res;
}
void CompressionCodecLZSSE4::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
UInt32 res;
res = LZSSE4_Decompress(source, source_size, dest, uncompressed_size);
if (res < uncompressed_size)
throw Exception("Cannot decompress block with LZSSE4", ErrorCodes::CANNOT_DECOMPRESS);
}
void registerCodecLZSSE4(CompressionCodecFactory & factory)
{
UInt8 method_code = UInt8(CompressionMethodByte::LZSSE4);
factory.registerCompressionCodec(
"LZSSE4",
method_code,
[&](const ASTPtr & arguments) -> CompressionCodecPtr
{
int level = 1;
if (arguments && !arguments->children.empty())
{
if (arguments->children.size() != 1)
throw Exception(
"LZSSE4 codec must have 1 parameter, given " + std::to_string(arguments->children.size()),
ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE);
const auto children = arguments->children;
const auto * level_literal = children[0]->as<ASTLiteral>();
if (!level_literal)
throw Exception("LZSSE4 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER);
level = level_literal->value.safeGet<UInt64>();
}
return std::make_shared<CompressionCodecLZSSE4>(level);
});
}
}

33
src/Compression/CompressionCodecLZSSE4.h
View File

@ -1,33 +0,0 @@
#pragma once
#include <Compression/ICompressionCodec.h>
namespace DB
{
class CompressionCodecLZSSE4 : public ICompressionCodec
{
public:
CompressionCodecLZSSE4(int level_);
uint8_t getMethodByte() const override;
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
void updateHash(SipHash & hash) const override;
protected:
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
bool isCompression() const override { return true; }
bool isGenericCompression() const override { return true; }
private:
const int level;
};
}

90
src/Compression/CompressionCodecLZSSE8.cpp
View File

@ -1,90 +0,0 @@
#include <Compression/CompressionCodecLZSSE8.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressionInfo.h>
#include <Parsers/ASTLiteral.h>
#include <lzsse8/lzsse8.h>
#include <Common/ErrorCodes.h>
namespace DB
{
namespace ErrorCodes
{
extern const int CANNOT_COMPRESS;
extern const int CANNOT_DECOMPRESS;
extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE;
extern const int ILLEGAL_CODEC_PARAMETER;
}
CompressionCodecLZSSE8::CompressionCodecLZSSE8(int level_) : level(level_)
{
setCodecDescription("LZSSE8", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
}
uint8_t CompressionCodecLZSSE8::getMethodByte() const
{
return static_cast<uint8_t>(CompressionMethodByte::LZSSE8);
}
void CompressionCodecLZSSE8::updateHash(SipHash & hash) const
{
getCodecDesc()->updateTreeHash(hash);
}
UInt32 CompressionCodecLZSSE8::getMaxCompressedDataSize(UInt32 uncompressed_size) const
{
return uncompressed_size;
}
UInt32 CompressionCodecLZSSE8::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
UInt32 res;
LZSSE8_OptimalParseState * state = LZSSE8_MakeOptimalParseState(source_size);
res = LZSSE8_CompressOptimalParse(state, source, source_size, dest, source_size, level);
LZSSE8_FreeOptimalParseState(state);
if (res == 0)
throw Exception("Cannot compress block with LZSSE", ErrorCodes::CANNOT_COMPRESS);
return res;
}
void CompressionCodecLZSSE8::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const
{
UInt32 res;
res = LZSSE8_Decompress(source, source_size, dest, uncompressed_size);
if (res < uncompressed_size)
throw Exception("Cannot decompress block with LZSSE", ErrorCodes::CANNOT_DECOMPRESS);
}
void registerCodecLZSSE8(CompressionCodecFactory & factory)
{
UInt8 method_code = UInt8(CompressionMethodByte::LZSSE8);
factory.registerCompressionCodec(
"LZSSE8",
method_code,
[&](const ASTPtr & arguments) -> CompressionCodecPtr
{
int level = 1;
if (arguments && !arguments->children.empty())
{
if (arguments->children.size() != 1)
throw Exception(
"LZSSE8 codec must have 1 parameter, given " + std::to_string(arguments->children.size()),
ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE);
const auto children = arguments->children;
const auto * level_literal = children[0]->as<ASTLiteral>();
if (!level_literal)
throw Exception("LZSSE8 first codec argument must be integer", ErrorCodes::ILLEGAL_CODEC_PARAMETER);
level = level_literal->value.safeGet<UInt64>();
}
return std::make_shared<CompressionCodecLZSSE8>(level);
});
}
}

1
src/Compression/CompressionCodecLizard.cpp
View File

@ -4,7 +4,6 @@
#include <Parsers/ASTLiteral.h>
#include <lib/lizard_compress.h>
#include <lib/lizard_decompress.h>
#include <Common/ErrorCodes.h>
namespace DB

27
src/Compression/CompressionFactory.cpp
View File

@ -61,7 +61,8 @@ CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std
}
}
void CompressionCodecFactory::validateCodec(const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs) const
void CompressionCodecFactory::validateCodec(
const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs) const
{
if (family_name.empty())
throw Exception("Compression codec name cannot be empty", ErrorCodes::BAD_ARGUMENTS);
@ -69,12 +70,14 @@ void CompressionCodecFactory::validateCodec(const String & family_name, std::opt
if (level)
{
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), {}, sanity_check, allow_experimental_codecs);
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)),
{}, sanity_check, allow_experimental_codecs);
}
else
{
auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name));
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), {}, sanity_check, allow_experimental_codecs);
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier),
{}, sanity_check, allow_experimental_codecs);
}
}
@ -140,7 +143,8 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
if (column_type)
{
CompressionCodecPtr prev_codec;
IDataType::StreamCallbackWithType callback = [&](const ISerialization::SubstreamPath & substream_path, const IDataType & substream_type)
IDataType::StreamCallbackWithType callback = [&](
const ISerialization::SubstreamPath & substream_path, const IDataType & substream_type)
{
if (ISerialization::isSpecialCompressionAllowed(substream_path))
{
@ -229,7 +233,9 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC);
}
CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const
CompressionCodecPtr CompressionCodecFactory::get(
const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const
{
if (current_default == nullptr)
current_default = default_codec;
@ -280,6 +286,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IData
throw Exception("Unexpected AST structure for compression codec: " + queryToString(ast), ErrorCodes::UNEXPECTED_AST_STRUCTURE);
}
CompressionCodecPtr CompressionCodecFactory::get(const uint8_t byte_code) const
{
const auto family_code_and_creator = family_code_with_codec.find(byte_code);
@ -337,7 +344,7 @@ void CompressionCodecFactory::registerSimpleCompressionCodec(
registerCompressionCodec(family_name, byte_code, [family_name, creator](const ASTPtr & ast)
{
if (ast)
throw Exception("Compression codec " + family_name + " cannot have arguments", ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS);
throw Exception(ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS, "Compression codec {} cannot have arguments", family_name);
return creator();
});
}
@ -354,9 +361,7 @@ void registerCodecGorilla(CompressionCodecFactory & factory);
void registerCodecMultiple(CompressionCodecFactory & factory);
void registerCodecLizard(CompressionCodecFactory & factory);
void registerCodecDensity(CompressionCodecFactory & factory);
void registerCodecLZSSE2(CompressionCodecFactory & factory);
void registerCodecLZSSE4(CompressionCodecFactory & factory);
void registerCodecLZSSE8(CompressionCodecFactory & factory);
void registerCodecsLZSSE(CompressionCodecFactory & factory);
CompressionCodecFactory::CompressionCodecFactory()
{
@ -371,9 +376,7 @@ CompressionCodecFactory::CompressionCodecFactory()
registerCodecMultiple(*this);
registerCodecLizard(*this);
registerCodecDensity(*this);
registerCodecLZSSE2(*this);
registerCodecLZSSE4(*this);
registerCodecLZSSE8(*this);
registerCodecsLZSSE(*this);
default_codec = get("LZ4", {});
}

2
src/Compression/CompressionInfo.h
View File

@ -47,7 +47,7 @@ enum class CompressionMethodByte : uint8_t
Density = 0x97,
LZSSE2 = 0x98,
LZSSE4 = 0x99,
LZSSE8 = 0xa0,
LZSSE8 = 0xA0,
};

3
src/Storages/System/StorageSystemErrors.cpp
View File

@ -3,8 +3,9 @@
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeArray.h>
#include <Storages/System/StorageSystemErrors.h>
#include <Common/ErrorCodes.h>
#include <Interpreters/Context.h>
#include <Common/ErrorCodes.h>
namespace DB
{