thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 08:32:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

refine ZSTDQAT code name

Browse Source
This commit is contained in:
jinjunzh 2023-12-13 11:02:06 -05:00
parent b4a945ec89
commit 5e1b048e52
4 changed files with 40 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
contrib/CMakeLists.txt vendored
View File

@ -188,17 +188,17 @@ elseif(ENABLE_QATLIB)
message (${RECONFIGURE_MESSAGE_LEVEL} "QAT library is only supported on x86_64 arch")
endif()
if (ENABLE_QATLIB)
option (ENABLE_USDM_DRV "A User Space DMA-able Memory (USDM) component which allocates/frees DMA-able memory" OFF)
option (ENABLE_QAT_OUTOFTREE "Using out-of-tree driver, user needs customize ICP_ROOT variable" OFF)
option (ENABLE_QAT_USDM_DRIVER "A User Space DMA-able Memory (USDM) component which allocates/frees DMA-able memory" OFF)
option (ENABLE_QAT_OUT_OF_TREE_BUILD "Using out-of-tree driver, user needs customize ICP_ROOT variable" OFF)
set(ICP_ROOT "" CACHE STRING "ICP_ROOT variable to define the path of out-of-tree driver package")
if (ENABLE_QAT_OUTOFTREE)
if (ICP_ROOT STREQUAL "")
message(FATAL_ERROR "Please define the path of out-of-tree driver package with -DICP_ROOT=xxx or disable out-of-tree build with -DENABLE_QAT_OUTOFTREE=OFF; \
If you want out-of-tree build but have no package available, please download and build ICP package from: https://www.intel.com/content/www/us/en/download/765501.html")
endif ()
else()
add_contrib (qatlib-cmake qatlib)
endif ()
if (ENABLE_QAT_OUT_OF_TREE_BUILD)
if (ICP_ROOT STREQUAL "")
message(FATAL_ERROR "Please define the path of out-of-tree driver package with -DICP_ROOT=xxx or disable out-of-tree build with -DENABLE_QAT_OUT_OF_TREE_BUILD=OFF; \
If you want out-of-tree build but have no package available, please download and build ICP package from: https://www.intel.com/content/www/us/en/download/765501.html")
endif ()
else()
add_contrib (qatlib-cmake qatlib) # requires: isa-l
endif ()
add_contrib (QAT-ZSTD-Plugin-cmake QAT-ZSTD-Plugin)
else()
message(STATUS "Not using QAT")

6
contrib/QAT-ZSTD-Plugin-cmake/CMakeLists.txt
View File

@ -3,7 +3,7 @@
# meanwhile, user need to set ICP_ROOT environment variable which point to the root directory of QAT driver source tree.
# in-tree means kernel has built-in qat driver, QAT-ZSTD-PLUGIN just has dependency on qatlib.
if (ENABLE_QAT_OUTOFTREE)
if (ENABLE_QAT_OUT_OF_TREE_BUILD)
# Intel QATZSTD out-of-tree build
message(STATUS "Intel QATZSTD out-of-tree build, ICP_ROOT:${ICP_ROOT}")
@ -16,7 +16,7 @@ set(QAT_AL_INCLUDE_DIR "${ICP_ROOT}/quickassist/lookaside/access_layer/include")
set(QAT_USDM_INCLUDE_DIR "${ICP_ROOT}/quickassist/utilities/libusdm_drv")
set(USDM_LIBRARY "${ICP_ROOT}/build/libusdm_drv_s.so")
set(QAT_S_LIBRARY "${ICP_ROOT}/build/libqat_s.so")
if (ENABLE_USDM_DRV)
if (ENABLE_QAT_USDM_DRIVER)
add_definitions(-DENABLE_USDM_DRV)
endif()
add_library(_qatzstd_plugin ${QATZSTD_SRC})
@ -69,7 +69,7 @@ file(COPY "${LIBQAT_ROOT_DIR}/quickassist/utilities/libusdm_drv/qae_mem.h"
DESTINATION "${LIBQAT_HEADER_DIR}/qat/"
)
if (ENABLE_USDM_DRV)
if (ENABLE_QAT_USDM_DRIVER)
add_definitions(-DENABLE_USDM_DRV)
endif()

42
src/Compression/CompressionCodecQATZSTD.cpp → src/Compression/CompressionCodecZSTDQAT.cpp
View File

@ -1,4 +1,5 @@
#ifdef ENABLE_QATZSTD_COMPRESSION
#ifdef ENABLE_ZSTDQAT_COMPRESSION
#include <CompressionCodecZSTD.h>
#include <Compression/CompressionInfo.h>
#include <Compression/CompressionFactory.h>
#include <zstd.h>
@ -8,8 +9,7 @@
#include <Common/typeid_cast.h>
#include <Poco/Logger.h>
#include <Common/logger_useful.h>
#include "qatseqprod.h"
#include "CompressionCodecZSTD.h"
#include <qatseqprod.h>
namespace DB
{
@ -20,13 +20,13 @@ namespace ErrorCodes
extern const int ILLEGAL_CODEC_PARAMETER;
}
class CompressionCodecQATZSTD : public CompressionCodecZSTD
class CompressionCodecZSTDQAT : public CompressionCodecZSTD
{
public:
static constexpr auto QATZSTD_SUPPORTED_MIN_LEVEL = 1;
static constexpr auto QATZSTD_SUPPORTED_MAX_LEVEL = 12;
explicit CompressionCodecQATZSTD(int level_);
~CompressionCodecQATZSTD() override;
static constexpr auto ZSTDQAT_SUPPORTED_MIN_LEVEL = 1;
static constexpr auto ZSTDQAT_SUPPORTED_MAX_LEVEL = 12;
explicit CompressionCodecZSTDQAT(int level_);
~CompressionCodecZSTDQAT() override;
protected:
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
@ -39,7 +39,7 @@ private:
Poco::Logger * log;
};
UInt32 CompressionCodecQATZSTD::doCompressData(const char * source, UInt32 source_size, char * dest) const
UInt32 CompressionCodecZSTDQAT::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
if (!initialized)
{
@ -68,39 +68,39 @@ UInt32 CompressionCodecQATZSTD::doCompressData(const char * source, UInt32 sourc
return static_cast<UInt32>(compressed_size);
}
void registerCodecQATZSTD(CompressionCodecFactory & factory)
void registerCodecZSTDQAT(CompressionCodecFactory & factory)
{
factory.registerCompressionCodec("QATZSTD", {}, [&](const ASTPtr & arguments) -> CompressionCodecPtr
factory.registerCompressionCodec("ZSTDQAT", {}, [&](const ASTPtr & arguments) -> CompressionCodecPtr
{
int level = CompressionCodecZSTD::ZSTD_DEFAULT_LEVEL;
if (arguments && !arguments->children.empty())
{
if (arguments->children.size() > 1)
throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "QATZSTD codec must have 1 parameter, given {}", arguments->children.size());
throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "ZSTDQAT codec must have 1 parameter, given {}", arguments->children.size());
const auto children = arguments->children;
const auto * literal = children[0]->as<ASTLiteral>();
if (!literal)
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "QATZSTD codec argument must be integer");
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "ZSTDQAT codec argument must be integer");
level = static_cast<int>(literal->value.safeGet<UInt64>());
if (level > CompressionCodecQATZSTD::QATZSTD_SUPPORTED_MAX_LEVEL || level < CompressionCodecQATZSTD::QATZSTD_SUPPORTED_MIN_LEVEL)
if (level > CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL || level < CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL)
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER,
"QATZSTD codec doesn't support level more than {} and lower than {} , given {}",
CompressionCodecQATZSTD::QATZSTD_SUPPORTED_MAX_LEVEL, CompressionCodecQATZSTD::QATZSTD_SUPPORTED_MIN_LEVEL, level);
"ZSTDQAT codec doesn't support level more than {} and lower than {} , given {}",
CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL, CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL, level);
}
return std::make_shared<CompressionCodecQATZSTD>(level);
return std::make_shared<CompressionCodecZSTDQAT>(level);
});
}
CompressionCodecQATZSTD::CompressionCodecQATZSTD(int level_)
: CompressionCodecZSTD(level_), level(level_), initialized(false), log(&Poco::Logger::get("CompressionCodecQATZSTD"))
CompressionCodecZSTDQAT::CompressionCodecZSTDQAT(int level_)
: CompressionCodecZSTD(level_), level(level_), initialized(false), log(&Poco::Logger::get("CompressionCodecZSTDQAT"))
{
setCodecDescription("QATZSTD", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
setCodecDescription("ZSTDQAT", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
}
CompressionCodecQATZSTD::~CompressionCodecQATZSTD()
CompressionCodecZSTDQAT::~CompressionCodecZSTDQAT()
{
if (initialized)
{

12
src/Compression/CompressionFactory.cpp
View File

@ -167,13 +167,13 @@ void registerCodecNone(CompressionCodecFactory & factory);
void registerCodecLZ4(CompressionCodecFactory & factory);
void registerCodecLZ4HC(CompressionCodecFactory & factory);
void registerCodecZSTD(CompressionCodecFactory & factory);
#ifdef ENABLE_ZSTDQAT_COMPRESSION
void registerCodecZSTDQAT(CompressionCodecFactory & factory);
#endif
void registerCodecMultiple(CompressionCodecFactory & factory);
#ifdef ENABLE_QPL_COMPRESSION
void registerCodecDeflateQpl(CompressionCodecFactory & factory);
#endif
#ifdef ENABLE_QATZSTD_COMPRESSION
void registerCodecQATZSTD(CompressionCodecFactory & factory);
#endif
/// Keeper use only general-purpose codecs, so we don't need these special codecs
/// in standalone build
@ -192,6 +192,9 @@ CompressionCodecFactory::CompressionCodecFactory()
registerCodecNone(*this);
registerCodecLZ4(*this);
registerCodecZSTD(*this);
#ifdef ENABLE_ZSTDQAT_COMPRESSION
registerCodecZSTDQAT(*this);
#endif
registerCodecLZ4HC(*this);
registerCodecMultiple(*this);
#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
@ -203,9 +206,6 @@ CompressionCodecFactory::CompressionCodecFactory()
registerCodecFPC(*this);
#ifdef ENABLE_QPL_COMPRESSION
registerCodecDeflateQpl(*this);
#endif
#ifdef ENABLE_QATZSTD_COMPRESSION
registerCodecQATZSTD(*this);
#endif
registerCodecGCD(*this);
#endif