Rename from ZSTDQPL to ZSTD_QPL

This commit is contained in:
Robert Schulze 2024-01-07 17:47:31 +00:00
parent 21d14e488b
commit 0f51e32bb5
No known key found for this signature in database
GPG Key ID: 26703B55FB13728A
12 changed files with 98 additions and 60 deletions

View File

@ -78,7 +78,7 @@ else () # In-tree build
${QAT_USDM_INCLUDE_DIR}
${ZSTD_LIBRARY_DIR}
${LIBQAT_HEADER_DIR})
target_compile_definitions(_qatzstd_plugin PRIVATE -DDEBUGLEVEL=0 PUBLIC -DENABLE_ZSTDQAT_COMPRESSION -DINTREE)
target_compile_definitions(_qatzstd_plugin PRIVATE -DDEBUGLEVEL=0 PUBLIC -DENABLE_ZSTD_QAT_CODEC -DINTREE)
target_include_directories(_qatzstd_plugin SYSTEM PUBLIC $<BUILD_INTERFACE:${QATZSTD_SRC_DIR}> $<INSTALL_INTERFACE:include>)
add_library (ch_contrib::qatzstd_plugin ALIAS _qatzstd_plugin)
endif ()

View File

@ -4773,6 +4773,28 @@ Type: Int64
Default: 0
## enable_deflate_qpl_codec {#enable_deflate_qpl_codec}
If turned on, the DEFLATE_QPL codec may be used to compress columns.
Possible values:
- 0 - Disabled
- 1 - Enabled
Type: Bool
## enable_zstd_qat_codec {#enable_zstd_qat_codec}
If turned on, the ZSTD_QAT codec may be used to compress columns.
Possible values:
- 0 - Disabled
- 1 - Enabled
Type: Bool
## rewrite_count_distinct_if_with_count_distinct_implementation
Allows you to rewrite `countDistcintIf` with [count_distinct_implementation](#count_distinct_implementation) setting.
@ -5157,4 +5179,4 @@ The value 0 means that you can delete all tables without any restrictions.
:::note
This query setting overwrites its server setting equivalent, see [max_table_size_to_drop](/docs/en/operations/server-configuration-parameters/settings.md/#max-table-size-to-drop)
:::
:::

View File

@ -372,22 +372,23 @@ ClickHouse supports general purpose codecs and specialized codecs.
#### ZSTD
`ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: \[1, 22\]. Default value: 1.
`ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: \[1, 22\]. Default level: 1.
High compression levels are useful for asymmetric scenarios, like compress once, decompress repeatedly. Higher levels mean better compression and higher CPU usage.
#### ZSTDQAT
#### ZSTD_QAT
`ZSTDQAT[(level)]` — ZSTD QAT (hardware-accelerated compression) implemented by [QAT-ZSTD-Plugin](https://github.com/intel/QAT-ZSTD-Plugin) with configurable level. Default level: 1. Setting `level <= 0` applies the default level. Possible levels: \[1, 12\]. Recommended level range: \[6, 12\].
`ZSTD_QAT[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable level, implemented by [Intel® QATlib](https://github.com/intel/qatlib) and [Intel® QAT ZSTD Plugin](https://github.com/intel/QAT-ZSTD-Plugin). Possible levels: \[1, 12\]. Default level: 1. Recommended level range: \[6, 12\]. Some limitations apply:
- ZSTDQAT is disabled by default and can only be used after setting configuration parameter `enable_zstd_qat_codec = 1`.
- ZSTDQAT tries to use an Intel® QAT offloading device ([QuickAssist Technology](https://www.intel.com/content/www/us/en/developer/topic-technology/open/quick-assist-technology/overview.html)). If no such device was found, it will fallback to ZSTD compression in software.
- ZSTD_QAT is disabled by default and can only be used after enabling configuration setting [enable_zstd_qat_codec](../../../operations/settings/settings.md#enable_zstd_qat_codec).
- For compression, ZSTD_QAT tries to use an Intel® QAT offloading device ([QuickAssist Technology](https://www.intel.com/content/www/us/en/developer/topic-technology/open/quick-assist-technology/overview.html)). If no such device was found, it will fallback to ZSTD compression in software.
- Decompression is always performed in software.
#### DEFLATE_QPL
`DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_deflate_qpl_codec = 1`.
- DEFLATE_QPL is disabled by default and can only be used after enabling configuration setting [enable_deflate_qpl_codec](../../../operations/settings/settings.md#enable_deflate_qpl_codec).
- DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
- DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.

View File

@ -51,7 +51,7 @@ UInt32 CompressionCodecZSTD::doCompressData(const char * source, UInt32 source_s
ZSTD_freeCCtx(cctx);
if (ZSTD_isError(compressed_size))
throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD codec: {}", std::string(ZSTD_getErrorName(compressed_size)));
throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD codec: {}", ZSTD_getErrorName(compressed_size));
return static_cast<UInt32>(compressed_size);
}
@ -65,13 +65,19 @@ void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_s
throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress ZSTD-encoded data: {}", std::string(ZSTD_getErrorName(res)));
}
CompressionCodecZSTD::CompressionCodecZSTD(int level_, int window_log_) : level(level_), enable_long_range(true), window_log(window_log_)
CompressionCodecZSTD::CompressionCodecZSTD(int level_, int window_log_)
: level(level_)
, enable_long_range(true)
, window_log(window_log_)
{
setCodecDescription(
"ZSTD", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level)), std::make_shared<ASTLiteral>(static_cast<UInt64>(window_log))});
}
CompressionCodecZSTD::CompressionCodecZSTD(int level_) : level(level_), enable_long_range(false), window_log(0)
CompressionCodecZSTD::CompressionCodecZSTD(int level_)
: level(level_)
, enable_long_range(false)
, window_log(0)
{
setCodecDescription("ZSTD", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
}

View File

@ -1,15 +1,12 @@
#ifdef ENABLE_ZSTDQAT_COMPRESSION
#include <Compression/CompressionCodecZSTD.h>
#include <Compression/CompressionInfo.h>
#include <Compression/CompressionFactory.h>
#include <zstd.h>
#include <Parsers/IAST.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <Common/typeid_cast.h>
#include <Poco/Logger.h>
#ifdef ENABLE_ZSTD_QAT_CODEC
#include <Common/logger_useful.h>
#include <Compression/CompressionCodecZSTD.h>
#include <Compression/CompressionFactory.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/IAST.h>
#include <qatseqprod.h>
#include <zstd.h>
namespace DB
{
@ -20,18 +17,18 @@ namespace ErrorCodes
extern const int ILLEGAL_CODEC_PARAMETER;
}
/// Hardware-accelerated ZSTD. Supports only compression so far.
class CompressionCodecZSTDQAT : public CompressionCodecZSTD
{
public:
/// QAT Hardware only supports compression levels L1 to L12
static constexpr auto ZSTDQAT_SUPPORTED_MIN_LEVEL = 1;
static constexpr auto ZSTDQAT_SUPPORTED_MAX_LEVEL = 12;
explicit CompressionCodecZSTDQAT(int level_);
~CompressionCodecZSTDQAT() override;
protected:
bool isZSTDQAT() const override { return true; }
/// TODO: So far, QAT hardware only support compression. For next generation in future, it will support decompression as well.
bool isZstdQat() const override { return true; }
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
private:
@ -46,28 +43,29 @@ UInt32 CompressionCodecZSTDQAT::doCompressData(const char * source, UInt32 sourc
size_t compressed_size = ZSTD_compress2(cctx, dest, ZSTD_compressBound(source_size), source, source_size);
if (ZSTD_isError(compressed_size))
throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD codec: {}", std::string(ZSTD_getErrorName(compressed_size)));
throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD_QAT codec: {}", ZSTD_getErrorName(compressed_size));
return static_cast<UInt32>(compressed_size);
}
void registerCodecZSTDQAT(CompressionCodecFactory & factory)
{
factory.registerCompressionCodec("ZSTDQAT", {}, [&](const ASTPtr & arguments) -> CompressionCodecPtr
factory.registerCompressionCodec("ZSTD_QAT", {}, [&](const ASTPtr & arguments) -> CompressionCodecPtr
{
int level = CompressionCodecZSTD::ZSTD_DEFAULT_LEVEL;
if (arguments && !arguments->children.empty())
{
if (arguments->children.size() > 1)
throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "ZSTDQAT codec must have 1 parameter, given {}", arguments->children.size());
throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "ZSTD_QAT codec must have 1 parameter, given {}", arguments->children.size());
const auto children = arguments->children;
const auto * literal = children[0]->as<ASTLiteral>();
if (!literal)
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "ZSTDQAT codec argument must be integer");
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "ZSTD_QAT codec argument must be integer");
level = static_cast<int>(literal->value.safeGet<UInt64>());
if (level > CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL || level < CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL)
if (level < CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL || level > CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL )
/// that's a hardware limitation
throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER,
"ZSTDQAT codec doesn't support level more than {} and lower than {} , given {}",
CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL, CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL, level);
@ -78,39 +76,42 @@ void registerCodecZSTDQAT(CompressionCodecFactory & factory)
}
CompressionCodecZSTDQAT::CompressionCodecZSTDQAT(int level_)
: CompressionCodecZSTD(level_), level(level_), log(&Poco::Logger::get("CompressionCodecZSTDQAT"))
: CompressionCodecZSTD(level_)
, level(level_)
, log(&Poco::Logger::get("CompressionCodecZSTDQAT"))
{
setCodecDescription("ZSTDQAT", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
setCodecDescription("ZSTD_QAT", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
cctx = ZSTD_createCCtx();
/// Start QAT device, start QAT device at any time before compression job started
int res = QZSTD_startQatDevice();
LOG_WARNING(log, "Initialization of hardware-assisted(QAT) ZSTD codec result: {} ", static_cast<UInt32>(res));
/// Create sequence producer state for QAT sequence producer
LOG_DEBUG(log, "Initialization of ZSTD_QAT codec, status: {} ", res);
sequenceProducerState = QZSTD_createSeqProdState();
/// register qatSequenceProducer
ZSTD_registerSequenceProducer(
cctx,
sequenceProducerState,
qatSequenceProducer
);
/// Enable sequence producer fallback
ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableSeqProducerFallback, 1);
ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level);
}
CompressionCodecZSTDQAT::~CompressionCodecZSTDQAT()
{
/// Free sequence producer state
if (sequenceProducerState != nullptr)
{
QZSTD_freeSeqProdState(sequenceProducerState);
sequenceProducerState = nullptr;
}
if (cctx != nullptr)
{
auto status = ZSTD_freeCCtx(cctx);
size_t status = ZSTD_freeCCtx(cctx);
if (status != 0)
LOG_WARNING(log, "ZSTD_freeCCtx failed with status: {} ", static_cast<UInt32>(status));
LOG_WARNING(log, "ZSTD_freeCCtx failed with status: {} ", status);
cctx = nullptr;
}
}

View File

@ -167,7 +167,7 @@ void registerCodecNone(CompressionCodecFactory & factory);
void registerCodecLZ4(CompressionCodecFactory & factory);
void registerCodecLZ4HC(CompressionCodecFactory & factory);
void registerCodecZSTD(CompressionCodecFactory & factory);
#ifdef ENABLE_ZSTDQAT_COMPRESSION
#ifdef ENABLE_ZSTD_QAT_CODEC
void registerCodecZSTDQAT(CompressionCodecFactory & factory);
#endif
void registerCodecMultiple(CompressionCodecFactory & factory);
@ -192,7 +192,7 @@ CompressionCodecFactory::CompressionCodecFactory()
registerCodecNone(*this);
registerCodecLZ4(*this);
registerCodecZSTD(*this);
#ifdef ENABLE_ZSTDQAT_COMPRESSION
#ifdef ENABLE_ZSTD_QAT_CODEC
registerCodecZSTDQAT(*this);
#endif
registerCodecLZ4HC(*this);

View File

@ -165,7 +165,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
" You can enable it with the 'enable_deflate_qpl_codec' setting.",
codec_family_name);
if (!enable_zstd_qat_codec && result_codec->isZSTDQAT())
if (!enable_zstd_qat_codec && result_codec->isZstdQat())
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Codec {} is disabled by default."
" You can enable it with the 'enable_zstd_qat_codec' setting.",

View File

@ -121,8 +121,8 @@ public:
/// Is this the DEFLATE_QPL codec?
virtual bool isDeflateQpl() const { return false; }
/// Is this the ZSTDQAT codec?
virtual bool isZSTDQAT() const { return false; }
/// Is this the ZSTD_QAT codec?
virtual bool isZstdQat() const { return false; }
/// If it does nothing.
virtual bool isNone() const { return false; }

View File

@ -352,7 +352,7 @@ class IColumn;
M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \
M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \
M(Bool, enable_zstd_qat_codec, false, "Enable/disable the ZSTDQAT codec.", 0) \
M(Bool, enable_zstd_qat_codec, false, "Enable/disable the ZSTD_QAT codec.", 0) \
M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \

View File

@ -0,0 +1,6 @@
CREATE TABLE default.compression_codec\n(\n `id` UInt64 CODEC(ZSTD_QAT(1)),\n `data` String CODEC(ZSTD_QAT(1)),\n `ddd` Date CODEC(ZSTD_QAT(1)),\n `ddd32` Date32 CODEC(ZSTD_QAT(1)),\n `somenum` Float64 CODEC(ZSTD_QAT(1)),\n `somestr` FixedString(3) CODEC(ZSTD_QAT(1)),\n `othernum` Int64 CODEC(ZSTD_QAT(1)),\n `somearray` Array(UInt8) CODEC(ZSTD_QAT(1)),\n `somemap` Map(String, UInt32) CODEC(ZSTD_QAT(1)),\n `sometuple` Tuple(UInt16, UInt64) CODEC(ZSTD_QAT(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
1 hello 2018-12-14 2018-12-14 1.1 aaa 5 [1,2,3] {'k1':1,'k2':2} (1,2)
2 world 2018-12-15 2018-12-15 2.2 bbb 6 [4,5,6] {'k3':3,'k4':4} (3,4)
3 ! 2018-12-16 2018-12-16 3.3 ccc 7 [7,8,9] {'k5':5,'k6':6} (5,6)
2
10001

View File

@ -1,8 +1,6 @@
--Tags: no-fasttest, no-cpu-aarch64, no-cpu-s390x
-- no-fasttest because ZSTDQAT isn't available in fasttest
-- no-cpu-aarch64 and no-cpu-s390x because ZSTDQAT is x86-only
-- A bunch of random DDLs to test the ZSTDQAT codec.
-- no-fasttest because ZSTD_QAT isn't available in fasttest
-- no-cpu-aarch64 and no-cpu-s390x because ZSTD_QAT is x86-only
SET enable_zstd_qat_codec = 1;
@ -12,17 +10,21 @@ SET send_logs_level = 'fatal';
DROP TABLE IF EXISTS compression_codec;
-- negative test
CREATE TABLE compression_codec(id UInt64 CODEC(ZSTD_QAT(0))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER }
CREATE TABLE compression_codec(id UInt64 CODEC(ZSTD_QAT(13))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER }
CREATE TABLE compression_codec(
id UInt64 CODEC(ZSTDQAT),
data String CODEC(ZSTDQAT),
ddd Date CODEC(ZSTDQAT),
ddd32 Date32 CODEC(ZSTDQAT),
somenum Float64 CODEC(ZSTDQAT),
somestr FixedString(3) CODEC(ZSTDQAT),
othernum Int64 CODEC(ZSTDQAT),
somearray Array(UInt8) CODEC(ZSTDQAT),
somemap Map(String, UInt32) CODEC(ZSTDQAT),
sometuple Tuple(UInt16, UInt64) CODEC(ZSTDQAT),
id UInt64 CODEC(ZSTD_QAT),
data String CODEC(ZSTD_QAT),
ddd Date CODEC(ZSTD_QAT),
ddd32 Date32 CODEC(ZSTD_QAT),
somenum Float64 CODEC(ZSTD_QAT),
somestr FixedString(3) CODEC(ZSTD_QAT),
othernum Int64 CODEC(ZSTD_QAT),
somearray Array(UInt8) CODEC(ZSTD_QAT),
somemap Map(String, UInt32) CODEC(ZSTD_QAT),
sometuple Tuple(UInt16, UInt64) CODEC(ZSTD_QAT),
) ENGINE = MergeTree() ORDER BY tuple();
SHOW CREATE TABLE compression_codec;

View File

@ -1001,7 +1001,7 @@ ZooKeeperRequest
ZooKeeperSession
ZooKeeperWatch
ZooKeepers
ZSTDQAT
ZSTD_QAT
aarch
accurateCast
accurateCastOrDefault