Rename from ZSTDQPL to ZSTD_QPL

contrib/QAT-ZSTD-Plugin-cmake/CMakeLists.txt

@@ -78,7 +78,7 @@ else () # In-tree build
         ${QAT_USDM_INCLUDE_DIR}
         ${ZSTD_LIBRARY_DIR}
         ${LIBQAT_HEADER_DIR})
-    target_compile_definitions(_qatzstd_plugin PRIVATE -DDEBUGLEVEL=0 PUBLIC -DENABLE_ZSTDQAT_COMPRESSION -DINTREE)
+    target_compile_definitions(_qatzstd_plugin PRIVATE -DDEBUGLEVEL=0 PUBLIC -DENABLE_ZSTD_QAT_CODEC -DINTREE)
     target_include_directories(_qatzstd_plugin SYSTEM PUBLIC $<BUILD_INTERFACE:${QATZSTD_SRC_DIR}> $<INSTALL_INTERFACE:include>)
     add_library (ch_contrib::qatzstd_plugin ALIAS _qatzstd_plugin)
 endif ()

docs/en/operations/settings/settings.md

@@ -4773,6 +4773,28 @@ Type: Int64
 
 Default: 0
 
+## enable_deflate_qpl_codec {#enable_deflate_qpl_codec}
+
+If turned on, the DEFLATE_QPL codec may be used to compress columns.
+
+Possible values:
+
+- 0 - Disabled
+- 1 - Enabled
+
+Type: Bool
+
+## enable_zstd_qat_codec {#enable_zstd_qat_codec}
+
+If turned on, the ZSTD_QAT codec may be used to compress columns.
+
+Possible values:
+
+- 0 - Disabled
+- 1 - Enabled
+
+Type: Bool
+
 ## rewrite_count_distinct_if_with_count_distinct_implementation
 
 Allows you to rewrite `countDistcintIf` with [count_distinct_implementation](#count_distinct_implementation) setting.

docs/en/sql-reference/statements/create/table.md

@@ -372,22 +372,23 @@ ClickHouse supports general purpose codecs and specialized codecs.
 
 #### ZSTD
 
-`ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: \[1, 22\]. Default value: 1.
+`ZSTD[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable `level`. Possible levels: \[1, 22\]. Default level: 1.
 
 High compression levels are useful for asymmetric scenarios, like compress once, decompress repeatedly. Higher levels mean better compression and higher CPU usage.
 
-#### ZSTDQAT
+#### ZSTD_QAT
 
-`ZSTDQAT[(level)]` — ZSTD QAT (hardware-accelerated compression) implemented by [QAT-ZSTD-Plugin](https://github.com/intel/QAT-ZSTD-Plugin) with configurable level. Default level: 1. Setting `level <= 0` applies the default level. Possible levels: \[1, 12\]. Recommended level range: \[6, 12\].
+`ZSTD_QAT[(level)]` — [ZSTD compression algorithm](https://en.wikipedia.org/wiki/Zstandard) with configurable level, implemented by [Intel® QATlib](https://github.com/intel/qatlib) and [Intel® QAT ZSTD Plugin](https://github.com/intel/QAT-ZSTD-Plugin). Possible levels: \[1, 12\]. Default level: 1. Recommended level range: \[6, 12\]. Some limitations apply:
 
-- ZSTDQAT is disabled by default and can only be used after setting configuration parameter `enable_zstd_qat_codec = 1`.
+- ZSTD_QAT is disabled by default and can only be used after enabling configuration setting [enable_zstd_qat_codec](../../../operations/settings/settings.md#enable_zstd_qat_codec).
-- ZSTDQAT tries to use an Intel® QAT offloading device ([QuickAssist Technology](https://www.intel.com/content/www/us/en/developer/topic-technology/open/quick-assist-technology/overview.html)). If no such device was found, it will fallback to ZSTD compression in software.
+- For compression, ZSTD_QAT tries to use an Intel® QAT offloading device ([QuickAssist Technology](https://www.intel.com/content/www/us/en/developer/topic-technology/open/quick-assist-technology/overview.html)). If no such device was found, it will fallback to ZSTD compression in software.
+- Decompression is always performed in software.
 
 #### DEFLATE_QPL
 
 `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
 
-- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_deflate_qpl_codec = 1`.
+- DEFLATE_QPL is disabled by default and can only be used after enabling configuration setting [enable_deflate_qpl_codec](../../../operations/settings/settings.md#enable_deflate_qpl_codec).
 - DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
 - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
 - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.

src/Compression/CompressionCodecZSTD.cpp

@@ -51,7 +51,7 @@ UInt32 CompressionCodecZSTD::doCompressData(const char * source, UInt32 source_s
     ZSTD_freeCCtx(cctx);
 
     if (ZSTD_isError(compressed_size))
-        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD codec: {}", std::string(ZSTD_getErrorName(compressed_size)));
+        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD codec: {}", ZSTD_getErrorName(compressed_size));
 
     return static_cast<UInt32>(compressed_size);
 }
@@ -65,13 +65,19 @@ void CompressionCodecZSTD::doDecompressData(const char * source, UInt32 source_s
         throw Exception(ErrorCodes::CANNOT_DECOMPRESS, "Cannot decompress ZSTD-encoded data: {}", std::string(ZSTD_getErrorName(res)));
 }
 
-CompressionCodecZSTD::CompressionCodecZSTD(int level_, int window_log_) : level(level_), enable_long_range(true), window_log(window_log_)
+CompressionCodecZSTD::CompressionCodecZSTD(int level_, int window_log_)
+    : level(level_)
+    , enable_long_range(true)
+    , window_log(window_log_)
 {
     setCodecDescription(
         "ZSTD", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level)), std::make_shared<ASTLiteral>(static_cast<UInt64>(window_log))});
 }
 
-CompressionCodecZSTD::CompressionCodecZSTD(int level_) : level(level_), enable_long_range(false), window_log(0)
+CompressionCodecZSTD::CompressionCodecZSTD(int level_)
+    : level(level_)
+    , enable_long_range(false)
+    , window_log(0)
 {
     setCodecDescription("ZSTD", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
 }

src/Compression/CompressionCodecZSTDQAT.cpp

@@ -1,15 +1,12 @@
-#ifdef ENABLE_ZSTDQAT_COMPRESSION
+#ifdef ENABLE_ZSTD_QAT_CODEC
-#include <Compression/CompressionCodecZSTD.h>
-#include <Compression/CompressionInfo.h>
-#include <Compression/CompressionFactory.h>
-#include <zstd.h>
-#include <Parsers/IAST.h>
-#include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTFunction.h>
-#include <Common/typeid_cast.h>
-#include <Poco/Logger.h>
 #include <Common/logger_useful.h>
+#include <Compression/CompressionCodecZSTD.h>
+#include <Compression/CompressionFactory.h>
+#include <Parsers/ASTLiteral.h>
+#include <Parsers/IAST.h>
+
 #include <qatseqprod.h>
+#include <zstd.h>
 
 namespace DB
 {
@@ -20,18 +17,18 @@ namespace ErrorCodes
     extern const int ILLEGAL_CODEC_PARAMETER;
 }
 
+/// Hardware-accelerated ZSTD. Supports only compression so far.
 class CompressionCodecZSTDQAT : public CompressionCodecZSTD
 {
 public:
-    /// QAT Hardware only supports compression levels L1 to L12
     static constexpr auto ZSTDQAT_SUPPORTED_MIN_LEVEL = 1;
     static constexpr auto ZSTDQAT_SUPPORTED_MAX_LEVEL = 12;
+
     explicit CompressionCodecZSTDQAT(int level_);
     ~CompressionCodecZSTDQAT() override;
 
 protected:
-    bool isZSTDQAT() const override { return true; }
+    bool isZstdQat() const override { return true; }
-    /// TODO: So far, QAT hardware only support compression. For next generation in future, it will support decompression as well.
     UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
 
 private:
@@ -46,28 +43,29 @@ UInt32 CompressionCodecZSTDQAT::doCompressData(const char * source, UInt32 sourc
     size_t compressed_size = ZSTD_compress2(cctx, dest, ZSTD_compressBound(source_size), source, source_size);
 
     if (ZSTD_isError(compressed_size))
-        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD codec: {}", std::string(ZSTD_getErrorName(compressed_size)));
+        throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot compress with ZSTD_QAT codec: {}", ZSTD_getErrorName(compressed_size));
 
     return static_cast<UInt32>(compressed_size);
 }
 
 void registerCodecZSTDQAT(CompressionCodecFactory & factory)
 {
-    factory.registerCompressionCodec("ZSTDQAT", {}, [&](const ASTPtr & arguments) -> CompressionCodecPtr
+    factory.registerCompressionCodec("ZSTD_QAT", {}, [&](const ASTPtr & arguments) -> CompressionCodecPtr
     {
         int level = CompressionCodecZSTD::ZSTD_DEFAULT_LEVEL;
         if (arguments && !arguments->children.empty())
         {
             if (arguments->children.size() > 1)
-                throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "ZSTDQAT codec must have 1 parameter, given {}", arguments->children.size());
+                throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_CODEC_TYPE, "ZSTD_QAT codec must have 1 parameter, given {}", arguments->children.size());
 
             const auto children = arguments->children;
             const auto * literal = children[0]->as<ASTLiteral>();
             if (!literal)
-                throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "ZSTDQAT codec argument must be integer");
+                throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER, "ZSTD_QAT codec argument must be integer");
 
             level = static_cast<int>(literal->value.safeGet<UInt64>());
-            if (level > CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL || level < CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL)
+            if (level < CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL || level > CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL )
+                /// that's a hardware limitation
                 throw Exception(ErrorCodes::ILLEGAL_CODEC_PARAMETER,
                     "ZSTDQAT codec doesn't support level more than {} and lower than {} , given {}",
                     CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MAX_LEVEL, CompressionCodecZSTDQAT::ZSTDQAT_SUPPORTED_MIN_LEVEL, level);
@@ -78,39 +76,42 @@ void registerCodecZSTDQAT(CompressionCodecFactory & factory)
 }
 
 CompressionCodecZSTDQAT::CompressionCodecZSTDQAT(int level_)
-    : CompressionCodecZSTD(level_), level(level_), log(&Poco::Logger::get("CompressionCodecZSTDQAT"))
+    : CompressionCodecZSTD(level_)
+    , level(level_)
+    , log(&Poco::Logger::get("CompressionCodecZSTDQAT"))
 {
-    setCodecDescription("ZSTDQAT", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
+    setCodecDescription("ZSTD_QAT", {std::make_shared<ASTLiteral>(static_cast<UInt64>(level))});
+
     cctx = ZSTD_createCCtx();
-    /// Start QAT device, start QAT device at any time before compression job started
+
     int res = QZSTD_startQatDevice();
-    LOG_WARNING(log, "Initialization of hardware-assisted(QAT) ZSTD codec result: {} ", static_cast<UInt32>(res));
+    LOG_DEBUG(log, "Initialization of ZSTD_QAT codec, status: {} ", res);
-    /// Create sequence producer state for QAT sequence producer
+
     sequenceProducerState = QZSTD_createSeqProdState();
-    /// register qatSequenceProducer
+
     ZSTD_registerSequenceProducer(
         cctx,
         sequenceProducerState,
         qatSequenceProducer
     );
-    /// Enable sequence producer fallback
+
     ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableSeqProducerFallback, 1);
     ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level);
 }
 
 CompressionCodecZSTDQAT::~CompressionCodecZSTDQAT()
 {
-    /// Free sequence producer state
     if (sequenceProducerState != nullptr)
     {
         QZSTD_freeSeqProdState(sequenceProducerState);
         sequenceProducerState = nullptr;
     }
+
     if (cctx != nullptr)
     {
-        auto status = ZSTD_freeCCtx(cctx);
+        size_t status = ZSTD_freeCCtx(cctx);
         if (status != 0)
-            LOG_WARNING(log, "ZSTD_freeCCtx failed with status: {} ", static_cast<UInt32>(status));
+            LOG_WARNING(log, "ZSTD_freeCCtx failed with status: {} ", status);
         cctx = nullptr;
     }
 }

src/Compression/CompressionFactory.cpp

@@ -167,7 +167,7 @@ void registerCodecNone(CompressionCodecFactory & factory);
 void registerCodecLZ4(CompressionCodecFactory & factory);
 void registerCodecLZ4HC(CompressionCodecFactory & factory);
 void registerCodecZSTD(CompressionCodecFactory & factory);
-#ifdef ENABLE_ZSTDQAT_COMPRESSION
+#ifdef ENABLE_ZSTD_QAT_CODEC
 void registerCodecZSTDQAT(CompressionCodecFactory & factory);
 #endif
 void registerCodecMultiple(CompressionCodecFactory & factory);
@@ -192,7 +192,7 @@ CompressionCodecFactory::CompressionCodecFactory()
     registerCodecNone(*this);
     registerCodecLZ4(*this);
     registerCodecZSTD(*this);
-#ifdef ENABLE_ZSTDQAT_COMPRESSION
+#ifdef ENABLE_ZSTD_QAT_CODEC
     registerCodecZSTDQAT(*this);
 #endif
     registerCodecLZ4HC(*this);

src/Compression/CompressionFactoryAdditions.cpp

@@ -165,7 +165,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
                         " You can enable it with the 'enable_deflate_qpl_codec' setting.",
                         codec_family_name);
 
-                if (!enable_zstd_qat_codec && result_codec->isZSTDQAT())
+                if (!enable_zstd_qat_codec && result_codec->isZstdQat())
                     throw Exception(ErrorCodes::BAD_ARGUMENTS,
                         "Codec {} is disabled by default."
                         " You can enable it with the 'enable_zstd_qat_codec' setting.",

src/Compression/ICompressionCodec.h

@@ -121,8 +121,8 @@ public:
     /// Is this the DEFLATE_QPL codec?
     virtual bool isDeflateQpl() const { return false; }
 
-    /// Is this the ZSTDQAT codec?
+    /// Is this the ZSTD_QAT codec?
-    virtual bool isZSTDQAT() const { return false; }
+    virtual bool isZstdQat() const { return false; }
 
     /// If it does nothing.
     virtual bool isNone() const { return false; }

src/Core/Settings.h

@@ -352,7 +352,7 @@ class IColumn;
     M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
     M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \
     M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \
-    M(Bool, enable_zstd_qat_codec, false, "Enable/disable the ZSTDQAT codec.", 0) \
+    M(Bool, enable_zstd_qat_codec, false, "Enable/disable the ZSTD_QAT codec.", 0) \
     M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
     M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \

tests/queries/0_stateless/00804_test_zstd_qat_codec_compression.reference

@@ -0,0 +1,6 @@
+CREATE TABLE default.compression_codec\n(\n    `id` UInt64 CODEC(ZSTD_QAT(1)),\n    `data` String CODEC(ZSTD_QAT(1)),\n    `ddd` Date CODEC(ZSTD_QAT(1)),\n    `ddd32` Date32 CODEC(ZSTD_QAT(1)),\n    `somenum` Float64 CODEC(ZSTD_QAT(1)),\n    `somestr` FixedString(3) CODEC(ZSTD_QAT(1)),\n    `othernum` Int64 CODEC(ZSTD_QAT(1)),\n    `somearray` Array(UInt8) CODEC(ZSTD_QAT(1)),\n    `somemap` Map(String, UInt32) CODEC(ZSTD_QAT(1)),\n    `sometuple` Tuple(UInt16, UInt64) CODEC(ZSTD_QAT(1))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192
+1	hello	2018-12-14	2018-12-14	1.1	aaa	5	[1,2,3]	{'k1':1,'k2':2}	(1,2)
+2	world	2018-12-15	2018-12-15	2.2	bbb	6	[4,5,6]	{'k3':3,'k4':4}	(3,4)
+3	!	2018-12-16	2018-12-16	3.3	ccc	7	[7,8,9]	{'k5':5,'k6':6}	(5,6)
+2
+10001

tests/queries/0_stateless/00804_test_zstd_qat_codec_compression.sql

@@ -1,8 +1,6 @@
 --Tags: no-fasttest, no-cpu-aarch64, no-cpu-s390x
--- no-fasttest because ZSTDQAT isn't available in fasttest
+-- no-fasttest because ZSTD_QAT isn't available in fasttest
--- no-cpu-aarch64 and no-cpu-s390x because ZSTDQAT is x86-only
+-- no-cpu-aarch64 and no-cpu-s390x because ZSTD_QAT is x86-only
-
--- A bunch of random DDLs to test the ZSTDQAT codec.
 
 SET enable_zstd_qat_codec = 1;
 
@@ -12,17 +10,21 @@ SET send_logs_level = 'fatal';
 
 DROP TABLE IF EXISTS compression_codec;
 
+-- negative test
+CREATE TABLE compression_codec(id UInt64 CODEC(ZSTD_QAT(0))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER }
+CREATE TABLE compression_codec(id UInt64 CODEC(ZSTD_QAT(13))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError ILLEGAL_CODEC_PARAMETER }
+
 CREATE TABLE compression_codec(
-    id UInt64 CODEC(ZSTDQAT),
+    id UInt64 CODEC(ZSTD_QAT),
-    data String CODEC(ZSTDQAT),
+    data String CODEC(ZSTD_QAT),
-    ddd Date CODEC(ZSTDQAT),
+    ddd Date CODEC(ZSTD_QAT),
-    ddd32 Date32 CODEC(ZSTDQAT),
+    ddd32 Date32 CODEC(ZSTD_QAT),
-    somenum Float64 CODEC(ZSTDQAT),
+    somenum Float64 CODEC(ZSTD_QAT),
-    somestr FixedString(3) CODEC(ZSTDQAT),
+    somestr FixedString(3) CODEC(ZSTD_QAT),
-    othernum Int64 CODEC(ZSTDQAT),
+    othernum Int64 CODEC(ZSTD_QAT),
-    somearray Array(UInt8) CODEC(ZSTDQAT),
+    somearray Array(UInt8) CODEC(ZSTD_QAT),
-    somemap Map(String, UInt32) CODEC(ZSTDQAT),
+    somemap Map(String, UInt32) CODEC(ZSTD_QAT),
-    sometuple Tuple(UInt16, UInt64) CODEC(ZSTDQAT),
+    sometuple Tuple(UInt16, UInt64) CODEC(ZSTD_QAT),
 ) ENGINE = MergeTree() ORDER BY tuple();
 
 SHOW CREATE TABLE compression_codec;

utils/check-style/aspell-ignore/en/aspell-dict.txt

@@ -1001,7 +1001,7 @@ ZooKeeperRequest
 ZooKeeperSession
 ZooKeeperWatch
 ZooKeepers
-ZSTDQAT
+ZSTD_QAT
 aarch
 accurateCast
 accurateCastOrDefault