Merge remote-tracking branch 'upstream/master' into #31363_format_template_configure_in_settings

src/Coordination/KeeperStateMachine.cpp

@@ -136,12 +136,12 @@ namespace
 {
 
 void assertDigest(
-    const KeeperStorage::Digest & first,
-    const KeeperStorage::Digest & second,
+    const KeeperStorage::Digest & expected,
+    const KeeperStorage::Digest & actual,
     const Coordination::ZooKeeperRequest & request,
     bool committing)
 {
-    if (!KeeperStorage::checkDigest(first, second))
+    if (!KeeperStorage::checkDigest(expected, actual))
     {
         LOG_FATAL(
             getLogger("KeeperStateMachine"),
@@ -149,9 +149,9 @@ void assertDigest(
             "{}). Keeper will terminate to avoid inconsistencies.\nExtra information about the request:\n{}",
             committing ? "committing" : "preprocessing",
             request.getOpNum(),
-            first.value,
-            second.value,
-            first.version,
+            expected.value,
+            actual.value,
+            expected.version,
             request.toString());
         std::terminate();
     }

src/Coordination/KeeperStorage.cpp

@@ -174,7 +174,6 @@ uint64_t calculateDigest(std::string_view path, std::string_view data, const Kee
 
     hash.update(data);
 
-    hash.update(stat.czxid);
     hash.update(stat.czxid);
     hash.update(stat.mzxid);
     hash.update(stat.ctime);
@@ -183,7 +182,6 @@ uint64_t calculateDigest(std::string_view path, std::string_view data, const Kee
     hash.update(stat.cversion);
     hash.update(stat.aversion);
     hash.update(stat.ephemeralOwner);
-    hash.update(data.length());
     hash.update(stat.numChildren);
     hash.update(stat.pzxid);
 
@@ -2531,6 +2529,17 @@ void KeeperStorage::recalculateStats()
     container.recalculateDataSize();
 }
 
+bool KeeperStorage::checkDigest(const Digest & first, const Digest & second)
+{
+    if (first.version != second.version)
+        return true;
+
+    if (first.version == DigestVersion::NO_DIGEST)
+        return true;
+
+    return first.value == second.value;
+}
+
 String KeeperStorage::generateDigest(const String & userdata)
 {
     std::vector<String> user_password;

src/Coordination/KeeperStorage.h

@@ -95,10 +95,11 @@ public:
     {
         NO_DIGEST = 0,
         V1 = 1,
-        V2 = 2  // added system nodes that modify the digest on startup so digest from V0 is invalid
+        V2 = 2, // added system nodes that modify the digest on startup so digest from V0 is invalid
+        V3 = 3  // fixed bug with casting, removed duplicate czxid usage
     };
 
-    static constexpr auto CURRENT_DIGEST_VERSION = DigestVersion::V2;
+    static constexpr auto CURRENT_DIGEST_VERSION = DigestVersion::V3;
 
     struct ResponseForSession
     {
@@ -113,16 +114,7 @@ public:
         uint64_t value{0};
     };
 
-    static bool checkDigest(const Digest & first, const Digest & second)
-    {
-        if (first.version != second.version)
-            return true;
-
-        if (first.version == DigestVersion::NO_DIGEST)
-            return true;
-
-        return first.value == second.value;
-    }
+    static bool checkDigest(const Digest & first, const Digest & second);
 
     static String generateDigest(const String & userdata);
 

src/IO/ReadHelpers.cpp

@@ -6,6 +6,7 @@
 #include <Formats/FormatSettings.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/BufferWithOwnMemory.h>
+#include <IO/PeekableReadBuffer.h>
 #include <IO/readFloatText.h>
 #include <IO/Operators.h>
 #include <base/find_symbols.h>

src/IO/ReadHelpers.h

@@ -38,7 +38,6 @@
 #include <IO/CompressionMethod.h>
 #include <IO/ReadBuffer.h>
 #include <IO/ReadBufferFromMemory.h>
-#include <IO/PeekableReadBuffer.h>
 #include <IO/VarInt.h>
 
 #include <pcg_random.hpp>
@@ -51,6 +50,7 @@ namespace DB
 
 template <typename Allocator>
 struct Memory;
+class PeekableReadBuffer;
 
 namespace ErrorCodes
 {

src/Processors/Formats/Impl/CSVRowInputFormat.h

@@ -7,6 +7,7 @@
 #include <Processors/Formats/RowInputFormatWithNamesAndTypes.h>
 #include <Processors/Formats/ISchemaReader.h>
 #include <Formats/FormatSettings.h>
+#include <IO/PeekableReadBuffer.h>
 
 
 namespace DB

src/Processors/Formats/Impl/JSONRowInputFormat.cpp

@@ -2,6 +2,7 @@
 #include <Formats/JSONUtils.h>
 #include <Formats/FormatFactory.h>
 #include <Formats/EscapingRuleUtils.h>
+#include <IO/PeekableReadBuffer.h>
 #include <IO/ReadHelpers.h>
 
 namespace DB

src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h

@@ -4,6 +4,7 @@
 #include <Formats/FormatSettings.h>
 #include <Processors/Formats/RowInputFormatWithNamesAndTypes.h>
 #include <Processors/Formats/ISchemaReader.h>
+#include <IO/PeekableReadBuffer.h>
 
 
 namespace DB

src/Processors/Formats/RowInputFormatWithNamesAndTypes.cpp

@@ -7,6 +7,7 @@
 #include <IO/ReadHelpers.h>
 #include <IO/Operators.h>
 #include <IO/ReadBufferFromString.h>
+#include <IO/PeekableReadBuffer.h>
 #include <Formats/EscapingRuleUtils.h>
 
 

src/Storages/StorageFile.cpp

@@ -25,6 +25,7 @@
 #include <IO/WriteHelpers.h>
 #include <IO/Archives/createArchiveReader.h>
 #include <IO/Archives/IArchiveReader.h>
+#include <IO/PeekableReadBuffer.h>
 
 #include <Formats/FormatFactory.h>
 #include <Formats/ReadSchemaUtils.h>

src/Storages/VirtualColumnUtils.cpp

@@ -1,5 +1,6 @@
 #include <algorithm>
 #include <memory>
+#include <stack>
 #include <Core/NamesAndTypes.h>
 #include <Core/TypeId.h>
 

tests/config/config.d/keeper_port.xml

@@ -4,6 +4,7 @@
         <server_id>1</server_id>
 
         <create_snapshot_on_exit>1</create_snapshot_on_exit>
+        <digest_enabled>1</digest_enabled>
 
         <coordination_settings>
             <operation_timeout_ms>10000</operation_timeout_ms>

tests/queries/0_stateless/02720_row_policy_column_with_dots.reference

@@ -1 +1 @@
-2024-01-01	Hello	World
+1

tests/queries/0_stateless/02720_row_policy_column_with_dots.sql

@@ -1,6 +1,6 @@
-CREATE table if not exists table_with_dot_column (date Date, regular_column String, `other_column.2` String) ENGINE = MergeTree() ORDER BY date;
-INSERT INTO table_with_dot_column select '2020-01-01', 'Hello', 'World';
-INSERT INTO table_with_dot_column select '2024-01-01', 'Hello', 'World';
+CREATE TABLE IF NOT EXISTS table_with_dot_column (date Date, regular_column String, `other_column.2` String) ENGINE = MergeTree() ORDER BY date;
+INSERT INTO table_with_dot_column SELECT '2020-01-01', 'Hello', 'World';
+INSERT INTO table_with_dot_column SELECT toDate(now() + 48*3600), 'Hello', 'World';
 CREATE ROW POLICY IF NOT EXISTS row_policy ON table_with_dot_column USING toDate(date) >= today() - 30 TO ALL;
-SELECT * FROM table_with_dot_column;
+SELECT count(*) FROM table_with_dot_column;
 DROP TABLE table_with_dot_column;