Merge pull request #66475 from rschu1ze/bump-rocksdb

Bump RocksDB from v6.29.5 to v7.10.2

contrib/rocksdb

@@ -1 +1 @@
-Subproject commit be366233921293bd07a84dc4ea6991858665f202
+Subproject commit 01e43568fa9f3f7bf107b2b66c00b286b456f33e

contrib/rocksdb-cmake/CMakeLists.txt

@@ -5,6 +5,9 @@ if (NOT ENABLE_ROCKSDB)
   return()
 endif()
 
+# not in original build system, otherwise xxHash.cc fails to compile with ClickHouse C++23 default
+set (CMAKE_CXX_STANDARD 20)
+
 # Always disable jemalloc for rocksdb by default because it introduces non-standard jemalloc APIs
 option(WITH_JEMALLOC "build with JeMalloc" OFF)
 
@@ -16,14 +19,6 @@ option(WITH_LZ4 "build with lz4" ON)
 option(WITH_ZLIB "build with zlib" ON)
 option(WITH_ZSTD "build with zstd" ON)
 
-# third-party/folly is only validated to work on Linux and Windows for now.
-# So only turn it on there by default.
-if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
-  option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" ON)
-else()
-  option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
-endif()
-
 if(WITH_SNAPPY)
   add_definitions(-DSNAPPY)
   list(APPEND THIRDPARTY_LIBS ch_contrib::snappy)
@@ -44,7 +39,7 @@ if(WITH_ZSTD)
   list(APPEND THIRDPARTY_LIBS ch_contrib::zstd)
 endif()
 
-option(PORTABLE "build a portable binary" ON)
+add_definitions(-DROCKSDB_PORTABLE)
 
 if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ)
   add_definitions(-DHAVE_SSE42)
@@ -59,11 +54,6 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
   # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
 endif()
 
-set (HAVE_THREAD_LOCAL 1)
-if(HAVE_THREAD_LOCAL)
-  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
-endif()
-
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
@@ -89,19 +79,21 @@ set(ROCKSDB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rocksdb")
 
 include_directories(${ROCKSDB_SOURCE_DIR})
 include_directories("${ROCKSDB_SOURCE_DIR}/include")
-if(WITH_FOLLY_DISTRIBUTED_MUTEX)
-  include_directories("${ROCKSDB_SOURCE_DIR}/third-party/folly")
-endif()
 
 set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/cache/cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/cache_entry_roles.cc
     ${ROCKSDB_SOURCE_DIR}/cache/cache_key.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/cache_helpers.cc
     ${ROCKSDB_SOURCE_DIR}/cache/cache_reservation_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/charged_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/clock_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/compressed_secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
     ${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_contents.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_builder.cc
@@ -113,6 +105,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_format.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_sequential_reader.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_source.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/prefetch_buffer_collection.cc
     ${ROCKSDB_SOURCE_DIR}/db/builder.cc
     ${ROCKSDB_SOURCE_DIR}/db/c.cc
@@ -124,7 +117,11 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_fifo.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_level.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_universal.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_service_job.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_state.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_outputs.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/sst_partitioner.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/subcompaction_state.cc
     ${ROCKSDB_SOURCE_DIR}/db/convenience.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_filesnapshot.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/compacted_db_impl.cc
@@ -159,10 +156,11 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc
     ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc
     ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc
-    ${ROCKSDB_SOURCE_DIR}/db/periodic_work_scheduler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/periodic_task_scheduler.cc
     ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc
     ${ROCKSDB_SOURCE_DIR}/db/range_tombstone_fragmenter.cc
     ${ROCKSDB_SOURCE_DIR}/db/repair.cc
+    ${ROCKSDB_SOURCE_DIR}/db/seqno_to_time_mapping.cc
     ${ROCKSDB_SOURCE_DIR}/db/snapshot_impl.cc
     ${ROCKSDB_SOURCE_DIR}/db/table_cache.cc
     ${ROCKSDB_SOURCE_DIR}/db/table_properties_collector.cc
@@ -174,6 +172,8 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/version_set.cc
     ${ROCKSDB_SOURCE_DIR}/db/wal_edit.cc
     ${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wide/wide_column_serialization.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wide/wide_columns.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_batch.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_controller.cc
@@ -182,7 +182,6 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/env/env.cc
     ${ROCKSDB_SOURCE_DIR}/env/env_chroot.cc
     ${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc
-    ${ROCKSDB_SOURCE_DIR}/env/env_hdfs.cc
     ${ROCKSDB_SOURCE_DIR}/env/file_system.cc
     ${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc
     ${ROCKSDB_SOURCE_DIR}/env/fs_remap.cc
@@ -233,16 +232,17 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/options/options.cc
     ${ROCKSDB_SOURCE_DIR}/options/options_helper.cc
     ${ROCKSDB_SOURCE_DIR}/options/options_parser.cc
+    ${ROCKSDB_SOURCE_DIR}/port/mmap.cc
     ${ROCKSDB_SOURCE_DIR}/port/stack_trace.cc
     ${ROCKSDB_SOURCE_DIR}/table/adaptive/adaptive_table_factory.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/binary_search_index_reader.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block.cc
-    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_filter_block.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_builder.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_factory.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_iterator.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_reader.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_cache.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefetcher.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefix_index.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_hash_index.cc
@@ -300,9 +300,12 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_record_result.cc
     ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_record.cc
     ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_replay.cc
+    ${ROCKSDB_SOURCE_DIR}/util/async_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/util/cleanable.cc
     ${ROCKSDB_SOURCE_DIR}/util/coding.cc
     ${ROCKSDB_SOURCE_DIR}/util/compaction_job_stats_impl.cc
     ${ROCKSDB_SOURCE_DIR}/util/comparator.cc
+    ${ROCKSDB_SOURCE_DIR}/util/compression.cc
     ${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc
     ${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc
     ${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
@@ -311,16 +314,17 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc
     ${ROCKSDB_SOURCE_DIR}/util/random.cc
     ${ROCKSDB_SOURCE_DIR}/util/rate_limiter.cc
-    ${ROCKSDB_SOURCE_DIR}/util/regex.cc
     ${ROCKSDB_SOURCE_DIR}/util/ribbon_config.cc
     ${ROCKSDB_SOURCE_DIR}/util/slice.cc
     ${ROCKSDB_SOURCE_DIR}/util/file_checksum_helper.cc
     ${ROCKSDB_SOURCE_DIR}/util/status.cc
+    ${ROCKSDB_SOURCE_DIR}/util/stderr_logger.cc
     ${ROCKSDB_SOURCE_DIR}/util/string_util.cc
     ${ROCKSDB_SOURCE_DIR}/util/thread_local.cc
     ${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc
     ${ROCKSDB_SOURCE_DIR}/util/xxhash.cc
-    ${ROCKSDB_SOURCE_DIR}/utilities/backupable/backupable_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/agg_merge/agg_merge.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/backup/backup_engine.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_compaction_filter.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl.cc
@@ -335,6 +339,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/utilities/checkpoint/checkpoint_impl.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/counted_fs.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/debug.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/env_mirror.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/env_timed.cc
@@ -422,15 +427,6 @@ list(APPEND SOURCES
     "${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc"
     "${ROCKSDB_SOURCE_DIR}/env/io_posix.cc")
 
-if(WITH_FOLLY_DISTRIBUTED_MUTEX)
-  list(APPEND SOURCES
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/detail/Futex.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/AtomicNotification.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/DistributedMutex.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/ParkingLot.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/WaitOptions.cpp")
-endif()
-
 add_library(_rocksdb ${SOURCES})
 add_library(ch_contrib::rocksdb ALIAS _rocksdb)
 target_link_libraries(_rocksdb PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})

contrib/rocksdb-cmake/build_version.cc

@@ -1,16 +1,33 @@
 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-/// This file was edited for ClickHouse.
 
 #include <memory>
 
 #include "rocksdb/version.h"
+#include "rocksdb/utilities/object_registry.h"
 #include "util/string_util.h"
 
 // The build script may replace these values with real values based
 // on whether or not GIT is available and the platform settings
-static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:0";
+static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:72438a678872544809393b831c7273794c074215";
-static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:master";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:main";
-static const std::string rocksdb_build_date = "rocksdb_build_date:2000-01-01";
+#define HAS_GIT_CHANGES 0
+#if HAS_GIT_CHANGES == 0
+// If HAS_GIT_CHANGES is 0, the GIT date is used.
+// Use the time the branch/tag was last modified
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-07-12 16:01:57";
+#else
+// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
+// Use the time the build was created.
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-07-13 17:15:50";
+#endif
+
+extern "C" {
+
+} // extern "C"
+
+std::unordered_map<std::string, ROCKSDB_NAMESPACE::RegistrarFunc> ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = {
+  
+};
 
 namespace ROCKSDB_NAMESPACE {
 static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
@@ -39,12 +56,12 @@ const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
 }
 
 std::string GetRocksVersionAsString(bool with_patch) {
-  std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
+  std::string version = std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR);
   if (with_patch) {
-    return version + "." + ToString(ROCKSDB_PATCH);
+    return version + "." + std::to_string(ROCKSDB_PATCH);
   } else {
     return version;
-  }
+ }
 }
 
 std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {