mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge pull request #15073 from sundy-li/storage-rocksdb
StorageEmbeddedRocksDB
This commit is contained in:
alexey-milovidov
GitHub
commit
ff29b2d2ee
4
.gitmodules
vendored
4
.gitmodules
vendored
@ -193,6 +193,10 @@
|
||||
[submodule "contrib/miniselect"]
|
||||
path = contrib/miniselect
|
||||
url = https://github.com/danlark1/miniselect
|
||||
[submodule "contrib/rocksdb"]
|
||||
path = contrib/rocksdb
|
||||
url = https://github.com/facebook/rocksdb
|
||||
branch = v6.11.4
|
||||
[submodule "contrib/xz"]
|
||||
path = contrib/xz
|
||||
url = https://github.com/xz-mirror/xz
|
||||
|
||||
@ -456,6 +456,8 @@ include (cmake/find/simdjson.cmake)
|
||||
include (cmake/find/rapidjson.cmake)
|
||||
include (cmake/find/fastops.cmake)
|
||||
include (cmake/find/odbc.cmake)
|
||||
include (cmake/find/rocksdb.cmake)
|
||||
|
||||
|
||||
if(NOT USE_INTERNAL_PARQUET_LIBRARY)
|
||||
set (ENABLE_ORC OFF CACHE INTERNAL "")
|
||||
|
||||
21
base/glibc-compatibility/musl/sync_file_range.c
Normal file
21
base/glibc-compatibility/musl/sync_file_range.c
Normal file
@ -0,0 +1,21 @@
|
||||
#define _GNU_SOURCE
|
||||
#include <fcntl.h>
|
||||
#include <errno.h>
|
||||
#include "syscall.h"
|
||||
|
||||
// works same in x86_64 && aarch64
|
||||
#define __SYSCALL_LL_E(x) (x)
|
||||
#define __SYSCALL_LL_O(x) (x)
|
||||
|
||||
int sync_file_range(int fd, off_t pos, off_t len, unsigned flags)
|
||||
{
|
||||
#if defined(SYS_sync_file_range2)
|
||||
return syscall(SYS_sync_file_range2, fd, flags,
|
||||
__SYSCALL_LL_E(pos), __SYSCALL_LL_E(len));
|
||||
#elif defined(SYS_sync_file_range)
|
||||
return __syscall(SYS_sync_file_range, fd,
|
||||
__SYSCALL_LL_O(pos), __SYSCALL_LL_E(len), flags);
|
||||
#else
|
||||
return __syscall_ret(-ENOSYS);
|
||||
#endif
|
||||
}
|
||||
67
cmake/find/rocksdb.cmake
Normal file
67
cmake/find/rocksdb.cmake
Normal file
@ -0,0 +1,67 @@
|
||||
option(ENABLE_ROCKSDB "Enable ROCKSDB" ${ENABLE_LIBRARIES})
|
||||
|
||||
if (NOT ENABLE_ROCKSDB)
|
||||
if (USE_INTERNAL_ROCKSDB_LIBRARY)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal rocksdb library with ENABLE_ROCKSDB=OFF")
|
||||
endif()
|
||||
return()
|
||||
endif()
|
||||
|
||||
option(USE_INTERNAL_ROCKSDB_LIBRARY "Set to FALSE to use system ROCKSDB library instead of bundled" ${NOT_UNBUNDLED})
|
||||
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/CMakeLists.txt")
|
||||
if (USE_INTERNAL_ROCKSDB_LIBRARY)
|
||||
message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal rocksdb")
|
||||
endif()
|
||||
set (MISSING_INTERNAL_ROCKSDB 1)
|
||||
endif ()
|
||||
|
||||
if (NOT USE_INTERNAL_ROCKSDB_LIBRARY)
|
||||
find_library (ROCKSDB_LIBRARY rocksdb)
|
||||
find_path (ROCKSDB_INCLUDE_DIR NAMES rocksdb/db.h PATHS ${ROCKSDB_INCLUDE_PATHS})
|
||||
if (NOT ROCKSDB_LIBRARY OR NOT ROCKSDB_INCLUDE_DIR)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system rocksdb library")
|
||||
endif()
|
||||
|
||||
if (NOT SNAPPY_LIBRARY)
|
||||
include(cmake/find/snappy.cmake)
|
||||
endif()
|
||||
if (NOT ZLIB_LIBRARY)
|
||||
include(cmake/find/zlib.cmake)
|
||||
endif()
|
||||
|
||||
find_package(BZip2)
|
||||
find_library(ZSTD_LIBRARY zstd)
|
||||
find_library(LZ4_LIBRARY lz4)
|
||||
find_library(GFLAGS_LIBRARY gflags)
|
||||
|
||||
if(SNAPPY_LIBRARY AND ZLIB_LIBRARY AND LZ4_LIBRARY AND BZIP2_FOUND AND ZSTD_LIBRARY AND GFLAGS_LIBRARY)
|
||||
list (APPEND ROCKSDB_LIBRARY ${SNAPPY_LIBRARY})
|
||||
list (APPEND ROCKSDB_LIBRARY ${ZLIB_LIBRARY})
|
||||
list (APPEND ROCKSDB_LIBRARY ${LZ4_LIBRARY})
|
||||
list (APPEND ROCKSDB_LIBRARY ${BZIP2_LIBRARY})
|
||||
list (APPEND ROCKSDB_LIBRARY ${ZSTD_LIBRARY})
|
||||
list (APPEND ROCKSDB_LIBRARY ${GFLAGS_LIBRARY})
|
||||
else()
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL}
|
||||
"Can't find system rocksdb: snappy=${SNAPPY_LIBRARY} ;"
|
||||
" zlib=${ZLIB_LIBRARY} ;"
|
||||
" lz4=${LZ4_LIBRARY} ;"
|
||||
" bz2=${BZIP2_LIBRARY} ;"
|
||||
" zstd=${ZSTD_LIBRARY} ;"
|
||||
" gflags=${GFLAGS_LIBRARY} ;")
|
||||
endif()
|
||||
endif ()
|
||||
|
||||
if(ROCKSDB_LIBRARY AND ROCKSDB_INCLUDE_DIR)
|
||||
set(USE_ROCKSDB 1)
|
||||
elseif (NOT MISSING_INTERNAL_ROCKSDB)
|
||||
set (USE_INTERNAL_ROCKSDB_LIBRARY 1)
|
||||
|
||||
set (ROCKSDB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/include")
|
||||
set (ROCKSDB_LIBRARY "rocksdb")
|
||||
set (USE_ROCKSDB 1)
|
||||
endif ()
|
||||
|
||||
message (STATUS "Using ROCKSDB=${USE_ROCKSDB}: ${ROCKSDB_INCLUDE_DIR} : ${ROCKSDB_LIBRARY}")
|
||||
4
contrib/CMakeLists.txt
vendored
4
contrib/CMakeLists.txt
vendored
@ -321,3 +321,7 @@ if (USE_KRB5)
|
||||
add_subdirectory (cyrus-sasl-cmake)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (USE_INTERNAL_ROCKSDB_LIBRARY)
|
||||
add_subdirectory(rocksdb-cmake)
|
||||
endif()
|
||||
|
||||
1
contrib/rocksdb
vendored
Submodule
1
contrib/rocksdb
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 963314ffd681596ef2738a95249fe4c1163ef87a
|
||||
668
contrib/rocksdb-cmake/CMakeLists.txt
Normal file
668
contrib/rocksdb-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,668 @@
|
||||
## this file is extracted from `contrib/rocksdb/CMakeLists.txt`
|
||||
set(ROCKSDB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rocksdb")
|
||||
list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_SOURCE_DIR}/cmake/modules/")
|
||||
|
||||
find_program(CCACHE_FOUND ccache)
|
||||
if(CCACHE_FOUND)
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
|
||||
endif(CCACHE_FOUND)
|
||||
|
||||
if (SANITIZE STREQUAL "undefined")
|
||||
set(WITH_UBSAN ON)
|
||||
elseif (SANITIZE STREQUAL "address")
|
||||
set(WITH_ASAN ON)
|
||||
elseif (SANITIZE STREQUAL "thread")
|
||||
set(WITH_TSAN ON)
|
||||
endif()
|
||||
|
||||
|
||||
set(PORTABLE ON)
|
||||
## always disable jemalloc for rocksdb by default
|
||||
## because it introduces non-standard jemalloc APIs
|
||||
option(WITH_JEMALLOC "build with JeMalloc" OFF)
|
||||
option(WITH_SNAPPY "build with SNAPPY" ${USE_SNAPPY})
|
||||
## lz4, zlib, zstd is enabled in ClickHouse by default
|
||||
option(WITH_LZ4 "build with lz4" ON)
|
||||
option(WITH_ZLIB "build with zlib" ON)
|
||||
option(WITH_ZSTD "build with zstd" ON)
|
||||
|
||||
# third-party/folly is only validated to work on Linux and Windows for now.
|
||||
# So only turn it on there by default.
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
|
||||
if(MSVC AND MSVC_VERSION LESS 1910)
|
||||
# Folly does not compile with MSVC older than VS2017
|
||||
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
|
||||
else()
|
||||
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" ON)
|
||||
endif()
|
||||
else()
|
||||
option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
|
||||
endif()
|
||||
|
||||
if( NOT DEFINED CMAKE_CXX_STANDARD )
|
||||
set(CMAKE_CXX_STANDARD 11)
|
||||
endif()
|
||||
|
||||
if(MSVC)
|
||||
option(WITH_XPRESS "build with windows built in compression" OFF)
|
||||
include(${ROCKSDB_SOURCE_DIR}/thirdparty.inc)
|
||||
else()
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
|
||||
# FreeBSD has jemalloc as default malloc
|
||||
# but it does not have all the jemalloc files in include/...
|
||||
set(WITH_JEMALLOC ON)
|
||||
else()
|
||||
if(WITH_JEMALLOC)
|
||||
add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
|
||||
list(APPEND THIRDPARTY_LIBS jemalloc)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(WITH_SNAPPY)
|
||||
add_definitions(-DSNAPPY)
|
||||
list(APPEND THIRDPARTY_LIBS snappy)
|
||||
endif()
|
||||
|
||||
if(WITH_ZLIB)
|
||||
add_definitions(-DZLIB)
|
||||
list(APPEND THIRDPARTY_LIBS zlib)
|
||||
endif()
|
||||
|
||||
if(WITH_LZ4)
|
||||
add_definitions(-DLZ4)
|
||||
list(APPEND THIRDPARTY_LIBS lz4)
|
||||
endif()
|
||||
|
||||
if(WITH_ZSTD)
|
||||
add_definitions(-DZSTD)
|
||||
include_directories(${ZSTD_INCLUDE_DIR})
|
||||
include_directories(${ZSTD_INCLUDE_DIR}/common)
|
||||
include_directories(${ZSTD_INCLUDE_DIR}/dictBuilder)
|
||||
include_directories(${ZSTD_INCLUDE_DIR}/deprecated)
|
||||
|
||||
list(APPEND THIRDPARTY_LIBS zstd)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
string(TIMESTAMP TS "%Y/%m/%d %H:%M:%S" UTC)
|
||||
set(GIT_DATE_TIME "${TS}" CACHE STRING "the time we first built rocksdb")
|
||||
|
||||
find_package(Git)
|
||||
|
||||
if(GIT_FOUND AND EXISTS "${ROCKSDB_SOURCE_DIR}/.git")
|
||||
if(WIN32)
|
||||
execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${ROCKSDB_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
||||
else()
|
||||
execute_process(COMMAND ${GIT_EXECUTABLE} -C ${ROCKSDB_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
|
||||
endif()
|
||||
else()
|
||||
set(GIT_SHA 0)
|
||||
endif()
|
||||
|
||||
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
|
||||
|
||||
set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/rocksdb_build_version.cc)
|
||||
configure_file(${ROCKSDB_SOURCE_DIR}/util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
|
||||
add_library(rocksdb_build_version OBJECT ${BUILD_VERSION_CC})
|
||||
target_include_directories(rocksdb_build_version PRIVATE
|
||||
${ROCKSDB_SOURCE_DIR}/util)
|
||||
if(MSVC)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
|
||||
if(MINGW)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format -fno-asynchronous-unwind-tables")
|
||||
add_definitions(-D_POSIX_C_SOURCE=1)
|
||||
endif()
|
||||
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
|
||||
include(CheckCXXCompilerFlag)
|
||||
CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
|
||||
if(HAVE_OMIT_LEAF_FRAME_POINTER)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
include(CheckCCompilerFlag)
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
|
||||
CHECK_C_COMPILER_FLAG("-mcpu=power9" HAS_POWER9)
|
||||
if(HAS_POWER9)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power9 -mtune=power9")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power9 -mtune=power9")
|
||||
else()
|
||||
CHECK_C_COMPILER_FLAG("-mcpu=power8" HAS_POWER8)
|
||||
if(HAS_POWER8)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcpu=power8 -mtune=power8")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=power8 -mtune=power8")
|
||||
endif(HAS_POWER8)
|
||||
endif(HAS_POWER9)
|
||||
CHECK_C_COMPILER_FLAG("-maltivec" HAS_ALTIVEC)
|
||||
if(HAS_ALTIVEC)
|
||||
message(STATUS " HAS_ALTIVEC yes")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maltivec")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maltivec")
|
||||
endif(HAS_ALTIVEC)
|
||||
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
|
||||
if(HAS_ARMV8_CRC)
|
||||
message(STATUS " HAS_ARMV8_CRC yes")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
|
||||
endif(HAS_ARMV8_CRC)
|
||||
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
|
||||
|
||||
|
||||
include(CheckCXXSourceCompiles)
|
||||
if(NOT MSVC)
|
||||
set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
|
||||
endif()
|
||||
|
||||
CHECK_CXX_SOURCE_COMPILES("
|
||||
#include <cstdint>
|
||||
#include <nmmintrin.h>
|
||||
#include <wmmintrin.h>
|
||||
int main() {
|
||||
volatile uint32_t x = _mm_crc32_u32(0, 0);
|
||||
const auto a = _mm_set_epi64x(0, 0);
|
||||
const auto b = _mm_set_epi64x(0, 0);
|
||||
const auto c = _mm_clmulepi64_si128(a, b, 0x00);
|
||||
auto d = _mm_cvtsi128_si64(c);
|
||||
}
|
||||
" HAVE_SSE42)
|
||||
unset(CMAKE_REQUIRED_FLAGS)
|
||||
if(HAVE_SSE42)
|
||||
add_definitions(-DHAVE_SSE42)
|
||||
add_definitions(-DHAVE_PCLMUL)
|
||||
elseif(FORCE_SSE42)
|
||||
message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
|
||||
endif()
|
||||
|
||||
CHECK_CXX_SOURCE_COMPILES("
|
||||
#if defined(_MSC_VER) && !defined(__thread)
|
||||
#define __thread __declspec(thread)
|
||||
#endif
|
||||
int main() {
|
||||
static __thread int tls;
|
||||
}
|
||||
" HAVE_THREAD_LOCAL)
|
||||
if(HAVE_THREAD_LOCAL)
|
||||
add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
|
||||
endif()
|
||||
|
||||
option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
|
||||
if(FAIL_ON_WARNINGS)
|
||||
if(MSVC)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX")
|
||||
else() # assume GCC
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
option(WITH_ASAN "build with ASAN" OFF)
|
||||
if(WITH_ASAN)
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
|
||||
if(WITH_JEMALLOC)
|
||||
message(FATAL "ASAN does not work well with JeMalloc")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
option(WITH_TSAN "build with TSAN" OFF)
|
||||
if(WITH_TSAN)
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=thread -pie")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=thread -fPIC")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=thread -fPIC")
|
||||
if(WITH_JEMALLOC)
|
||||
message(FATAL "TSAN does not work well with JeMalloc")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
option(WITH_UBSAN "build with UBSAN" OFF)
|
||||
if(WITH_UBSAN)
|
||||
add_definitions(-DROCKSDB_UBSAN_RUN)
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=undefined")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=undefined")
|
||||
if(WITH_JEMALLOC)
|
||||
message(FATAL "UBSAN does not work well with JeMalloc")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
|
||||
add_definitions(-fno-builtin-memcmp -DCYGWIN)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
|
||||
add_definitions(-DOS_MACOSX)
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
|
||||
add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
|
||||
# no debug info for IOS, that will make our library big
|
||||
add_definitions(-DNDEBUG)
|
||||
endif()
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
|
||||
add_definitions(-DOS_LINUX)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
|
||||
add_definitions(-DOS_SOLARIS)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
|
||||
add_definitions(-DOS_GNU_KFREEBSD)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
||||
add_definitions(-DOS_FREEBSD)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
|
||||
add_definitions(-DOS_NETBSD)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "OpenBSD")
|
||||
add_definitions(-DOS_OPENBSD)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "DragonFly")
|
||||
add_definitions(-DOS_DRAGONFLYBSD)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "Android")
|
||||
add_definitions(-DOS_ANDROID)
|
||||
elseif(CMAKE_SYSTEM_NAME MATCHES "Windows")
|
||||
add_definitions(-DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX)
|
||||
if(MINGW)
|
||||
add_definitions(-D_WIN32_WINNT=_WIN32_WINNT_VISTA)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(NOT WIN32)
|
||||
add_definitions(-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX)
|
||||
endif()
|
||||
|
||||
option(WITH_FALLOCATE "build with fallocate" ON)
|
||||
if(WITH_FALLOCATE)
|
||||
CHECK_CXX_SOURCE_COMPILES("
|
||||
#include <fcntl.h>
|
||||
#include <linux/falloc.h>
|
||||
int main() {
|
||||
int fd = open(\"/dev/null\", 0);
|
||||
fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
|
||||
}
|
||||
" HAVE_FALLOCATE)
|
||||
if(HAVE_FALLOCATE)
|
||||
add_definitions(-DROCKSDB_FALLOCATE_PRESENT)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
CHECK_CXX_SOURCE_COMPILES("
|
||||
#include <fcntl.h>
|
||||
int main() {
|
||||
int fd = open(\"/dev/null\", 0);
|
||||
sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
|
||||
}
|
||||
" HAVE_SYNC_FILE_RANGE_WRITE)
|
||||
if(HAVE_SYNC_FILE_RANGE_WRITE)
|
||||
add_definitions(-DROCKSDB_RANGESYNC_PRESENT)
|
||||
endif()
|
||||
|
||||
CHECK_CXX_SOURCE_COMPILES("
|
||||
#include <pthread.h>
|
||||
int main() {
|
||||
(void) PTHREAD_MUTEX_ADAPTIVE_NP;
|
||||
}
|
||||
" HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
|
||||
if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
|
||||
add_definitions(-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX)
|
||||
endif()
|
||||
|
||||
include(CheckCXXSymbolExists)
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "^FreeBSD")
|
||||
check_cxx_symbol_exists(malloc_usable_size ${ROCKSDB_SOURCE_DIR}/malloc_np.h HAVE_MALLOC_USABLE_SIZE)
|
||||
else()
|
||||
check_cxx_symbol_exists(malloc_usable_size ${ROCKSDB_SOURCE_DIR}/malloc.h HAVE_MALLOC_USABLE_SIZE)
|
||||
endif()
|
||||
if(HAVE_MALLOC_USABLE_SIZE)
|
||||
add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
|
||||
endif()
|
||||
|
||||
check_cxx_symbol_exists(sched_getcpu sched.h HAVE_SCHED_GETCPU)
|
||||
if(HAVE_SCHED_GETCPU)
|
||||
add_definitions(-DROCKSDB_SCHED_GETCPU_PRESENT)
|
||||
endif()
|
||||
|
||||
check_cxx_symbol_exists(getauxval auvx.h HAVE_AUXV_GETAUXVAL)
|
||||
if(HAVE_AUXV_GETAUXVAL)
|
||||
add_definitions(-DROCKSDB_AUXV_GETAUXVAL_PRESENT)
|
||||
endif()
|
||||
|
||||
include_directories(${ROCKSDB_SOURCE_DIR})
|
||||
include_directories(${ROCKSDB_SOURCE_DIR}/include)
|
||||
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
|
||||
include_directories(${ROCKSDB_SOURCE_DIR}/third-party/folly)
|
||||
endif()
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
# Main library source code
|
||||
|
||||
set(SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/cache/cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/cache/clock_cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_garbage.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_meta.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_format.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_writer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/c.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/column_family.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compacted_db_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_job.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_fifo.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_level.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_universal.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/compaction/sst_partitioner.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/convenience.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_filesnapshot.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_write.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_compaction_flush.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_files.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_open.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_debug.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_experimental.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_readonly.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_secondary.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_info_dumper.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/db_iter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/dbformat.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/error_handler.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/event_helpers.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/experimental.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/external_sst_file_ingestion_job.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/file_indexer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/flush_job.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/flush_scheduler.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/forward_iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/import_column_family_job.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/internal_stats.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/logs_with_prep_tracker.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/log_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/log_writer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/malloc_stats.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/memtable.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/range_tombstone_fragmenter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/repair.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/snapshot_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/table_cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/table_properties_collector.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/transaction_log_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/trim_history_scheduler.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/version_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/version_edit.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/version_edit_handler.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/version_set.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/wal_edit.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/write_batch.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/write_controller.cc
|
||||
${ROCKSDB_SOURCE_DIR}/db/write_thread.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/env.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/env_chroot.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/env_hdfs.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/file_system.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/mock_env.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/delete_scheduler.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/file_prefetch_buffer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/file_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/filename.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/random_access_file_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/read_write_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/readahead_raf.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/sequence_file_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/sst_file_manager_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/file/writable_file_writer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/logging/auto_roll_logger.cc
|
||||
${ROCKSDB_SOURCE_DIR}/logging/event_logger.cc
|
||||
${ROCKSDB_SOURCE_DIR}/logging/log_buffer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memory/arena.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memory/concurrent_arena.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memory/jemalloc_nodump_allocator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memory/memkind_kmem_allocator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memtable/alloc_tracker.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memtable/hash_linklist_rep.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memtable/hash_skiplist_rep.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memtable/skiplistrep.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memtable/vectorrep.cc
|
||||
${ROCKSDB_SOURCE_DIR}/memtable/write_buffer_manager.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/histogram.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/histogram_windowing.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/in_memory_stats_history.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/instrumented_mutex.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/iostats_context.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/perf_context.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/perf_level.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/persistent_stats_history.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/statistics.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/stats_dump_scheduler.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_updater.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/monitoring/thread_status_util_debug.cc
|
||||
${ROCKSDB_SOURCE_DIR}/options/cf_options.cc
|
||||
${ROCKSDB_SOURCE_DIR}/options/db_options.cc
|
||||
${ROCKSDB_SOURCE_DIR}/options/options.cc
|
||||
${ROCKSDB_SOURCE_DIR}/options/options_helper.cc
|
||||
${ROCKSDB_SOURCE_DIR}/options/options_parser.cc
|
||||
${ROCKSDB_SOURCE_DIR}/port/stack_trace.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/adaptive/adaptive_table_factory.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/binary_search_index_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_filter_block.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_factory.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefetcher.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefix_index.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_hash_index.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_footer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/filter_block_reader_common.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/filter_policy.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/flush_block_policy.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/full_filter_block.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/hash_index_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/index_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/index_reader_common.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/parsed_full_filter_block.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_filter_block.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_index_iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/partitioned_index_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/reader_common.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_based/uncompression_dict_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/block_fetcher.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_factory.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/cuckoo/cuckoo_table_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/format.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/get_context.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/merging_iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/meta_blocks.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/persistent_cache_helper.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_bloom.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_builder.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_factory.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_index.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_key_coding.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/sst_file_dumper.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/sst_file_reader.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/sst_file_writer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/table_properties.cc
|
||||
${ROCKSDB_SOURCE_DIR}/table/two_level_iterator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/test_util/sync_point.cc
|
||||
${ROCKSDB_SOURCE_DIR}/test_util/sync_point_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/test_util/testutil.cc
|
||||
${ROCKSDB_SOURCE_DIR}/test_util/transaction_test_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/tools/block_cache_analyzer/block_cache_trace_analyzer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/tools/dump/db_dump_tool.cc
|
||||
${ROCKSDB_SOURCE_DIR}/tools/ldb_cmd.cc
|
||||
${ROCKSDB_SOURCE_DIR}/tools/ldb_tool.cc
|
||||
${ROCKSDB_SOURCE_DIR}/tools/sst_dump_tool.cc
|
||||
${ROCKSDB_SOURCE_DIR}/tools/trace_analyzer_tool.cc
|
||||
${ROCKSDB_SOURCE_DIR}/trace_replay/trace_replay.cc
|
||||
${ROCKSDB_SOURCE_DIR}/trace_replay/block_cache_tracer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/trace_replay/io_tracer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/coding.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/compaction_job_stats_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/comparator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/dynamic_bloom.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/hash.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/random.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/rate_limiter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/slice.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/file_checksum_helper.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/status.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/string_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/thread_local.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc
|
||||
${ROCKSDB_SOURCE_DIR}/util/xxhash.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/backupable/backupable_db.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_compaction_filter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl_filesnapshot.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_dump_tool.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_file.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/cassandra/cassandra_compaction_filter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/cassandra/format.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/cassandra/merge_operator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/checkpoint/checkpoint_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/debug.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/env_mirror.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/env_timed.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/fault_injection_env.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/fault_injection_fs.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/leveldb_options/leveldb_options.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/memory/memory_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/bytesxor.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/max.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/put.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/sortlist.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/string_append/stringappend.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/string_append/stringappend2.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/merge_operators/uint64add.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/object_registry.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/option_change_migration/option_change_migration.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/options/options_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier_file.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/block_cache_tier_metadata.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/persistent_cache_tier.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/volatile_tier_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/cache_simulator.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/sim_cache.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_on_deletion_collector.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/trace/file_trace_reader_writer.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/lock_tracker.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/point_lock_tracker.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/optimistic_transaction_db_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/optimistic_transaction.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/pessimistic_transaction.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/pessimistic_transaction_db.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/snapshot_checker.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_base.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_db_mutex_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_lock_mgr.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/transaction_util.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn_db.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn_db.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/ttl/db_ttl_impl.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index.cc
|
||||
${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index_internal.cc
|
||||
$<TARGET_OBJECTS:rocksdb_build_version>)
|
||||
|
||||
if(HAVE_SSE42 AND NOT MSVC)
|
||||
set_source_files_properties(
|
||||
${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
|
||||
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/util/crc32c_ppc.c
|
||||
${ROCKSDB_SOURCE_DIR}/util/crc32c_ppc_asm.S)
|
||||
endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
|
||||
|
||||
if(HAS_ARMV8_CRC)
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/util/crc32c_arm64.cc)
|
||||
endif(HAS_ARMV8_CRC)
|
||||
|
||||
if(WIN32)
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/io_win.cc
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/env_win.cc
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/env_default.cc
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/port_win.cc
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/win_logger.cc)
|
||||
if(NOT MINGW)
|
||||
# Mingw only supports std::thread when using
|
||||
# posix threads.
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/win_thread.cc)
|
||||
endif()
|
||||
if(WITH_XPRESS)
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/xpress_win.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_JEMALLOC)
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/port/win/win_jemalloc.cc)
|
||||
endif()
|
||||
|
||||
else()
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/port/port_posix.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/env_posix.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc
|
||||
${ROCKSDB_SOURCE_DIR}/env/io_posix.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_FOLLY_DISTRIBUTED_MUTEX)
|
||||
list(APPEND SOURCES
|
||||
${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/detail/Futex.cpp
|
||||
${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/AtomicNotification.cpp
|
||||
${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/DistributedMutex.cpp
|
||||
${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/ParkingLot.cpp
|
||||
${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/WaitOptions.cpp)
|
||||
endif()
|
||||
|
||||
set(ROCKSDB_STATIC_LIB rocksdb)
|
||||
|
||||
if(WIN32)
|
||||
set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib)
|
||||
else()
|
||||
set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
|
||||
endif()
|
||||
|
||||
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
|
||||
target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
|
||||
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
||||
@ -64,6 +64,8 @@ RUN apt-get update \
|
||||
libbz2-dev \
|
||||
libavro-dev \
|
||||
libfarmhash-dev \
|
||||
librocksdb-dev \
|
||||
libgflags-dev \
|
||||
libmysqlclient-dev \
|
||||
--yes --no-install-recommends
|
||||
|
||||
|
||||
@ -275,6 +275,9 @@ TESTS_TO_SKIP=(
|
||||
00646_url_engine
|
||||
00974_query_profiler
|
||||
|
||||
# In fasttest, ENABLE_LIBRARIES=0, so rocksdb engine is not enabled by default
|
||||
01504_rocksdb
|
||||
|
||||
# Look at DistributedFilesToInsert, so cannot run in parallel.
|
||||
01460_DistributedFilesToInsert
|
||||
|
||||
|
||||
@ -43,6 +43,8 @@ RUN apt-get --allow-unauthenticated update -y \
|
||||
libreadline-dev \
|
||||
libsasl2-dev \
|
||||
libzstd-dev \
|
||||
librocksdb-dev \
|
||||
libgflags-dev \
|
||||
lsof \
|
||||
moreutils \
|
||||
ncdu \
|
||||
|
||||
@ -0,0 +1,45 @@
|
||||
---
|
||||
toc_priority: 6
|
||||
toc_title: EmbeddedRocksDB
|
||||
---
|
||||
|
||||
# EmbeddedRocksDB Engine {#EmbeddedRocksDB-engine}
|
||||
|
||||
This engine allows integrating ClickHouse with [rocksdb](http://rocksdb.org/).
|
||||
|
||||
`EmbeddedRocksDB` lets you:
|
||||
|
||||
## Creating a Table {#table_engine-EmbeddedRocksDB-creating-a-table}
|
||||
|
||||
``` sql
|
||||
CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
|
||||
(
|
||||
name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1],
|
||||
name2 [type2] [DEFAULT|MATERIALIZED|ALIAS expr2],
|
||||
...
|
||||
) ENGINE = EmbeddedRocksDB PRIMARY KEY(primary_key_name)
|
||||
```
|
||||
|
||||
Required parameters:
|
||||
|
||||
- `primary_key_name` – any column name in the column list.
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test
|
||||
(
|
||||
`key` String,
|
||||
`v1` UInt32,
|
||||
`v2` String,
|
||||
`v3` Float32,
|
||||
)
|
||||
ENGINE = EmbeddedRocksDB
|
||||
PRIMARY KEY key
|
||||
```
|
||||
|
||||
## Description {#description}
|
||||
|
||||
- `primary key` must be specified, it only supports one column in primary key. The primary key will serialized in binary as rocksdb key.
|
||||
- columns other than the primary key will be serialized in binary as rocksdb value in corresponding order.
|
||||
- queries with key `equals` or `in` filtering will be optimized to multi keys lookup from rocksdb.
|
||||
@ -78,6 +78,10 @@ if (USE_AMQPCPP)
|
||||
add_headers_and_sources(dbms Storages/RabbitMQ)
|
||||
endif()
|
||||
|
||||
if (USE_ROCKSDB)
|
||||
add_headers_and_sources(dbms Storages/RocksDB)
|
||||
endif()
|
||||
|
||||
if (USE_AWS_S3)
|
||||
add_headers_and_sources(dbms Common/S3)
|
||||
add_headers_and_sources(dbms Disks/S3)
|
||||
@ -294,6 +298,7 @@ if (USE_KRB5)
|
||||
dbms_target_link_libraries(PRIVATE ${KRB5_LIBRARY})
|
||||
endif()
|
||||
|
||||
|
||||
if(RE2_INCLUDE_DIR)
|
||||
target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR})
|
||||
endif()
|
||||
@ -409,6 +414,11 @@ if (USE_ORC)
|
||||
dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/contrib/orc/c++/include)
|
||||
endif ()
|
||||
|
||||
if (USE_ROCKSDB)
|
||||
dbms_target_link_libraries(PUBLIC ${ROCKSDB_LIBRARY})
|
||||
dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ROCKSDB_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (ENABLE_TESTS AND USE_GTEST)
|
||||
macro (grep_gtest_sources BASE_DIR DST_VAR)
|
||||
# Cold match files that are not in tests/ directories
|
||||
|
||||
@ -519,9 +519,9 @@
|
||||
M(550, CONDITIONAL_TREE_PARENT_NOT_FOUND) \
|
||||
M(551, ILLEGAL_PROJECTION_MANIPULATOR) \
|
||||
M(552, UNRECOGNIZED_ARGUMENTS) \
|
||||
M(553, ROCKSDB_ERROR) \
|
||||
M(553, LZMA_STREAM_ENCODER_FAILED) \
|
||||
M(554, LZMA_STREAM_DECODER_FAILED) \
|
||||
\
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
M(1000, POCO_EXCEPTION) \
|
||||
M(1001, STD_EXCEPTION) \
|
||||
|
||||
@ -11,3 +11,5 @@
|
||||
#cmakedefine01 USE_SSL
|
||||
#cmakedefine01 USE_OPENCL
|
||||
#cmakedefine01 USE_LDAP
|
||||
#cmakedefine01 USE_ROCKSDB
|
||||
|
||||
|
||||
65
src/Storages/RocksDB/EmbeddedRocksDBBlockInputStream.cpp
Normal file
65
src/Storages/RocksDB/EmbeddedRocksDBBlockInputStream.cpp
Normal file
@ -0,0 +1,65 @@
|
||||
#include <DataStreams/IBlockInputStream.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <IO/ReadBufferFromString.h>
|
||||
#include <Storages/RocksDB/StorageEmbeddedRocksDB.h>
|
||||
#include <Storages/RocksDB/EmbeddedRocksDBBlockInputStream.h>
|
||||
|
||||
#include <rocksdb/db.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ROCKSDB_ERROR;
|
||||
}
|
||||
|
||||
EmbeddedRocksDBBlockInputStream::EmbeddedRocksDBBlockInputStream(
|
||||
StorageEmbeddedRocksDB & storage_,
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
size_t max_block_size_)
|
||||
: storage(storage_)
|
||||
, metadata_snapshot(metadata_snapshot_)
|
||||
, max_block_size(max_block_size_)
|
||||
{
|
||||
sample_block = metadata_snapshot->getSampleBlock();
|
||||
primary_key_pos = sample_block.getPositionByName(storage.primary_key);
|
||||
}
|
||||
|
||||
Block EmbeddedRocksDBBlockInputStream::readImpl()
|
||||
{
|
||||
if (finished)
|
||||
return {};
|
||||
|
||||
if (!iterator)
|
||||
{
|
||||
iterator = std::unique_ptr<rocksdb::Iterator>(storage.rocksdb_ptr->NewIterator(rocksdb::ReadOptions()));
|
||||
iterator->SeekToFirst();
|
||||
}
|
||||
|
||||
MutableColumns columns = sample_block.cloneEmptyColumns();
|
||||
|
||||
for (size_t rows = 0; iterator->Valid() && rows < max_block_size; ++rows, iterator->Next())
|
||||
{
|
||||
ReadBufferFromString key_buffer(iterator->key());
|
||||
ReadBufferFromString value_buffer(iterator->value());
|
||||
|
||||
size_t idx = 0;
|
||||
for (const auto & elem : sample_block)
|
||||
{
|
||||
elem.type->deserializeBinary(*columns[idx], idx == primary_key_pos ? key_buffer : value_buffer);
|
||||
++idx;
|
||||
}
|
||||
}
|
||||
|
||||
finished = !iterator->Valid();
|
||||
if (!iterator->status().ok())
|
||||
{
|
||||
throw Exception("Engine " + getName() + " got error while seeking key value datas: " + iterator->status().ToString(),
|
||||
ErrorCodes::ROCKSDB_ERROR);
|
||||
}
|
||||
return sample_block.cloneWithColumns(std::move(columns));
|
||||
}
|
||||
|
||||
}
|
||||
39
src/Storages/RocksDB/EmbeddedRocksDBBlockInputStream.h
Normal file
39
src/Storages/RocksDB/EmbeddedRocksDBBlockInputStream.h
Normal file
@ -0,0 +1,39 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataStreams/IBlockInputStream.h>
|
||||
|
||||
|
||||
namespace rocksdb
|
||||
{
|
||||
class Iterator;
|
||||
}
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class StorageEmbeddedRocksDB;
|
||||
struct StorageInMemoryMetadata;
|
||||
using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
|
||||
|
||||
class EmbeddedRocksDBBlockInputStream : public IBlockInputStream
|
||||
{
|
||||
|
||||
public:
|
||||
EmbeddedRocksDBBlockInputStream(
|
||||
StorageEmbeddedRocksDB & storage_, const StorageMetadataPtr & metadata_snapshot_, size_t max_block_size_);
|
||||
|
||||
String getName() const override { return "EmbeddedRocksDB"; }
|
||||
Block getHeader() const override { return sample_block; }
|
||||
Block readImpl() override;
|
||||
|
||||
private:
|
||||
StorageEmbeddedRocksDB & storage;
|
||||
StorageMetadataPtr metadata_snapshot;
|
||||
const size_t max_block_size;
|
||||
|
||||
Block sample_block;
|
||||
std::unique_ptr<rocksdb::Iterator> iterator;
|
||||
size_t primary_key_pos;
|
||||
bool finished = false;
|
||||
};
|
||||
}
|
||||
67
src/Storages/RocksDB/EmbeddedRocksDBBlockOutputStream.cpp
Normal file
67
src/Storages/RocksDB/EmbeddedRocksDBBlockOutputStream.cpp
Normal file
@ -0,0 +1,67 @@
|
||||
#include <Storages/RocksDB/EmbeddedRocksDBBlockOutputStream.h>
|
||||
#include <Storages/RocksDB/StorageEmbeddedRocksDB.h>
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
|
||||
#include <rocksdb/db.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ROCKSDB_ERROR;
|
||||
}
|
||||
|
||||
EmbeddedRocksDBBlockOutputStream::EmbeddedRocksDBBlockOutputStream(
|
||||
StorageEmbeddedRocksDB & storage_,
|
||||
const StorageMetadataPtr & metadata_snapshot_)
|
||||
: storage(storage_)
|
||||
, metadata_snapshot(metadata_snapshot_)
|
||||
{
|
||||
Block sample_block = metadata_snapshot->getSampleBlock();
|
||||
for (const auto & elem : sample_block)
|
||||
{
|
||||
if (elem.name == storage.primary_key)
|
||||
break;
|
||||
++primary_key_pos;
|
||||
}
|
||||
}
|
||||
|
||||
Block EmbeddedRocksDBBlockOutputStream::getHeader() const
|
||||
{
|
||||
return metadata_snapshot->getSampleBlock();
|
||||
}
|
||||
|
||||
void EmbeddedRocksDBBlockOutputStream::write(const Block & block)
|
||||
{
|
||||
metadata_snapshot->check(block, true);
|
||||
auto rows = block.rows();
|
||||
|
||||
WriteBufferFromOwnString wb_key;
|
||||
WriteBufferFromOwnString wb_value;
|
||||
|
||||
rocksdb::WriteBatch batch;
|
||||
rocksdb::Status status;
|
||||
for (size_t i = 0; i < rows; i++)
|
||||
{
|
||||
wb_key.restart();
|
||||
wb_value.restart();
|
||||
|
||||
size_t idx = 0;
|
||||
for (const auto & elem : block)
|
||||
{
|
||||
elem.type->serializeBinary(*elem.column, i, idx == primary_key_pos ? wb_key : wb_value);
|
||||
++idx;
|
||||
}
|
||||
status = batch.Put(wb_key.str(), wb_value.str());
|
||||
if (!status.ok())
|
||||
throw Exception("RocksDB write error: " + status.ToString(), ErrorCodes::ROCKSDB_ERROR);
|
||||
}
|
||||
|
||||
status = storage.rocksdb_ptr->Write(rocksdb::WriteOptions(), &batch);
|
||||
if (!status.ok())
|
||||
throw Exception("RocksDB write error: " + status.ToString(), ErrorCodes::ROCKSDB_ERROR);
|
||||
}
|
||||
|
||||
}
|
||||
29
src/Storages/RocksDB/EmbeddedRocksDBBlockOutputStream.h
Normal file
29
src/Storages/RocksDB/EmbeddedRocksDBBlockOutputStream.h
Normal file
@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
|
||||
#include <DataStreams/IBlockOutputStream.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class StorageEmbeddedRocksDB;
|
||||
struct StorageInMemoryMetadata;
|
||||
using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
|
||||
|
||||
class EmbeddedRocksDBBlockOutputStream : public IBlockOutputStream
|
||||
{
|
||||
public:
|
||||
EmbeddedRocksDBBlockOutputStream(
|
||||
StorageEmbeddedRocksDB & storage_,
|
||||
const StorageMetadataPtr & metadata_snapshot_);
|
||||
|
||||
Block getHeader() const override;
|
||||
void write(const Block & block) override;
|
||||
|
||||
private:
|
||||
StorageEmbeddedRocksDB & storage;
|
||||
StorageMetadataPtr metadata_snapshot;
|
||||
size_t primary_key_pos = 0;
|
||||
};
|
||||
|
||||
}
|
||||
374
src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
Normal file
374
src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
Normal file
@ -0,0 +1,374 @@
|
||||
#include <Storages/RocksDB/StorageEmbeddedRocksDB.h>
|
||||
#include <Storages/RocksDB/EmbeddedRocksDBBlockOutputStream.h>
|
||||
#include <Storages/RocksDB/EmbeddedRocksDBBlockInputStream.h>
|
||||
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Storages/StorageFactory.h>
|
||||
|
||||
#include <Parsers/ASTSelectQuery.h>
|
||||
#include <Parsers/ASTIdentifier.h>
|
||||
#include <Parsers/ASTLiteral.h>
|
||||
#include <Parsers/ASTSubquery.h>
|
||||
#include <Parsers/ASTExpressionList.h>
|
||||
#include <Parsers/ASTFunction.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
|
||||
#include <IO/WriteBufferFromString.h>
|
||||
|
||||
#include <Processors/Sources/SourceFromInputStream.h>
|
||||
#include <Processors/Pipe.h>
|
||||
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Interpreters/Set.h>
|
||||
#include <Interpreters/PreparedSets.h>
|
||||
#include <Interpreters/TreeRewriter.h>
|
||||
|
||||
#include <Poco/File.h>
|
||||
#include <Poco/Path.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <rocksdb/db.h>
|
||||
#include <rocksdb/table.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int BAD_ARGUMENTS;
|
||||
extern const int ROCKSDB_ERROR;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
|
||||
// returns keys may be filter by condition
|
||||
static bool traverseASTFilter(const String & primary_key, const DataTypePtr & primary_key_type, const ASTPtr & elem, const PreparedSets & sets, FieldVector & res)
|
||||
{
|
||||
const auto * function = elem->as<ASTFunction>();
|
||||
if (!function)
|
||||
return false;
|
||||
|
||||
if (function->name == "and")
|
||||
{
|
||||
// one child has the key filter condition is ok
|
||||
for (const auto & child : function->arguments->children)
|
||||
if (traverseASTFilter(primary_key, primary_key_type, child, sets, res))
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
else if (function->name == "or")
|
||||
{
|
||||
// make sure every child has the key filter condition
|
||||
FieldVector child_res;
|
||||
for (const auto & child : function->arguments->children)
|
||||
{
|
||||
if (!traverseASTFilter(primary_key, primary_key_type, child, sets, child_res))
|
||||
return false;
|
||||
}
|
||||
res.insert(res.end(), child_res.begin(), child_res.end());
|
||||
return true;
|
||||
}
|
||||
else if (function->name == "equals" || function->name == "in")
|
||||
{
|
||||
const auto & args = function->arguments->as<ASTExpressionList &>();
|
||||
const ASTIdentifier * ident;
|
||||
const IAST * value;
|
||||
|
||||
if (args.children.size() != 2)
|
||||
return false;
|
||||
|
||||
if (function->name == "in")
|
||||
{
|
||||
ident = args.children.at(0)->as<ASTIdentifier>();
|
||||
if (!ident)
|
||||
return false;
|
||||
|
||||
if (ident->name() != primary_key)
|
||||
return false;
|
||||
value = args.children.at(1).get();
|
||||
|
||||
PreparedSetKey set_key;
|
||||
if ((value->as<ASTSubquery>() || value->as<ASTIdentifier>()))
|
||||
set_key = PreparedSetKey::forSubquery(*value);
|
||||
else
|
||||
set_key = PreparedSetKey::forLiteral(*value, {primary_key_type});
|
||||
|
||||
auto set_it = sets.find(set_key);
|
||||
if (set_it == sets.end())
|
||||
return false;
|
||||
SetPtr prepared_set = set_it->second;
|
||||
|
||||
if (!prepared_set->hasExplicitSetElements())
|
||||
return false;
|
||||
|
||||
prepared_set->checkColumnsNumber(1);
|
||||
const auto & set_column = *prepared_set->getSetElements()[0];
|
||||
for (size_t row = 0; row < set_column.size(); ++row)
|
||||
{
|
||||
res.push_back(set_column[row]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
if ((ident = args.children.at(0)->as<ASTIdentifier>()))
|
||||
value = args.children.at(1).get();
|
||||
else if ((ident = args.children.at(1)->as<ASTIdentifier>()))
|
||||
value = args.children.at(0).get();
|
||||
else
|
||||
return false;
|
||||
|
||||
if (ident->name() != primary_key)
|
||||
return false;
|
||||
|
||||
//function->name == "equals"
|
||||
if (const auto * literal = value->as<ASTLiteral>())
|
||||
{
|
||||
res.push_back(literal->value);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
/** Retrieve from the query a condition of the form `key = 'key'`, `key in ('xxx_'), from conjunctions in the WHERE clause.
|
||||
* TODO support key like search
|
||||
*/
|
||||
static std::pair<FieldVector, bool> getFilterKeys(const String & primary_key, const DataTypePtr & primary_key_type, const SelectQueryInfo & query_info)
|
||||
{
|
||||
const auto & select = query_info.query->as<ASTSelectQuery &>();
|
||||
if (!select.where())
|
||||
{
|
||||
return std::make_pair(FieldVector{}, true);
|
||||
}
|
||||
FieldVector res;
|
||||
auto matched_keys = traverseASTFilter(primary_key, primary_key_type, select.where(), query_info.sets, res);
|
||||
return std::make_pair(res, !matched_keys);
|
||||
}
|
||||
|
||||
|
||||
class EmbeddedRocksDBSource : public SourceWithProgress
|
||||
{
|
||||
public:
|
||||
EmbeddedRocksDBSource(
|
||||
const StorageEmbeddedRocksDB & storage_,
|
||||
const StorageMetadataPtr & metadata_snapshot_,
|
||||
const FieldVector & keys_,
|
||||
const size_t start_,
|
||||
const size_t end_,
|
||||
const size_t max_block_size_)
|
||||
: SourceWithProgress(metadata_snapshot_->getSampleBlock())
|
||||
, storage(storage_)
|
||||
, metadata_snapshot(metadata_snapshot_)
|
||||
, start(start_)
|
||||
, end(end_)
|
||||
, max_block_size(max_block_size_)
|
||||
{
|
||||
// slice the keys
|
||||
if (end > start)
|
||||
{
|
||||
keys.resize(end - start);
|
||||
std::copy(keys_.begin() + start, keys_.begin() + end, keys.begin());
|
||||
}
|
||||
}
|
||||
|
||||
String getName() const override
|
||||
{
|
||||
return storage.getName();
|
||||
}
|
||||
|
||||
Chunk generate() override
|
||||
{
|
||||
if (processed_keys >= keys.size() || (start == end))
|
||||
return {};
|
||||
|
||||
std::vector<rocksdb::Slice> slices_keys;
|
||||
slices_keys.reserve(keys.size());
|
||||
std::vector<String> values;
|
||||
std::vector<WriteBufferFromOwnString> wbs(keys.size());
|
||||
|
||||
const auto & sample_block = metadata_snapshot->getSampleBlock();
|
||||
const auto & key_column = sample_block.getByName(storage.primary_key);
|
||||
auto columns = sample_block.cloneEmptyColumns();
|
||||
size_t primary_key_pos = sample_block.getPositionByName(storage.primary_key);
|
||||
|
||||
for (size_t i = processed_keys; i < std::min(keys.size(), processed_keys + max_block_size); ++i)
|
||||
{
|
||||
key_column.type->serializeBinary(keys[i], wbs[i]);
|
||||
auto str_ref = wbs[i].stringRef();
|
||||
slices_keys.emplace_back(str_ref.data, str_ref.size);
|
||||
}
|
||||
|
||||
auto statuses = storage.rocksdb_ptr->MultiGet(rocksdb::ReadOptions(), slices_keys, &values);
|
||||
for (size_t i = 0; i < statuses.size(); ++i)
|
||||
{
|
||||
if (statuses[i].ok())
|
||||
{
|
||||
ReadBufferFromString key_buffer(slices_keys[i]);
|
||||
ReadBufferFromString value_buffer(values[i]);
|
||||
|
||||
size_t idx = 0;
|
||||
for (const auto & elem : sample_block)
|
||||
{
|
||||
elem.type->deserializeBinary(*columns[idx], idx == primary_key_pos ? key_buffer : value_buffer);
|
||||
++idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
processed_keys += max_block_size;
|
||||
|
||||
UInt64 num_rows = columns.at(0)->size();
|
||||
return Chunk(std::move(columns), num_rows);
|
||||
}
|
||||
|
||||
private:
|
||||
const StorageEmbeddedRocksDB & storage;
|
||||
|
||||
const StorageMetadataPtr metadata_snapshot;
|
||||
const size_t start;
|
||||
const size_t end;
|
||||
const size_t max_block_size;
|
||||
FieldVector keys;
|
||||
|
||||
size_t processed_keys = 0;
|
||||
};
|
||||
|
||||
|
||||
StorageEmbeddedRocksDB::StorageEmbeddedRocksDB(const StorageID & table_id_,
|
||||
const String & relative_data_path_,
|
||||
const StorageInMemoryMetadata & metadata_,
|
||||
bool attach,
|
||||
Context & context_,
|
||||
const String & primary_key_)
|
||||
: IStorage(table_id_), primary_key{primary_key_}
|
||||
{
|
||||
setInMemoryMetadata(metadata_);
|
||||
rocksdb_dir = context_.getPath() + relative_data_path_;
|
||||
if (!attach)
|
||||
{
|
||||
Poco::File(rocksdb_dir).createDirectories();
|
||||
}
|
||||
initDb();
|
||||
}
|
||||
|
||||
void StorageEmbeddedRocksDB::truncate(const ASTPtr &, const StorageMetadataPtr & , const Context &, TableExclusiveLockHolder &)
|
||||
{
|
||||
rocksdb_ptr->Close();
|
||||
Poco::File(rocksdb_dir).remove(true);
|
||||
Poco::File(rocksdb_dir).createDirectories();
|
||||
initDb();
|
||||
}
|
||||
|
||||
void StorageEmbeddedRocksDB::initDb()
|
||||
{
|
||||
rocksdb::Options options;
|
||||
rocksdb::DB * db;
|
||||
options.create_if_missing = true;
|
||||
options.compression = rocksdb::CompressionType::kZSTD;
|
||||
rocksdb::Status status = rocksdb::DB::Open(options, rocksdb_dir, &db);
|
||||
|
||||
if (status != rocksdb::Status::OK())
|
||||
throw Exception("Fail to open rocksdb path at: " + rocksdb_dir + ": " + status.ToString(), ErrorCodes::ROCKSDB_ERROR);
|
||||
rocksdb_ptr = std::unique_ptr<rocksdb::DB>(db);
|
||||
}
|
||||
|
||||
|
||||
Pipe StorageEmbeddedRocksDB::read(
|
||||
const Names & column_names,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
SelectQueryInfo & query_info,
|
||||
const Context & /*context*/,
|
||||
QueryProcessingStage::Enum /*processed_stage*/,
|
||||
size_t max_block_size,
|
||||
unsigned num_streams)
|
||||
{
|
||||
metadata_snapshot->check(column_names, getVirtuals(), getStorageID());
|
||||
FieldVector keys;
|
||||
bool all_scan = false;
|
||||
|
||||
auto primary_key_data_type = metadata_snapshot->getSampleBlock().getByName(primary_key).type;
|
||||
std::tie(keys, all_scan) = getFilterKeys(primary_key, primary_key_data_type, query_info);
|
||||
if (all_scan)
|
||||
{
|
||||
auto reader = std::make_shared<EmbeddedRocksDBBlockInputStream>(
|
||||
*this, metadata_snapshot, max_block_size);
|
||||
return Pipe(std::make_shared<SourceFromInputStream>(reader));
|
||||
}
|
||||
else
|
||||
{
|
||||
if (keys.empty())
|
||||
return {};
|
||||
|
||||
std::sort(keys.begin(), keys.end());
|
||||
auto unique_iter = std::unique(keys.begin(), keys.end());
|
||||
if (unique_iter != keys.end())
|
||||
keys.erase(unique_iter, keys.end());
|
||||
|
||||
Pipes pipes;
|
||||
size_t start = 0;
|
||||
size_t end;
|
||||
|
||||
const size_t num_threads = std::min(size_t(num_streams), keys.size());
|
||||
const size_t batch_per_size = ceil(keys.size() * 1.0 / num_threads);
|
||||
|
||||
for (size_t t = 0; t < num_threads; ++t)
|
||||
{
|
||||
if (start >= keys.size())
|
||||
start = end = 0;
|
||||
else
|
||||
end = start + batch_per_size > keys.size() ? keys.size() : start + batch_per_size;
|
||||
|
||||
pipes.emplace_back(
|
||||
std::make_shared<EmbeddedRocksDBSource>(*this, metadata_snapshot, keys, start, end, max_block_size));
|
||||
start += batch_per_size;
|
||||
}
|
||||
return Pipe::unitePipes(std::move(pipes));
|
||||
}
|
||||
}
|
||||
|
||||
BlockOutputStreamPtr StorageEmbeddedRocksDB::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/)
|
||||
{
|
||||
return std::make_shared<EmbeddedRocksDBBlockOutputStream>(*this, metadata_snapshot);
|
||||
}
|
||||
|
||||
|
||||
static StoragePtr create(const StorageFactory::Arguments & args)
|
||||
{
|
||||
// TODO custom RocksDBSettings, table function
|
||||
if (!args.engine_args.empty())
|
||||
throw Exception(
|
||||
"Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)",
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
|
||||
StorageInMemoryMetadata metadata;
|
||||
metadata.setColumns(args.columns);
|
||||
metadata.setConstraints(args.constraints);
|
||||
|
||||
if (!args.storage_def->primary_key)
|
||||
throw Exception("StorageEmbeddedRocksDB must require one column in primary key", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
metadata.primary_key = KeyDescription::getKeyFromAST(args.storage_def->primary_key->ptr(), metadata.columns, args.context);
|
||||
auto primary_key_names = metadata.getColumnsRequiredForPrimaryKey();
|
||||
if (primary_key_names.size() != 1)
|
||||
{
|
||||
throw Exception("StorageEmbeddedRocksDB must require one column in primary key", ErrorCodes::BAD_ARGUMENTS);
|
||||
}
|
||||
return StorageEmbeddedRocksDB::create(args.table_id, args.relative_data_path, metadata, args.attach, args.context, primary_key_names[0]);
|
||||
}
|
||||
|
||||
|
||||
void registerStorageEmbeddedRocksDB(StorageFactory & factory)
|
||||
{
|
||||
StorageFactory::StorageFeatures features{
|
||||
.supports_sort_order = true,
|
||||
};
|
||||
|
||||
factory.registerStorage("EmbeddedRocksDB", create, features);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
64
src/Storages/RocksDB/StorageEmbeddedRocksDB.h
Normal file
64
src/Storages/RocksDB/StorageEmbeddedRocksDB.h
Normal file
@ -0,0 +1,64 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <ext/shared_ptr_helper.h>
|
||||
#include <Storages/IStorage.h>
|
||||
|
||||
|
||||
namespace rocksdb
|
||||
{
|
||||
class DB;
|
||||
}
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
class Context;
|
||||
|
||||
class StorageEmbeddedRocksDB final : public ext::shared_ptr_helper<StorageEmbeddedRocksDB>, public IStorage
|
||||
{
|
||||
friend struct ext::shared_ptr_helper<StorageEmbeddedRocksDB>;
|
||||
friend class EmbeddedRocksDBSource;
|
||||
friend class EmbeddedRocksDBBlockOutputStream;
|
||||
friend class EmbeddedRocksDBBlockInputStream;
|
||||
public:
|
||||
std::string getName() const override { return "EmbeddedRocksDB"; }
|
||||
|
||||
Pipe read(
|
||||
const Names & column_names,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
SelectQueryInfo & query_info,
|
||||
const Context & context,
|
||||
QueryProcessingStage::Enum processed_stage,
|
||||
size_t max_block_size,
|
||||
unsigned num_streams) override;
|
||||
|
||||
BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, const Context & context) override;
|
||||
void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &) override;
|
||||
|
||||
bool supportsParallelInsert() const override { return true; }
|
||||
bool supportsIndexForIn() const override { return true; }
|
||||
bool mayBenefitFromIndexForIn(
|
||||
const ASTPtr & node, const Context & /*query_context*/, const StorageMetadataPtr & /*metadata_snapshot*/) const override
|
||||
{
|
||||
return node->getColumnName() == primary_key;
|
||||
}
|
||||
|
||||
protected:
|
||||
StorageEmbeddedRocksDB(const StorageID & table_id_,
|
||||
const String & relative_data_path_,
|
||||
const StorageInMemoryMetadata & metadata,
|
||||
bool attach,
|
||||
Context & context_,
|
||||
const String & primary_key_);
|
||||
|
||||
private:
|
||||
const String primary_key;
|
||||
using RocksDBPtr = std::unique_ptr<rocksdb::DB>;
|
||||
RocksDBPtr rocksdb_ptr;
|
||||
String rocksdb_dir;
|
||||
|
||||
void initDb();
|
||||
};
|
||||
}
|
||||
@ -1,10 +1,64 @@
|
||||
#include <Storages/registerStorages.h>
|
||||
#include <Storages/StorageFactory.h>
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
# include "config_core.h"
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void registerStorageLog(StorageFactory & factory);
|
||||
void registerStorageTinyLog(StorageFactory & factory);
|
||||
void registerStorageStripeLog(StorageFactory & factory);
|
||||
void registerStorageMergeTree(StorageFactory & factory);
|
||||
void registerStorageNull(StorageFactory & factory);
|
||||
void registerStorageMerge(StorageFactory & factory);
|
||||
void registerStorageBuffer(StorageFactory & factory);
|
||||
void registerStorageDistributed(StorageFactory & factory);
|
||||
void registerStorageMemory(StorageFactory & factory);
|
||||
void registerStorageFile(StorageFactory & factory);
|
||||
void registerStorageURL(StorageFactory & factory);
|
||||
void registerStorageDictionary(StorageFactory & factory);
|
||||
void registerStorageSet(StorageFactory & factory);
|
||||
void registerStorageJoin(StorageFactory & factory);
|
||||
void registerStorageView(StorageFactory & factory);
|
||||
void registerStorageMaterializedView(StorageFactory & factory);
|
||||
void registerStorageLiveView(StorageFactory & factory);
|
||||
void registerStorageGenerateRandom(StorageFactory & factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
void registerStorageS3(StorageFactory & factory);
|
||||
void registerStorageCOS(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
void registerStorageHDFS(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
void registerStorageODBC(StorageFactory & factory);
|
||||
void registerStorageJDBC(StorageFactory & factory);
|
||||
|
||||
#if USE_MYSQL
|
||||
void registerStorageMySQL(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
void registerStorageMongoDB(StorageFactory & factory);
|
||||
|
||||
#if USE_RDKAFKA
|
||||
void registerStorageKafka(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_AMQPCPP
|
||||
void registerStorageRabbitMQ(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_ROCKSDB
|
||||
void registerStorageEmbeddedRocksDB(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
|
||||
void registerStorages()
|
||||
{
|
||||
auto & factory = StorageFactory::instance();
|
||||
@ -28,7 +82,7 @@ void registerStorages()
|
||||
registerStorageLiveView(factory);
|
||||
registerStorageGenerateRandom(factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
#if USE_AWS_S3
|
||||
registerStorageS3(factory);
|
||||
registerStorageCOS(factory);
|
||||
#endif
|
||||
@ -53,6 +107,10 @@ void registerStorages()
|
||||
#if USE_AMQPCPP
|
||||
registerStorageRabbitMQ(factory);
|
||||
#endif
|
||||
|
||||
#if USE_ROCKSDB
|
||||
registerStorageEmbeddedRocksDB(factory);
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -1,59 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include <Common/config.h>
|
||||
# include "config_core.h"
|
||||
#endif
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class StorageFactory;
|
||||
|
||||
void registerStorageLog(StorageFactory & factory);
|
||||
void registerStorageTinyLog(StorageFactory & factory);
|
||||
void registerStorageStripeLog(StorageFactory & factory);
|
||||
void registerStorageMergeTree(StorageFactory & factory);
|
||||
void registerStorageNull(StorageFactory & factory);
|
||||
void registerStorageMerge(StorageFactory & factory);
|
||||
void registerStorageBuffer(StorageFactory & factory);
|
||||
void registerStorageDistributed(StorageFactory & factory);
|
||||
void registerStorageMemory(StorageFactory & factory);
|
||||
void registerStorageFile(StorageFactory & factory);
|
||||
void registerStorageURL(StorageFactory & factory);
|
||||
void registerStorageDictionary(StorageFactory & factory);
|
||||
void registerStorageSet(StorageFactory & factory);
|
||||
void registerStorageJoin(StorageFactory & factory);
|
||||
void registerStorageView(StorageFactory & factory);
|
||||
void registerStorageMaterializedView(StorageFactory & factory);
|
||||
void registerStorageLiveView(StorageFactory & factory);
|
||||
void registerStorageGenerateRandom(StorageFactory & factory);
|
||||
|
||||
#if USE_AWS_S3
|
||||
void registerStorageS3(StorageFactory & factory);
|
||||
void registerStorageCOS(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
void registerStorageHDFS(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
void registerStorageODBC(StorageFactory & factory);
|
||||
void registerStorageJDBC(StorageFactory & factory);
|
||||
|
||||
#if USE_MYSQL
|
||||
void registerStorageMySQL(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
void registerStorageMongoDB(StorageFactory & factory);
|
||||
|
||||
#if USE_RDKAFKA
|
||||
void registerStorageKafka(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_AMQPCPP
|
||||
void registerStorageRabbitMQ(StorageFactory & factory);
|
||||
#endif
|
||||
|
||||
void registerStorages();
|
||||
|
||||
}
|
||||
|
||||
@ -8,7 +8,7 @@ PEERDIR(
|
||||
|
||||
|
||||
SRCS(
|
||||
<? find . -name '*.cpp' | grep -v -F tests | grep -v -P 'Kafka|RabbitMQ|S3|HDFS|Licenses|TimeZones' | sed 's/^\.\// /' | sort ?>
|
||||
<? find . -name '*.cpp' | grep -v -F tests | grep -v -P 'Kafka|RabbitMQ|S3|HDFS|Licenses|TimeZones|RocksDB' | sed 's/^\.\// /' | sort ?>
|
||||
)
|
||||
|
||||
END()
|
||||
|
||||
10
tests/queries/0_stateless/01504_rocksdb.reference
Normal file
10
tests/queries/0_stateless/01504_rocksdb.reference
Normal file
@ -0,0 +1,10 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1 1 1 1 1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
47
tests/queries/0_stateless/01504_rocksdb.sql
Normal file
47
tests/queries/0_stateless/01504_rocksdb.sql
Normal file
@ -0,0 +1,47 @@
|
||||
DROP TABLE IF EXISTS test;
|
||||
|
||||
CREATE TABLE test (key String, value UInt32) Engine=EmbeddedRocksDB; -- { serverError 36 }
|
||||
CREATE TABLE test (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key2); -- { serverError 47 }
|
||||
CREATE TABLE test (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key, value); -- { serverError 36 }
|
||||
CREATE TABLE test (key Tuple(String, UInt32), value UInt64) Engine=EmbeddedRocksDB PRIMARY KEY(key);
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (key String, value UInt32) Engine=EmbeddedRocksDB PRIMARY KEY(key);
|
||||
|
||||
INSERT INTO test SELECT '1_1', number FROM numbers(10000);
|
||||
SELECT COUNT(1) == 1 FROM test;
|
||||
|
||||
INSERT INTO test SELECT concat(toString(number), '_1'), number FROM numbers(10000);
|
||||
SELECT COUNT(1) == 10000 FROM test;
|
||||
SELECT uniqExact(key) == 32 FROM (SELECT * FROM test LIMIT 32 SETTINGS max_block_size = 1);
|
||||
SELECT SUM(value) == 1 + 99 + 900 FROM test WHERE key IN ('1_1', '99_1', '900_1');
|
||||
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
DROP TABLE IF EXISTS test_memory;
|
||||
|
||||
CREATE TABLE test (k UInt32, value UInt64, dummy Tuple(UInt32, Float64), bm AggregateFunction(groupBitmap, UInt64)) Engine=EmbeddedRocksDB PRIMARY KEY(k);
|
||||
CREATE TABLE test_memory AS test Engine = Memory;
|
||||
|
||||
INSERT INTO test SELECT number % 77 AS k, SUM(number) AS value, (1, 1.2), bitmapBuild(groupArray(number)) FROM numbers(10000000) group by k;
|
||||
|
||||
INSERT INTO test_memory SELECT number % 77 AS k, SUM(number) AS value, (1, 1.2), bitmapBuild(groupArray(number)) FROM numbers(10000000) group by k;
|
||||
|
||||
|
||||
SELECT A.a = B.a, A.b = B.b, A.c = B.c, A.d = B.d, A.e = B.e FROM ( SELECT 0 AS a, groupBitmapMerge(bm) AS b , SUM(k) AS c, SUM(value) AS d, SUM(dummy.1) AS e FROM test) A ANY LEFT JOIN (SELECT 0 AS a, groupBitmapMerge(bm) AS b , SUM(k) AS c, SUM(value) AS d, SUM(dummy.1) AS e FROM test_memory) B USING a ORDER BY a;
|
||||
|
||||
CREATE TEMPORARY TABLE keys AS SELECT * FROM numbers(1000);
|
||||
|
||||
SET max_rows_to_read = 2;
|
||||
SELECT dummy == (1,1.2) FROM test WHERE k IN (1, 3) OR k IN (1) OR k IN (3, 1) OR k IN [1] OR k IN [1, 3] ;
|
||||
SELECT k == 4 FROM test WHERE k = 4 OR k IN [4] OR k in (4, 10000001, 10000002) AND value > 0;
|
||||
SELECT k == 4 FROM test WHERE k IN (SELECT toUInt32(number) FROM keys WHERE number = 4);
|
||||
SELECT k, value FROM test WHERE k = 0 OR value > 0; -- { serverError 158 }
|
||||
SELECT k, value FROM test WHERE k = 0 AND k IN (1, 3) OR k > 8; -- { serverError 158 }
|
||||
|
||||
TRUNCATE TABLE test;
|
||||
SELECT 0 == COUNT(1) FROM test;
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
DROP TABLE IF EXISTS test_memory;
|
||||
|
||||
@ -150,6 +150,7 @@
|
||||
00609_mv_index_in_in
|
||||
00510_materizlized_view_and_deduplication_zookeeper
|
||||
00738_lock_for_inner_table
|
||||
01504_rocksdb
|
||||
01515_force_data_skipping_indices
|
||||
01526_complex_key_dict_direct_layout
|
||||
01509_dictionary_preallocate
|
||||
|
||||
Loading…
Reference in New Issue
Block a user