thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master'

Browse Source
This commit is contained in:
BayoNet 2017-11-28 15:27:33 +03:00
commit 92fabedfa3
124 changed files with 861 additions and 440 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
CMakeLists.txt
View File

@ -75,7 +75,7 @@ if (USE_STATIC_LIBRARIES)
list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
endif ()
if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64.*|x86_64.*|AMD64.*")
if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "amd64.*|x86_64.*|AMD64.*")
option (USE_INTERNAL_MEMCPY "Use internal implementation of 'memcpy' function instead of provided by libc. Only for x86_64." ON)
if (CMAKE_SYSTEM MATCHES "Linux")
@ -123,15 +123,6 @@ if (ARCHNATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
endif ()
if (CMAKE_SYSTEM MATCHES "Linux" AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux with Clang)" ON)
if (USE_LIBCXX)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -pthread") # NOTE: Why this is not the default and why this is needed only with libc++?
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=1") # More checks in debug build.
endif ()
endif ()
# Special options for better optimized code with clang
#if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
# set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -Wno-unused-command-line-argument -mllvm -inline-threshold=10000")
@ -156,20 +147,29 @@ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fn
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3")
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline")
if (NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_SYSTEM MATCHES "FreeBSD"))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libstdc++")
if (MAKE_STATIC_LIBRARIES AND NOT APPLE AND NOT (CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_SYSTEM MATCHES "FreeBSD"))
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
endif ()
if (NOT APPLE)
set(THREADS_PREFER_PTHREAD_FLAG ON)
include (cmake/test_compiler.cmake)
if (CMAKE_SYSTEM MATCHES "Linux" AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${GLIBC_COMPATIBILITY_LINK_FLAGS} ${CXX11_ABI_FLAGS}")
if (USE_LIBCXX AND (CMAKE_CXX_COMPILER_ID STREQUAL "Clang"))
link_libraries (-Wl,-Bstatic -stdlib=libc++ c++ c++abi -Wl,-Bdynamic)
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++ (only make sense on Linux with Clang)" ${HAVE_LIBCXX})
if (USE_LIBCXX)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") # Ok for clang6, for older can cause 'not used option' warning
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=1") # More checks in debug build.
if (MAKE_STATIC_LIBRARIES)
link_libraries (-Wl,-Bstatic -stdlib=libc++ c++ c++abi -Wl,-Bdynamic)
else ()
link_libraries (-stdlib=libc++ c++ c++abi)
endif ()
endif ()
endif ()
include (cmake/test_compiler.cmake)
if (USE_STATIC_LIBRARIES AND HAVE_NO_PIE)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FLAG_NO_PIE}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FLAG_NO_PIE}")
@ -278,9 +278,10 @@ include (libs/libdaemon/cmake/find_unwind.cmake)
set (FULL_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
set (FULL_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
message (STATUS "C_FLAGS = ${FULL_C_FLAGS}")
message (STATUS "CXX_FLAGS = ${FULL_CXX_FLAGS}")
message (STATUS "LINK_FLAGS = ${CMAKE_EXE_LINKER_FLAGS}")
set (FULL_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${CMAKE_BUILD_TYPE_UC}}")
message (STATUS "C_FLAGS = ${FULL_C_FLAGS}")
message (STATUS "CXX_FLAGS = ${FULL_CXX_FLAGS}")
message (STATUS "LINKER_FLAGS = ${FULL_EXE_LINKER_FLAGS}")
# Directory for Yandex specific files
set (CLICKHOUSE_PRIVATE_DIR ${ClickHouse_SOURCE_DIR}/private/)

4
cmake/find_poco.cmake
View File

@ -62,8 +62,8 @@ else ()
if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY)
list (APPEND Poco_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/libzlib-ng/"
"${ClickHouse_BINARY_DIR}/contrib/libzlib-ng/"
"${ClickHouse_SOURCE_DIR}/contrib/zlib-ng/"
"${ClickHouse_BINARY_DIR}/contrib/zlib-ng/"
)
endif ()

6
cmake/find_zlib.cmake
View File

@ -7,7 +7,11 @@ endif ()
if (NOT ZLIB_FOUND)
set (USE_INTERNAL_ZLIB_LIBRARY 1)
set (ZLIB_COMPAT 1) # for zlib-ng, also enables WITH_GZFILEOP
set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libzlib-ng" "${ClickHouse_BINARY_DIR}/contrib/libzlib-ng") # generated zconf.h
set (WITH_NATIVE_INSTRUCTIONS ${ARCHNATIVE})
if (CMAKE_SYSTEM MATCHES "FreeBSD" OR ARCH_I386)
set (WITH_OPTIM 0 CACHE INTERNAL "") # Bug in assembler
endif ()
set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/zlib-ng" "${ClickHouse_BINARY_DIR}/contrib/zlib-ng") # generated zconf.h
set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco
set (ZLIB_FOUND 1) # for poco
if (USE_STATIC_LIBRARIES)

18
cmake/test_compiler.cmake
View File

@ -1,9 +1,24 @@
include (CheckCXXSourceCompiles)
include (CMakePushCheckState)
cmake_push_check_state ()
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
# clang4 : -no-pie cause error
# clang6 : -no-pie cause warning
set (TEST_FLAG "-lc++ -lc++abi")
set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG}")
check_cxx_source_compiles("
#include <iostream>
int main() {
std::cerr << std::endl;
return 0;
}
" HAVE_LIBCXX)
else ()
cmake_push_check_state ()
@ -21,6 +36,7 @@ else ()
set (FLAG_NO_PIE ${TEST_FLAG})
endif ()
cmake_pop_check_state ()
endif ()
cmake_pop_check_state ()

8
contrib/CMakeLists.txt vendored
View File

@ -50,6 +50,14 @@ if (USE_INTERNAL_ZLIB_LIBRARY)
# We should use same defines when including zlib.h as used when zlib compiled
target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP)
set_target_properties(example PROPERTIES EXCLUDE_FROM_ALL 1)
set_target_properties(minigzip PROPERTIES EXCLUDE_FROM_ALL 1)
if (TARGET example64)
set_target_properties(example64 PROPERTIES EXCLUDE_FROM_ALL 1)
endif ()
if (TARGET minigzip64)
set_target_properties(minigzip64 PROPERTIES EXCLUDE_FROM_ALL 1)
endif ()
endif ()
if (USE_INTERNAL_CCTZ_LIBRARY)

6
contrib/libtcmalloc/CMakeLists.txt
View File

@ -37,7 +37,9 @@ add_library (tcmalloc_minimal_internal
./src/memory_region_map.cc
)
target_compile_options (tcmalloc_minimal_internal PUBLIC
target_compile_options (tcmalloc_minimal_internal
PRIVATE
-DNO_TCMALLOC_SAMPLES
-DNDEBUG
-DNO_FRAME_POINTER
@ -47,6 +49,8 @@ target_compile_options (tcmalloc_minimal_internal PUBLIC
-Wno-deprecated-declarations
-Wno-unused-function
-Wno-unused-private-field
PUBLIC
-fno-builtin-malloc
-fno-builtin-free
-fno-builtin-realloc

2
contrib/zookeeper vendored

@ -1 +1 @@
Subproject commit d2f05a6946820a6e1a3ba326da446f0a8f9546ed
Subproject commit 5aa9e889fe9e739af3c2a00222d9a3a0a57179dd

6
dbms/CMakeLists.txt
View File

@ -157,11 +157,12 @@ target_link_libraries (clickhouse_common_io
${ZOOKEEPER_LIBRARY}
${DOUBLE_CONVERSION_LIBRARIES}
${Poco_Net_LIBRARY}
${PLATFORM_LIBS}
)
target_link_libraries (dbms
clickhouse_common_io
clickhouse_parsers
clickhouse_common_io
${MYSQLXX_LIBRARY}
${FARMHASH_LIBRARIES}
${METROHASH_LIBRARIES}
@ -197,9 +198,8 @@ if (USE_CAPNP)
endif ()
target_link_libraries (dbms
${PLATFORM_LIBS}
${CMAKE_DL_LIBS}
${CMAKE_THREAD_LIBS_INIT}
Threads::Threads
)
target_include_directories (dbms BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})

4
dbms/cmake/version.cmake
View File

@ -1,6 +1,6 @@
# This strings autochanged from release_lib.sh:
set(VERSION_DESCRIBE v1.1.54312-testing)
set(VERSION_REVISION 54312)
set(VERSION_DESCRIBE v1.1.54314-testing)
set(VERSION_REVISION 54314)
# end of autochange
set (VERSION_MAJOR 1)

2
dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h
View File

@ -9,7 +9,7 @@
#include <Columns/ColumnArray.h>
#include <Columns/ColumnVector.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Interpreters/convertFieldToType.h>
#include <AggregateFunctions/IBinaryAggregateFunction.h>

2
dbms/src/AggregateFunctions/AggregateFunctionQuantile.h
View File

@ -2,7 +2,7 @@
#include <AggregateFunctions/ReservoirSampler.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>

2
dbms/src/AggregateFunctions/AggregateFunctionQuantileDeterministic.h
View File

@ -2,7 +2,7 @@
#include <AggregateFunctions/ReservoirSamplerDeterministic.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>

2
dbms/src/AggregateFunctions/AggregateFunctionSumMap.h
View File

@ -9,7 +9,7 @@
#include <Columns/ColumnArray.h>
#include <Columns/ColumnTuple.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <AggregateFunctions/IBinaryAggregateFunction.h>
#include <Functions/FunctionHelpers.h>
#include <map>

2
dbms/src/AggregateFunctions/AggregateFunctionTopK.h
View File

@ -11,7 +11,7 @@
#include <Common/SpaceSaving.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <AggregateFunctions/AggregateFunctionGroupArray.h>

2
dbms/src/AggregateFunctions/AggregateFunctionUniqUpTo.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <AggregateFunctions/IUnaryAggregateFunction.h>
#include <AggregateFunctions/UniqVariadicHash.h>
#include <DataTypes/DataTypesNumber.h>

2
dbms/src/AggregateFunctions/CMakeLists.txt
View File

@ -26,6 +26,6 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
parseAggregateFunctionParameters.h
)
add_library(clickhouse_aggregate_functions ${SPLIT_SHARED} ${clickhouse_aggregate_functions_sources})
add_library(clickhouse_aggregate_functions ${clickhouse_aggregate_functions_sources})
target_link_libraries(clickhouse_aggregate_functions dbms)
target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

2
dbms/src/AggregateFunctions/QuantilesCommon.h
View File

@ -3,7 +3,7 @@
#include <vector>
#include <Core/Field.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
namespace DB

2
dbms/src/Analyzers/TypeAndConstantInference.cpp
View File

@ -15,7 +15,7 @@
#include <Parsers/ASTSubquery.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/FieldToDataType.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeExpression.h>

2
dbms/src/Client/Connection.h
View File

@ -8,7 +8,7 @@
#include <Core/Block.h>
#include <Core/Defines.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <Core/Protocol.h>
#include <Core/QueryProcessingStage.h>

2
dbms/src/Columns/ColumnConst.cpp
View File

@ -3,7 +3,7 @@
#include <Columns/ColumnConst.h>
#include <Columns/ColumnNullable.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/typeid_cast.h>

74
dbms/src/Common/ConfigProcessor.cpp
View File

@ -35,8 +35,21 @@ static std::string numberFromHost(const std::string & s)
return "";
}
ConfigProcessor::ConfigProcessor(bool throw_on_bad_incl_, bool log_to_console, const Substitutions & substitutions_)
: throw_on_bad_incl(throw_on_bad_incl_)
static std::string preprocessedConfigPath(const std::string & path)
{
Poco::Path preprocessed_path(path);
preprocessed_path.setBaseName(preprocessed_path.getBaseName() + "-preprocessed");
return preprocessed_path.toString();
}
ConfigProcessor::ConfigProcessor(
const std::string & path_,
bool throw_on_bad_incl_,
bool log_to_console,
const Substitutions & substitutions_)
: path(path_)
, preprocessed_path(preprocessedConfigPath(path))
, throw_on_bad_incl(throw_on_bad_incl_)
, substitutions(substitutions_)
/// We need larger name pool to allow to support vast amount of users in users.xml files for ClickHouse.
/// Size is prime because Poco::XML::NamePool uses bad (inefficient, low quality)
@ -118,13 +131,6 @@ static bool allWhitespace(const std::string & s)
return s.find_first_not_of(" \t\n\r") == std::string::npos;
}
static std::string preprocessedConfigPath(const std::string & path)
{
Poco::Path preprocessed_path(path);
preprocessed_path.setBaseName(preprocessed_path.getBaseName() + "-preprocessed");
return preprocessed_path.toString();
}
void ConfigProcessor::mergeRecursive(XMLDocumentPtr config, Node * config_root, Node * with_root)
{
NodeListPtr with_nodes = with_root->childNodes();
@ -373,16 +379,15 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
}
XMLDocumentPtr ConfigProcessor::processConfig(
const std::string & path_str,
bool * has_zk_includes,
zkutil::ZooKeeperNodeCache * zk_node_cache)
bool * has_zk_includes,
zkutil::ZooKeeperNodeCache * zk_node_cache)
{
XMLDocumentPtr config = dom_parser.parse(path_str);
XMLDocumentPtr config = dom_parser.parse(path);
std::vector<std::string> contributing_files;
contributing_files.push_back(path_str);
contributing_files.push_back(path);
for (auto & merge_file : getConfigMergeFiles(path_str))
for (auto & merge_file : getConfigMergeFiles(path))
{
try
{
@ -392,7 +397,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
}
catch (Poco::Exception & e)
{
throw Poco::Exception("Failed to merge config with " + merge_file + ": " + e.displayText());
throw Poco::Exception("Failed to merge config with '" + merge_file + "': " + e.displayText());
}
}
@ -422,7 +427,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
}
catch (Poco::Exception & e)
{
throw Poco::Exception("Failed to preprocess config `" + path_str + "': " + e.displayText(), e);
throw Poco::Exception("Failed to preprocess config '" + path + "': " + e.displayText(), e);
}
if (has_zk_includes)
@ -432,15 +437,15 @@ XMLDocumentPtr ConfigProcessor::processConfig(
comment << " This file was generated automatically.\n";
comment << " Do not edit it: it is likely to be discarded and generated again before it's read next time.\n";
comment << " Files used to generate this file:";
for (const std::string & path : contributing_files)
for (const std::string & contributing_file : contributing_files)
{
comment << "\n " << path;
comment << "\n " << contributing_file;
}
if (zk_node_cache && !contributing_zk_paths.empty())
{
comment << "\n ZooKeeper nodes used to generate this file:";
for (const std::string & path : contributing_zk_paths)
comment << "\n " << path;
for (const std::string & contributing_zk_path : contributing_zk_paths)
comment << "\n " << contributing_zk_path;
}
comment << " ";
@ -452,39 +457,29 @@ XMLDocumentPtr ConfigProcessor::processConfig(
return config;
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(const std::string & path, bool allow_zk_includes)
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfig(bool allow_zk_includes)
{
bool has_zk_includes;
XMLDocumentPtr config_xml = processConfig(path, &has_zk_includes);
XMLDocumentPtr config_xml = processConfig(&has_zk_includes);
if (has_zk_includes && !allow_zk_includes)
throw Poco::Exception("Error while loading config `" + path + "': from_zk includes are not allowed!");
bool preprocessed_written = false;
if (!has_zk_includes)
{
savePreprocessedConfig(config_xml, preprocessedConfigPath(path));
preprocessed_written = true;
}
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, preprocessed_written};
return LoadedConfig{configuration, has_zk_includes, /* loaded_from_preprocessed = */ false, config_xml};
}
ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
const std::string & path,
zkutil::ZooKeeperNodeCache & zk_node_cache,
bool fallback_to_preprocessed)
{
std::string preprocessed_path = preprocessedConfigPath(path);
XMLDocumentPtr config_xml;
bool has_zk_includes;
bool processed_successfully = false;
try
{
config_xml = processConfig(path, &has_zk_includes, &zk_node_cache);
config_xml = processConfig(&has_zk_includes, &zk_node_cache);
processed_successfully = true;
}
catch (const Poco::Exception & ex)
@ -504,19 +499,16 @@ ConfigProcessor::LoadedConfig ConfigProcessor::loadConfigWithZooKeeperIncludes(
config_xml = dom_parser.parse(preprocessed_path);
}
if (processed_successfully)
savePreprocessedConfig(config_xml, preprocessed_path);
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return LoadedConfig{configuration, has_zk_includes, !processed_successfully, processed_successfully};
return LoadedConfig{configuration, has_zk_includes, !processed_successfully, config_xml};
}
void ConfigProcessor::savePreprocessedConfig(const XMLDocumentPtr & config, const std::string & preprocessed_path)
void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config)
{
try
{
DOMWriter().writeNode(preprocessed_path, config);
DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml);
}
catch (Poco::Exception & e)
{

28
dbms/src/Common/ConfigProcessor.h
View File

@ -32,7 +32,11 @@ public:
using Substitutions = std::vector<std::pair<std::string, std::string>>;
/// Set log_to_console to true if the logging subsystem is not initialized yet.
ConfigProcessor(bool throw_on_bad_incl = false, bool log_to_console = false, const Substitutions & substitutions = Substitutions());
explicit ConfigProcessor(
const std::string & path,
bool throw_on_bad_incl = false,
bool log_to_console = false,
const Substitutions & substitutions = Substitutions());
~ConfigProcessor();
@ -52,9 +56,8 @@ public:
/// If has_zk_includes is non-NULL and there are such elements, set has_zk_includes to true.
/// 5) (Yandex.Metrika-specific) Substitute "<layer/>" with "<layer>layer number from the hostname</layer>".
XMLDocumentPtr processConfig(
const std::string & path,
bool * has_zk_includes = nullptr,
zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr);
bool * has_zk_includes = nullptr,
zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr);
/// loadConfig* functions apply processConfig and create Poco::Util::XMLConfiguration.
@ -67,28 +70,31 @@ public:
ConfigurationPtr configuration;
bool has_zk_includes;
bool loaded_from_preprocessed;
bool preprocessed_written;
XMLDocumentPtr preprocessed_xml;
};
/// If allow_zk_includes is true, expect that the configuration XML can contain from_zk nodes.
/// If it is the case, set has_zk_includes to true and don't write config-preprocessed.xml,
/// expecting that config would be reloaded with zookeeper later.
LoadedConfig loadConfig(const std::string & path, bool allow_zk_includes = false);
LoadedConfig loadConfig(bool allow_zk_includes = false);
/// If fallback_to_preprocessed is true, then if KeeperException is thrown during config
/// processing, load the configuration from the preprocessed file.
LoadedConfig loadConfigWithZooKeeperIncludes(
const std::string & path,
zkutil::ZooKeeperNodeCache & zk_node_cache,
bool fallback_to_preprocessed = false);
zkutil::ZooKeeperNodeCache & zk_node_cache,
bool fallback_to_preprocessed = false);
void savePreprocessedConfig(const LoadedConfig & loaded_config);
public:
using Files = std::list<std::string>;
static Files getConfigMergeFiles(const std::string & config_path);
private:
const std::string path;
const std::string preprocessed_path;
bool throw_on_bad_incl;
Logger * log;
@ -114,6 +120,4 @@ private:
Poco::XML::Node * node,
zkutil::ZooKeeperNodeCache * zk_node_cache,
std::unordered_set<std::string> & contributing_zk_paths);
void savePreprocessedConfig(const XMLDocumentPtr & config, const std::string & preprocessed_path);
};

2
dbms/src/Core/FieldVisitors.cpp → dbms/src/Common/FieldVisitors.cpp
View File

@ -4,7 +4,7 @@
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/SipHash.h>

0
dbms/src/Core/FieldVisitors.h → dbms/src/Common/FieldVisitors.h
View File

4
dbms/src/Common/ZooKeeper/tests/zk_many_watches_reconnect.cpp
View File

@ -23,8 +23,8 @@ int main(int argc, char ** argv)
return 3;
}
ConfigProcessor processor(false, true);
auto config = processor.loadConfig(argv[1]).configuration;
ConfigProcessor processor(argv[1], false, true);
auto config = processor.loadConfig().configuration;
zkutil::ZooKeeper zk(*config, "zookeeper");
zkutil::EventPtr watch = std::make_shared<Poco::Event>();

4
dbms/src/Common/ZooKeeper/tests/zkutil_test_multi_exception.cpp
View File

@ -2,7 +2,11 @@
#include <Common/Exception.h>
#include <iostream>
#include <chrono>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
using namespace DB;

4
dbms/src/Common/tests/CMakeLists.txt
View File

@ -63,5 +63,5 @@ add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cp
target_include_directories (integer_hash_tables_and_hashes BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
target_link_libraries (integer_hash_tables_and_hashes clickhouse_common_io)
add_executable (allocactor allocator.cpp)
target_link_libraries (allocactor dbms)
add_executable (allocator allocator.cpp)
target_link_libraries (allocator clickhouse_common_io)

6
dbms/src/Common/tests/allocator.cpp
View File

@ -15,7 +15,7 @@ int main()
p = alloc.realloc(p, old_size, size);
old_size = size;
}
alloc.free(p, size);
alloc.free(p, old_size);
std::cerr << "50mb+50mb+.. ok.\n";
}
@ -38,7 +38,7 @@ int main()
size = old_size;
std::cerr << "ok. impossible catch.\n";
} */
alloc.free(p, size);
alloc.free(p, old_size);
std::cerr << "1gb,512mb,128mb,.. ok.\n";
}
@ -53,7 +53,7 @@ int main()
p = alloc.realloc(p, old_size, size);
old_size = size;
}
alloc.free(p, size);
alloc.free(p, old_size);
std::cerr << "1,2,4,8,..,1G ok.\n";
}

4
dbms/src/Common/tests/gtest_rw_lock_fifo.cpp.cpp
View File

@ -1,4 +1,8 @@
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
#include <Common/RWLockFIFO.h>
#include <Common/Stopwatch.h>
#include <common/Types.h>

4
dbms/src/Common/tests/gtest_unescapeForFileName.cpp
View File

@ -1,5 +1,9 @@
#include <Common/escapeForFileName.h>
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
using namespace DB;

2
dbms/src/Core/Field.cpp
View File

@ -4,7 +4,7 @@
#include <IO/WriteHelpers.h>
#include <Core/Field.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
namespace DB

1
dbms/src/Core/tests/CMakeLists.txt
View File

@ -12,6 +12,7 @@ add_executable (move_field move_field.cpp)
target_link_libraries (move_field clickhouse_common_io)
add_executable (rvo_test rvo_test.cpp)
target_link_libraries (rvo_test Threads::Threads)
add_executable (string_ref_hash string_ref_hash.cpp)
target_link_libraries (string_ref_hash clickhouse_common_io)

2
dbms/src/Core/tests/field.cpp
View File

@ -3,7 +3,7 @@
#include <sstream>
#include <Core/Field.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/Stopwatch.h>
#include <DataStreams/TabSeparatedRowOutputStream.h>

2
dbms/src/DataStreams/CollapsingSortedBlockInputStream.cpp
View File

@ -1,4 +1,4 @@
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataStreams/CollapsingSortedBlockInputStream.h>
#include <Columns/ColumnsNumber.h>

2
dbms/src/DataStreams/IProfilingBlockInputStream.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <Interpreters/Limits.h>

2
dbms/src/DataStreams/JSONRowOutputStream.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Core/Block.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <IO/WriteBuffer.h>
#include <Common/Stopwatch.h>
#include <DataStreams/IRowOutputStream.h>

31
dbms/src/DataStreams/SummingSortedBlockInputStream.cpp
View File

@ -5,7 +5,7 @@
#include <DataTypes/DataTypeArray.h>
#include <Columns/ColumnTuple.h>
#include <Common/StringUtils.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <common/logger_useful.h>
#include <Common/typeid_cast.h>
@ -62,12 +62,10 @@ void SummingSortedBlockInputStream::insertCurrentRowIfNeeded(ColumnPlainPtrs & m
}
catch (...)
{
desc.function->destroy(desc.state.data());
desc.created = false;
desc.destroyState();
throw;
}
desc.function->destroy(desc.state.data());
desc.created = false;
desc.destroyState();
}
else
desc.merged_column->insertDefault();
@ -123,8 +121,6 @@ Block SummingSortedBlockInputStream::readImpl()
/// Additional initialization.
if (current_row.empty())
{
auto & factory = AggregateFunctionFactory::instance();
current_row.resize(num_columns);
next_key.columns.resize(description.size());
@ -178,12 +174,9 @@ Block SummingSortedBlockInputStream::readImpl()
std::find(column_names_to_sum.begin(), column_names_to_sum.end(), column.name))
{
// Create aggregator to sum this column
auto desc = AggregateDescription{};
AggregateDescription desc;
desc.column_numbers = {i};
desc.function = factory.get("sumWithOverflow", {column.type});
desc.function->setArguments({column.type});
desc.add_function = desc.function->getAddressOfAddFunction();
desc.state.resize(desc.function->sizeOfData());
desc.init("sumWithOverflow", {column.type});
columns_to_aggregate.emplace_back(std::move(desc));
}
else
@ -218,8 +211,8 @@ Block SummingSortedBlockInputStream::readImpl()
}
DataTypes argument_types = {};
auto desc = AggregateDescription{};
auto map_desc = MapDescription{};
AggregateDescription desc;
MapDescription map_desc;
column_num_it = map.second.begin();
for (; column_num_it != map.second.end(); ++column_num_it)
@ -263,10 +256,7 @@ Block SummingSortedBlockInputStream::readImpl()
if (map_desc.key_col_nums.size() == 1)
{
// Create summation for all value columns in the map
desc.function = factory.get("sumMap", argument_types);
desc.function->setArguments(argument_types);
desc.add_function = desc.function->getAddressOfAddFunction();
desc.state.resize(desc.function->sizeOfData());
desc.init("sumMap", argument_types);
columns_to_aggregate.emplace_back(std::move(desc));
}
else
@ -345,10 +335,7 @@ void SummingSortedBlockInputStream::merge(ColumnPlainPtrs & merged_columns, std:
/// Reset aggregation states for next row
for (auto & desc : columns_to_aggregate)
{
desc.function->create(desc.state.data());
desc.created = true;
}
desc.createState();
// Start aggregations with current row
addRow(current_row, current);

33
dbms/src/DataStreams/SummingSortedBlockInputStream.h
View File

@ -4,6 +4,8 @@
#include <Core/ColumnNumbers.h>
#include <DataStreams/MergingSortedBlockInputStream.h>
#include <AggregateFunctions/IAggregateFunction.h>
#include <AggregateFunctions/AggregateFunctionFactory.h>
namespace DB
{
@ -79,12 +81,39 @@ private:
std::vector<char> state;
bool created = false;
void init(const char * function_name, const DataTypes & argument_types)
{
function = AggregateFunctionFactory::instance().get(function_name, argument_types);
function->setArguments(argument_types);
add_function = function->getAddressOfAddFunction();
state.resize(function->sizeOfData());
}
void createState()
{
if (created)
return;
function->create(state.data());
created = true;
}
void destroyState()
{
if (!created)
return;
function->destroy(state.data());
created = false;
}
/// Explicitly destroy aggregation state if the stream is terminated
~AggregateDescription()
{
if (created)
function->destroy(state.data());
destroyState();
}
AggregateDescription() = default;
AggregateDescription(AggregateDescription &&) = default;
AggregateDescription(const AggregateDescription &) = delete;
};
/// Stores numbers of key-columns and value-columns.

2
dbms/src/DataStreams/ValuesRowInputStream.cpp
View File

@ -5,7 +5,7 @@
#include <Parsers/ExpressionListParsers.h>
#include <DataStreams/ValuesRowInputStream.h>
#include <DataTypes/DataTypeArray.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Core/Block.h>
#include <Common/typeid_cast.h>

2
dbms/src/DataStreams/XMLRowOutputStream.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Core/Block.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <IO/WriteBuffer.h>
#include <Common/Stopwatch.h>
#include <DataStreams/IRowOutputStream.h>

2
dbms/src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -1,4 +1,4 @@
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>

2
dbms/src/DataTypes/FieldToDataType.cpp
View File

@ -1,4 +1,4 @@
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/FieldToDataType.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeNull.h>

2
dbms/src/DataTypes/FieldToDataType.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
namespace DB

2
dbms/src/Dictionaries/CatBoostModel.cpp
View File

@ -1,5 +1,5 @@
#include <Dictionaries/CatBoostModel.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <mutex>
#include <Columns/ColumnString.h>
#include <Columns/ColumnFixedString.h>

21
dbms/src/Dictionaries/DictionarySourceFactory.cpp
View File

@ -9,7 +9,7 @@
#include <Dictionaries/LibraryDictionarySource.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDate.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Columns/ColumnsNumber.h>
#include <IO/HTTPCommon.h>
#include <memory>
@ -27,7 +27,9 @@
#include <Dictionaries/MySQLDictionarySource.h>
#endif
#include <Poco/Logger.h>
#include <common/logger_useful.h>
namespace DB
{
@ -82,12 +84,20 @@ Block createSampleBlock(const DictionaryStructure & dict_struct)
DictionarySourceFactory::DictionarySourceFactory()
: log(&Poco::Logger::get("DictionarySourceFactory"))
{
#if Poco_DataODBC_FOUND
Poco::Data::ODBC::Connector::registerConnector();
#endif
}
void DictionarySourceFactory::registerSource(const std::string & source_type, Creator create_source)
{
LOG_DEBUG(log, "Register dictionary source type `" + source_type + "`");
if (!registered_sources.emplace(source_type, std::move(create_source)).second)
throw Exception("DictionarySourceFactory: the source name '" + source_type + "' is not unique",
ErrorCodes::LOGICAL_ERROR);
}
DictionarySourcePtr DictionarySourceFactory::create(
const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
@ -176,6 +186,15 @@ DictionarySourcePtr DictionarySourceFactory::create(
{
return std::make_unique<LibraryDictionarySource>(dict_struct, config, config_prefix + ".library", sample_block, context);
}
else
{
const auto found = registered_sources.find(source_type);
if (found != registered_sources.end())
{
const auto & create_source = found->second;
return create_source(dict_struct, config, config_prefix, sample_block, context);
}
}
throw Exception{
name + ": unknown dictionary source type: " + source_type,

18
dbms/src/Dictionaries/DictionarySourceFactory.h
View File

@ -3,6 +3,7 @@
#include <Dictionaries/IDictionarySource.h>
#include <ext/singleton.h>
#include <unordered_map>
namespace Poco
{
@ -10,6 +11,8 @@ namespace Poco
{
class AbstractConfiguration;
}
class Logger;
}
namespace DB
@ -22,11 +25,26 @@ struct DictionaryStructure;
class DictionarySourceFactory : public ext::singleton<DictionarySourceFactory>
{
public:
using Creator = std::function<DictionarySourcePtr(
const DictionaryStructure & dict_struct,
const Poco::Util::AbstractConfiguration & config,
const std::string & config_prefix,
Block & sample_block,
const Context & context)>;
DictionarySourceFactory();
void registerSource(const std::string & source_type, Creator create_source);
DictionarySourcePtr create(
const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix,
const DictionaryStructure & dict_struct, Context & context) const;
private:
using SourceRegistry = std::unordered_map<std::string, Creator>;
SourceRegistry registered_sources;
Poco::Logger * log;
};
}

2
dbms/src/Dictionaries/MongoDBBlockInputStream.cpp
View File

@ -18,7 +18,7 @@
#include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h>
#include <ext/range.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
namespace DB

2
dbms/src/Dictionaries/MongoDBDictionarySource.cpp
View File

@ -14,7 +14,7 @@
// dbms/src/IO/WriteHelpers.h:146 #define writeCString(s, buf)
#include <Dictionaries/MongoDBDictionarySource.h>
#include <Dictionaries/MongoDBBlockInputStream.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <ext/enumerate.h>

4
dbms/src/Functions/CMakeLists.txt
View File

@ -24,6 +24,8 @@ generate_function_register(Arithmetic
FunctionBitTest
FunctionBitTestAny
FunctionBitTestAll
FunctionGCD
FunctionLCM
)
generate_function_register(Array
@ -68,7 +70,7 @@ add_headers_and_sources(clickhouse_functions ${ClickHouse_BINARY_DIR}/dbms/src/F
list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp DataTypeTraits.cpp FunctionHelpers.cpp)
list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h DataTypeTraits.h FunctionHelpers.h)
add_library(clickhouse_functions ${SPLIT_SHARED} ${clickhouse_functions_sources})
add_library(clickhouse_functions ${clickhouse_functions_sources})
target_link_libraries(clickhouse_functions dbms)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)

43
dbms/src/Functions/FunctionsArithmetic.h
View File

@ -11,11 +11,12 @@
#include <Functions/FunctionFactory.h>
#include <DataTypes/NumberTraits.h>
#include <Core/AccurateComparison.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/typeid_cast.h>
#include <IO/WriteHelpers.h>
#include <Interpreters/ExpressionActions.h>
#include <ext/range.h>
#include <boost/math/common_factor.hpp>
namespace DB
@ -209,9 +210,9 @@ struct ModuloImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<A>::Type(b));
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
return typename NumberTraits::ToInteger<A>::Type(a)
% typename NumberTraits::ToInteger<A>::Type(b);
% typename NumberTraits::ToInteger<B>::Type(b);
}
};
@ -430,6 +431,38 @@ struct AbsImpl
}
};
template <typename A, typename B>
struct GCDImpl
{
using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<B>::Type(b), typename NumberTraits::ToInteger<A>::Type(a));
return boost::math::gcd(
typename NumberTraits::ToInteger<Result>::Type(a),
typename NumberTraits::ToInteger<Result>::Type(b));
}
};
template <typename A, typename B>
struct LCMImpl
{
using ResultType = typename NumberTraits::ResultOfAdditionMultiplication<A, B>::Type;
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<B>::Type(b), typename NumberTraits::ToInteger<A>::Type(a));
return boost::math::lcm(
typename NumberTraits::ToInteger<Result>::Type(a),
typename NumberTraits::ToInteger<Result>::Type(b));
}
};
/// this one is just for convenience
template <bool B, typename T1, typename T2> using If = typename std::conditional<B, T1, T2>::type;
/// these ones for better semantics
@ -1006,6 +1039,8 @@ struct NameBitTestAny { static constexpr auto name = "bitTestAny"; };
struct NameBitTestAll { static constexpr auto name = "bitTestAll"; };
struct NameLeast { static constexpr auto name = "least"; };
struct NameGreatest { static constexpr auto name = "greatest"; };
struct NameGCD { static constexpr auto name = "gcd"; };
struct NameLCM { static constexpr auto name = "lcm"; };
using FunctionPlus = FunctionBinaryArithmetic<PlusImpl, NamePlus>;
using FunctionMinus = FunctionBinaryArithmetic<MinusImpl, NameMinus>;
@ -1027,6 +1062,8 @@ using FunctionBitRotateRight = FunctionBinaryArithmetic<BitRotateRightImpl, Name
using FunctionBitTest = FunctionBinaryArithmetic<BitTestImpl, NameBitTest>;
using FunctionLeast = FunctionBinaryArithmetic<LeastImpl, NameLeast>;
using FunctionGreatest = FunctionBinaryArithmetic<GreatestImpl, NameGreatest>;
using FunctionGCD = FunctionBinaryArithmetic<GCDImpl, NameGCD>;
using FunctionLCM = FunctionBinaryArithmetic<LCMImpl, NameLCM>;
/// Monotonicity properties for some functions.

2
dbms/src/Functions/FunctionsArray.h
View File

@ -1,6 +1,6 @@
#pragma once
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesNumber.h>

2
dbms/src/Functions/FunctionsConversion.h
View File

@ -26,7 +26,7 @@
#include <Columns/ColumnArray.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnTuple.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Interpreters/ExpressionActions.h>
#include <Functions/IFunction.h>
#include <Functions/FunctionsMiscellaneous.h>

103
dbms/src/Functions/FunctionsExternalDictionaries.h
View File

@ -761,13 +761,14 @@ DECLARE_DICT_GET_TRAITS(UInt32, DataTypeDateTime)
DECLARE_DICT_GET_TRAITS(UInt128, DataTypeUUID)
#undef DECLARE_DICT_GET_TRAITS
template <typename DataType>
template <typename DataType, typename Name>
class FunctionDictGet final : public IFunction
{
using Type = typename DataType::FieldType;
public:
static const std::string name;
static constexpr auto name = Name::name;
static FunctionPtr create(const Context & context)
{
@ -1060,32 +1061,42 @@ private:
const ExternalDictionaries & dictionaries;
};
template <typename DataType>
const std::string FunctionDictGet<DataType>::name = "dictGet" + DataType{}.getName();
struct NameDictGetUInt8 { static constexpr auto name = "dictGetUInt8"; };
struct NameDictGetUInt16 { static constexpr auto name = "dictGetUInt16"; };
struct NameDictGetUInt32 { static constexpr auto name = "dictGetUInt32"; };
struct NameDictGetUInt64 { static constexpr auto name = "dictGetUInt64"; };
struct NameDictGetInt8 { static constexpr auto name = "dictGetInt8"; };
struct NameDictGetInt16 { static constexpr auto name = "dictGetInt16"; };
struct NameDictGetInt32 { static constexpr auto name = "dictGetInt32"; };
struct NameDictGetInt64 { static constexpr auto name = "dictGetInt64"; };
struct NameDictGetFloat32 { static constexpr auto name = "dictGetFloat32"; };
struct NameDictGetFloat64 { static constexpr auto name = "dictGetFloat64"; };
struct NameDictGetDate { static constexpr auto name = "dictGetDate"; };
struct NameDictGetDateTime { static constexpr auto name = "dictGetDateTime"; };
struct NameDictGetUUID { static constexpr auto name = "dictGetUUID"; };
using FunctionDictGetUInt8 = FunctionDictGet<DataTypeUInt8, NameDictGetUInt8>;
using FunctionDictGetUInt16 = FunctionDictGet<DataTypeUInt16, NameDictGetUInt16>;
using FunctionDictGetUInt32 = FunctionDictGet<DataTypeUInt32, NameDictGetUInt32>;
using FunctionDictGetUInt64 = FunctionDictGet<DataTypeUInt64, NameDictGetUInt64>;
using FunctionDictGetInt8 = FunctionDictGet<DataTypeInt8, NameDictGetInt8>;
using FunctionDictGetInt16 = FunctionDictGet<DataTypeInt16, NameDictGetInt16>;
using FunctionDictGetInt32 = FunctionDictGet<DataTypeInt32, NameDictGetInt32>;
using FunctionDictGetInt64 = FunctionDictGet<DataTypeInt64, NameDictGetInt64>;
using FunctionDictGetFloat32 = FunctionDictGet<DataTypeFloat32, NameDictGetFloat32>;
using FunctionDictGetFloat64 = FunctionDictGet<DataTypeFloat64, NameDictGetFloat64>;
using FunctionDictGetDate = FunctionDictGet<DataTypeDate, NameDictGetDate>;
using FunctionDictGetDateTime = FunctionDictGet<DataTypeDateTime, NameDictGetDateTime>;
using FunctionDictGetUUID = FunctionDictGet<DataTypeUUID, NameDictGetUUID>;
using FunctionDictGetUInt8 = FunctionDictGet<DataTypeUInt8>;
using FunctionDictGetUInt16 = FunctionDictGet<DataTypeUInt16>;
using FunctionDictGetUInt32 = FunctionDictGet<DataTypeUInt32>;
using FunctionDictGetUInt64 = FunctionDictGet<DataTypeUInt64>;
using FunctionDictGetInt8 = FunctionDictGet<DataTypeInt8>;
using FunctionDictGetInt16 = FunctionDictGet<DataTypeInt16>;
using FunctionDictGetInt32 = FunctionDictGet<DataTypeInt32>;
using FunctionDictGetInt64 = FunctionDictGet<DataTypeInt64>;
using FunctionDictGetFloat32 = FunctionDictGet<DataTypeFloat32>;
using FunctionDictGetFloat64 = FunctionDictGet<DataTypeFloat64>;
using FunctionDictGetDate = FunctionDictGet<DataTypeDate>;
using FunctionDictGetDateTime = FunctionDictGet<DataTypeDateTime>;
using FunctionDictGetUUID = FunctionDictGet<DataTypeUUID>;
template <typename DataType>
template <typename DataType, typename Name>
class FunctionDictGetOrDefault final : public IFunction
{
using Type = typename DataType::FieldType;
public:
static const std::string name;
static constexpr auto name = Name::name;
static FunctionPtr create(const Context & context)
{
@ -1130,7 +1141,7 @@ private:
{
throw Exception{
"Illegal type " + arguments[3]->getName() + " of fourth argument of function " + getName()
+ ", must be " + DataType{}.getName() + ".",
+ ", must be " + String(DataType{}.getFamilyName()) + ".",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
}
@ -1221,7 +1232,7 @@ private:
}
else
throw Exception{
"Fourth argument of function " + getName() + " must be " + DataType{}.getName(),
"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()),
ErrorCodes::ILLEGAL_COLUMN};
}
@ -1259,7 +1270,7 @@ private:
}
else
throw Exception{
"Fourth argument of function " + getName() + " must be " + DataType{}.getName(),
"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()),
ErrorCodes::ILLEGAL_COLUMN};
}
@ -1309,7 +1320,7 @@ private:
}
else
throw Exception{
"Fourth argument of function " + getName() + " must be " + DataType{}.getName(),
"Fourth argument of function " + getName() + " must be " + String(DataType{}.getFamilyName()),
ErrorCodes::ILLEGAL_COLUMN};
return true;
@ -1318,23 +1329,33 @@ private:
const ExternalDictionaries & dictionaries;
};
template <typename DataType>
const std::string FunctionDictGetOrDefault<DataType>::name = "dictGet" + DataType{}.getName() + "OrDefault";
struct NameDictGetUInt8OrDefault { static constexpr auto name = "dictGetUInt8OrDefault"; };
struct NameDictGetUInt16OrDefault { static constexpr auto name = "dictGetUInt16OrDefault"; };
struct NameDictGetUInt32OrDefault { static constexpr auto name = "dictGetUInt32OrDefault"; };
struct NameDictGetUInt64OrDefault { static constexpr auto name = "dictGetUInt64OrDefault"; };
struct NameDictGetInt8OrDefault { static constexpr auto name = "dictGetInt8OrDefault"; };
struct NameDictGetInt16OrDefault { static constexpr auto name = "dictGetInt16OrDefault"; };
struct NameDictGetInt32OrDefault { static constexpr auto name = "dictGetInt32OrDefault"; };
struct NameDictGetInt64OrDefault { static constexpr auto name = "dictGetInt64OrDefault"; };
struct NameDictGetFloat32OrDefault { static constexpr auto name = "dictGetFloat32OrDefault"; };
struct NameDictGetFloat64OrDefault { static constexpr auto name = "dictGetFloat64OrDefault"; };
struct NameDictGetDateOrDefault { static constexpr auto name = "dictGetDateOrDefault"; };
struct NameDictGetDateTimeOrDefault { static constexpr auto name = "dictGetDateTimeOrDefault"; };
struct NameDictGetUUIDOrDefault { static constexpr auto name = "dictGetUUIDOrDefault"; };
using FunctionDictGetUInt8OrDefault = FunctionDictGetOrDefault<DataTypeUInt8>;
using FunctionDictGetUInt16OrDefault = FunctionDictGetOrDefault<DataTypeUInt16>;
using FunctionDictGetUInt32OrDefault = FunctionDictGetOrDefault<DataTypeUInt32>;
using FunctionDictGetUInt64OrDefault = FunctionDictGetOrDefault<DataTypeUInt64>;
using FunctionDictGetInt8OrDefault = FunctionDictGetOrDefault<DataTypeInt8>;
using FunctionDictGetInt16OrDefault = FunctionDictGetOrDefault<DataTypeInt16>;
using FunctionDictGetInt32OrDefault = FunctionDictGetOrDefault<DataTypeInt32>;
using FunctionDictGetInt64OrDefault = FunctionDictGetOrDefault<DataTypeInt64>;
using FunctionDictGetFloat32OrDefault = FunctionDictGetOrDefault<DataTypeFloat32>;
using FunctionDictGetFloat64OrDefault = FunctionDictGetOrDefault<DataTypeFloat64>;
using FunctionDictGetDateOrDefault = FunctionDictGetOrDefault<DataTypeDate>;
using FunctionDictGetDateTimeOrDefault = FunctionDictGetOrDefault<DataTypeDateTime>;
using FunctionDictGetUUIDOrDefault = FunctionDictGetOrDefault<DataTypeUUID>;
using FunctionDictGetUInt8OrDefault = FunctionDictGet<DataTypeUInt8, NameDictGetUInt8OrDefault>;
using FunctionDictGetUInt16OrDefault = FunctionDictGet<DataTypeUInt16, NameDictGetUInt16OrDefault>;
using FunctionDictGetUInt32OrDefault = FunctionDictGet<DataTypeUInt32, NameDictGetUInt32OrDefault>;
using FunctionDictGetUInt64OrDefault = FunctionDictGet<DataTypeUInt64, NameDictGetUInt64OrDefault>;
using FunctionDictGetInt8OrDefault = FunctionDictGet<DataTypeInt8, NameDictGetInt8OrDefault>;
using FunctionDictGetInt16OrDefault = FunctionDictGet<DataTypeInt16, NameDictGetInt16OrDefault>;
using FunctionDictGetInt32OrDefault = FunctionDictGet<DataTypeInt32, NameDictGetInt32OrDefault>;
using FunctionDictGetInt64OrDefault = FunctionDictGet<DataTypeInt64, NameDictGetInt64OrDefault>;
using FunctionDictGetFloat32OrDefault = FunctionDictGet<DataTypeFloat32, NameDictGetFloat32OrDefault>;
using FunctionDictGetFloat64OrDefault = FunctionDictGet<DataTypeFloat64, NameDictGetFloat64OrDefault>;
using FunctionDictGetDateOrDefault = FunctionDictGet<DataTypeDate, NameDictGetDateOrDefault>;
using FunctionDictGetDateTimeOrDefault = FunctionDictGet<DataTypeDateTime, NameDictGetDateTimeOrDefault>;
using FunctionDictGetUUIDOrDefault = FunctionDictGet<DataTypeUUID, NameDictGetUUIDOrDefault>;
/// Functions to work with hierarchies.

2
dbms/src/Functions/FunctionsFindCluster.h
View File

@ -2,7 +2,7 @@
#include <cmath>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeArray.h>

18
dbms/src/Functions/FunctionsHigherOrder.h
View File

@ -885,16 +885,16 @@ public:
};
struct NameArrayMap { static constexpr auto name = "arrayMap"; };
struct NameArrayFilter { static constexpr auto name = "arrayFilter"; };
struct NameArrayCount { static constexpr auto name = "arrayCount"; };
struct NameArrayExists { static constexpr auto name = "arrayExists"; };
struct NameArrayAll { static constexpr auto name = "arrayAll"; };
struct NameArraySum { static constexpr auto name = "arraySum"; };
struct NameArrayFirst { static constexpr auto name = "arrayFirst"; };
struct NameArrayFirstIndex { static constexpr auto name = "arrayFirstIndex"; };
struct NameArrayMap { static constexpr auto name = "arrayMap"; };
struct NameArrayFilter { static constexpr auto name = "arrayFilter"; };
struct NameArrayCount { static constexpr auto name = "arrayCount"; };
struct NameArrayExists { static constexpr auto name = "arrayExists"; };
struct NameArrayAll { static constexpr auto name = "arrayAll"; };
struct NameArraySum { static constexpr auto name = "arraySum"; };
struct NameArrayFirst { static constexpr auto name = "arrayFirst"; };
struct NameArrayFirstIndex { static constexpr auto name = "arrayFirstIndex"; };
struct NameArraySort { static constexpr auto name = "arraySort"; };
struct NameArrayReverseSort { static constexpr auto name = "arrayReverseSort"; };
struct NameArrayReverseSort { static constexpr auto name = "arrayReverseSort"; };
using FunctionArrayMap = FunctionArrayMapped<ArrayMapImpl, NameArrayMap>;
using FunctionArrayFilter = FunctionArrayMapped<ArrayFilterImpl, NameArrayFilter>;

2
dbms/src/Functions/FunctionsMiscellaneous.cpp
View File

@ -8,7 +8,7 @@
#include <Columns/ColumnSet.h>
#include <Common/UnicodeBar.h>
#include <Common/UTF8Helpers.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeAggregateFunction.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDate.h>

3
dbms/src/Functions/FunctionsTransform.h
View File

@ -1,8 +1,7 @@
#pragma once
#include <mutex>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeArray.h>

2
dbms/src/Functions/GatherUtils.h
View File

@ -16,7 +16,7 @@
#include <Common/memcpySmall.h>
#include <ext/range.h>
#include <Core/TypeListNumber.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeTraits.h>

2
dbms/src/Core/Progress.cpp → dbms/src/IO/Progress.cpp
View File

@ -1,4 +1,4 @@
#include <Core/Progress.h>
#include "Progress.h"
#include <IO/ReadBuffer.h>
#include <IO/WriteBuffer.h>

0
dbms/src/Core/Progress.h → dbms/src/IO/Progress.h
View File

2
dbms/src/IO/WriteBufferFromHTTPServerResponse.cpp
View File

@ -7,7 +7,7 @@
#include <IO/HTTPCommon.h>
#include <Common/NetException.h>
#include <Common/Stopwatch.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
namespace DB
{

2
dbms/src/IO/WriteBufferFromHTTPServerResponse.h
View File

@ -12,7 +12,7 @@
#include <IO/HTTPCommon.h>
#include <Common/NetException.h>
#include <Common/Stopwatch.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
namespace Poco

7
dbms/src/IO/tests/gtest_cascade_and_memory_write_buffer.cpp
View File

@ -1,18 +1,17 @@
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
#include <stdexcept>
#include <Poco/File.h>
#include <IO/CascadeWriteBuffer.h>
#include <IO/MemoryReadWriteBuffer.h>
#include <IO/WriteBufferFromTemporaryFile.h>
#include <IO/ReadBufferFromString.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ConcatReadBuffer.h>
#include <IO/copyData.h>
#include <Common/typeid_cast.h>
using namespace DB;

35
dbms/src/Interpreters/Context.cpp
View File

@ -25,7 +25,8 @@
#include <Storages/MergeTree/MergeTreeSettings.h>
#include <Storages/CompressionSettingsSelector.h>
#include <Interpreters/Settings.h>
#include <Interpreters/Users.h>
#include <Interpreters/RuntimeComponentsFactory.h>
#include <Interpreters/ISecurityManager.h>
#include <Interpreters/Quota.h>
#include <Interpreters/EmbeddedDictionaries.h>
#include <Interpreters/ExternalDictionaries.h>
@ -90,6 +91,8 @@ struct ContextShared
{
Logger * log = &Logger::get("Context");
std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory;
/// For access of most of shared objects. Recursive mutex.
mutable Poco::Mutex mutex;
/// Separate mutex for access of dictionaries. Separate mutex to avoid locks when server doing request to itself.
@ -115,7 +118,7 @@ struct ContextShared
mutable std::shared_ptr<ExternalDictionaries> external_dictionaries;
mutable std::shared_ptr<ExternalModels> external_models;
String default_profile_name; /// Default profile name used for default values.
Users users; /// Known users.
std::shared_ptr<ISecurityManager> security_manager; /// Known users.
Quotas quotas; /// Known quotas for resource use.
mutable UncompressedCachePtr uncompressed_cache; /// The cache of decompressed blocks.
mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files.
@ -181,7 +184,8 @@ struct ContextShared
pcg64 rng{randomSeed()};
ContextShared()
ContextShared(std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory_)
: runtime_components_factory(std::move(runtime_components_factory_))
{
/// TODO: make it singleton (?)
static std::atomic<size_t> num_calls{0};
@ -191,6 +195,8 @@ struct ContextShared
std::cerr.flush();
std::terminate();
}
initialize();
}
@ -236,21 +242,32 @@ struct ContextShared
databases.clear();
}
}
private:
void initialize()
{
security_manager = runtime_components_factory->createSecurityManager();
}
};
Context::Context() = default;
Context Context::createGlobal()
Context Context::createGlobal(std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory)
{
Context res;
res.shared = std::make_shared<ContextShared>();
res.runtime_components_factory = runtime_components_factory;
res.shared = std::make_shared<ContextShared>(runtime_components_factory);
res.quota = std::make_shared<QuotaForIntervals>();
res.system_logs = std::make_shared<SystemLogs>();
return res;
}
Context Context::createGlobal()
{
return createGlobal(std::make_unique<RuntimeComponentsFactory>());
}
Context::~Context()
{
@ -512,7 +529,7 @@ void Context::setUsersConfig(const ConfigurationPtr & config)
{
auto lock = getLock();
shared->users_config = config;
shared->users.loadFromConfig(*shared->users_config);
shared->security_manager->loadFromConfig(*shared->users_config);
shared->quotas.loadFromConfig(*shared->users_config);
}
@ -526,7 +543,7 @@ void Context::calculateUserSettings()
{
auto lock = getLock();
String profile = shared->users.get(client_info.current_user).profile;
String profile = shared->security_manager->getUser(client_info.current_user).profile;
/// 1) Set default settings (hardcoded values)
/// NOTE: we ignore global_context settings (from which it is usually copied)
@ -547,7 +564,7 @@ void Context::setUser(const String & name, const String & password, const Poco::
{
auto lock = getLock();
const User & user_props = shared->users.get(name, password, address.host());
const User & user_props = shared->security_manager->authorizeAndGetUser(name, password, address.host());
client_info.current_user = name;
client_info.current_address = address;
@ -582,7 +599,7 @@ void Context::checkDatabaseAccessRights(const std::string & database_name) const
/// All users have access to the database system.
return;
}
if (!shared->users.isAllowedDatabase(client_info.current_user, database_name))
if (!shared->security_manager->hasAccessToDatabase(client_info.current_user, database_name))
throw Exception("Access denied to database " + database_name, ErrorCodes::DATABASE_ACCESS_DENIED);
}

4
dbms/src/Interpreters/Context.h
View File

@ -32,6 +32,7 @@ namespace DB
{
struct ContextShared;
class IRuntimeComponentsFactory;
class QuotaForIntervals;
class EmbeddedDictionaries;
class ExternalDictionaries;
@ -89,6 +90,8 @@ private:
using Shared = std::shared_ptr<ContextShared>;
Shared shared;
std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory;
ClientInfo client_info;
std::shared_ptr<QuotaForIntervals> quota; /// Current quota. By default - empty quota, that have no limits.
@ -116,6 +119,7 @@ private:
public:
/// Create initial Context with ContextShared and etc.
static Context createGlobal(std::shared_ptr<IRuntimeComponentsFactory> runtime_components_factory);
static Context createGlobal();
~Context();

5
dbms/src/Interpreters/ExternalLoader.cpp
View File

@ -278,8 +278,9 @@ void ExternalLoader::reloadFromConfigFile(const std::string & config_path, const
const auto last_modified = config_file.getLastModified();
if (force_reload || last_modified > config_last_modified)
{
ConfigProcessor config_processor = ConfigProcessor();
ConfigProcessor::LoadedConfig cfg = config_processor.loadConfig(config_path);
ConfigProcessor config_processor = ConfigProcessor(config_path);
ConfigProcessor::LoadedConfig cfg = config_processor.loadConfig();
config_processor.savePreprocessedConfig(cfg);
Poco::AutoPtr<Poco::Util::AbstractConfiguration> config = cfg.configuration;

22
dbms/src/Interpreters/IRuntimeComponentsFactory.h Normal file
View File

@ -0,0 +1,22 @@
#pragma once
#include <Interpreters/ISecurityManager.h>
#include <memory>
namespace DB
{
/** Factory of query engine runtime components / services.
* Helps to host query engine in external applications
* by replacing or reconfiguring its components.
*/
class IRuntimeComponentsFactory
{
public:
virtual std::unique_ptr<ISecurityManager> createSecurityManager() = 0;
virtual ~IRuntimeComponentsFactory() {}
};
}

33
dbms/src/Interpreters/ISecurityManager.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#include <Interpreters/Users.h>
namespace DB
{
/** Duties of security manager:
* 1) Authenticate users
* 2) Provide user settings (profile, quota, ACLs)
* 3) Grant access to databases
*/
class ISecurityManager
{
public:
virtual void loadFromConfig(Poco::Util::AbstractConfiguration & config) = 0;
/// Find user and make authorize checks
virtual const User & authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address) const = 0;
/// Just find user
virtual const User & getUser(const String & user_name) const = 0;
/// Check if the user has access to the database.
virtual bool hasAccessToDatabase(const String & user_name, const String & database_name) const = 0;
virtual ~ISecurityManager() {}
};
}

2
dbms/src/Interpreters/InterpreterSetQuery.cpp
View File

@ -2,7 +2,7 @@
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterSetQuery.h>
#include <Common/typeid_cast.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
namespace DB
{

2
dbms/src/Interpreters/ProcessList.h
View File

@ -8,7 +8,7 @@
#include <Poco/Condition.h>
#include <Common/Stopwatch.h>
#include <Core/Defines.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <Common/MemoryTracker.h>
#include <Interpreters/QueryPriorities.h>
#include <Interpreters/ClientInfo.h>

21
dbms/src/Interpreters/RuntimeComponentsFactory.h Normal file
View File

@ -0,0 +1,21 @@
#pragma once
#include <Interpreters/IRuntimeComponentsFactory.h>
#include <Interpreters/SecurityManager.h>
namespace DB
{
/** Default implementation of runtime components factory
* used by native server application.
*/
class RuntimeComponentsFactory : public IRuntimeComponentsFactory
{
public:
std::unique_ptr<ISecurityManager> createSecurityManager() override
{
return std::make_unique<SecurityManager>();
}
};
}

114
dbms/src/Interpreters/SecurityManager.cpp Normal file
View File

@ -0,0 +1,114 @@
#include <Interpreters/SecurityManager.h>
#include <Poco/Net/IPAddress.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Poco/String.h>
#include <Common/Exception.h>
#include <IO/HexWriteBuffer.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <openssl/sha.h>
#include <common/logger_useful.h>
namespace DB
{
namespace ErrorCodes
{
extern const int DNS_ERROR;
extern const int UNKNOWN_ADDRESS_PATTERN_TYPE;
extern const int UNKNOWN_USER;
extern const int REQUIRED_PASSWORD;
extern const int WRONG_PASSWORD;
extern const int IP_ADDRESS_NOT_ALLOWED;
extern const int BAD_ARGUMENTS;
}
void SecurityManager::loadFromConfig(Poco::Util::AbstractConfiguration & config)
{
Container new_users;
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys("users", config_keys);
for (const std::string & key : config_keys)
new_users.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(key, "users." + key, config));
users = std::move(new_users);
}
const User & SecurityManager::authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address) const
{
auto it = users.find(user_name);
if (users.end() == it)
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
if (!it->second.addresses.contains(address))
throw Exception("User " + user_name + " is not allowed to connect from address " + address.toString(), ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
auto on_wrong_password = [&]()
{
if (password.empty())
throw Exception("Password required for user " + user_name, ErrorCodes::REQUIRED_PASSWORD);
else
throw Exception("Wrong password for user " + user_name, ErrorCodes::WRONG_PASSWORD);
};
if (!it->second.password_sha256_hex.empty())
{
unsigned char hash[32];
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, reinterpret_cast<const unsigned char *>(password.data()), password.size());
SHA256_Final(hash, &ctx);
String hash_hex;
{
WriteBufferFromString buf(hash_hex);
HexWriteBuffer hex_buf(buf);
hex_buf.write(reinterpret_cast<const char *>(hash), sizeof(hash));
}
Poco::toLowerInPlace(hash_hex);
if (hash_hex != it->second.password_sha256_hex)
on_wrong_password();
}
else if (password != it->second.password)
{
on_wrong_password();
}
return it->second;
}
const User & SecurityManager::getUser(const String & user_name) const
{
auto it = users.find(user_name);
if (users.end() == it)
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
return it->second;
}
bool SecurityManager::hasAccessToDatabase(const std::string & user_name, const std::string & database_name) const
{
auto it = users.find(user_name);
if (users.end() == it)
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
const auto & user = it->second;
return user.databases.empty() || user.databases.count(database_name);
}
}

32
dbms/src/Interpreters/SecurityManager.h Normal file
View File

@ -0,0 +1,32 @@
#pragma once
#include <Interpreters/ISecurityManager.h>
#include <map>
namespace DB
{
/** Default implementation of security manager used by native server application.
* Manages fixed set of users listed in 'Users' configuration file.
*/
class SecurityManager : public ISecurityManager
{
private:
using Container = std::map<String, User>;
Container users;
public:
void loadFromConfig(Poco::Util::AbstractConfiguration & config) override;
const User & authorizeAndGetUser(
const String & user_name,
const String & password,
const Poco::Net::IPAddress & address) const override;
const User & getUser(const String & user_name) const override;
bool hasAccessToDatabase(const String & user_name, const String & database_name) const override;
};
}

2
dbms/src/Interpreters/Set.cpp
View File

@ -1,5 +1,5 @@
#include <Core/Field.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Core/Row.h>
#include <Columns/ColumnsNumber.h>

83
dbms/src/Interpreters/Users.cpp
View File

@ -306,87 +306,4 @@ User::User(const String & name_, const String & config_elem, Poco::Util::Abstrac
}
void Users::loadFromConfig(Poco::Util::AbstractConfiguration & config)
{
Container new_cont;
Poco::Util::AbstractConfiguration::Keys config_keys;
config.keys("users", config_keys);
for (const std::string & key : config_keys)
new_cont.emplace(std::piecewise_construct, std::forward_as_tuple(key), std::forward_as_tuple(key, "users." + key, config));
cont = std::move(new_cont);
}
const User & Users::get(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const
{
auto it = cont.find(user_name);
if (cont.end() == it)
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
if (!it->second.addresses.contains(address))
throw Exception("User " + user_name + " is not allowed to connect from address " + address.toString(), ErrorCodes::IP_ADDRESS_NOT_ALLOWED);
auto on_wrong_password = [&]()
{
if (password.empty())
throw Exception("Password required for user " + user_name, ErrorCodes::REQUIRED_PASSWORD);
else
throw Exception("Wrong password for user " + user_name, ErrorCodes::WRONG_PASSWORD);
};
if (!it->second.password_sha256_hex.empty())
{
unsigned char hash[32];
SHA256_CTX ctx;
SHA256_Init(&ctx);
SHA256_Update(&ctx, reinterpret_cast<const unsigned char *>(password.data()), password.size());
SHA256_Final(hash, &ctx);
String hash_hex;
{
WriteBufferFromString buf(hash_hex);
HexWriteBuffer hex_buf(buf);
hex_buf.write(reinterpret_cast<const char *>(hash), sizeof(hash));
}
Poco::toLowerInPlace(hash_hex);
if (hash_hex != it->second.password_sha256_hex)
on_wrong_password();
}
else if (password != it->second.password)
{
on_wrong_password();
}
return it->second;
}
const User & Users::get(const String & user_name)
{
auto it = cont.find(user_name);
if (cont.end() == it)
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
return it->second;
}
bool Users::isAllowedDatabase(const std::string & user_name, const std::string & database_name) const
{
auto it = cont.find(user_name);
if (it == cont.end())
throw Exception("Unknown user " + user_name, ErrorCodes::UNKNOWN_USER);
const auto & user = it->second;
return user.databases.empty() || user.databases.count(database_name);
}
}

24
dbms/src/Interpreters/Users.h
View File

@ -2,7 +2,6 @@
#include <Core/Types.h>
#include <map>
#include <vector>
#include <unordered_set>
#include <memory>
@ -38,7 +37,7 @@ public:
class AddressPatterns
{
private:
using Container = std::vector<std::unique_ptr<IAddressPattern>>;
using Container = std::vector<std::shared_ptr<IAddressPattern>>;
Container patterns;
public:
@ -70,25 +69,4 @@ struct User
};
/// Known users.
class Users
{
private:
using Container = std::map<String, User>;
Container cont;
public:
void loadFromConfig(Poco::Util::AbstractConfiguration & config);
/// Find user and make authorize checks
const User & get(const String & user_name, const String & password, const Poco::Net::IPAddress & address) const;
/// Just find user
const User & get(const String & user_name);
/// Check if the user has access to the database.
bool isAllowedDatabase(const String & user_name, const String & database_name) const;
};
}

2
dbms/src/Interpreters/convertFieldToType.cpp
View File

@ -14,7 +14,7 @@
#include <DataTypes/DataTypeTraits.h>
#include <Core/AccurateComparison.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeUUID.h>

10
dbms/src/Interpreters/tests/users.cpp
View File

@ -1,5 +1,5 @@
#include <Common/ConfigProcessor.h>
#include <Interpreters/Users.h>
#include <Interpreters/SecurityManager.h>
#include <boost/filesystem.hpp>
@ -195,7 +195,7 @@ void runOneTest(size_t test_num, const TestDescriptor & test_descriptor)
try
{
config = ConfigProcessor{}.loadConfig(path_name).configuration;
config = ConfigProcessor(path_name).loadConfig().configuration;
}
catch (const Poco::Exception & ex)
{
@ -204,11 +204,11 @@ void runOneTest(size_t test_num, const TestDescriptor & test_descriptor)
throw std::runtime_error(os.str());
}
DB::Users users;
DB::SecurityManager security_manager;
try
{
users.loadFromConfig(*config);
security_manager.loadFromConfig(*config);
}
catch (const Poco::Exception & ex)
{
@ -223,7 +223,7 @@ void runOneTest(size_t test_num, const TestDescriptor & test_descriptor)
try
{
res = users.isAllowedDatabase(entry.user_name, entry.database_name);
res = security_manager.hasAccessToDatabase(entry.user_name, entry.database_name);
}
catch (const Poco::Exception &)
{

2
dbms/src/Parsers/ASTLiteral.cpp
View File

@ -1,5 +1,5 @@
#include <Common/SipHash.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Parsers/ASTLiteral.h>
#include <IO/WriteHelpers.h>

2
dbms/src/Parsers/ASTLiteral.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Core/Field.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Parsers/ASTWithAlias.h>

2
dbms/src/Parsers/ASTSelectQuery.cpp
View File

@ -1,4 +1,4 @@
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Parsers/ASTSetQuery.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTAsterisk.h>

2
dbms/src/Parsers/ASTSetQuery.h
View File

@ -1,7 +1,7 @@
#pragma once
#include <Core/Field.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Parsers/IAST.h>

6
dbms/src/Parsers/tests/CMakeLists.txt
View File

@ -3,11 +3,11 @@ include_directories (${CMAKE_CURRENT_BINARY_DIR})
set(SRCS )
add_executable (lexer lexer.cpp ${SRCS})
target_link_libraries (lexer dbms clickhouse_parsers)
target_link_libraries (lexer clickhouse_parsers)
add_executable (select_parser select_parser.cpp ${SRCS})
target_link_libraries (select_parser dbms clickhouse_parsers)
target_link_libraries (select_parser clickhouse_parsers)
add_executable (create_parser create_parser.cpp ${SRCS})
target_link_libraries (create_parser dbms clickhouse_parsers)
target_link_libraries (create_parser clickhouse_parsers)

30
dbms/src/Server/CMakeLists.txt
View File

@ -16,32 +16,32 @@ add_library(clickhouse-server
TCPHandler.cpp
)
target_link_libraries(clickhouse-server daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
target_link_libraries(clickhouse-server clickhouse_common_io daemon clickhouse_storages_system clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
target_include_directories (clickhouse-server PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
add_library(clickhouse-local LocalServer.cpp)
target_link_libraries(clickhouse-local dbms clickhouse-server clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
target_link_libraries(clickhouse-local clickhouse-server clickhouse_functions clickhouse_aggregate_functions clickhouse_table_functions)
add_library(clickhouse-extract-from-config ExtractFromConfig.cpp)
target_link_libraries(clickhouse-extract-from-config dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library(clickhouse-extract-from-config ${SPLIT_SHARED} ExtractFromConfig.cpp)
target_link_libraries(clickhouse-extract-from-config clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library (clickhouse-client Client.cpp)
target_link_libraries (clickhouse-client dbms clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-client clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
install (FILES clickhouse-client.xml DESTINATION ${CLICKHOUSE_ETC_DIR}/clickhouse-client COMPONENT clickhouse-client RENAME config.xml)
add_library (clickhouse-benchmark Benchmark.cpp)
target_link_libraries (clickhouse-benchmark dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library (clickhouse-benchmark ${SPLIT_SHARED} Benchmark.cpp)
target_link_libraries (clickhouse-benchmark clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-benchmark PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
add_library (clickhouse-performance-test PerformanceTest.cpp)
target_link_libraries (clickhouse-performance-test dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library (clickhouse-performance-test ${SPLIT_SHARED} PerformanceTest.cpp)
target_link_libraries (clickhouse-performance-test clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_include_directories (clickhouse-performance-test PRIVATE ${PCG_RANDOM_INCLUDE_DIR})
add_library (clickhouse-compressor-lib Compressor.cpp)
target_link_libraries (clickhouse-compressor-lib dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library (clickhouse-compressor-lib ${SPLIT_SHARED} Compressor.cpp)
target_link_libraries (clickhouse-compressor-lib clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library (clickhouse-format Format.cpp)
target_link_libraries (clickhouse-format dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
add_library (clickhouse-format ${SPLIT_SHARED} Format.cpp)
target_link_libraries (clickhouse-format clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (USE_EMBEDDED_COMPILER)
link_directories (${LLVM_LIBRARY_DIRS})
@ -57,7 +57,9 @@ target_link_libraries(clickhouse
clickhouse-performance-test
clickhouse-extract-from-config
clickhouse-compressor-lib
clickhouse-format)
clickhouse-format
dbms
)
if (USE_EMBEDDED_COMPILER)
add_subdirectory (Compiler)

8
dbms/src/Server/ConfigReloader.cpp
View File

@ -66,15 +66,16 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
FilesChangesTracker new_files = getNewFileList();
if (force || new_files.isDifferOrNewerThan(files))
{
ConfigProcessor config_processor(path);
ConfigProcessor::LoadedConfig loaded_config;
try
{
LOG_DEBUG(log, "Loading config `" << path << "'");
loaded_config = ConfigProcessor().loadConfig(path, /* allow_zk_includes = */ true);
loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
if (loaded_config.has_zk_includes)
loaded_config = ConfigProcessor().loadConfigWithZooKeeperIncludes(
path, zk_node_cache, fallback_to_preprocessed);
loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
zk_node_cache, fallback_to_preprocessed);
}
catch (...)
{
@ -84,6 +85,7 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
tryLogCurrentException(log, "Error loading config from `" + path + "'");
return;
}
config_processor.savePreprocessedConfig(loaded_config);
/** We should remember last modification time if and only if config was sucessfully loaded
* Otherwise a race condition could occur during config files update:

6
dbms/src/Server/ExtractFromConfig.cpp
View File

@ -26,16 +26,16 @@ static void setupLogging(const std::string & log_level)
static std::string extractFromConfig(
const std::string & config_path, const std::string & key, bool process_zk_includes)
{
ConfigProcessor processor(/* throw_on_bad_incl = */ false, /* log_to_console = */ false);
ConfigProcessor processor(config_path, /* throw_on_bad_incl = */ false, /* log_to_console = */ false);
bool has_zk_includes;
XMLDocumentPtr config_xml = processor.processConfig(config_path, &has_zk_includes);
XMLDocumentPtr config_xml = processor.processConfig(&has_zk_includes);
if (has_zk_includes && process_zk_includes)
{
ConfigurationPtr bootstrap_configuration(new Poco::Util::XMLConfiguration(config_xml));
zkutil::ZooKeeperPtr zookeeper = std::make_shared<zkutil::ZooKeeper>(
*bootstrap_configuration, "zookeeper");
zkutil::ZooKeeperNodeCache zk_node_cache([&] { return zookeeper; });
config_xml = processor.processConfig(config_path, &has_zk_includes, &zk_node_cache);
config_xml = processor.processConfig(&has_zk_includes, &zk_node_cache);
}
ConfigurationPtr configuration(new Poco::Util::XMLConfiguration(config_xml));
return configuration->getString(key);

15
dbms/src/Server/LocalServer.cpp
View File

@ -247,10 +247,10 @@ try
/// Load config files if exists
if (config().has("config-file") || Poco::File("config.xml").exists())
{
ConfigurationPtr processed_config = ConfigProcessor(false, true)
.loadConfig(config().getString("config-file", "config.xml"))
.configuration;
config().add(processed_config.duplicate(), PRIO_DEFAULT, false);
ConfigProcessor config_processor(config().getString("config-file", "config.xml"), false, true);
auto loaded_config = config_processor.loadConfig();
config_processor.savePreprocessedConfig(loaded_config);
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}
context = std::make_unique<Context>(Context::createGlobal());
@ -452,8 +452,11 @@ void LocalServer::setupUsers()
if (config().has("users_config") || config().has("config-file") || Poco::File("config.xml").exists())
{
auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
users_config = ConfigProcessor().loadConfig(users_config_path).configuration;
const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
ConfigProcessor config_processor(users_config_path);
const auto loaded_config = config_processor.loadConfig();
config_processor.savePreprocessedConfig(loaded_config);
users_config = loaded_config.configuration;
}
else
{

6
dbms/src/Server/Server.cpp
View File

@ -108,8 +108,10 @@ int Server::main(const std::vector<std::string> & args)
if (loaded_config.has_zk_includes)
{
auto old_configuration = loaded_config.configuration;
loaded_config = ConfigProcessor().loadConfigWithZooKeeperIncludes(
config_path, main_config_zk_node_cache, /* fallback_to_preprocessed = */ true);
ConfigProcessor config_processor(config_path);
loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
main_config_zk_node_cache, /* fallback_to_preprocessed = */ true);
config_processor.savePreprocessedConfig(loaded_config);
config().removeConfiguration(old_configuration.get());
config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
}

2
dbms/src/Server/TCPHandler.cpp
View File

@ -6,7 +6,7 @@
#include <Common/Stopwatch.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <IO/CompressedReadBuffer.h>
#include <IO/CompressedWriteBuffer.h>

2
dbms/src/Server/TCPHandler.h
View File

@ -4,7 +4,7 @@
#include <Common/CurrentMetrics.h>
#include <Common/Stopwatch.h>
#include <Core/Progress.h>
#include <IO/Progress.h>
#include <Core/Protocol.h>
#include <Core/QueryProcessingStage.h>
#include <DataStreams/BlockIO.h>

21
dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -1,26 +1,7 @@
/// Compatibility with clang, in which std::numeric_limits (from libstdc++ from gcc) for some reason does not specialize for __uint128_t.
#if __clang__ && __clang_major__ < 4
#include <limits>
namespace std
{
template <>
struct numeric_limits<__uint128_t>
{
static constexpr bool is_specialized = true;
static constexpr bool is_signed = false;
static constexpr bool is_integer = true;
static constexpr int radix = 2;
static constexpr int digits = 8 * sizeof(char) * 2;
static constexpr __uint128_t min () { return 0; } // used in boost 1.65.1+
};
}
#endif
#include <boost/rational.hpp> /// For calculations related to sampling coefficients.
#include <optional>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Storages/MergeTree/MergeTreeDataSelectExecutor.h>
#include <Storages/MergeTree/MergeTreeBlockInputStream.h>
#include <Storages/MergeTree/MergeTreeReadPool.h>

2
dbms/src/Storages/MergeTree/MergeTreePartition.cpp
View File

@ -3,7 +3,7 @@
#include <Storages/MergeTree/MergeTreeDataPart.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/HashingWriteBuffer.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <DataTypes/DataTypeDate.h>
#include <Common/SipHash.h>
#include <Common/typeid_cast.h>

2
dbms/src/Storages/MergeTree/PKCondition.cpp
View File

@ -11,7 +11,7 @@
#include <Columns/ColumnTuple.h>
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/typeid_cast.h>
#include <Interpreters/convertFieldToType.h>
#include <Interpreters/Set.h>

2
dbms/src/Storages/StorageFactory.cpp
View File

@ -1,7 +1,7 @@
#include <unistd.h>
#include <Poco/Util/Application.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Common/StringUtils.h>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeTuple.h>

2
dbms/src/Storages/StorageMergeTree.cpp
View File

@ -1,5 +1,5 @@
#include <optional>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Storages/StorageMergeTree.h>
#include <Storages/MergeTree/MergeTreeBlockOutputStream.h>
#include <Storages/MergeTree/DiskSpaceMonitor.h>

2
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -1,7 +1,7 @@
#include <Common/ZooKeeper/Types.h>
#include <Common/ZooKeeper/KeeperException.h>
#include <Core/FieldVisitors.h>
#include <Common/FieldVisitors.h>
#include <Storages/ColumnsDescription.h>
#include <Storages/StorageReplicatedMergeTree.h>

2
dbms/src/Storages/System/CMakeLists.txt
View File

@ -1,5 +1,5 @@
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
add_headers_and_sources(storages_system .)
add_library(clickhouse_storages_system ${SPLIT_SHARED} ${storages_system_headers} ${storages_system_sources})
add_library(clickhouse_storages_system ${storages_system_headers} ${storages_system_sources})
target_link_libraries(clickhouse_storages_system dbms)

4
dbms/src/Storages/tests/gtest_range_filtered.cpp
View File

@ -1,4 +1,8 @@
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
#include <common/RangeFiltered.h>
#include <vector>
#include <set>

4
dbms/src/Storages/tests/gtest_row_source_bits_test.cpp
View File

@ -1,4 +1,8 @@
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wsign-compare"
#include <gtest/gtest.h>
#pragma GCC diagnostic pop
#include <DataStreams/ColumnGathererStream.h>
using DB::RowSourcePart;

2
dbms/src/TableFunctions/CMakeLists.txt
View File

@ -4,5 +4,5 @@ add_headers_and_sources(clickhouse_table_functions .)
list(REMOVE_ITEM clickhouse_table_functions_sources TableFunctionFactory.cpp)
list(REMOVE_ITEM clickhouse_table_functions_headers ITableFunction.h TableFunctionFactory.h)
add_library(clickhouse_table_functions ${SPLIT_SHARED} ${clickhouse_table_functions_sources})
add_library(clickhouse_table_functions ${clickhouse_table_functions_sources})
target_link_libraries(clickhouse_table_functions dbms clickhouse_storages_system ${Poco_Foundation_LIBRARY})

Some files were not shown because too many files have changed in this diff Show More