thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moved code to cpp [#CLICKHOUSE-2].

Browse Source
This commit is contained in:
Alexey Milovidov 2017-04-08 04:32:05 +03:00
parent 82766247e2
commit a0b9bf5e97
33 changed files with 329 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h
View File

@ -25,6 +25,7 @@ namespace ErrorCodes
extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION;
extern const int TOO_MUCH_ARGUMENTS_FOR_FUNCTION;
extern const int SYNTAX_ERROR;
extern const int BAD_ARGUMENTS;
}
/// helper type for comparing `std::pair`s using solely the .first member

1
dbms/src/AggregateFunctions/AggregateFunctionUniq.h
View File

@ -16,6 +16,7 @@
#include <Common/HashTable/HashSet.h>
#include <Common/HyperLogLogWithSmallSetOptimization.h>
#include <Common/CombinedCardinalityEstimator.h>
#include <Common/MemoryTracker.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnTuple.h>

1
dbms/src/Client/Benchmark.cpp
View File

@ -52,6 +52,7 @@ namespace ErrorCodes
extern const int POCO_EXCEPTION;
extern const int STD_EXCEPTION;
extern const int UNKNOWN_EXCEPTION;
extern const int BAD_ARGUMENTS;
}
class Benchmark

165
dbms/src/Common/Allocator.cpp Normal file
View File

@ -0,0 +1,165 @@
#if !defined(__APPLE__) && !defined(__FreeBSD__)
#include <malloc.h>
#endif
#include <cstdlib>
#include <sys/mman.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
#include <Common/Allocator.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int CANNOT_ALLOCATE_MEMORY;
extern const int CANNOT_MUNMAP;
extern const int CANNOT_MREMAP;
}
}
/** Многие современные аллокаторы (например, tcmalloc) не умеют делать mremap для realloc,
* даже в случае достаточно больших кусков памяти.
* Хотя это позволяет увеличить производительность и уменьшить потребление памяти во время realloc-а.
* Чтобы это исправить, делаем mremap самостоятельно, если кусок памяти достаточно большой.
* Порог (64 МБ) выбран достаточно большим, так как изменение адресного пространства
* довольно сильно тормозит, особенно в случае наличия большого количества потоков.
* Рассчитываем, что набор операций mmap/что-то сделать/mremap может выполняться всего лишь около 1000 раз в секунду.
*
* PS. Также это требуется, потому что tcmalloc не может выделить кусок памяти больше 16 GB.
*/
static constexpr size_t MMAP_THRESHOLD = 64 * (1 << 20);
static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
template <bool clear_memory_>
void * Allocator<clear_memory_>::alloc(size_t size, size_t alignment)
{
if (current_memory_tracker)
current_memory_tracker->alloc(size);
void * buf;
if (size >= MMAP_THRESHOLD)
{
if (alignment > MMAP_MIN_ALIGNMENT)
throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (MAP_FAILED == buf)
DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
/// No need for zero-fill, because mmap guarantees it.
}
else
{
if (alignment <= MALLOC_MIN_ALIGNMENT)
{
if (clear_memory)
buf = ::calloc(size, 1);
else
buf = ::malloc(size);
if (nullptr == buf)
DB::throwFromErrno("Allocator: Cannot malloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
}
else
{
buf = nullptr;
int res = posix_memalign(&buf, alignment, size);
if (0 != res)
DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res);
if (clear_memory)
memset(buf, 0, size);
}
}
return buf;
}
template <bool clear_memory_>
void Allocator<clear_memory_>::free(void * buf, size_t size)
{
if (size >= MMAP_THRESHOLD)
{
if (0 != munmap(buf, size))
DB::throwFromErrno("Allocator: Cannot munmap.", DB::ErrorCodes::CANNOT_MUNMAP);
}
else
{
::free(buf);
}
if (current_memory_tracker)
current_memory_tracker->free(size);
}
template <bool clear_memory_>
void * Allocator<clear_memory_>::realloc(void * buf, size_t old_size, size_t new_size, size_t alignment)
{
#if !defined(__APPLE__) && !defined(__FreeBSD__)
if (old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT)
{
if (current_memory_tracker)
current_memory_tracker->realloc(old_size, new_size);
buf = ::realloc(buf, new_size);
if (nullptr == buf)
DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
if (clear_memory)
memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
}
else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD)
{
if (current_memory_tracker)
current_memory_tracker->realloc(old_size, new_size);
buf = mremap(buf, old_size, new_size, MREMAP_MAYMOVE);
if (MAP_FAILED == buf)
DB::throwFromErrno("Allocator: Cannot mremap.", DB::ErrorCodes::CANNOT_MREMAP);
/// No need for zero-fill, because mmap guarantees it.
}
#else
// TODO: We need to use mmap/calloc on Apple too.
if ((old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT) ||
(old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD))
{
if (current_memory_tracker)
current_memory_tracker->realloc(old_size, new_size);
buf = ::realloc(buf, new_size);
if (nullptr == buf)
DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
if (clear_memory)
memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
}
#endif
else
{
void * new_buf = alloc(new_size, alignment);
memcpy(new_buf, buf, old_size);
free(buf, old_size);
buf = new_buf;
}
return buf;
}
/// Explicit template instantinations.
template class Allocator<true>;
template class Allocator<false>;

189
dbms/src/Common/Allocator.h
View File

@ -1,36 +1,6 @@
#pragma once
#if !defined(__APPLE__) && !defined(__FreeBSD__)
#include <malloc.h>
#endif
#include <cstdlib>
#include <string.h>
#include <sys/mman.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int CANNOT_ALLOCATE_MEMORY;
extern const int CANNOT_MUNMAP;
extern const int CANNOT_MREMAP;
}
}
/** При использовании AllocatorWithStackMemory, размещённом на стеке,
* GCC 4.9 ошибочно делает предположение, что мы можем вызывать free от указателя на стек.
* На самом деле, комбинация условий внутри AllocatorWithStackMemory этого не допускает.
*/
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
/** Отвечает за выделение/освобождение памяти. Используется, например, в PODArray, Arena.
@ -47,144 +17,18 @@ class Allocator
protected:
static constexpr bool clear_memory = clear_memory_;
private:
/** Многие современные аллокаторы (например, tcmalloc) не умеют делать mremap для realloc,
* даже в случае достаточно больших кусков памяти.
* Хотя это позволяет увеличить производительность и уменьшить потребление памяти во время realloc-а.
* Чтобы это исправить, делаем mremap самостоятельно, если кусок памяти достаточно большой.
* Порог (64 МБ) выбран достаточно большим, так как изменение адресного пространства
* довольно сильно тормозит, особенно в случае наличия большого количества потоков.
* Рассчитываем, что набор операций mmap/что-то сделать/mremap может выполняться всего лишь около 1000 раз в секунду.
*
* PS. Также это требуется, потому что tcmalloc не может выделить кусок памяти больше 16 GB.
*/
static constexpr size_t MMAP_THRESHOLD = 64 * (1 << 20);
static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
public:
/// Выделить кусок памяти.
void * alloc(size_t size, size_t alignment = 0)
{
if (current_memory_tracker)
current_memory_tracker->alloc(size);
/// Allocate memory range.
void * alloc(size_t size, size_t alignment = 0);
void * buf;
/// Free memory range.
void free(void * buf, size_t size);
if (size >= MMAP_THRESHOLD)
{
if (alignment > MMAP_MIN_ALIGNMENT)
throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS);
buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
if (MAP_FAILED == buf)
DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
/// Заполнение нулями не нужно - mmap сам это делает.
}
else
{
if (alignment <= MALLOC_MIN_ALIGNMENT)
{
if (clear_memory)
buf = ::calloc(size, 1);
else
buf = ::malloc(size);
if (nullptr == buf)
DB::throwFromErrno("Allocator: Cannot malloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
}
else
{
buf = nullptr;
int res = posix_memalign(&buf, alignment, size);
if (0 != res)
DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res);
if (clear_memory)
memset(buf, 0, size);
}
}
return buf;
}
/// Освободить память.
void free(void * buf, size_t size)
{
if (size >= MMAP_THRESHOLD)
{
if (0 != munmap(buf, size))
DB::throwFromErrno("Allocator: Cannot munmap.", DB::ErrorCodes::CANNOT_MUNMAP);
}
else
{
::free(buf);
}
if (current_memory_tracker)
current_memory_tracker->free(size);
}
/** Увеличить размер куска памяти.
* Содержимое старого куска памяти переезжает в начало нового.
* Положение куска памяти может измениться.
/** Enlarge memory range.
* Data from old range is moved to the beginning of new range.
* Address of memory range could change.
*/
void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0)
{
#if !defined(__APPLE__) && !defined(__FreeBSD__)
if (old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT)
{
if (current_memory_tracker)
current_memory_tracker->realloc(old_size, new_size);
buf = ::realloc(buf, new_size);
if (nullptr == buf)
DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
if (clear_memory)
memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
}
else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD)
{
if (current_memory_tracker)
current_memory_tracker->realloc(old_size, new_size);
buf = mremap(buf, old_size, new_size, MREMAP_MAYMOVE);
if (MAP_FAILED == buf)
DB::throwFromErrno("Allocator: Cannot mremap.", DB::ErrorCodes::CANNOT_MREMAP);
/// Заполнение нулями не нужно.
}
#else
// TODO: We need to use mmap/calloc on Apple too.
if ((old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT) ||
(old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD))
{
if (current_memory_tracker)
current_memory_tracker->realloc(old_size, new_size);
buf = ::realloc(buf, new_size);
if (nullptr == buf)
DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
if (clear_memory)
memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
}
#endif
else
{
void * new_buf = alloc(new_size, alignment);
memcpy(new_buf, buf, old_size);
free(buf, old_size);
buf = new_buf;
}
return buf;
}
void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0);
protected:
static constexpr size_t getStackThreshold()
@ -194,7 +38,16 @@ protected:
};
/** Аллокатор с оптимизацией для маленьких кусков памяти.
/** При использовании AllocatorWithStackMemory, размещённом на стеке,
* GCC 4.9 ошибочно делает предположение, что мы можем вызывать free от указателя на стек.
* На самом деле, комбинация условий внутри AllocatorWithStackMemory этого не допускает.
*/
#if !__clang__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
#endif
/** Allocator with optimization to place small memory ranges in automatic memory.
*/
template <typename Base, size_t N = 64>
class AllocatorWithStackMemory : private Base
@ -223,15 +76,15 @@ public:
void * realloc(void * buf, size_t old_size, size_t new_size)
{
/// Было в stack_memory, там и останется.
/// Was in stack_memory, will remain there.
if (new_size <= N)
return buf;
/// Уже не помещалось в stack_memory.
/// Already was big enough to not fit in stack_memory.
if (old_size > N)
return Base::realloc(buf, old_size, new_size);
/// Было в stack_memory, но теперь не помещается.
/// Was in stack memory, but now will not fit there.
void * new_buf = Base::alloc(new_size);
memcpy(new_buf, buf, old_size);
return new_buf;

1
dbms/src/Common/CombinedCardinalityEstimator.h
View File

@ -3,6 +3,7 @@
#include <Common/HashTable/SmallTable.h>
#include <Common/HashTable/HashSet.h>
#include <Common/HyperLogLogCounter.h>
#include <Common/MemoryTracker.h>
#include <Core/Defines.h>

6
dbms/src/Common/ExternalTable.h
View File

@ -19,6 +19,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
/// Базовый класс содержащий основную информацию о внешней таблице и
/// основные функции для извлечения этой информации из текстовых полей.
class BaseExternalTable

5
dbms/src/Common/HyperLogLogWithSmallSetOptimization.h
View File

@ -2,6 +2,8 @@
#include <Common/HyperLogLogCounter.h>
#include <Common/HashTable/SmallTable.h>
#include <Common/MemoryTracker.h>
namespace DB
{
@ -12,7 +14,8 @@ namespace DB
* Смотрите также более практичную реализацию в CombinedCardinalityEstimator.h,
* где используется также хэш-таблица для множеств среднего размера.
*/
template <
template
<
typename Key,
UInt8 small_set_size,
UInt8 K,

1
dbms/src/DataStreams/AsynchronousBlockInputStream.h
View File

@ -6,6 +6,7 @@
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/ThreadPool.h>
#include <Common/MemoryTracker.h>
namespace CurrentMetrics

1
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp
View File

@ -1,6 +1,7 @@
#include <future>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h>

2
dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h
View File

@ -7,6 +7,8 @@
#include <condition_variable>
class MemoryTracker;
namespace DB
{

1
dbms/src/DataStreams/ParallelInputsProcessor.h
View File

@ -11,6 +11,7 @@
#include <DataStreams/IProfilingBlockInputStream.h>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
/** Allows to process multiple block input streams (sources) in parallel, using specified number of threads.

22
dbms/src/Dictionaries/CacheDictionary.cpp
View File

@ -6,11 +6,33 @@
#include <Common/HashTable/Hash.h>
#include <Common/Stopwatch.h>
#include <Common/ProfilingScopedRWLock.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <ext/size.hpp>
#include <ext/range.hpp>
#include <ext/map.hpp>
namespace ProfileEvents
{
extern const Event DictCacheKeysRequested;
extern const Event DictCacheKeysRequestedMiss;
extern const Event DictCacheKeysRequestedFound;
extern const Event DictCacheKeysExpired;
extern const Event DictCacheKeysNotFound;
extern const Event DictCacheKeysHit;
extern const Event DictCacheRequestTimeNs;
extern const Event DictCacheRequests;
extern const Event DictCacheLockWriteNs;
extern const Event DictCacheLockReadNs;
}
namespace CurrentMetrics
{
extern const Metric DictCacheRequests;
}
namespace DB
{

19
dbms/src/Dictionaries/CacheDictionary.h
View File

@ -4,6 +4,7 @@
#include <Dictionaries/IDictionarySource.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Common/ArenaWithFreeLists.h>
#include <Common/CurrentMetrics.h>
#include <Columns/ColumnString.h>
#include <ext/bit_cast.hpp>
#include <Poco/RWLock.h>
@ -15,24 +16,6 @@
#include <tuple>
#include <random>
namespace ProfileEvents
{
extern const Event DictCacheKeysRequested;
extern const Event DictCacheKeysRequestedMiss;
extern const Event DictCacheKeysRequestedFound;
extern const Event DictCacheKeysExpired;
extern const Event DictCacheKeysNotFound;
extern const Event DictCacheKeysHit;
extern const Event DictCacheRequestTimeNs;
extern const Event DictCacheRequests;
extern const Event DictCacheLockWriteNs;
extern const Event DictCacheLockReadNs;
}
namespace CurrentMetrics
{
extern const Metric DictCacheRequests;
}
namespace DB
{

22
dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp
View File

@ -3,8 +3,30 @@
#include <Common/randomSeed.h>
#include <Common/Stopwatch.h>
#include <Common/ProfilingScopedRWLock.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <ext/range.hpp>
namespace ProfileEvents
{
extern const Event DictCacheKeysRequested;
extern const Event DictCacheKeysRequestedMiss;
extern const Event DictCacheKeysRequestedFound;
extern const Event DictCacheKeysExpired;
extern const Event DictCacheKeysNotFound;
extern const Event DictCacheKeysHit;
extern const Event DictCacheRequestTimeNs;
extern const Event DictCacheLockWriteNs;
extern const Event DictCacheLockReadNs;
}
namespace CurrentMetrics
{
extern const Metric DictCacheRequests;
}
namespace DB
{

17
dbms/src/Dictionaries/ComplexKeyCacheDictionary.h
View File

@ -20,23 +20,6 @@
#include <tuple>
#include <random>
namespace ProfileEvents
{
extern const Event DictCacheKeysRequested;
extern const Event DictCacheKeysRequestedMiss;
extern const Event DictCacheKeysRequestedFound;
extern const Event DictCacheKeysExpired;
extern const Event DictCacheKeysNotFound;
extern const Event DictCacheKeysHit;
extern const Event DictCacheRequestTimeNs;
extern const Event DictCacheLockWriteNs;
extern const Event DictCacheLockReadNs;
}
namespace CurrentMetrics
{
extern const Metric DictCacheRequests;
}
namespace DB
{

1
dbms/src/Dictionaries/DictionaryStructure.cpp
View File

@ -11,6 +11,7 @@ namespace ErrorCodes
extern const int UNKNOWN_TYPE;
extern const int ARGUMENT_OUT_OF_BOUND;
extern const int TYPE_MISMATCH;
extern const int BAD_ARGUMENTS;
}

6
dbms/src/Functions/FunctionsStringSearch.cpp
View File

@ -23,6 +23,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
/** Implementation details for functions of 'position' family depending on ASCII/UTF8 and case sensitiveness.
*/
struct PositionCaseSensitiveASCII

6
dbms/src/Functions/FunctionsTransform.h
View File

@ -19,6 +19,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
/** transform(x, from_array, to_array[, default]) - преобразовать x согласно переданному явным образом соответствию.
*/

1
dbms/src/IO/CompressedWriteBuffer.cpp
View File

@ -10,6 +10,7 @@
#include <zstd.h>
#include <Common/unaligned.h>
#include <Core/Types.h>
#include <IO/CompressedWriteBuffer.h>

2
dbms/src/IO/MemoryReadWriteBuffer.cpp
View File

@ -1,5 +1,7 @@
#include <IO/MemoryReadWriteBuffer.h>
#include <common/likely.h>
#include <boost/noncopyable.hpp>
namespace DB
{

2
dbms/src/IO/MemoryReadWriteBuffer.h
View File

@ -5,6 +5,8 @@
#include <IO/IReadableWriteBuffer.h>
#include <Common/Allocator.h>
#include <Core/Defines.h>
#include <boost/noncopyable.hpp>
namespace DB
{

5
dbms/src/IO/ReadWriteBufferFromHTTP.cpp
View File

@ -7,13 +7,15 @@
#include <Poco/Net/HTTPResponse.h>
#include <IO/ReadBufferFromIStream.h>
#include <Common/SimpleCache.h>
#include <Common/config.h>
#include <Core/Types.h>
#include <common/logger_useful.h>
#include <Common/config.h>
#if Poco_NetSSL_FOUND
#include <Poco/Net/HTTPSClientSession.h>
#endif
namespace DB
{
@ -33,7 +35,6 @@ static Poco::Net::IPAddress resolveHost(const String & host)
static SimpleCache<decltype(resolveHostImpl), &resolveHostImpl> cache;
return cache(host);
}
// ==========
ReadWriteBufferFromHTTP::ReadWriteBufferFromHTTP(

1
dbms/src/Interpreters/Aggregator.cpp
View File

@ -20,6 +20,7 @@
#include <Interpreters/Aggregator.h>
#include <Common/ClickHouseRevision.h>
#include <Common/MemoryTracker.h>
#include <Interpreters/config_compile.h>

1
dbms/src/Interpreters/DictionaryFactory.cpp
View File

@ -19,6 +19,7 @@ namespace ErrorCodes
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
extern const int UNSUPPORTED_METHOD;
extern const int TOO_SMALL_BUFFER_SIZE;
extern const int BAD_ARGUMENTS;
}

41
dbms/src/Interpreters/ExternalDictionaries.cpp
View File

@ -3,6 +3,7 @@
#include <Dictionaries/DictionaryStructure.h>
#include <Dictionaries/IDictionarySource.h>
#include <Common/StringUtils.h>
#include <Common/MemoryTracker.h>
#include <ext/scope_guard.hpp>
#include <Poco/Util/Application.h>
#include <Poco/Glob.h>
@ -11,7 +12,7 @@
namespace
{
/// 5 seconds
const auto check_period_sec = 5;
const auto backoff_initial_sec = 5;
/// 10 minutes
const auto backoff_max_sec = 10 * 60;
@ -27,6 +28,44 @@ namespace ErrorCodes
}
void ExternalDictionaries::reloadPeriodically()
{
setThreadName("ExterDictReload");
while (true)
{
if (destroy.tryWait(check_period_sec * 1000))
return;
reloadImpl();
}
}
ExternalDictionaries::ExternalDictionaries(Context & context, const bool throw_on_error)
: context(context), log(&Logger::get("ExternalDictionaries"))
{
{
/** During synchronous loading of external dictionaries at moment of query execution,
* we should not use per query memory limit.
*/
TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker;
reloadImpl(throw_on_error);
}
reloading_thread = std::thread{&ExternalDictionaries::reloadPeriodically, this};
}
ExternalDictionaries::~ExternalDictionaries()
{
destroy.set();
reloading_thread.join();
}
namespace
{
std::set<std::string> getDictionariesConfigPaths(const Poco::Util::AbstractConfiguration & config)

37
dbms/src/Interpreters/ExternalDictionaries.h
View File

@ -37,8 +37,6 @@ class Context;
class ExternalDictionaries
{
private:
static const auto check_period_sec = 5;
friend class StorageSystemDictionaries;
mutable std::mutex dictionaries_mutex;
@ -85,41 +83,12 @@ private:
void reloadImpl(bool throw_on_error = false);
void reloadFromFile(const std::string & config_path, bool throw_on_error);
void reloadPeriodically()
{
setThreadName("ExterDictReload");
while (true)
{
if (destroy.tryWait(check_period_sec * 1000))
return;
reloadImpl();
}
}
void reloadPeriodically();
public:
/// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
ExternalDictionaries(Context & context, const bool throw_on_error)
: context(context), log(&Logger::get("ExternalDictionaries"))
{
{
/** During synchronous loading of external dictionaries at moment of query execution,
* we should not use per query memory limit.
*/
TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker;
reloadImpl(throw_on_error);
}
reloading_thread = std::thread{&ExternalDictionaries::reloadPeriodically, this};
}
~ExternalDictionaries()
{
destroy.set();
reloading_thread.join();
}
ExternalDictionaries(Context & context, const bool throw_on_error);
~ExternalDictionaries();
MultiVersion<IDictionaryBase>::Version getDictionary(const std::string & name) const;
};

6
dbms/src/Interpreters/InterpreterOptimizeQuery.cpp
View File

@ -6,6 +6,12 @@
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
BlockIO InterpreterOptimizeQuery::execute()
{
const ASTOptimizeQuery & ast = typeid_cast<const ASTOptimizeQuery &>(*query_ptr);

1
dbms/src/Interpreters/Users.cpp
View File

@ -35,6 +35,7 @@ namespace ErrorCodes
extern const int REQUIRED_PASSWORD;
extern const int WRONG_PASSWORD;
extern const int IP_ADDRESS_NOT_ALLOWED;
extern const int BAD_ARGUMENTS;
}

1
dbms/src/Storages/MergeTree/MergeTreeReader.cpp
View File

@ -3,6 +3,7 @@
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeNullable.h>
#include <Common/escapeForFileName.h>
#include <Common/MemoryTracker.h>
#include <IO/CachedCompressedReadBuffer.h>
#include <IO/CompressedReadBufferFromFile.h>
#include <Columns/ColumnArray.h>

1
dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp
View File

@ -7,6 +7,7 @@
#include <Columns/ColumnArray.h>
#include <Columns/ColumnNullable.h>
#include <Common/StringUtils.h>
#include <Common/MemoryTracker.h>
#include <Poco/File.h>

1
dbms/src/Storages/StorageBuffer.cpp
View File

@ -9,6 +9,7 @@
#include <Parsers/ASTExpressionList.h>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/MemoryTracker.h>
#include <common/logger_useful.h>
#include <Poco/Ext/ThreadNumber.h>

3
dbms/src/Storages/StorageFile.h
View File

@ -8,6 +8,9 @@
#include <common/logger_useful.h>
#include <atomic>
namespace DB
{