Moved code to cpp [#CLICKHOUSE-2].

dbms/src/AggregateFunctions/AggregateFunctionSequenceMatch.h

@@ -25,6 +25,7 @@ namespace ErrorCodes
     extern const int TOO_LESS_ARGUMENTS_FOR_FUNCTION;
     extern const int TOO_MUCH_ARGUMENTS_FOR_FUNCTION;
     extern const int SYNTAX_ERROR;
+    extern const int BAD_ARGUMENTS;
 }
 
 /// helper type for comparing `std::pair`s using solely the .first member

dbms/src/AggregateFunctions/AggregateFunctionUniq.h

@@ -16,6 +16,7 @@
 #include <Common/HashTable/HashSet.h>
 #include <Common/HyperLogLogWithSmallSetOptimization.h>
 #include <Common/CombinedCardinalityEstimator.h>
+#include <Common/MemoryTracker.h>
 
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnTuple.h>

dbms/src/Client/Benchmark.cpp

@@ -52,6 +52,7 @@ namespace ErrorCodes
     extern const int POCO_EXCEPTION;
     extern const int STD_EXCEPTION;
     extern const int UNKNOWN_EXCEPTION;
+    extern const int BAD_ARGUMENTS;
 }
 
 class Benchmark

dbms/src/Common/Allocator.cpp

@@ -0,0 +1,165 @@
+#if !defined(__APPLE__) && !defined(__FreeBSD__)
+#include <malloc.h>
+#endif
+
+#include <cstdlib>
+#include <sys/mman.h>
+
+#include <Common/MemoryTracker.h>
+#include <Common/Exception.h>
+#include <Common/Allocator.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+    extern const int CANNOT_ALLOCATE_MEMORY;
+    extern const int CANNOT_MUNMAP;
+    extern const int CANNOT_MREMAP;
+}
+}
+
+
+/** Многие современные аллокаторы (например, tcmalloc) не умеют делать mremap для realloc,
+  *  даже в случае достаточно больших кусков памяти.
+  * Хотя это позволяет увеличить производительность и уменьшить потребление памяти во время realloc-а.
+  * Чтобы это исправить, делаем mremap самостоятельно, если кусок памяти достаточно большой.
+  * Порог (64 МБ) выбран достаточно большим, так как изменение адресного пространства
+  *  довольно сильно тормозит, особенно в случае наличия большого количества потоков.
+  * Рассчитываем, что набор операций mmap/что-то сделать/mremap может выполняться всего лишь около 1000 раз в секунду.
+  *
+  * PS. Также это требуется, потому что tcmalloc не может выделить кусок памяти больше 16 GB.
+  */
+static constexpr size_t MMAP_THRESHOLD = 64 * (1 << 20);
+static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
+static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
+
+
+template <bool clear_memory_>
+void * Allocator<clear_memory_>::alloc(size_t size, size_t alignment)
+{
+    if (current_memory_tracker)
+        current_memory_tracker->alloc(size);
+
+    void * buf;
+
+    if (size >= MMAP_THRESHOLD)
+    {
+        if (alignment > MMAP_MIN_ALIGNMENT)
+            throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS);
+
+        buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+        if (MAP_FAILED == buf)
+            DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+
+        /// No need for zero-fill, because mmap guarantees it.
+    }
+    else
+    {
+        if (alignment <= MALLOC_MIN_ALIGNMENT)
+        {
+            if (clear_memory)
+                buf = ::calloc(size, 1);
+            else
+                buf = ::malloc(size);
+
+            if (nullptr == buf)
+                DB::throwFromErrno("Allocator: Cannot malloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+        }
+        else
+        {
+            buf = nullptr;
+            int res = posix_memalign(&buf, alignment, size);
+
+            if (0 != res)
+                DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res);
+
+            if (clear_memory)
+                memset(buf, 0, size);
+        }
+    }
+
+    return buf;
+}
+
+
+template <bool clear_memory_>
+void Allocator<clear_memory_>::free(void * buf, size_t size)
+{
+    if (size >= MMAP_THRESHOLD)
+    {
+        if (0 != munmap(buf, size))
+            DB::throwFromErrno("Allocator: Cannot munmap.", DB::ErrorCodes::CANNOT_MUNMAP);
+    }
+    else
+    {
+        ::free(buf);
+    }
+
+    if (current_memory_tracker)
+        current_memory_tracker->free(size);
+}
+
+
+template <bool clear_memory_>
+void * Allocator<clear_memory_>::realloc(void * buf, size_t old_size, size_t new_size, size_t alignment)
+{
+#if !defined(__APPLE__) && !defined(__FreeBSD__)
+    if (old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT)
+    {
+        if (current_memory_tracker)
+            current_memory_tracker->realloc(old_size, new_size);
+
+        buf = ::realloc(buf, new_size);
+
+        if (nullptr == buf)
+            DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+
+        if (clear_memory)
+            memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
+    }
+    else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD)
+    {
+        if (current_memory_tracker)
+            current_memory_tracker->realloc(old_size, new_size);
+
+        buf = mremap(buf, old_size, new_size, MREMAP_MAYMOVE);
+        if (MAP_FAILED == buf)
+            DB::throwFromErrno("Allocator: Cannot mremap.", DB::ErrorCodes::CANNOT_MREMAP);
+
+        /// No need for zero-fill, because mmap guarantees it.
+    }
+#else
+    // TODO: We need to use mmap/calloc on Apple too.
+    if ((old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT) ||
+        (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD))
+    {
+        if (current_memory_tracker)
+            current_memory_tracker->realloc(old_size, new_size);
+
+        buf = ::realloc(buf, new_size);
+
+        if (nullptr == buf)
+            DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
+
+        if (clear_memory)
+            memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
+    }
+#endif
+    else
+    {
+        void * new_buf = alloc(new_size, alignment);
+        memcpy(new_buf, buf, old_size);
+        free(buf, old_size);
+        buf = new_buf;
+    }
+
+    return buf;
+}
+
+
+/// Explicit template instantinations.
+template class Allocator<true>;
+template class Allocator<false>;

dbms/src/Common/Allocator.h

@@ -1,36 +1,6 @@
 #pragma once
 
-#if !defined(__APPLE__) && !defined(__FreeBSD__)
-#include <malloc.h>
-#endif
-#include <cstdlib>
 #include <string.h>
-#include <sys/mman.h>
-
-#include <Common/MemoryTracker.h>
-#include <Common/Exception.h>
-
-
-namespace DB
-{
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-    extern const int CANNOT_ALLOCATE_MEMORY;
-    extern const int CANNOT_MUNMAP;
-    extern const int CANNOT_MREMAP;
-}
-}
-
-
-/** При использовании AllocatorWithStackMemory, размещённом на стеке,
-  *  GCC 4.9 ошибочно делает предположение, что мы можем вызывать free от указателя на стек.
-  * На самом деле, комбинация условий внутри AllocatorWithStackMemory этого не допускает.
-  */
-#if !__clang__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
-#endif
 
 
 /** Отвечает за выделение/освобождение памяти. Используется, например, в PODArray, Arena.
@@ -47,144 +17,18 @@ class Allocator
 protected:
     static constexpr bool clear_memory = clear_memory_;
 
-private:
-    /** Многие современные аллокаторы (например, tcmalloc) не умеют делать mremap для realloc,
-      *  даже в случае достаточно больших кусков памяти.
-      * Хотя это позволяет увеличить производительность и уменьшить потребление памяти во время realloc-а.
-      * Чтобы это исправить, делаем mremap самостоятельно, если кусок памяти достаточно большой.
-      * Порог (64 МБ) выбран достаточно большим, так как изменение адресного пространства
-      *  довольно сильно тормозит, особенно в случае наличия большого количества потоков.
-      * Рассчитываем, что набор операций mmap/что-то сделать/mremap может выполняться всего лишь около 1000 раз в секунду.
-      *
-      * PS. Также это требуется, потому что tcmalloc не может выделить кусок памяти больше 16 GB.
-      */
-    static constexpr size_t MMAP_THRESHOLD = 64 * (1 << 20);
-    static constexpr size_t MMAP_MIN_ALIGNMENT = 4096;
-    static constexpr size_t MALLOC_MIN_ALIGNMENT = 8;
-
 public:
-    /// Выделить кусок памяти.
-    void * alloc(size_t size, size_t alignment = 0)
-    {
-        if (current_memory_tracker)
-            current_memory_tracker->alloc(size);
+    /// Allocate memory range.
+    void * alloc(size_t size, size_t alignment = 0);
 
-        void * buf;
+    /// Free memory range.
+    void free(void * buf, size_t size);
 
-        if (size >= MMAP_THRESHOLD)
-        {
-            if (alignment > MMAP_MIN_ALIGNMENT)
-                throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS);
-
-            buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-            if (MAP_FAILED == buf)
-                DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-
-            /// Заполнение нулями не нужно - mmap сам это делает.
-        }
-        else
-        {
-            if (alignment <= MALLOC_MIN_ALIGNMENT)
-            {
-                if (clear_memory)
-                    buf = ::calloc(size, 1);
-                else
-                    buf = ::malloc(size);
-
-                if (nullptr == buf)
-                    DB::throwFromErrno("Allocator: Cannot malloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-            }
-            else
-            {
-                buf = nullptr;
-                int res = posix_memalign(&buf, alignment, size);
-
-                if (0 != res)
-                    DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res);
-
-                if (clear_memory)
-                    memset(buf, 0, size);
-            }
-        }
-
-        return buf;
-    }
-
-    /// Освободить память.
-    void free(void * buf, size_t size)
-    {
-        if (size >= MMAP_THRESHOLD)
-        {
-            if (0 != munmap(buf, size))
-                DB::throwFromErrno("Allocator: Cannot munmap.", DB::ErrorCodes::CANNOT_MUNMAP);
-        }
-        else
-        {
-            ::free(buf);
-        }
-
-        if (current_memory_tracker)
-            current_memory_tracker->free(size);
-    }
-
-    /** Увеличить размер куска памяти.
-      * Содержимое старого куска памяти переезжает в начало нового.
-      * Положение куска памяти может измениться.
+    /** Enlarge memory range.
+      * Data from old range is moved to the beginning of new range.
+      * Address of memory range could change.
       */
-    void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0)
-    {
-    #if !defined(__APPLE__) && !defined(__FreeBSD__)
-        if (old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT)
-        {
-            if (current_memory_tracker)
-                current_memory_tracker->realloc(old_size, new_size);
-
-            buf = ::realloc(buf, new_size);
-
-            if (nullptr == buf)
-                DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-
-            if (clear_memory)
-                memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
-        }
-        else if (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD)
-        {
-            if (current_memory_tracker)
-                current_memory_tracker->realloc(old_size, new_size);
-
-            buf = mremap(buf, old_size, new_size, MREMAP_MAYMOVE);
-            if (MAP_FAILED == buf)
-                DB::throwFromErrno("Allocator: Cannot mremap.", DB::ErrorCodes::CANNOT_MREMAP);
-
-            /// Заполнение нулями не нужно.
-        }
-    #else
-        // TODO: We need to use mmap/calloc on Apple too.
-        if ((old_size < MMAP_THRESHOLD && new_size < MMAP_THRESHOLD && alignment <= MALLOC_MIN_ALIGNMENT) ||
-            (old_size >= MMAP_THRESHOLD && new_size >= MMAP_THRESHOLD))
-        {
-            if (current_memory_tracker)
-                current_memory_tracker->realloc(old_size, new_size);
-
-            buf = ::realloc(buf, new_size);
-
-            if (nullptr == buf)
-                DB::throwFromErrno("Allocator: Cannot realloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
-
-            if (clear_memory)
-                memset(reinterpret_cast<char *>(buf) + old_size, 0, new_size - old_size);
-        }
-    #endif
-        else
-        {
-            void * new_buf = alloc(new_size, alignment);
-            memcpy(new_buf, buf, old_size);
-            free(buf, old_size);
-            buf = new_buf;
-        }
-
-        return buf;
-    }
+    void * realloc(void * buf, size_t old_size, size_t new_size, size_t alignment = 0);
 
 protected:
     static constexpr size_t getStackThreshold()
@@ -194,7 +38,16 @@ protected:
 };
 
 
-/** Аллокатор с оптимизацией для маленьких кусков памяти.
+/** При использовании AllocatorWithStackMemory, размещённом на стеке,
+  *  GCC 4.9 ошибочно делает предположение, что мы можем вызывать free от указателя на стек.
+  * На самом деле, комбинация условий внутри AllocatorWithStackMemory этого не допускает.
+  */
+#if !__clang__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wfree-nonheap-object"
+#endif
+
+/** Allocator with optimization to place small memory ranges in automatic memory.
   */
 template <typename Base, size_t N = 64>
 class AllocatorWithStackMemory : private Base
@@ -223,15 +76,15 @@ public:
 
     void * realloc(void * buf, size_t old_size, size_t new_size)
     {
-        /// Было в stack_memory, там и останется.
+        /// Was in stack_memory, will remain there.
         if (new_size <= N)
             return buf;
 
-        /// Уже не помещалось в stack_memory.
+        /// Already was big enough to not fit in stack_memory.
         if (old_size > N)
             return Base::realloc(buf, old_size, new_size);
 
-        /// Было в stack_memory, но теперь не помещается.
+        /// Was in stack memory, but now will not fit there.
         void * new_buf = Base::alloc(new_size);
         memcpy(new_buf, buf, old_size);
         return new_buf;

dbms/src/Common/CombinedCardinalityEstimator.h

@@ -3,6 +3,7 @@
 #include <Common/HashTable/SmallTable.h>
 #include <Common/HashTable/HashSet.h>
 #include <Common/HyperLogLogCounter.h>
+#include <Common/MemoryTracker.h>
 #include <Core/Defines.h>
 
 

dbms/src/Common/ExternalTable.h

@@ -19,6 +19,12 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+
 /// Базовый класс содержащий основную информацию о внешней таблице и
 /// основные функции для извлечения этой информации из текстовых полей.
 class BaseExternalTable

dbms/src/Common/HyperLogLogWithSmallSetOptimization.h

@@ -2,6 +2,8 @@
 
 #include <Common/HyperLogLogCounter.h>
 #include <Common/HashTable/SmallTable.h>
+#include <Common/MemoryTracker.h>
+
 
 namespace DB
 {
@@ -12,7 +14,8 @@ namespace DB
   * Смотрите также более практичную реализацию в CombinedCardinalityEstimator.h,
   *  где используется также хэш-таблица для множеств среднего размера.
   */
-template <
+template
+<
     typename Key,
     UInt8 small_set_size,
     UInt8 K,

dbms/src/DataStreams/AsynchronousBlockInputStream.h

@@ -6,6 +6,7 @@
 #include <Common/setThreadName.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/ThreadPool.h>
+#include <Common/MemoryTracker.h>
 
 
 namespace CurrentMetrics

dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp

@@ -1,6 +1,7 @@
 #include <future>
 #include <Common/setThreadName.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/MemoryTracker.h>
 #include <DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h>
 
 

dbms/src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h

@@ -7,6 +7,8 @@
 #include <condition_variable>
 
 
+class MemoryTracker;
+
 namespace DB
 {
 

dbms/src/DataStreams/ParallelInputsProcessor.h

@@ -11,6 +11,7 @@
 #include <DataStreams/IProfilingBlockInputStream.h>
 #include <Common/setThreadName.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/MemoryTracker.h>
 
 
 /** Allows to process multiple block input streams (sources) in parallel, using specified number of threads.

dbms/src/Dictionaries/CacheDictionary.cpp

@@ -6,11 +6,33 @@
 #include <Common/HashTable/Hash.h>
 #include <Common/Stopwatch.h>
 #include <Common/ProfilingScopedRWLock.h>
+#include <Common/ProfileEvents.h>
+#include <Common/CurrentMetrics.h>
 #include <ext/size.hpp>
 #include <ext/range.hpp>
 #include <ext/map.hpp>
 
 
+namespace ProfileEvents
+{
+    extern const Event DictCacheKeysRequested;
+    extern const Event DictCacheKeysRequestedMiss;
+    extern const Event DictCacheKeysRequestedFound;
+    extern const Event DictCacheKeysExpired;
+    extern const Event DictCacheKeysNotFound;
+    extern const Event DictCacheKeysHit;
+    extern const Event DictCacheRequestTimeNs;
+    extern const Event DictCacheRequests;
+    extern const Event DictCacheLockWriteNs;
+    extern const Event DictCacheLockReadNs;
+}
+
+namespace CurrentMetrics
+{
+    extern const Metric DictCacheRequests;
+}
+
+
 namespace DB
 {
 

dbms/src/Dictionaries/CacheDictionary.h

@@ -4,6 +4,7 @@
 #include <Dictionaries/IDictionarySource.h>
 #include <Dictionaries/DictionaryStructure.h>
 #include <Common/ArenaWithFreeLists.h>
+#include <Common/CurrentMetrics.h>
 #include <Columns/ColumnString.h>
 #include <ext/bit_cast.hpp>
 #include <Poco/RWLock.h>
@@ -15,24 +16,6 @@
 #include <tuple>
 #include <random>
 
-namespace ProfileEvents
-{
-    extern const Event DictCacheKeysRequested;
-    extern const Event DictCacheKeysRequestedMiss;
-    extern const Event DictCacheKeysRequestedFound;
-    extern const Event DictCacheKeysExpired;
-    extern const Event DictCacheKeysNotFound;
-    extern const Event DictCacheKeysHit;
-    extern const Event DictCacheRequestTimeNs;
-    extern const Event DictCacheRequests;
-    extern const Event DictCacheLockWriteNs;
-    extern const Event DictCacheLockReadNs;
-}
-
-namespace CurrentMetrics
-{
-    extern const Metric DictCacheRequests;
-}
 
 namespace DB
 {

dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp

@@ -3,8 +3,30 @@
 #include <Common/randomSeed.h>
 #include <Common/Stopwatch.h>
 #include <Common/ProfilingScopedRWLock.h>
+#include <Common/ProfileEvents.h>
+#include <Common/CurrentMetrics.h>
 #include <ext/range.hpp>
 
+
+namespace ProfileEvents
+{
+    extern const Event DictCacheKeysRequested;
+    extern const Event DictCacheKeysRequestedMiss;
+    extern const Event DictCacheKeysRequestedFound;
+    extern const Event DictCacheKeysExpired;
+    extern const Event DictCacheKeysNotFound;
+    extern const Event DictCacheKeysHit;
+    extern const Event DictCacheRequestTimeNs;
+    extern const Event DictCacheLockWriteNs;
+    extern const Event DictCacheLockReadNs;
+}
+
+namespace CurrentMetrics
+{
+    extern const Metric DictCacheRequests;
+}
+
+
 namespace DB
 {
 

dbms/src/Dictionaries/ComplexKeyCacheDictionary.h

@@ -20,23 +20,6 @@
 #include <tuple>
 #include <random>
 
-namespace ProfileEvents
-{
-    extern const Event DictCacheKeysRequested;
-    extern const Event DictCacheKeysRequestedMiss;
-    extern const Event DictCacheKeysRequestedFound;
-    extern const Event DictCacheKeysExpired;
-    extern const Event DictCacheKeysNotFound;
-    extern const Event DictCacheKeysHit;
-    extern const Event DictCacheRequestTimeNs;
-    extern const Event DictCacheLockWriteNs;
-    extern const Event DictCacheLockReadNs;
-}
-
-namespace CurrentMetrics
-{
-    extern const Metric DictCacheRequests;
-}
 
 namespace DB
 {

dbms/src/Dictionaries/DictionaryStructure.cpp

@@ -11,6 +11,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_TYPE;
     extern const int ARGUMENT_OUT_OF_BOUND;
     extern const int TYPE_MISMATCH;
+    extern const int BAD_ARGUMENTS;
 }
 
 

dbms/src/Functions/FunctionsStringSearch.cpp

@@ -23,6 +23,12 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+
 /** Implementation details for functions of 'position' family depending on ASCII/UTF8 and case sensitiveness.
   */
 struct PositionCaseSensitiveASCII

dbms/src/Functions/FunctionsTransform.h

@@ -19,6 +19,12 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+
 /** transform(x, from_array, to_array[, default]) - преобразовать x согласно переданному явным образом соответствию.
   */
 

dbms/src/IO/CompressedWriteBuffer.cpp

@@ -10,6 +10,7 @@
 #include <zstd.h>
 
 #include <Common/unaligned.h>
+#include <Core/Types.h>
 
 #include <IO/CompressedWriteBuffer.h>
 

dbms/src/IO/MemoryReadWriteBuffer.cpp

@@ -1,5 +1,7 @@
 #include <IO/MemoryReadWriteBuffer.h>
 #include <common/likely.h>
+#include <boost/noncopyable.hpp>
+
 
 namespace DB
 {

dbms/src/IO/MemoryReadWriteBuffer.h

@@ -5,6 +5,8 @@
 #include <IO/IReadableWriteBuffer.h>
 #include <Common/Allocator.h>
 #include <Core/Defines.h>
+#include <boost/noncopyable.hpp>
+
 
 namespace DB
 {

dbms/src/IO/ReadWriteBufferFromHTTP.cpp

@@ -7,13 +7,15 @@
 #include <Poco/Net/HTTPResponse.h>
 #include <IO/ReadBufferFromIStream.h>
 #include <Common/SimpleCache.h>
+#include <Common/config.h>
+#include <Core/Types.h>
 #include <common/logger_useful.h>
 
-#include <Common/config.h>
 #if Poco_NetSSL_FOUND
 #include <Poco/Net/HTTPSClientSession.h>
 #endif
 
+
 namespace DB
 {
 
@@ -33,7 +35,6 @@ static Poco::Net::IPAddress resolveHost(const String & host)
     static SimpleCache<decltype(resolveHostImpl), &resolveHostImpl> cache;
     return cache(host);
 }
-// ==========
 
 
 ReadWriteBufferFromHTTP::ReadWriteBufferFromHTTP(

dbms/src/Interpreters/Aggregator.cpp

@@ -20,6 +20,7 @@
 
 #include <Interpreters/Aggregator.h>
 #include <Common/ClickHouseRevision.h>
+#include <Common/MemoryTracker.h>
 #include <Interpreters/config_compile.h>
 
 

dbms/src/Interpreters/DictionaryFactory.cpp

@@ -19,6 +19,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_ELEMENT_IN_CONFIG;
     extern const int UNSUPPORTED_METHOD;
     extern const int TOO_SMALL_BUFFER_SIZE;
+    extern const int BAD_ARGUMENTS;
 }
 
 

dbms/src/Interpreters/ExternalDictionaries.cpp

@@ -3,6 +3,7 @@
 #include <Dictionaries/DictionaryStructure.h>
 #include <Dictionaries/IDictionarySource.h>
 #include <Common/StringUtils.h>
+#include <Common/MemoryTracker.h>
 #include <ext/scope_guard.hpp>
 #include <Poco/Util/Application.h>
 #include <Poco/Glob.h>
@@ -11,7 +12,7 @@
 
 namespace
 {
-    /// 5 seconds
+    const auto check_period_sec = 5;
     const auto backoff_initial_sec = 5;
     /// 10 minutes
     const auto backoff_max_sec = 10 * 60;
@@ -27,6 +28,44 @@ namespace ErrorCodes
 }
 
 
+void ExternalDictionaries::reloadPeriodically()
+{
+    setThreadName("ExterDictReload");
+
+    while (true)
+    {
+        if (destroy.tryWait(check_period_sec * 1000))
+            return;
+
+        reloadImpl();
+    }
+}
+
+
+ExternalDictionaries::ExternalDictionaries(Context & context, const bool throw_on_error)
+    : context(context), log(&Logger::get("ExternalDictionaries"))
+{
+    {
+        /** During synchronous loading of external dictionaries at moment of query execution,
+            *  we should not use per query memory limit.
+            */
+        TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker;
+
+        reloadImpl(throw_on_error);
+    }
+
+    reloading_thread = std::thread{&ExternalDictionaries::reloadPeriodically, this};
+}
+
+
+ExternalDictionaries::~ExternalDictionaries()
+{
+    destroy.set();
+    reloading_thread.join();
+}
+
+
+
 namespace
 {
     std::set<std::string> getDictionariesConfigPaths(const Poco::Util::AbstractConfiguration & config)

dbms/src/Interpreters/ExternalDictionaries.h

@@ -37,8 +37,6 @@ class Context;
 class ExternalDictionaries
 {
 private:
-    static const auto check_period_sec = 5;
-
     friend class StorageSystemDictionaries;
 
     mutable std::mutex dictionaries_mutex;
@@ -85,41 +83,12 @@ private:
     void reloadImpl(bool throw_on_error = false);
     void reloadFromFile(const std::string & config_path, bool throw_on_error);
 
-    void reloadPeriodically()
-    {
-        setThreadName("ExterDictReload");
-
-        while (true)
-        {
-            if (destroy.tryWait(check_period_sec * 1000))
-                return;
-
-            reloadImpl();
-        }
-    }
+    void reloadPeriodically();
 
 public:
     /// Dictionaries will be loaded immediately and then will be updated in separate thread, each 'reload_period' seconds.
-    ExternalDictionaries(Context & context, const bool throw_on_error)
-        : context(context), log(&Logger::get("ExternalDictionaries"))
-    {
-        {
-            /** During synchronous loading of external dictionaries at moment of query execution,
-              *  we should not use per query memory limit.
-              */
-            TemporarilyDisableMemoryTracker temporarily_disable_memory_tracker;
-
-            reloadImpl(throw_on_error);
-        }
-
-        reloading_thread = std::thread{&ExternalDictionaries::reloadPeriodically, this};
-    }
-
-    ~ExternalDictionaries()
-    {
-        destroy.set();
-        reloading_thread.join();
-    }
+    ExternalDictionaries(Context & context, const bool throw_on_error);
+    ~ExternalDictionaries();
 
     MultiVersion<IDictionaryBase>::Version getDictionary(const std::string & name) const;
 };

dbms/src/Interpreters/InterpreterOptimizeQuery.cpp

@@ -6,6 +6,12 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+}
+
+
 BlockIO InterpreterOptimizeQuery::execute()
 {
     const ASTOptimizeQuery & ast = typeid_cast<const ASTOptimizeQuery &>(*query_ptr);

dbms/src/Interpreters/Users.cpp

@@ -35,6 +35,7 @@ namespace ErrorCodes
     extern const int REQUIRED_PASSWORD;
     extern const int WRONG_PASSWORD;
     extern const int IP_ADDRESS_NOT_ALLOWED;
+    extern const int BAD_ARGUMENTS;
 }
 
 

dbms/src/Storages/MergeTree/MergeTreeReader.cpp

@@ -3,6 +3,7 @@
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <Common/escapeForFileName.h>
+#include <Common/MemoryTracker.h>
 #include <IO/CachedCompressedReadBuffer.h>
 #include <IO/CompressedReadBufferFromFile.h>
 #include <Columns/ColumnArray.h>

dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp

@@ -7,6 +7,7 @@
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnNullable.h>
 #include <Common/StringUtils.h>
+#include <Common/MemoryTracker.h>
 #include <Poco/File.h>
 
 

dbms/src/Storages/StorageBuffer.cpp

@@ -9,6 +9,7 @@
 #include <Parsers/ASTExpressionList.h>
 #include <Common/setThreadName.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/MemoryTracker.h>
 #include <common/logger_useful.h>
 #include <Poco/Ext/ThreadNumber.h>
 

dbms/src/Storages/StorageFile.h

@@ -8,6 +8,9 @@
 
 #include <common/logger_useful.h>
 
+#include <atomic>
+
+
 namespace DB
 {