From ae4694b70680a9d4e52b7169ba15f40bfef37477 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 27 Aug 2015 13:39:30 +0300 Subject: [PATCH 1/4] =?UTF-8?q?dbms::Allocator:=20support=20big=20alignmen?= =?UTF-8?q?ts=20for=20=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88?= =?UTF-8?q?=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88=20[#METR?= =?UTF-8?q?-17814]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbms/include/DB/Common/Allocator.h | 42 ++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/dbms/include/DB/Common/Allocator.h b/dbms/include/DB/Common/Allocator.h index 524d88f010f..0ef3ec17838 100644 --- a/dbms/include/DB/Common/Allocator.h +++ b/dbms/include/DB/Common/Allocator.h @@ -36,16 +36,21 @@ public: if (size >= MMAP_THRESHOLD) { - if (alignment > MMAP_MIN_ALIGNMENT) - throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS); + if (alignment <= MMAP_MIN_ALIGNMENT) + { + buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == buf) + DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (MAP_FAILED == buf) - DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - - /// См. комментарий в HashTableAllocator.h - if (size >= HUGE_PAGE_SIZE && 0 != madvise(buf, size, MADV_HUGEPAGE)) - DB::throwFromErrno("HashTableAllocator: Cannot madvise with MADV_HUGEPAGE.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + /// См. комментарий в HashTableAllocator.h + if (size >= HUGE_PAGE_SIZE && 0 != madvise(buf, size, MADV_HUGEPAGE)) + DB::throwFromErrno("HashTableAllocator: Cannot madvise with MADV_HUGEPAGE.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + else + { + /// если выравнивание больше, чем размер страницы используем posix_memalign + buf = allocateUsingPosixMemalign(size, alignment); + } } else { @@ -58,11 +63,7 @@ public: } else { - buf = nullptr; - int res = posix_memalign(&buf, alignment, size); - - if (0 != res) - DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); + buf = allocateUsingPosixMemalign(size, alignment); } } @@ -121,4 +122,17 @@ public: return buf; } + +private: + void * allocateUsingPosixMemalign(size_t size, size_t alignment) + { + void * buf = nullptr; + + int res = posix_memalign(&buf, alignment, size); + + if (0 != res) + DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); + + return buf; + } }; From 6e511e28684b5d8ccc314e86083f0f10e4a4f889 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 27 Aug 2015 13:39:30 +0300 Subject: [PATCH 2/4] =?UTF-8?q?dbms::Allocator:=20support=20big=20alignmen?= =?UTF-8?q?ts=20for=20=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88?= =?UTF-8?q?=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88=20[#METR?= =?UTF-8?q?-17814]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dbms/include/DB/Common/Allocator.h | 42 ++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/dbms/include/DB/Common/Allocator.h b/dbms/include/DB/Common/Allocator.h index 524d88f010f..0ef3ec17838 100644 --- a/dbms/include/DB/Common/Allocator.h +++ b/dbms/include/DB/Common/Allocator.h @@ -36,16 +36,21 @@ public: if (size >= MMAP_THRESHOLD) { - if (alignment > MMAP_MIN_ALIGNMENT) - throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS); + if (alignment <= MMAP_MIN_ALIGNMENT) + { + buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == buf) + DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (MAP_FAILED == buf) - DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - - /// См. комментарий в HashTableAllocator.h - if (size >= HUGE_PAGE_SIZE && 0 != madvise(buf, size, MADV_HUGEPAGE)) - DB::throwFromErrno("HashTableAllocator: Cannot madvise with MADV_HUGEPAGE.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + /// См. комментарий в HashTableAllocator.h + if (size >= HUGE_PAGE_SIZE && 0 != madvise(buf, size, MADV_HUGEPAGE)) + DB::throwFromErrno("HashTableAllocator: Cannot madvise with MADV_HUGEPAGE.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + } + else + { + /// если выравнивание больше, чем размер страницы используем posix_memalign + buf = allocateUsingPosixMemalign(size, alignment); + } } else { @@ -58,11 +63,7 @@ public: } else { - buf = nullptr; - int res = posix_memalign(&buf, alignment, size); - - if (0 != res) - DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); + buf = allocateUsingPosixMemalign(size, alignment); } } @@ -121,4 +122,17 @@ public: return buf; } + +private: + void * allocateUsingPosixMemalign(size_t size, size_t alignment) + { + void * buf = nullptr; + + int res = posix_memalign(&buf, alignment, size); + + if (0 != res) + DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); + + return buf; + } }; From 4a26a56d69e769b0d1a8655a3d8b15631b186f61 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Tue, 1 Sep 2015 13:41:22 +0300 Subject: [PATCH 3/4] =?UTF-8?q?Revert=20"dbms::Allocator:=20support=20big?= =?UTF-8?q?=20alignments=20for=20=E2=96=88=E2=96=88=E2=96=88=E2=96=88?= =?UTF-8?q?=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88=E2=96=88?= =?UTF-8?q?=20[#METR-17814]"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 8d5d4b6b9265e2b5a103a6d713df81d7fe232745. --- dbms/include/DB/Common/Allocator.h | 42 ++++++++++-------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/dbms/include/DB/Common/Allocator.h b/dbms/include/DB/Common/Allocator.h index 0ef3ec17838..524d88f010f 100644 --- a/dbms/include/DB/Common/Allocator.h +++ b/dbms/include/DB/Common/Allocator.h @@ -36,21 +36,16 @@ public: if (size >= MMAP_THRESHOLD) { - if (alignment <= MMAP_MIN_ALIGNMENT) - { - buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (MAP_FAILED == buf) - DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + if (alignment > MMAP_MIN_ALIGNMENT) + throw DB::Exception("Too large alignment: more than page size.", DB::ErrorCodes::BAD_ARGUMENTS); - /// См. комментарий в HashTableAllocator.h - if (size >= HUGE_PAGE_SIZE && 0 != madvise(buf, size, MADV_HUGEPAGE)) - DB::throwFromErrno("HashTableAllocator: Cannot madvise with MADV_HUGEPAGE.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - else - { - /// если выравнивание больше, чем размер страницы используем posix_memalign - buf = allocateUsingPosixMemalign(size, alignment); - } + buf = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == buf) + DB::throwFromErrno("Allocator: Cannot mmap.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); + + /// См. комментарий в HashTableAllocator.h + if (size >= HUGE_PAGE_SIZE && 0 != madvise(buf, size, MADV_HUGEPAGE)) + DB::throwFromErrno("HashTableAllocator: Cannot madvise with MADV_HUGEPAGE.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); } else { @@ -63,7 +58,11 @@ public: } else { - buf = allocateUsingPosixMemalign(size, alignment); + buf = nullptr; + int res = posix_memalign(&buf, alignment, size); + + if (0 != res) + DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); } } @@ -122,17 +121,4 @@ public: return buf; } - -private: - void * allocateUsingPosixMemalign(size_t size, size_t alignment) - { - void * buf = nullptr; - - int res = posix_memalign(&buf, alignment, size); - - if (0 != res) - DB::throwFromErrno("Cannot allocate memory (posix_memalign)", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY, res); - - return buf; - } }; From b82f504c37d13f5843c70af93b0897c75c0f3cff Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Tue, 8 Sep 2015 13:28:27 +0300 Subject: [PATCH 4/4] Merge --- dbms/include/DB/Common/HashTable/HashTable.h | 17 +++++++++------- dbms/include/DB/Common/HashTable/SmallTable.h | 20 +++++++++++-------- .../00212_aggregate_function_uniq.reference | 2 ++ .../00212_aggregate_function_uniq.sql | 4 ++++ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/dbms/include/DB/Common/HashTable/HashTable.h b/dbms/include/DB/Common/HashTable/HashTable.h index 02c157035a8..4d9edc52ad4 100644 --- a/dbms/include/DB/Common/HashTable/HashTable.h +++ b/dbms/include/DB/Common/HashTable/HashTable.h @@ -443,16 +443,18 @@ public: bool next() { + if (!is_initialized) + { + Cell::State::read(in); + DB::readVarUInt(size, in); + is_initialized = true; + } + if (read_count == size) { is_eof = true; return false; } - else if (read_count == 0) - { - Cell::State::read(in); - DB::readVarUInt(size, in); - } cell.read(in); ++read_count; @@ -462,18 +464,19 @@ public: inline const value_type & get() const { - if ((read_count == 0) || is_eof) + if (!is_initialized || is_eof) throw DB::Exception("No available data", DB::ErrorCodes::NO_AVAILABLE_DATA); return cell.getValue(); } private: - DB::ReadBuffer in; + DB::ReadBuffer & in; Cell cell; size_t read_count = 0; size_t size; bool is_eof = false; + bool is_initialized = false; }; class iterator diff --git a/dbms/include/DB/Common/HashTable/SmallTable.h b/dbms/include/DB/Common/HashTable/SmallTable.h index c68963a4798..3202a2a03f5 100644 --- a/dbms/include/DB/Common/HashTable/SmallTable.h +++ b/dbms/include/DB/Common/HashTable/SmallTable.h @@ -80,18 +80,21 @@ public: bool next() { - if (read_count == size) - { - is_eof = true; - return false; - } - else if (read_count == 0) + if (!is_initialized) { Cell::State::read(in); DB::readVarUInt(size, in); if (size > capacity) throw DB::Exception("Illegal size"); + + is_initialized = true; + } + + if (read_count == size) + { + is_eof = true; + return false; } cell.read(in); @@ -102,18 +105,19 @@ public: inline const value_type & get() const { - if ((read_count == 0) || is_eof) + if (!is_initialized || is_eof) throw DB::Exception("No available data", DB::ErrorCodes::NO_AVAILABLE_DATA); return cell.getValue(); } private: - DB::ReadBuffer in; + DB::ReadBuffer & in; Cell cell; size_t read_count = 0; size_t size; bool is_eof = false; + bool is_initialized = false; }; class iterator diff --git a/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.reference b/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.reference index b82ec5b6e3a..8c94b8a5d58 100644 --- a/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.reference +++ b/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.reference @@ -206,6 +206,7 @@ 31 53574 35 55022 36 53961 +1 1 1 3 1 6 1 @@ -414,3 +415,4 @@ 31 54074 35 54153 36 53999 +1 diff --git a/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.sql b/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.sql index 2886daeb3b3..b91ae03bf0a 100644 --- a/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.sql +++ b/dbms/tests/queries/0_stateless/00212_aggregate_function_uniq.sql @@ -16,6 +16,8 @@ SELECT Y, uniqHLL12(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS SELECT Y, uniqHLL12(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 3000) GROUP BY Y; SELECT Y, uniqHLL12(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y; +SELECT uniqHLL12(dummy) FROM remote('127.0.0.{1,2}', system.one); + /* uniqCombined */ SELECT Y, uniqCombined(X) FROM (SELECT number AS X, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 15) GROUP BY Y; @@ -33,3 +35,5 @@ SELECT Y, uniqCombined(X) FROM (SELECT number AS X, round(toFloat32(1/(1 + (3*X* SELECT Y, uniqCombined(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 15) GROUP BY Y; SELECT Y, uniqCombined(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 3000) GROUP BY Y; SELECT Y, uniqCombined(Z) FROM (SELECT number AS X, IPv4NumToString(toUInt32(X)) AS Z, (3*X*X - 7*X + 11) % 37 AS Y FROM system.numbers LIMIT 1000000) GROUP BY Y; + +SELECT uniqCombined(dummy) FROM remote('127.0.0.{1,2}', system.one);