From 6d71c367edfa0594e7404d16ea1fbc45dcc5b1b6 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 28 May 2015 17:08:31 +0300 Subject: [PATCH 1/8] WriteHelpers: add template specialization for const char * [#METR-15514] --- dbms/include/DB/IO/WriteHelpers.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/dbms/include/DB/IO/WriteHelpers.h b/dbms/include/DB/IO/WriteHelpers.h index 8e2b73dcd86..08e6e43da9a 100644 --- a/dbms/include/DB/IO/WriteHelpers.h +++ b/dbms/include/DB/IO/WriteHelpers.h @@ -252,13 +252,19 @@ void writeAnyEscapedString(const String & s, WriteBuffer & buf) } -inline void writeEscapedString(const String & s, WriteBuffer & buf) +inline void writeEscapedString(const char * str, size_t size, WriteBuffer & buf) { /// strpbrk в libc под Linux на процессорах с SSE 4.2 хорошо оптимизирована (этот if ускоряет код в 1.5 раза) - if (nullptr == strpbrk(s.data(), "\b\f\n\r\t\'\\") && strlen(s.data()) == s.size()) - writeString(s, buf); + if (nullptr == strpbrk(str, "\b\f\n\r\t\'\\") && strlen(str) == size) + writeString(str, size, buf); else - writeAnyEscapedString<'\''>(s, buf); + writeAnyEscapedString<'\''>(str, str + size, buf); +} + + +inline void writeEscapedString(const String & s, WriteBuffer & buf) +{ + writeEscapedString(s.data(), s.size(), buf); } @@ -470,6 +476,10 @@ inline void writeText(const Float32 & x, WriteBuffer & buf) { writeFloatText(x, inline void writeText(const Float64 & x, WriteBuffer & buf) { writeFloatText(x, buf); } inline void writeText(const String & x, WriteBuffer & buf) { writeEscapedString(x, buf); } inline void writeText(const bool & x, WriteBuffer & buf) { writeBoolText(x, buf); } +/// в отличие от метода для std::string +/// здесь предполагается, что x null-terminated строка. +inline void writeText(const char * x, WriteBuffer & buf) { writeEscapedString(x, strlen(x), buf); } +inline void writeText(const char * x, size_t size, WriteBuffer & buf) { writeEscapedString(x, size, buf); } inline void writeText(const VisitID_t & x, WriteBuffer & buf) { writeIntText(static_cast(x), buf); } inline void writeText(const mysqlxx::Date & x, WriteBuffer & buf) { writeDateText(x, buf); } From 4463a747dace754a1ffa3bae03f84166eb078dcb Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Wed, 3 Jun 2015 18:32:06 +0300 Subject: [PATCH 2/8] ReadHelpers: add assertChar function [#METR-15514] --- dbms/include/DB/IO/ReadHelpers.h | 1 + dbms/src/IO/ReadHelpers.cpp | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/dbms/include/DB/IO/ReadHelpers.h b/dbms/include/DB/IO/ReadHelpers.h index ef4a9eeed0b..12302a33436 100644 --- a/dbms/include/DB/IO/ReadHelpers.h +++ b/dbms/include/DB/IO/ReadHelpers.h @@ -105,6 +105,7 @@ void readVectorBinary(std::vector & v, ReadBuffer & buf, size_t MAX_VECTOR_SI void assertString(const char * s, ReadBuffer & buf); void assertEOF(ReadBuffer & buf); +void assertChar(char symbol, ReadBuffer & buf); inline void assertString(const String & s, ReadBuffer & buf) { diff --git a/dbms/src/IO/ReadHelpers.cpp b/dbms/src/IO/ReadHelpers.cpp index c392ba094c8..c16d547ca9f 100644 --- a/dbms/src/IO/ReadHelpers.cpp +++ b/dbms/src/IO/ReadHelpers.cpp @@ -36,6 +36,16 @@ void assertString(const char * s, ReadBuffer & buf) } } +void assertChar(char symbol, ReadBuffer & buf) +{ + if (buf.eof() || *buf.position() != symbol) + { + char err[2] = {symbol, '\0'}; + throwAtAssertionFailed(err, buf); + } + ++buf.position(); +} + void assertEOF(ReadBuffer & buf) { if (!buf.eof()) From 67944de74e3b5357adb37e779e078f28467b425d Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Tue, 9 Jun 2015 15:30:30 +0300 Subject: [PATCH 3/8] zookeeper: add createOrUpdate method [#METR-15514] --- libs/libzkutil/include/zkutil/ZooKeeper.h | 3 +++ libs/libzkutil/src/ZooKeeper.cpp | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/libs/libzkutil/include/zkutil/ZooKeeper.h b/libs/libzkutil/include/zkutil/ZooKeeper.h index 2eb6f8a12a0..a402470e8bf 100644 --- a/libs/libzkutil/include/zkutil/ZooKeeper.h +++ b/libs/libzkutil/include/zkutil/ZooKeeper.h @@ -126,6 +126,9 @@ public: void set(const std::string & path, const std::string & data, int32_t version = -1, Stat * stat = nullptr); + /** Создает ноду, если ее не существует. Иначе обновляет */ + void createOrUpdate(const std::string & path, const std::string & data, int32_t mode); + /** Не бросает исключение при следующих ошибках: * - Такой ноды нет. * - У ноды другая версия. diff --git a/libs/libzkutil/src/ZooKeeper.cpp b/libs/libzkutil/src/ZooKeeper.cpp index 506e4198e62..68fcd2269c1 100644 --- a/libs/libzkutil/src/ZooKeeper.cpp +++ b/libs/libzkutil/src/ZooKeeper.cpp @@ -418,6 +418,17 @@ void ZooKeeper::set(const std::string & path, const std::string & data, int32_t check(trySet(path, data, version, stat), path); } +void ZooKeeper::createOrUpdate(const std::string & path, const std::string & data, int32_t mode) +{ + int code = trySet(path, data, -1); + if (code == ZNONODE) + { + create(path, data, mode); + } + else + throw zkutil::KeeperException(code, path); +} + int32_t ZooKeeper::trySet(const std::string & path, const std::string & data, int32_t version, Stat * stat_) { From e201f39f7fd7523af6385414e12115b1fb8da402 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Wed, 10 Jun 2015 15:28:42 +0300 Subject: [PATCH 4/8] Zookeeper: style fix --- libs/libzkutil/src/ZooKeeper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/libzkutil/src/ZooKeeper.cpp b/libs/libzkutil/src/ZooKeeper.cpp index 68fcd2269c1..998e57e0a3a 100644 --- a/libs/libzkutil/src/ZooKeeper.cpp +++ b/libs/libzkutil/src/ZooKeeper.cpp @@ -390,7 +390,7 @@ bool ZooKeeper::tryGet(const std::string & path, std::string & res, Stat * stat_ { int32_t code = retry(std::bind(&ZooKeeper::getImpl, this, std::ref(path), std::ref(res), stat_, watch)); - if (!( code == ZOK || + if (!(code == ZOK || code == ZNONODE)) throw KeeperException(code, path); From dd05019fe2d9ac3b8ab9bf5086fb96b34a35ae53 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Wed, 10 Jun 2015 15:44:55 +0300 Subject: [PATCH 5/8] cloud: fix error [#METR-15514] --- libs/libzkutil/src/ZooKeeper.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/libzkutil/src/ZooKeeper.cpp b/libs/libzkutil/src/ZooKeeper.cpp index 998e57e0a3a..06ed54d65d4 100644 --- a/libs/libzkutil/src/ZooKeeper.cpp +++ b/libs/libzkutil/src/ZooKeeper.cpp @@ -425,7 +425,7 @@ void ZooKeeper::createOrUpdate(const std::string & path, const std::string & dat { create(path, data, mode); } - else + else if (code != ZOK) throw zkutil::KeeperException(code, path); } From 541f051cd146f1f5b80e8eeb7b1485290f14be3c Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Wed, 1 Jul 2015 11:55:31 +0300 Subject: [PATCH 6/8] zkutil: remove ctor [#METR-17045] --- libs/libzkutil/include/zkutil/ZooKeeper.h | 1 - libs/libzkutil/src/ZooKeeper.cpp | 7 ------- 2 files changed, 8 deletions(-) diff --git a/libs/libzkutil/include/zkutil/ZooKeeper.h b/libs/libzkutil/include/zkutil/ZooKeeper.h index a402470e8bf..d7c7c077d01 100644 --- a/libs/libzkutil/include/zkutil/ZooKeeper.h +++ b/libs/libzkutil/include/zkutil/ZooKeeper.h @@ -47,7 +47,6 @@ public: */ ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name); - ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std::string & config_name, int32_t session_timeout_ms); ~ZooKeeper(); diff --git a/libs/libzkutil/src/ZooKeeper.cpp b/libs/libzkutil/src/ZooKeeper.cpp index 06ed54d65d4..52b0a0eb545 100644 --- a/libs/libzkutil/src/ZooKeeper.cpp +++ b/libs/libzkutil/src/ZooKeeper.cpp @@ -128,13 +128,6 @@ ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration & config, const std init(args.hosts, args.session_timeout_ms); } -ZooKeeper::ZooKeeper(const Poco::Util::AbstractConfiguration& config, const std::string& config_name, int32_t session_timeout_ms_) -{ - ZooKeeperArgs args(config, config_name); - init(args.hosts, session_timeout_ms_); -} - - void * ZooKeeper::watchForEvent(EventPtr event) { if (event) From d71b7ca1b020004b5fa7074355a757529b2e332b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 1 Jul 2015 23:14:23 +0300 Subject: [PATCH 7/8] dbms: better diagnostics [#METR-12542]. --- dbms/src/Storages/StorageLog.cpp | 10 +++++++++- dbms/src/Storages/StorageTinyLog.cpp | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 8527fd32f54..f4bc9b587ae 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -233,7 +233,15 @@ Block LogBlockInputStream::readImpl() else column.column = column.type->createColumn(); - readData(*it, *column.type, *column.column, max_rows_to_read, 0, read_offsets); + try + { + readData(*it, *column.type, *column.column, max_rows_to_read, 0, read_offsets); + } + catch (Exception & e) + { + e.addMessage("while reading column " + *it + " at " + storage.path + escapeForFileName(storage.name)); + throw; + } if (column.column->size()) res.insert(column); diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index c8c357ddad5..acde326fb58 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -198,7 +198,15 @@ Block TinyLogBlockInputStream::readImpl() else column.column = column.type->createColumn(); - readData(*it, *column.type, *column.column, block_size, 0, read_offsets); + try + { + readData(*it, *column.type, *column.column, block_size, 0, read_offsets); + } + catch (Exception & e) + { + e.addMessage("while reading column " + *it + " at " + storage.full_path()); + throw; + } if (column.column->size()) res.insert(column); From efddabd212f0eaa9f0fb683c143d021f2a45f4ec Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 1 Jul 2015 23:42:42 +0300 Subject: [PATCH 8/8] Merge --- dbms/src/Storages/StorageChunkMerger.cpp | 15 ++++++++------- dbms/src/Storages/StorageChunks.cpp | 3 --- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/StorageChunkMerger.cpp b/dbms/src/Storages/StorageChunkMerger.cpp index 964d4af9a7e..e314f127b08 100644 --- a/dbms/src/Storages/StorageChunkMerger.cpp +++ b/dbms/src/Storages/StorageChunkMerger.cpp @@ -545,7 +545,8 @@ bool StorageChunkMerger::MergeTask::mergeChunks(const Storages & chunks) { LOG_INFO(log, "Shutdown requested while merging chunks."); output->writeSuffix(); - new_storage.removeReference(); /// После этого временные данные удалятся. + output = nullptr; + executeQuery("DROP TABLE IF EXISTS " + new_table_full_name, context, true); return false; } @@ -575,14 +576,16 @@ bool StorageChunkMerger::MergeTask::mergeChunks(const Storages & chunks) /// Отцепляем исходную таблицу. Ее данные и метаданные остаются на диске. tables_to_drop.push_back(context.detachTable(chunk_merger.source_database, src_name)); - /// Создаем на ее месте ChunkRef. Это возможно только потому что у ChunkRef нет ни, ни метаданных. + /// Создаем на ее месте ChunkRef. Это возможно только потому что у ChunkRef нет ни данных, ни метаданных. try { - context.addTable(chunk_merger.source_database, src_name, StorageChunkRef::create(src_name, context, chunk_merger.source_database, new_table_name, false)); + context.addTable(chunk_merger.source_database, src_name, + StorageChunkRef::create(src_name, context, chunk_merger.source_database, new_table_name, false)); } catch (...) { - LOG_ERROR(log, "Chunk " + src_name + " was removed but not replaced. Its data is stored in table " << new_table_name << ". You may need to resolve this manually."); + LOG_ERROR(log, "Chunk " + src_name + " was removed but not replaced. Its data is stored in table " + << new_table_name << ". You may need to resolve this manually."); throw; } @@ -601,9 +604,6 @@ bool StorageChunkMerger::MergeTask::mergeChunks(const Storages & chunks) /// что-нибудь может сломаться. } - /// Сейчас на new_storage ссылаются таблицы типа ChunkRef. Удалим лишнюю ссылку, которая была при создании. - new_storage.removeReference(); - LOG_TRACE(log, "Merged chunks."); return true; @@ -613,6 +613,7 @@ bool StorageChunkMerger::MergeTask::mergeChunks(const Storages & chunks) Poco::ScopedLock lock(context.getMutex()); currently_written_groups.erase(new_table_full_name); + executeQuery("DROP TABLE IF EXISTS " + new_table_full_name, context, true); throw; } diff --git a/dbms/src/Storages/StorageChunks.cpp b/dbms/src/Storages/StorageChunks.cpp index 68ec39e358d..aa55c1b43c0 100644 --- a/dbms/src/Storages/StorageChunks.cpp +++ b/dbms/src/Storages/StorageChunks.cpp @@ -161,9 +161,6 @@ StorageChunks::StorageChunks( context(context_), log(&Logger::get("StorageChunks")) { - if (!attach) - reference_counter.add(1, true); - _table_column_name = "_table" + VirtualColumnUtils::chooseSuffix(getColumnsList(), "_table"); try