diff --git a/dbms/include/DB/Core/Defines.h b/dbms/include/DB/Core/Defines.h index b1e1d961f37..f44dfa1f5c3 100644 --- a/dbms/include/DB/Core/Defines.h +++ b/dbms/include/DB/Core/Defines.h @@ -42,4 +42,4 @@ #define DBMS_MIN_REVISION_WITH_USER_PASSWORD 34482 #define DBMS_MIN_REVISION_WITH_TOTALS_EXTREMES 35265 #define DBMS_MIN_REVISION_WITH_STRING_QUERY_ID 39002 -#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES 50263 +#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES 50264 diff --git a/dbms/include/DB/Core/Exception.h b/dbms/include/DB/Core/Exception.h index b746b4f9c0f..047c156e491 100644 --- a/dbms/include/DB/Core/Exception.h +++ b/dbms/include/DB/Core/Exception.h @@ -3,40 +3,13 @@ #include #include -#include +#include #include -#include - namespace DB { -class Exception : public Poco::Exception -{ -public: - Exception(int code = 0); - Exception(const std::string & msg, int code = 0); - Exception(const std::string & msg, const std::string & arg, int code = 0); - Exception(const std::string & msg, const Exception & exc, int code = 0); - Exception(const Exception & exc); - explicit Exception(const Poco::Exception & exc); - ~Exception() throw(); - Exception & operator = (const Exception & exc); - const char * name() const throw(); - const char * className() const throw(); - Exception * clone() const; - void rethrow() const; - - /// Дописать к существующему сообщению что-нибудь ещё. - void addMessage(const std::string & arg); - - const StackTrace & getStackTrace() const { return trace; } - -private: - StackTrace trace; -}; - using Poco::SharedPtr; typedef SharedPtr ExceptionPtr; diff --git a/dbms/include/DB/Core/StackTrace.h b/dbms/include/DB/Core/StackTrace.h deleted file mode 100644 index 857f4ecd3f6..00000000000 --- a/dbms/include/DB/Core/StackTrace.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once - -#include -#include - -#define DBMS_STACK_TRACE_MAX_DEPTH 32 - - -namespace DB -{ - -/// Позволяет получить стек-трейс -class StackTrace -{ -public: - /// Стектрейс снимается в момент создания объекта - StackTrace(); - - /// Вывести в строку - std::string toString() const; - -private: - typedef void* Frame; - Frame frames[DBMS_STACK_TRACE_MAX_DEPTH]; - size_t frames_size; -}; - -} diff --git a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h index b63d78da613..e22656fde6b 100644 --- a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h @@ -5,6 +5,7 @@ namespace DB { +class ASTIdentifier; /** Позволяет добавить или удалить столбец в таблице. */ @@ -16,6 +17,8 @@ public: void execute(); private: + void dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns); + ASTPtr query_ptr; Context context; diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 616d7392697..c6e9dfb75cc 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -383,7 +383,7 @@ private: /// Загрузить множество кусков с данными с диска. Вызывается один раз - при создании объекта. void loadDataParts(); - void removeColumnFiles(String column_name); + void removeColumnFiles(String column_name, bool remove_array_size_files); /// Определить, не битые ли данные в директории. Проверяет индекс и засечеки, но не сами данные. bool isBrokenPart(const String & path); diff --git a/dbms/src/Core/Exception.cpp b/dbms/src/Core/Exception.cpp index 15aa3266242..1194d913d46 100644 --- a/dbms/src/Core/Exception.cpp +++ b/dbms/src/Core/Exception.cpp @@ -11,47 +11,6 @@ namespace DB { -Exception::Exception(int code): Poco::Exception(code) {} -Exception::Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {} -Exception::Exception(const std::string & msg, const std::string & arg, int code): Poco::Exception(msg, arg, code) {} -Exception::Exception(const std::string & msg, const Exception & exc, int code): Poco::Exception(msg, exc, code), trace(exc.trace) {} -Exception::Exception(const Exception & exc) : Poco::Exception(exc), trace(exc.trace) {} -Exception::Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {} -Exception::~Exception() throw() {} - -Exception & Exception::operator=(const Exception& exc) -{ - Poco::Exception::operator=(exc); - trace = exc.trace; - return *this; -} - -const char* Exception::name() const throw() -{ - return "DB::Exception"; -} - -const char* Exception::className() const throw() -{ - return "DB::Exception"; -} - -Exception * Exception::clone() const -{ - return new Exception(*this); -} - -void Exception::rethrow() const -{ - throw *this; -} - -void Exception::addMessage(const std::string & arg) -{ - extendedMessage(arg); -} - - void throwFromErrno(const std::string & s, int code, int e) { char buf[128]; diff --git a/dbms/src/Core/StackTrace.cpp b/dbms/src/Core/StackTrace.cpp deleted file mode 100644 index 5b5be938aa8..00000000000 --- a/dbms/src/Core/StackTrace.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include -#include -#include -#include - -#include - -#include - - -#define DBMS_STACK_TRACE_MAX_DEPTH 32 - - -namespace DB -{ - -StackTrace::StackTrace() -{ - frames_size = backtrace(frames, DBMS_STACK_TRACE_MAX_DEPTH); -} - -std::string StackTrace::toString() const -{ - char ** symbols = backtrace_symbols(frames, frames_size); - std::stringstream res; - - if (!symbols) - return "Cannot get symbols for stack trace.\n"; - - try - { - for (size_t i = 0, size = frames_size; i < size; ++i) - { - /// Делаем demangling имён. Имя находится в скобках, до символа '+'. - - char * name_start = NULL; - char * name_end = NULL; - char * demangled_name = NULL; - int status = 0; - - if (NULL != (name_start = strchr(symbols[i], '(')) - && NULL != (name_end = strchr(name_start, '+'))) - { - ++name_start; - *name_end = '\0'; - demangled_name = abi::__cxa_demangle(name_start, 0, 0, &status); - *name_end = '+'; - } - - try - { - res << i << ". "; - - if (NULL != demangled_name && 0 == status) - { - res.write(symbols[i], name_start - symbols[i]); - res << demangled_name << name_end; - } - else - res << symbols[i]; - - res << std::endl; - } - catch (...) - { - free(demangled_name); - throw; - } - free(demangled_name); - } - } - catch (...) - { - free(symbols); - throw; - } - - free(symbols); - return res.str(); -} - -} diff --git a/dbms/src/Core/tests/stack_trace.cpp b/dbms/src/Core/tests/stack_trace.cpp deleted file mode 100644 index 061fe0fecb4..00000000000 --- a/dbms/src/Core/tests/stack_trace.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include - -#include - - -int main(int argc, char ** argv) -{ - DB::StackTrace trace; - std::cerr << trace.toString(); - - return 0; -} diff --git a/dbms/src/IO/tests/o_direct_and_dirty_pages.cpp b/dbms/src/IO/tests/o_direct_and_dirty_pages.cpp index f372178f797..9ad09ee48a2 100644 --- a/dbms/src/IO/tests/o_direct_and_dirty_pages.cpp +++ b/dbms/src/IO/tests/o_direct_and_dirty_pages.cpp @@ -15,24 +15,58 @@ int main(int argc, char ** argv) try { + const size_t N = 100000; + const size_t BUF_SIZE = 1048576; + ReadBufferFromFile rand_in("/dev/urandom"); unsigned rand = 0; readBinary(rand, rand_in); String test = "Hello, world! " + toString(rand); + /// Пишем в файл как обычно, читаем с O_DIRECT. + { - WriteBufferFromFile wb("test", 4096); - writeStringBinary(test, wb); + WriteBufferFromFile wb("test1", BUF_SIZE); + for (size_t i = 0; i < N; ++i) + writeStringBinary(test, wb); wb.next(); } { - ReadBufferFromFile rb("test", 4096, O_RDONLY | O_DIRECT, nullptr, 4096); + ReadBufferFromFile rb("test1", BUF_SIZE, O_RDONLY | O_DIRECT, nullptr, 4096); String res; - readStringBinary(res, rb); + for (size_t i = 0; i < N; ++i) + readStringBinary(res, rb); - std::cerr << "test: " << test << ", res: " << res << std::endl; + std::cerr << "test: " << test << ", res: " << res << ", bytes: " << rb.count() << std::endl; + } + + /// Пишем в файл с O_DIRECT, читаем как обычно. + + { + WriteBufferFromFile wb("test2", BUF_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, 4096); + + for (size_t i = 0; i < N; ++i) + writeStringBinary(test, wb); + + if (wb.offset() % 4096 != 0) + { + size_t pad = 4096 - wb.offset() % 4096; + memset(wb.position(), 0, pad); + wb.position() += pad; + } + + wb.next(); + } + + { + ReadBufferFromFile rb("test2", BUF_SIZE); + String res; + for (size_t i = 0; i < N; ++i) + readStringBinary(res, rb); + + std::cerr << "test: " << test << ", res: " << res << ", bytes: " << rb.count() << std::endl; } } catch (const Exception & e) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index ae4808d338b..05ebe7cac5b 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -39,6 +39,65 @@ static bool namesEqualIgnoreAfterDot(const String & name_without_dot, const ASTP return (name_without_dot == name_type.name || name_with_dot == name_type.name.substr(0, name_with_dot.length())); } +void InterpreterAlterQuery::dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns) +{ + Exception e("Wrong column name. Cannot find column " + drop_column.name + " to drop", DB::ErrorCodes::ILLEGAL_COLUMN); + ASTs::iterator drop_it; + + size_t dot_pos = drop_column.name.find('.'); + /// случай удаления nested столбца + if (dot_pos != std::string::npos) + { + /// в Distributed таблицах столбцы имеют название "nested.column" + drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1)); + if (drop_it != columns.end()) + columns.erase(drop_it); + else + { + try + { + /// в MergeTree таблицах есть ASTFunction "nested" + /// в аргументах которой записаны столбцы + ASTs::iterator nested_it; + std::string nested_base_name = drop_column.name.substr(0, dot_pos); + nested_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, nested_base_name, _1)); + if (nested_it == columns.end()) + throw e; + + if ((**nested_it).children.size() != 1) + throw e; + + ASTFunction & f = dynamic_cast(*(**nested_it).children.back()); + if (f.name != "Nested") + throw e; + + ASTs & nested_columns = dynamic_cast(*f.arguments).children; + + drop_it = std::find_if(nested_columns.begin(), nested_columns.end(), boost::bind(namesEqual, drop_column.name.substr(dot_pos + 1), _1)); + if (drop_it == nested_columns.end()) + throw e; + else + nested_columns.erase(drop_it); + + if (nested_columns.empty()) + columns.erase(nested_it); + } + catch (std::bad_cast & bad_cast_err) + { + throw e; + } + } + } + else + { + drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1)); + if (drop_it == columns.end()) + throw e; + else + columns.erase(drop_it); + } +} + void InterpreterAlterQuery::execute() { ASTAlterQuery & alter = dynamic_cast(*query_ptr); @@ -62,7 +121,10 @@ void InterpreterAlterQuery::execute() ASTs & columns = dynamic_cast(*attach.columns).children; /// Различные проверки, на возможность выполнения запроса - ASTs columns_copy = columns; + ASTs columns_copy; + for (const auto & ast : columns) + columns_copy.push_back(ast->clone()); + IdentifierNameSet identifier_names; attach.storage->collectIdentifierNames(identifier_names); for (ASTAlterQuery::ParameterContainer::const_iterator alter_it = alter.parameters.begin(); @@ -101,11 +163,7 @@ void InterpreterAlterQuery::execute() if (identifier_names.find(drop_column.name) != identifier_names.end()) throw Exception("Cannot drop key column", DB::ErrorCodes::ILLEGAL_COLUMN); - ASTs::iterator drop_it = std::find_if(columns_copy.begin(), columns_copy.end(), boost::bind(namesEqual, drop_column.name, _1)); - if (drop_it == columns_copy.end()) - throw Exception("Wrong column name. Cannot find column " + drop_column.name +" to drop", DB::ErrorCodes::ILLEGAL_COLUMN); - else - columns_copy.erase(drop_it); + dropColumnFromAST(drop_column, columns_copy); } else if (params.type == ASTAlterQuery::MODIFY) { @@ -172,8 +230,7 @@ void InterpreterAlterQuery::execute() else if (params.type == ASTAlterQuery::DROP) { const ASTIdentifier & drop_column = dynamic_cast (*params.column); - ASTs::iterator drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1)); - columns.erase(drop_it); + dropColumnFromAST(drop_column, columns); } else if (params.type == ASTAlterQuery::MODIFY) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 9c8fc38fc44..f813419b1fd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -334,11 +334,21 @@ void MergeTreeData::dropAllData() Poco::File(full_path).remove(true); } -void MergeTreeData::removeColumnFiles(String column_name) +void MergeTreeData::removeColumnFiles(String column_name, bool remove_array_size_files) { Poco::ScopedLock lock(data_parts_mutex); Poco::ScopedLock lock_all(all_data_parts_mutex); + size_t dot_pos = column_name.find('.'); + if (dot_pos != std::string::npos) + { + std::string nested_column = column_name.substr(0, dot_pos); + column_name = nested_column + "%2E" + column_name.substr(dot_pos + 1); + + if (remove_array_size_files) + column_name = std::string("(?:") + nested_column + "|" + column_name + ")"; + } + /// Регэксп выбирает файлы столбца для удаления Poco::RegularExpression re(column_name + "(?:(?:\\.|\\%2E).+){0,1}" +"(?:\\.mrk|\\.bin|\\.size\\d+\\.bin|\\.size\\d+\\.mrk)"); /// Цикл по всем директориям кусочков @@ -388,6 +398,12 @@ static DataTypePtr getDataTypeByName(const String & name, const NamesAndTypesLis throw Exception("No column " + name + " in table", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE); } +/// одинаковыми считаются имена, вида "name.*" +static bool namesWithDotEqual(const String & name_with_dot, const DB::NameAndTypePair & name_type) +{ + return (name_with_dot == name_type.first.substr(0, name_with_dot.length())); +} + void MergeTreeData::alter(const ASTAlterQuery::Parameters & params) { { @@ -398,7 +414,15 @@ void MergeTreeData::alter(const ASTAlterQuery::Parameters & params) if (params.type == ASTAlterQuery::DROP) { String column_name = dynamic_cast(*params.column).name; - removeColumnFiles(column_name); + + /// Если нет колонок вида nested_name.*, то удалим столбцы размера массивов + bool remove_array_size_files = false; + size_t dot_pos = column_name.find('.'); + if (dot_pos != std::string::npos) + { + remove_array_size_files = (columns->end() == std::find_if(columns->begin(), columns->end(), boost::bind(namesWithDotEqual, column_name.substr(0, dot_pos), _1))); + } + removeColumnFiles(column_name, remove_array_size_files); context.getUncompressedCache()->reset(); context.getMarkCache()->reset(); diff --git a/dbms/tests/queries/0_stateless/00030_1_alter_table.reference b/dbms/tests/queries/0_stateless/00030_1_alter_table.reference new file mode 100644 index 00000000000..9fda58741f4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00030_1_alter_table.reference @@ -0,0 +1,10 @@ +CounterID UInt32 +StartDate Date +UserID UInt32 +VisitID UInt32 +NestedColumn.A Array(UInt8) +NestedColumn.S Array(String) +ToDrop UInt32 +Added0 UInt32 +Added1 UInt32 +Added2 UInt32 diff --git a/dbms/tests/queries/0_stateless/00030_1_alter_table.sql b/dbms/tests/queries/0_stateless/00030_1_alter_table.sql new file mode 100644 index 00000000000..77d25d15cb4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00030_1_alter_table.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS alter_test; + +CREATE TABLE alter_test (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192); + +INSERT INTO alter_test VALUES (1, '2014-01-01', 2, 3, [1,2,3], ['a','b','c'], 4); + +ALTER TABLE alter_test ADD COLUMN Added0 UInt32; +ALTER TABLE alter_test ADD COLUMN Added2 UInt32; +ALTER TABLE alter_test ADD COLUMN Added1 UInt32 AFTER Added0; + +DESC TABLE alter_test; diff --git a/dbms/tests/queries/0_stateless/00030_2_alter_table.reference b/dbms/tests/queries/0_stateless/00030_2_alter_table.reference new file mode 100644 index 00000000000..8c47f345849 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00030_2_alter_table.reference @@ -0,0 +1,7 @@ +CounterID UInt32 +StartDate Date +UserID UInt32 +VisitID UInt32 +Added0 String +Added1 UInt32 +Added2 UInt32 diff --git a/dbms/tests/queries/0_stateless/00030_2_alter_table.sql b/dbms/tests/queries/0_stateless/00030_2_alter_table.sql new file mode 100644 index 00000000000..d6fd3cf4a4a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00030_2_alter_table.sql @@ -0,0 +1,8 @@ +ALTER TABLE alter_test DROP COLUMN ToDrop; + +ALTER TABLE alter_test MODIFY COLUMN Added0 String; + +ALTER TABLE alter_test DROP COLUMN `NestedColumn.A`; +ALTER TABLE alter_test DROP COLUMN `NestedColumn.S`; + +DESC TABLE alter_test; diff --git a/dbms/tests/queries/0_stateless/00030_3_alter_table.reference b/dbms/tests/queries/0_stateless/00030_3_alter_table.reference new file mode 100644 index 00000000000..123652833e9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00030_3_alter_table.reference @@ -0,0 +1 @@ +1 2014-01-01 2 3 0 0 diff --git a/dbms/tests/queries/0_stateless/00030_3_alter_table.sql b/dbms/tests/queries/0_stateless/00030_3_alter_table.sql new file mode 100644 index 00000000000..d7ecb07f159 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00030_3_alter_table.sql @@ -0,0 +1,3 @@ +SELECT * FROM alter_test; + +DROP TABLE alter_test; diff --git a/libs/libzkutil/include/zkutil/KeeperException.h b/libs/libzkutil/include/zkutil/KeeperException.h index f4ec86fe958..ff5d96f6278 100644 --- a/libs/libzkutil/include/zkutil/KeeperException.h +++ b/libs/libzkutil/include/zkutil/KeeperException.h @@ -1,19 +1,19 @@ #pragma once -#include +#include #include namespace zkutil { -class KeeperException : public Poco::Exception +class KeeperException : public DB::Exception { public: - KeeperException(const std::string & msg) : Poco::Exception(msg), code(ReturnCode::Ok) {} + KeeperException(const std::string & msg) : DB::Exception(msg), code(ReturnCode::Ok) {} KeeperException(const std::string & msg, ReturnCode::type code_) - : Poco::Exception(msg + " (" + ReturnCode::toString(code_) + ")"), code(code_) {} + : DB::Exception(msg + " (" + ReturnCode::toString(code_) + ")"), code(code_) {} KeeperException(ReturnCode::type code_) - : Poco::Exception(ReturnCode::toString(code_)), code(code_) {} + : DB::Exception(ReturnCode::toString(code_)), code(code_) {} ReturnCode::type code; };