thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-25 17:12:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge

Browse Source
This commit is contained in:
Alexey Milovidov 2014-04-03 20:23:31 +04:00
commit 1246d54830
18 changed files with 181 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dbms/include/DB/Core/Defines.h
View File

@ -42,4 +42,4 @@
#define DBMS_MIN_REVISION_WITH_USER_PASSWORD 34482
#define DBMS_MIN_REVISION_WITH_TOTALS_EXTREMES 35265
#define DBMS_MIN_REVISION_WITH_STRING_QUERY_ID 39002
#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES 50263
#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES 50264

29
dbms/include/DB/Core/Exception.h
View File

@ -3,40 +3,13 @@
#include <cerrno>
#include <vector>
#include <Poco/Exception.h>
#include <statdaemons/Exception.h>
#include <Poco/SharedPtr.h>
#include <DB/Core/StackTrace.h>
namespace DB
{
class Exception : public Poco::Exception
{
public:
Exception(int code = 0);
Exception(const std::string & msg, int code = 0);
Exception(const std::string & msg, const std::string & arg, int code = 0);
Exception(const std::string & msg, const Exception & exc, int code = 0);
Exception(const Exception & exc);
explicit Exception(const Poco::Exception & exc);
~Exception() throw();
Exception & operator = (const Exception & exc);
const char * name() const throw();
const char * className() const throw();
Exception * clone() const;
void rethrow() const;
/// Дописать к существующему сообщению что-нибудь ещё.
void addMessage(const std::string & arg);
const StackTrace & getStackTrace() const { return trace; }
private:
StackTrace trace;
};
using Poco::SharedPtr;
typedef SharedPtr<Poco::Exception> ExceptionPtr;

28
dbms/include/DB/Core/StackTrace.h
View File

@ -1,28 +0,0 @@
#pragma once
#include <string>
#include <vector>
#define DBMS_STACK_TRACE_MAX_DEPTH 32
namespace DB
{
/// Позволяет получить стек-трейс
class StackTrace
{
public:
/// Стектрейс снимается в момент создания объекта
StackTrace();
/// Вывести в строку
std::string toString() const;
private:
typedef void* Frame;
Frame frames[DBMS_STACK_TRACE_MAX_DEPTH];
size_t frames_size;
};
}

3
dbms/include/DB/Interpreters/InterpreterAlterQuery.h
View File

@ -5,6 +5,7 @@
namespace DB
{
class ASTIdentifier;
/** Позволяет добавить или удалить столбец в таблице.
*/
@ -16,6 +17,8 @@ public:
void execute();
private:
void dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns);
ASTPtr query_ptr;
Context context;

2
dbms/include/DB/Storages/MergeTree/MergeTreeData.h
View File

@ -383,7 +383,7 @@ private:
/// Загрузить множество кусков с данными с диска. Вызывается один раз - при создании объекта.
void loadDataParts();
void removeColumnFiles(String column_name);
void removeColumnFiles(String column_name, bool remove_array_size_files);
/// Определить, не битые ли данные в директории. Проверяет индекс и засечеки, но не сами данные.
bool isBrokenPart(const String & path);

41
dbms/src/Core/Exception.cpp
View File

@ -11,47 +11,6 @@
namespace DB
{
Exception::Exception(int code): Poco::Exception(code) {}
Exception::Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
Exception::Exception(const std::string & msg, const std::string & arg, int code): Poco::Exception(msg, arg, code) {}
Exception::Exception(const std::string & msg, const Exception & exc, int code): Poco::Exception(msg, exc, code), trace(exc.trace) {}
Exception::Exception(const Exception & exc) : Poco::Exception(exc), trace(exc.trace) {}
Exception::Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
Exception::~Exception() throw() {}
Exception & Exception::operator=(const Exception& exc)
{
Poco::Exception::operator=(exc);
trace = exc.trace;
return *this;
}
const char* Exception::name() const throw()
{
return "DB::Exception";
}
const char* Exception::className() const throw()
{
return "DB::Exception";
}
Exception * Exception::clone() const
{
return new Exception(*this);
}
void Exception::rethrow() const
{
throw *this;
}
void Exception::addMessage(const std::string & arg)
{
extendedMessage(arg);
}
void throwFromErrno(const std::string & s, int code, int e)
{
char buf[128];

82
dbms/src/Core/StackTrace.cpp
View File

@ -1,82 +0,0 @@
#include <malloc.h>
#include <execinfo.h>
#include <cxxabi.h>
#include <string.h>
#include <sstream>
#include <DB/Core/StackTrace.h>
#define DBMS_STACK_TRACE_MAX_DEPTH 32
namespace DB
{
StackTrace::StackTrace()
{
frames_size = backtrace(frames, DBMS_STACK_TRACE_MAX_DEPTH);
}
std::string StackTrace::toString() const
{
char ** symbols = backtrace_symbols(frames, frames_size);
std::stringstream res;
if (!symbols)
return "Cannot get symbols for stack trace.\n";
try
{
for (size_t i = 0, size = frames_size; i < size; ++i)
{
/// Делаем demangling имён. Имя находится в скобках, до символа '+'.
char * name_start = NULL;
char * name_end = NULL;
char * demangled_name = NULL;
int status = 0;
if (NULL != (name_start = strchr(symbols[i], '('))
&& NULL != (name_end = strchr(name_start, '+')))
{
++name_start;
*name_end = '\0';
demangled_name = abi::__cxa_demangle(name_start, 0, 0, &status);
*name_end = '+';
}
try
{
res << i << ". ";
if (NULL != demangled_name && 0 == status)
{
res.write(symbols[i], name_start - symbols[i]);
res << demangled_name << name_end;
}
else
res << symbols[i];
res << std::endl;
}
catch (...)
{
free(demangled_name);
throw;
}
free(demangled_name);
}
}
catch (...)
{
free(symbols);
throw;
}
free(symbols);
return res.str();
}
}

12
dbms/src/Core/tests/stack_trace.cpp
View File

@ -1,12 +0,0 @@
#include <iostream>
#include <DB/Core/StackTrace.h>
int main(int argc, char ** argv)
{
DB::StackTrace trace;
std::cerr << trace.toString();
return 0;
}

44
dbms/src/IO/tests/o_direct_and_dirty_pages.cpp
View File

@ -15,24 +15,58 @@ int main(int argc, char ** argv)
try
{
const size_t N = 100000;
const size_t BUF_SIZE = 1048576;
ReadBufferFromFile rand_in("/dev/urandom");
unsigned rand = 0;
readBinary(rand, rand_in);
String test = "Hello, world! " + toString(rand);
/// Пишем в файл как обычно, читаем с O_DIRECT.
{
WriteBufferFromFile wb("test", 4096);
writeStringBinary(test, wb);
WriteBufferFromFile wb("test1", BUF_SIZE);
for (size_t i = 0; i < N; ++i)
writeStringBinary(test, wb);
wb.next();
}
{
ReadBufferFromFile rb("test", 4096, O_RDONLY | O_DIRECT, nullptr, 4096);
ReadBufferFromFile rb("test1", BUF_SIZE, O_RDONLY | O_DIRECT, nullptr, 4096);
String res;
readStringBinary(res, rb);
for (size_t i = 0; i < N; ++i)
readStringBinary(res, rb);
std::cerr << "test: " << test << ", res: " << res << std::endl;
std::cerr << "test: " << test << ", res: " << res << ", bytes: " << rb.count() << std::endl;
}
/// Пишем в файл с O_DIRECT, читаем как обычно.
{
WriteBufferFromFile wb("test2", BUF_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, 4096);
for (size_t i = 0; i < N; ++i)
writeStringBinary(test, wb);
if (wb.offset() % 4096 != 0)
{
size_t pad = 4096 - wb.offset() % 4096;
memset(wb.position(), 0, pad);
wb.position() += pad;
}
wb.next();
}
{
ReadBufferFromFile rb("test2", BUF_SIZE);
String res;
for (size_t i = 0; i < N; ++i)
readStringBinary(res, rb);
std::cerr << "test: " << test << ", res: " << res << ", bytes: " << rb.count() << std::endl;
}
}
catch (const Exception & e)

73
dbms/src/Interpreters/InterpreterAlterQuery.cpp
View File

@ -39,6 +39,65 @@ static bool namesEqualIgnoreAfterDot(const String & name_without_dot, const ASTP
return (name_without_dot == name_type.name || name_with_dot == name_type.name.substr(0, name_with_dot.length()));
}
void InterpreterAlterQuery::dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns)
{
Exception e("Wrong column name. Cannot find column " + drop_column.name + " to drop", DB::ErrorCodes::ILLEGAL_COLUMN);
ASTs::iterator drop_it;
size_t dot_pos = drop_column.name.find('.');
/// случай удаления nested столбца
if (dot_pos != std::string::npos)
{
/// в Distributed таблицах столбцы имеют название "nested.column"
drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
if (drop_it != columns.end())
columns.erase(drop_it);
else
{
try
{
/// в MergeTree таблицах есть ASTFunction "nested"
/// в аргументах которой записаны столбцы
ASTs::iterator nested_it;
std::string nested_base_name = drop_column.name.substr(0, dot_pos);
nested_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, nested_base_name, _1));
if (nested_it == columns.end())
throw e;
if ((**nested_it).children.size() != 1)
throw e;
ASTFunction & f = dynamic_cast<ASTFunction &>(*(**nested_it).children.back());
if (f.name != "Nested")
throw e;
ASTs & nested_columns = dynamic_cast<ASTExpressionList &>(*f.arguments).children;
drop_it = std::find_if(nested_columns.begin(), nested_columns.end(), boost::bind(namesEqual, drop_column.name.substr(dot_pos + 1), _1));
if (drop_it == nested_columns.end())
throw e;
else
nested_columns.erase(drop_it);
if (nested_columns.empty())
columns.erase(nested_it);
}
catch (std::bad_cast & bad_cast_err)
{
throw e;
}
}
}
else
{
drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
if (drop_it == columns.end())
throw e;
else
columns.erase(drop_it);
}
}
void InterpreterAlterQuery::execute()
{
ASTAlterQuery & alter = dynamic_cast<ASTAlterQuery &>(*query_ptr);
@ -62,7 +121,10 @@ void InterpreterAlterQuery::execute()
ASTs & columns = dynamic_cast<ASTExpressionList &>(*attach.columns).children;
/// Различные проверки, на возможность выполнения запроса
ASTs columns_copy = columns;
ASTs columns_copy;
for (const auto & ast : columns)
columns_copy.push_back(ast->clone());
IdentifierNameSet identifier_names;
attach.storage->collectIdentifierNames(identifier_names);
for (ASTAlterQuery::ParameterContainer::const_iterator alter_it = alter.parameters.begin();
@ -101,11 +163,7 @@ void InterpreterAlterQuery::execute()
if (identifier_names.find(drop_column.name) != identifier_names.end())
throw Exception("Cannot drop key column", DB::ErrorCodes::ILLEGAL_COLUMN);
ASTs::iterator drop_it = std::find_if(columns_copy.begin(), columns_copy.end(), boost::bind(namesEqual, drop_column.name, _1));
if (drop_it == columns_copy.end())
throw Exception("Wrong column name. Cannot find column " + drop_column.name +" to drop", DB::ErrorCodes::ILLEGAL_COLUMN);
else
columns_copy.erase(drop_it);
dropColumnFromAST(drop_column, columns_copy);
}
else if (params.type == ASTAlterQuery::MODIFY)
{
@ -172,8 +230,7 @@ void InterpreterAlterQuery::execute()
else if (params.type == ASTAlterQuery::DROP)
{
const ASTIdentifier & drop_column = dynamic_cast <const ASTIdentifier &>(*params.column);
ASTs::iterator drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
columns.erase(drop_it);
dropColumnFromAST(drop_column, columns);
}
else if (params.type == ASTAlterQuery::MODIFY)
{

28
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -334,11 +334,21 @@ void MergeTreeData::dropAllData()
Poco::File(full_path).remove(true);
}
void MergeTreeData::removeColumnFiles(String column_name)
void MergeTreeData::removeColumnFiles(String column_name, bool remove_array_size_files)
{
Poco::ScopedLock<Poco::FastMutex> lock(data_parts_mutex);
Poco::ScopedLock<Poco::FastMutex> lock_all(all_data_parts_mutex);
size_t dot_pos = column_name.find('.');
if (dot_pos != std::string::npos)
{
std::string nested_column = column_name.substr(0, dot_pos);
column_name = nested_column + "%2E" + column_name.substr(dot_pos + 1);
if (remove_array_size_files)
column_name = std::string("(?:") + nested_column + "|" + column_name + ")";
}
/// Регэксп выбирает файлы столбца для удаления
Poco::RegularExpression re(column_name + "(?:(?:\\.|\\%2E).+){0,1}" +"(?:\\.mrk|\\.bin|\\.size\\d+\\.bin|\\.size\\d+\\.mrk)");
/// Цикл по всем директориям кусочков
@ -388,6 +398,12 @@ static DataTypePtr getDataTypeByName(const String & name, const NamesAndTypesLis
throw Exception("No column " + name + " in table", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
}
/// одинаковыми считаются имена, вида "name.*"
static bool namesWithDotEqual(const String & name_with_dot, const DB::NameAndTypePair & name_type)
{
return (name_with_dot == name_type.first.substr(0, name_with_dot.length()));
}
void MergeTreeData::alter(const ASTAlterQuery::Parameters & params)
{
{
@ -398,7 +414,15 @@ void MergeTreeData::alter(const ASTAlterQuery::Parameters & params)
if (params.type == ASTAlterQuery::DROP)
{
String column_name = dynamic_cast<const ASTIdentifier &>(*params.column).name;
removeColumnFiles(column_name);
/// Если нет колонок вида nested_name.*, то удалим столбцы размера массивов
bool remove_array_size_files = false;
size_t dot_pos = column_name.find('.');
if (dot_pos != std::string::npos)
{
remove_array_size_files = (columns->end() == std::find_if(columns->begin(), columns->end(), boost::bind(namesWithDotEqual, column_name.substr(0, dot_pos), _1)));
}
removeColumnFiles(column_name, remove_array_size_files);
context.getUncompressedCache()->reset();
context.getMarkCache()->reset();

10
dbms/tests/queries/0_stateless/00030_1_alter_table.reference Normal file
View File

@ -0,0 +1,10 @@
CounterID UInt32
StartDate Date
UserID UInt32
VisitID UInt32
NestedColumn.A Array(UInt8)
NestedColumn.S Array(String)
ToDrop UInt32
Added0 UInt32
Added1 UInt32
Added2 UInt32

11
dbms/tests/queries/0_stateless/00030_1_alter_table.sql Normal file
View File

@ -0,0 +1,11 @@
DROP TABLE IF EXISTS alter_test;
CREATE TABLE alter_test (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);
INSERT INTO alter_test VALUES (1, '2014-01-01', 2, 3, [1,2,3], ['a','b','c'], 4);
ALTER TABLE alter_test ADD COLUMN Added0 UInt32;
ALTER TABLE alter_test ADD COLUMN Added2 UInt32;
ALTER TABLE alter_test ADD COLUMN Added1 UInt32 AFTER Added0;
DESC TABLE alter_test;

7
dbms/tests/queries/0_stateless/00030_2_alter_table.reference Normal file
View File

@ -0,0 +1,7 @@
CounterID UInt32
StartDate Date
UserID UInt32
VisitID UInt32
Added0 String
Added1 UInt32
Added2 UInt32

8
dbms/tests/queries/0_stateless/00030_2_alter_table.sql Normal file
View File

@ -0,0 +1,8 @@
ALTER TABLE alter_test DROP COLUMN ToDrop;
ALTER TABLE alter_test MODIFY COLUMN Added0 String;
ALTER TABLE alter_test DROP COLUMN `NestedColumn.A`;
ALTER TABLE alter_test DROP COLUMN `NestedColumn.S`;
DESC TABLE alter_test;

1
dbms/tests/queries/0_stateless/00030_3_alter_table.reference Normal file
View File

@ -0,0 +1 @@
1 2014-01-01 2 3 0 0

3
dbms/tests/queries/0_stateless/00030_3_alter_table.sql Normal file
View File

@ -0,0 +1,3 @@
SELECT * FROM alter_test;
DROP TABLE alter_test;

10
libs/libzkutil/include/zkutil/KeeperException.h
View File

@ -1,19 +1,19 @@
#pragma once
#include <Poco/Exception.h>
#include <statdaemons/Exception.h>
#include <zkutil/Types.h>
namespace zkutil
{
class KeeperException : public Poco::Exception
class KeeperException : public DB::Exception
{
public:
KeeperException(const std::string & msg) : Poco::Exception(msg), code(ReturnCode::Ok) {}
KeeperException(const std::string & msg) : DB::Exception(msg), code(ReturnCode::Ok) {}
KeeperException(const std::string & msg, ReturnCode::type code_)
: Poco::Exception(msg + " (" + ReturnCode::toString(code_) + ")"), code(code_) {}
: DB::Exception(msg + " (" + ReturnCode::toString(code_) + ")"), code(code_) {}
KeeperException(ReturnCode::type code_)
: Poco::Exception(ReturnCode::toString(code_)), code(code_) {}
: DB::Exception(ReturnCode::toString(code_)), code(code_) {}
ReturnCode::type code;
};