Merge

dbms/include/DB/Core/Defines.h

@@ -42,4 +42,4 @@
 #define DBMS_MIN_REVISION_WITH_USER_PASSWORD					34482
 #define DBMS_MIN_REVISION_WITH_TOTALS_EXTREMES					35265
 #define DBMS_MIN_REVISION_WITH_STRING_QUERY_ID					39002
-#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES					50263
+#define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES					50264

dbms/include/DB/Core/Exception.h

@@ -3,40 +3,13 @@
 #include <cerrno>
 #include <vector>
 
-#include <Poco/Exception.h>
+#include <statdaemons/Exception.h>
 #include <Poco/SharedPtr.h>
 
-#include <DB/Core/StackTrace.h>
-
 
 namespace DB
 {
 
-class Exception : public Poco::Exception
-{
-public:
-	Exception(int code = 0);
-	Exception(const std::string & msg, int code = 0);
-	Exception(const std::string & msg, const std::string & arg, int code = 0);
-	Exception(const std::string & msg, const Exception & exc, int code = 0);
-	Exception(const Exception & exc);
-	explicit Exception(const Poco::Exception & exc);
-	~Exception() throw();
-	Exception & operator = (const Exception & exc);
-	const char * name() const throw();
-	const char * className() const throw();
-	Exception * clone() const;
-	void rethrow() const;
-
-	/// Дописать к существующему сообщению что-нибудь ещё.
-	void addMessage(const std::string & arg);
-
-	const StackTrace & getStackTrace() const { return trace; }
-
-private:
-	StackTrace trace;
-};
-
 using Poco::SharedPtr;
 
 typedef SharedPtr<Poco::Exception> ExceptionPtr;

dbms/include/DB/Core/StackTrace.h

@@ -1,28 +0,0 @@
-#pragma once
-
-#include <string>
-#include <vector>
-
-#define DBMS_STACK_TRACE_MAX_DEPTH 32
-
-
-namespace DB
-{
-
-/// Позволяет получить стек-трейс
-class StackTrace
-{
-public:
-	/// Стектрейс снимается в момент создания объекта
-	StackTrace();
-
-	/// Вывести в строку
-	std::string toString() const;
-
-private:
-	typedef void* Frame;
-	Frame frames[DBMS_STACK_TRACE_MAX_DEPTH];
-	size_t frames_size;
-};
-
-}

dbms/include/DB/Interpreters/InterpreterAlterQuery.h

@@ -5,6 +5,7 @@
 
 namespace DB
 {
+class ASTIdentifier;
 
 /** Позволяет добавить или удалить столбец в таблице.
   */
@@ -16,6 +17,8 @@ public:
 	void execute();
 
 private:
+	void dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns);
+
 	ASTPtr query_ptr;
 	
 	Context context;

dbms/include/DB/Storages/MergeTree/MergeTreeData.h

@@ -383,7 +383,7 @@ private:
 	/// Загрузить множество кусков с данными с диска. Вызывается один раз - при создании объекта.
 	void loadDataParts();
 
-	void removeColumnFiles(String column_name);
+	void removeColumnFiles(String column_name, bool remove_array_size_files);
 
 	/// Определить, не битые ли данные в директории. Проверяет индекс и засечеки, но не сами данные.
 	bool isBrokenPart(const String & path);

dbms/src/Core/Exception.cpp

@@ -11,47 +11,6 @@
 namespace DB
 {
 
-Exception::Exception(int code): Poco::Exception(code) {}
-Exception::Exception(const std::string & msg, int code) : Poco::Exception(msg, code) {}
-Exception::Exception(const std::string & msg, const std::string & arg, int code): Poco::Exception(msg, arg, code) {}
-Exception::Exception(const std::string & msg, const Exception & exc, int code): Poco::Exception(msg, exc, code), trace(exc.trace) {}
-Exception::Exception(const Exception & exc) : Poco::Exception(exc), trace(exc.trace) {}
-Exception::Exception(const Poco::Exception & exc) : Poco::Exception(exc.displayText()) {}
-Exception::~Exception() throw() {}
-
-Exception & Exception::operator=(const Exception& exc)
-{
-	Poco::Exception::operator=(exc);
-	trace = exc.trace;	
-	return *this;
-}
-
-const char* Exception::name() const throw()
-{
-	return "DB::Exception";
-}
-
-const char* Exception::className() const throw()
-{
-	return "DB::Exception";
-}
-
-Exception * Exception::clone() const
-{
-	return new Exception(*this);
-}
-
-void Exception::rethrow() const
-{
-	throw *this;
-}
-
-void Exception::addMessage(const std::string & arg)
-{
-	extendedMessage(arg);
-}
-
-
 void throwFromErrno(const std::string & s, int code, int e)
 {
 	char buf[128];

dbms/src/Core/StackTrace.cpp

@@ -1,82 +0,0 @@
-#include <malloc.h>
-#include <execinfo.h>
-#include <cxxabi.h>
-#include <string.h>
-
-#include <sstream>
-
-#include <DB/Core/StackTrace.h>
-
-
-#define DBMS_STACK_TRACE_MAX_DEPTH 32
-
-
-namespace DB
-{
-
-StackTrace::StackTrace()
-{
-	frames_size = backtrace(frames, DBMS_STACK_TRACE_MAX_DEPTH);
-}
-
-std::string StackTrace::toString() const
-{
-	char ** symbols = backtrace_symbols(frames, frames_size);
-	std::stringstream res;
-
-	if (!symbols)
-		return "Cannot get symbols for stack trace.\n";
-
-	try
-	{
-		for (size_t i = 0, size = frames_size; i < size; ++i)
-		{
-			/// Делаем demangling имён. Имя находится в скобках, до символа '+'.
-			
-			char * name_start = NULL;
-			char * name_end = NULL;
-			char * demangled_name = NULL;
-			int status = 0;
-			
-			if (NULL != (name_start = strchr(symbols[i], '('))
-				&& NULL != (name_end = strchr(name_start, '+')))
-			{
-				++name_start;
-				*name_end = '\0';
-				demangled_name = abi::__cxa_demangle(name_start, 0, 0, &status);
-				*name_end = '+';
-			}
-
-			try
-			{
-				res << i << ". ";
-				
-				if (NULL != demangled_name && 0 == status)
-				{
-					res.write(symbols[i], name_start - symbols[i]);
-					res << demangled_name << name_end;
-				}
-				else
-					res << symbols[i];
-
-				res << std::endl;
-			}
-			catch (...)
-			{
-				free(demangled_name);
-				throw;
-			}
-			free(demangled_name);
-		}
-	}
-	catch (...)
-	{
-		free(symbols);
-		throw;
-	}
-	
-	free(symbols);
-	return res.str();
-}
-
-}

dbms/src/Core/tests/stack_trace.cpp

@@ -1,12 +0,0 @@
-#include <iostream>
-
-#include <DB/Core/StackTrace.h>
-
-
-int main(int argc, char ** argv)
-{
-	DB::StackTrace trace;
-	std::cerr << trace.toString();
-
-	return 0;
-}

dbms/src/IO/tests/o_direct_and_dirty_pages.cpp

@@ -15,24 +15,58 @@ int main(int argc, char ** argv)
 
 	try
 	{
+		const size_t N = 100000;
+		const size_t BUF_SIZE = 1048576;
+
 		ReadBufferFromFile rand_in("/dev/urandom");
 		unsigned rand = 0;
 		readBinary(rand, rand_in);
 
 		String test = "Hello, world! " + toString(rand);
 
+		/// Пишем в файл как обычно, читаем с O_DIRECT.
+
 		{
-			WriteBufferFromFile wb("test", 4096);
+			WriteBufferFromFile wb("test1", BUF_SIZE);
-			writeStringBinary(test, wb);
+			for (size_t i = 0; i < N; ++i)
+				writeStringBinary(test, wb);
 			wb.next();
 		}
 
 		{
-			ReadBufferFromFile rb("test", 4096, O_RDONLY | O_DIRECT, nullptr, 4096);
+			ReadBufferFromFile rb("test1", BUF_SIZE, O_RDONLY | O_DIRECT, nullptr, 4096);
 			String res;
-			readStringBinary(res, rb);
+			for (size_t i = 0; i < N; ++i)
+				readStringBinary(res, rb);
 
-			std::cerr << "test: " << test << ", res: " << res << std::endl;
+			std::cerr << "test: " << test << ", res: " << res << ", bytes: " << rb.count() << std::endl;
+		}
+
+		/// Пишем в файл с O_DIRECT, читаем как обычно.
+
+		{
+			WriteBufferFromFile wb("test2", BUF_SIZE, O_WRONLY | O_CREAT | O_TRUNC | O_DIRECT, 0666, nullptr, 4096);
+
+			for (size_t i = 0; i < N; ++i)
+				writeStringBinary(test, wb);
+
+			if (wb.offset() % 4096 != 0)
+			{
+				size_t pad = 4096 - wb.offset() % 4096;
+				memset(wb.position(), 0, pad);
+				wb.position() += pad;
+			}
+
+			wb.next();
+		}
+
+		{
+			ReadBufferFromFile rb("test2", BUF_SIZE);
+			String res;
+			for (size_t i = 0; i < N; ++i)
+				readStringBinary(res, rb);
+
+			std::cerr << "test: " << test << ", res: " << res << ", bytes: " << rb.count() << std::endl;
 		}
 	}
 	catch (const Exception & e)

dbms/src/Interpreters/InterpreterAlterQuery.cpp

@@ -39,6 +39,65 @@ static bool namesEqualIgnoreAfterDot(const String & name_without_dot, const ASTP
 	return (name_without_dot == name_type.name || name_with_dot == name_type.name.substr(0, name_with_dot.length()));
 }
 
+void InterpreterAlterQuery::dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns)
+{
+	Exception e("Wrong column name. Cannot find column " +  drop_column.name + " to drop", DB::ErrorCodes::ILLEGAL_COLUMN);
+	ASTs::iterator drop_it;
+
+	size_t dot_pos = drop_column.name.find('.');
+	/// случай удаления nested столбца
+	if (dot_pos != std::string::npos)
+	{
+		/// в Distributed таблицах столбцы имеют название "nested.column"
+		drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
+		if (drop_it != columns.end())
+			columns.erase(drop_it);
+		else
+		{
+			try
+			{
+				/// в MergeTree таблицах есть ASTFunction "nested"
+				/// в аргументах которой записаны столбцы
+				ASTs::iterator nested_it;
+				std::string nested_base_name = drop_column.name.substr(0, dot_pos);
+				nested_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, nested_base_name, _1));
+				if (nested_it == columns.end())
+					throw e;
+
+				if ((**nested_it).children.size() != 1)
+					throw e;
+
+				ASTFunction & f = dynamic_cast<ASTFunction &>(*(**nested_it).children.back());
+				if (f.name != "Nested")
+					throw e;
+
+				ASTs & nested_columns = dynamic_cast<ASTExpressionList &>(*f.arguments).children;
+
+				drop_it = std::find_if(nested_columns.begin(), nested_columns.end(), boost::bind(namesEqual, drop_column.name.substr(dot_pos + 1), _1));
+				if (drop_it == nested_columns.end())
+					throw e;
+				else
+					nested_columns.erase(drop_it);
+
+				if (nested_columns.empty())
+					columns.erase(nested_it);
+			}
+			catch (std::bad_cast & bad_cast_err)
+			{
+				throw e;
+			}
+		}
+	}
+	else
+	{
+		drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
+		if (drop_it == columns.end())
+			throw e;
+		else
+			columns.erase(drop_it);
+	}
+}
+
 void InterpreterAlterQuery::execute()
 {
 	ASTAlterQuery & alter = dynamic_cast<ASTAlterQuery &>(*query_ptr);
@@ -62,7 +121,10 @@ void InterpreterAlterQuery::execute()
 	ASTs & columns = dynamic_cast<ASTExpressionList &>(*attach.columns).children;
 
 	/// Различные проверки, на возможность выполнения запроса
-	ASTs columns_copy = columns;
+	ASTs columns_copy;
+	for (const auto & ast : columns)
+		columns_copy.push_back(ast->clone());
+
 	IdentifierNameSet identifier_names;
 	attach.storage->collectIdentifierNames(identifier_names);
 	for (ASTAlterQuery::ParameterContainer::const_iterator alter_it = alter.parameters.begin();
@@ -101,11 +163,7 @@ void InterpreterAlterQuery::execute()
 			if (identifier_names.find(drop_column.name) != identifier_names.end())
 				throw Exception("Cannot drop key column", DB::ErrorCodes::ILLEGAL_COLUMN);
 
-			ASTs::iterator drop_it = std::find_if(columns_copy.begin(), columns_copy.end(), boost::bind(namesEqual, drop_column.name, _1));
+			dropColumnFromAST(drop_column, columns_copy);
-			if (drop_it == columns_copy.end())
-				throw Exception("Wrong column name. Cannot find column " +  drop_column.name +" to drop", DB::ErrorCodes::ILLEGAL_COLUMN);
-			else
-				columns_copy.erase(drop_it);
 		}
 		else if (params.type == ASTAlterQuery::MODIFY)
 		{
@@ -172,8 +230,7 @@ void InterpreterAlterQuery::execute()
 		else if (params.type == ASTAlterQuery::DROP)
 		{
 			const ASTIdentifier & drop_column = dynamic_cast <const ASTIdentifier &>(*params.column);
-			ASTs::iterator drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
+			dropColumnFromAST(drop_column, columns);
-			columns.erase(drop_it);	
 		}
 		else if (params.type == ASTAlterQuery::MODIFY)
 		{

dbms/src/Storages/MergeTree/MergeTreeData.cpp

@@ -334,11 +334,21 @@ void MergeTreeData::dropAllData()
 	Poco::File(full_path).remove(true);
 }
 
-void MergeTreeData::removeColumnFiles(String column_name)
+void MergeTreeData::removeColumnFiles(String column_name, bool remove_array_size_files)
 {
 	Poco::ScopedLock<Poco::FastMutex> lock(data_parts_mutex);
 	Poco::ScopedLock<Poco::FastMutex> lock_all(all_data_parts_mutex);
 
+	size_t dot_pos = column_name.find('.');
+	if (dot_pos != std::string::npos)
+	{
+		std::string nested_column = column_name.substr(0, dot_pos);
+		column_name = nested_column + "%2E" + column_name.substr(dot_pos + 1);
+
+		if (remove_array_size_files)
+			column_name = std::string("(?:") + nested_column + "|" + column_name + ")";
+	}
+
 	/// Регэксп выбирает файлы столбца для удаления
 	Poco::RegularExpression re(column_name + "(?:(?:\\.|\\%2E).+){0,1}" +"(?:\\.mrk|\\.bin|\\.size\\d+\\.bin|\\.size\\d+\\.mrk)");
 	/// Цикл по всем директориям кусочков
@@ -388,6 +398,12 @@ static DataTypePtr getDataTypeByName(const String & name, const NamesAndTypesLis
 	throw Exception("No column " + name + " in table", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
 }
 
+/// одинаковыми считаются имена, вида "name.*"
+static bool namesWithDotEqual(const String & name_with_dot, const DB::NameAndTypePair & name_type)
+{
+	return (name_with_dot == name_type.first.substr(0, name_with_dot.length()));
+}
+
 void MergeTreeData::alter(const ASTAlterQuery::Parameters & params)
 {
 	{
@@ -398,7 +414,15 @@ void MergeTreeData::alter(const ASTAlterQuery::Parameters & params)
 	if (params.type == ASTAlterQuery::DROP)
 	{
 		String column_name = dynamic_cast<const ASTIdentifier &>(*params.column).name;
-		removeColumnFiles(column_name);
+
+		/// Если нет колонок вида nested_name.*, то удалим столбцы размера массивов
+		bool remove_array_size_files = false;
+		size_t dot_pos = column_name.find('.');
+		if (dot_pos != std::string::npos)
+		{
+			remove_array_size_files = (columns->end() == std::find_if(columns->begin(), columns->end(), boost::bind(namesWithDotEqual, column_name.substr(0, dot_pos), _1)));
+		}
+		removeColumnFiles(column_name, remove_array_size_files);
 
 		context.getUncompressedCache()->reset();
 		context.getMarkCache()->reset();

dbms/tests/queries/0_stateless/00030_1_alter_table.reference

@@ -0,0 +1,10 @@
+CounterID	UInt32
+StartDate	Date
+UserID	UInt32
+VisitID	UInt32
+NestedColumn.A	Array(UInt8)
+NestedColumn.S	Array(String)
+ToDrop	UInt32
+Added0	UInt32
+Added1	UInt32
+Added2	UInt32

dbms/tests/queries/0_stateless/00030_1_alter_table.sql

@@ -0,0 +1,11 @@
+DROP TABLE IF EXISTS alter_test;
+
+CREATE TABLE alter_test (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);
+
+INSERT INTO alter_test VALUES (1, '2014-01-01', 2, 3, [1,2,3], ['a','b','c'], 4);
+
+ALTER TABLE alter_test ADD COLUMN Added0 UInt32;
+ALTER TABLE alter_test ADD COLUMN Added2 UInt32;
+ALTER TABLE alter_test ADD COLUMN Added1 UInt32 AFTER Added0;
+
+DESC TABLE alter_test;

dbms/tests/queries/0_stateless/00030_2_alter_table.reference

@@ -0,0 +1,7 @@
+CounterID	UInt32
+StartDate	Date
+UserID	UInt32
+VisitID	UInt32
+Added0	String
+Added1	UInt32
+Added2	UInt32

dbms/tests/queries/0_stateless/00030_2_alter_table.sql

@@ -0,0 +1,8 @@
+ALTER TABLE alter_test DROP COLUMN ToDrop;
+
+ALTER TABLE alter_test MODIFY COLUMN Added0 String;
+
+ALTER TABLE alter_test DROP COLUMN `NestedColumn.A`;
+ALTER TABLE alter_test DROP COLUMN `NestedColumn.S`;
+
+DESC TABLE alter_test;

dbms/tests/queries/0_stateless/00030_3_alter_table.reference

@@ -0,0 +1 @@
+1	2014-01-01	2	3		0	0

dbms/tests/queries/0_stateless/00030_3_alter_table.sql

@@ -0,0 +1,3 @@
+SELECT * FROM alter_test;
+
+DROP TABLE alter_test;

libs/libzkutil/include/zkutil/KeeperException.h

@@ -1,19 +1,19 @@
 #pragma once
-#include <Poco/Exception.h>
+#include <statdaemons/Exception.h>
 #include <zkutil/Types.h>
 
 
 namespace zkutil
 {
 
-class KeeperException : public Poco::Exception
+class KeeperException : public DB::Exception
 {
 public:
-	KeeperException(const std::string & msg) : Poco::Exception(msg), code(ReturnCode::Ok) {}
+	KeeperException(const std::string & msg) : DB::Exception(msg), code(ReturnCode::Ok) {}
 	KeeperException(const std::string & msg, ReturnCode::type code_)
-		: Poco::Exception(msg + " (" + ReturnCode::toString(code_) + ")"), code(code_) {}
+		: DB::Exception(msg + " (" + ReturnCode::toString(code_) + ")"), code(code_) {}
 	KeeperException(ReturnCode::type code_)
-		: Poco::Exception(ReturnCode::toString(code_)), code(code_) {}
+		: DB::Exception(ReturnCode::toString(code_)), code(code_) {}
 
 	ReturnCode::type code;
 };