Merge

2024-09-23 02:00:49 +00:00 · 2014-04-04 19:22:53 +04:00 · 2014-04-04 19:22:53 +04:00 · 2a7c31cc66
commit 2a7c31cc66
parent 0f26b3819a 90200cccf5
18 changed files with 172 additions and 30 deletions
--- a/dbms/include/DB/Common/PODArray.h
+++ b/dbms/include/DB/Common/PODArray.h
@ -90,8 +90,6 @@ private:
 		size_t bytes_to_alloc = to_size(n);
 		c_start = c_end = Allocator::allocate(bytes_to_alloc);
 		c_end_of_storage = c_start + bytes_to_alloc;
-
-		//memset(c_start, 0, bytes_to_alloc);
 	}

 	void dealloc()
@ -107,8 +105,6 @@ private:

 	void realloc(size_t n)
 	{
-//		std::cerr << "realloc" << std::endl;
-
 		if (c_start == NULL)
 		{
 			alloc(n);
@ -138,9 +134,7 @@ private:
 			memcpy(c_start, old_c_start, old_c_end_of_storage - old_c_start);
 			Allocator::deallocate(old_c_start, old_c_end_of_storage - old_c_start);
 		}
-		
-		//memset(c_start + (old_c_end_of_storage - old_c_start), 0, bytes_to_alloc - (old_c_end_of_storage - old_c_start));
-				
+
 		c_end = c_start + end_diff;
 		c_end_of_storage = c_start + bytes_to_alloc;

--- a/dbms/include/DB/Core/Defines.h
+++ b/dbms/include/DB/Core/Defines.h
@ -20,7 +20,7 @@
 /// То же самое, но для операций слияния. Меньше DEFAULT_BLOCK_SIZE для экономии оперативки (так как читаются все столбцы).
 #define DEFAULT_MERGE_BLOCK_SIZE 								10000

-#define DEFAULT_MAX_QUERY_SIZE 									1048576
+#define DEFAULT_MAX_QUERY_SIZE 									65536
 #define SHOW_CHARS_ON_SYNTAX_ERROR 								160L
 #define DEFAULT_MAX_THREADS 									8
 #define DEFAULT_MAX_DISTRIBUTED_CONNECTIONS						1024
--- a/dbms/include/DB/Core/ErrorCodes.h
+++ b/dbms/include/DB/Core/ErrorCodes.h
@ -230,6 +230,7 @@ namespace ErrorCodes
 		NO_FILE_IN_DATA_PART,
 		UNEXPECTED_FILE_IN_DATA_PART,
 		BAD_SIZE_OF_FILE_IN_DATA_PART,
+		QUERY_IS_TOO_LARGE,

 		POCO_EXCEPTION = 1000,
 		STD_EXCEPTION,
--- a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h
+++ b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h
@ -5,6 +5,7 @@

 namespace DB
 {
+class ASTIdentifier;

 /** Позволяет добавить или удалить столбец в таблице.
  */
@ -16,6 +17,8 @@ public:
 	void execute();

 private:
+	void dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns);
+
 	ASTPtr query_ptr;
 	
 	Context context;
--- a/dbms/include/DB/Parsers/CommonParsers.h
+++ b/dbms/include/DB/Parsers/CommonParsers.h
@ -67,7 +67,7 @@ protected:
 	bool parseImpl(Pos & pos, Pos end, ASTPtr & node, const char *& expected)
 	{
 		Pos begin = pos;
-		while (*pos == ' ' || *pos == '\t' || (allow_newlines && *pos == '\n') || *pos == '\r' || *pos == '\f')
+		while (pos < end && (*pos == ' ' || *pos == '\t' || (allow_newlines && *pos == '\n') || *pos == '\r' || *pos == '\f'))
 			++pos;

 		return pos != begin;
--- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h
+++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h
@ -383,7 +383,7 @@ private:
 	/// Загрузить множество кусков с данными с диска. Вызывается один раз - при создании объекта.
 	void loadDataParts();

-	void removeColumnFiles(String column_name);
+	void removeColumnFiles(String column_name, bool remove_array_size_files);

 	/// Определить, не битые ли данные в директории. Проверяет индекс и засечеки, но не сами данные.
 	bool isBrokenPart(const String & path);
--- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp
@ -39,6 +39,65 @@ static bool namesEqualIgnoreAfterDot(const String & name_without_dot, const ASTP
 	return (name_without_dot == name_type.name || name_with_dot == name_type.name.substr(0, name_with_dot.length()));
 }

+void InterpreterAlterQuery::dropColumnFromAST(const ASTIdentifier & drop_column, ASTs & columns)
+{
+	Exception e("Wrong column name. Cannot find column " +  drop_column.name + " to drop", DB::ErrorCodes::ILLEGAL_COLUMN);
+	ASTs::iterator drop_it;
+
+	size_t dot_pos = drop_column.name.find('.');
+	/// случай удаления nested столбца
+	if (dot_pos != std::string::npos)
+	{
+		/// в Distributed таблицах столбцы имеют название "nested.column"
+		drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
+		if (drop_it != columns.end())
+			columns.erase(drop_it);
+		else
+		{
+			try
+			{
+				/// в MergeTree таблицах есть ASTFunction "nested"
+				/// в аргументах которой записаны столбцы
+				ASTs::iterator nested_it;
+				std::string nested_base_name = drop_column.name.substr(0, dot_pos);
+				nested_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, nested_base_name, _1));
+				if (nested_it == columns.end())
+					throw e;
+
+				if ((**nested_it).children.size() != 1)
+					throw e;
+
+				ASTFunction & f = dynamic_cast<ASTFunction &>(*(**nested_it).children.back());
+				if (f.name != "Nested")
+					throw e;
+
+				ASTs & nested_columns = dynamic_cast<ASTExpressionList &>(*f.arguments).children;
+
+				drop_it = std::find_if(nested_columns.begin(), nested_columns.end(), boost::bind(namesEqual, drop_column.name.substr(dot_pos + 1), _1));
+				if (drop_it == nested_columns.end())
+					throw e;
+				else
+					nested_columns.erase(drop_it);
+
+				if (nested_columns.empty())
+					columns.erase(nested_it);
+			}
+			catch (std::bad_cast & bad_cast_err)
+			{
+				throw e;
+			}
+		}
+	}
+	else
+	{
+		drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
+		if (drop_it == columns.end())
+			throw e;
+		else
+			columns.erase(drop_it);
+	}
+}
+
 void InterpreterAlterQuery::execute()
 {
 	ASTAlterQuery & alter = dynamic_cast<ASTAlterQuery &>(*query_ptr);
@ -62,7 +121,10 @@ void InterpreterAlterQuery::execute()
 	ASTs & columns = dynamic_cast<ASTExpressionList &>(*attach.columns).children;

 	/// Различные проверки, на возможность выполнения запроса
-	ASTs columns_copy = columns;
+	ASTs columns_copy;
+	for (const auto & ast : columns)
+		columns_copy.push_back(ast->clone());
+
 	IdentifierNameSet identifier_names;
 	attach.storage->collectIdentifierNames(identifier_names);
 	for (ASTAlterQuery::ParameterContainer::const_iterator alter_it = alter.parameters.begin();
@ -101,11 +163,7 @@ void InterpreterAlterQuery::execute()
 			if (identifier_names.find(drop_column.name) != identifier_names.end())
 				throw Exception("Cannot drop key column", DB::ErrorCodes::ILLEGAL_COLUMN);

-			ASTs::iterator drop_it = std::find_if(columns_copy.begin(), columns_copy.end(), boost::bind(namesEqual, drop_column.name, _1));
-			if (drop_it == columns_copy.end())
-				throw Exception("Wrong column name. Cannot find column " +  drop_column.name +" to drop", DB::ErrorCodes::ILLEGAL_COLUMN);
-			else
-				columns_copy.erase(drop_it);
+			dropColumnFromAST(drop_column, columns_copy);
 		}
 		else if (params.type == ASTAlterQuery::MODIFY)
 		{
@ -172,8 +230,7 @@ void InterpreterAlterQuery::execute()
 		else if (params.type == ASTAlterQuery::DROP)
 		{
 			const ASTIdentifier & drop_column = dynamic_cast <const ASTIdentifier &>(*params.column);
-			ASTs::iterator drop_it = std::find_if(columns.begin(), columns.end(), boost::bind(namesEqual, drop_column.name, _1));
-			columns.erase(drop_it);	
+			dropColumnFromAST(drop_column, columns);
 		}
 		else if (params.type == ASTAlterQuery::MODIFY)
 		{
--- a/dbms/src/Interpreters/executeQuery.cpp
+++ b/dbms/src/Interpreters/executeQuery.cpp
@ -33,7 +33,7 @@ void executeQuery(
 	ASTPtr ast;
 	const char * expected = "";

-	std::vector<char> parse_buf;
+	PODArray<char> parse_buf;
 	const char * begin;
 	const char * end;

@ -76,7 +76,13 @@ void executeQuery(

 	/// Засунем запрос в строку. Она выводится в лог и в processlist. Если запрос INSERT, то не будем включать данные для вставки.
 	auto insert = dynamic_cast<const ASTInsertQuery *>(&*ast);
-	String query(begin, (insert && insert->data) ? (insert->data - begin) : (pos - begin));
+	size_t query_size = (insert && insert->data) ? (insert->data - begin) : (pos - begin);
+
+	if (query_size > max_query_size)
+		throw Exception("Query is too large (" + toString(query_size) + ")."
+			" max_query_size = " + toString(max_query_size), ErrorCodes::QUERY_IS_TOO_LARGE);
+
+	String query(begin, query_size);

 	LOG_DEBUG(&Logger::get("executeQuery"), query);

--- a/dbms/src/Parsers/ExpressionElementParsers.cpp
+++ b/dbms/src/Parsers/ExpressionElementParsers.cpp
@ -280,33 +280,44 @@ bool ParserNumber::parseImpl(Pos & pos, Pos end, ASTPtr & node, const char *& ex
 	if (pos == end)
 		return false;

+	/** Максимальная длина числа. 319 символов достаточно, чтобы записать максимальный double в десятичной форме.
+	  * Лишнее копирование нужно, чтобы воспользоваться функциями strto*, которым нужна 0-терминированная строка.
+	  */
+	char buf[320];
+
+	size_t bytes_to_copy = end - pos < 319 ? end - pos : 319;
+	memcpy(buf, pos, bytes_to_copy);
+	buf[bytes_to_copy] = 0;
+
+	char * pos_double = buf;
 	errno = 0;	/// Функции strto* не очищают errno.
-	Float64 float_value = std::strtod(pos, const_cast<char**>(&pos));
-	if (pos == begin || errno == ERANGE)
+	Float64 float_value = std::strtod(buf, &pos_double);
+	if (pos_double == buf || errno == ERANGE)
 	{
-		expected = "number (this cause range error)";
+		expected = "number";
 		return false;
 	}
 	res = float_value;

 	/// попробуем использовать более точный тип - UInt64 или Int64

-	Pos pos_integer = begin;
+	char * pos_integer = buf;
 	if (float_value < 0)
 	{
 		errno = 0;
-		Int64 int_value = std::strtoll(pos_integer, const_cast<char**>(&pos_integer), 0);
-		if (pos_integer == pos && errno != ERANGE)
+		Int64 int_value = std::strtoll(buf, &pos_integer, 0);
+		if (pos_integer == pos_double && errno != ERANGE)
 			res = int_value;
 	}
 	else
 	{
 		errno = 0;
-		UInt64 uint_value = std::strtoull(pos_integer, const_cast<char**>(&pos_integer), 0);
-		if (pos_integer == pos && errno != ERANGE)
+		UInt64 uint_value = std::strtoull(buf, &pos_integer, 0);
+		if (pos_integer == pos_double && errno != ERANGE)
 			res = uint_value;
 	}

+	pos += pos_double - buf;
 	node = new ASTLiteral(StringRange(begin, pos), res);
 	return true;
 }
--- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp
@ -334,11 +334,21 @@ void MergeTreeData::dropAllData()
 	Poco::File(full_path).remove(true);
 }

-void MergeTreeData::removeColumnFiles(String column_name)
+void MergeTreeData::removeColumnFiles(String column_name, bool remove_array_size_files)
 {
 	Poco::ScopedLock<Poco::FastMutex> lock(data_parts_mutex);
 	Poco::ScopedLock<Poco::FastMutex> lock_all(all_data_parts_mutex);

+	size_t dot_pos = column_name.find('.');
+	if (dot_pos != std::string::npos)
+	{
+		std::string nested_column = column_name.substr(0, dot_pos);
+		column_name = nested_column + "%2E" + column_name.substr(dot_pos + 1);
+
+		if (remove_array_size_files)
+			column_name = std::string("(?:") + nested_column + "|" + column_name + ")";
+	}
+
 	/// Регэксп выбирает файлы столбца для удаления
 	Poco::RegularExpression re(column_name + "(?:(?:\\.|\\%2E).+){0,1}" +"(?:\\.mrk|\\.bin|\\.size\\d+\\.bin|\\.size\\d+\\.mrk)");
 	/// Цикл по всем директориям кусочков
@ -388,6 +398,12 @@ static DataTypePtr getDataTypeByName(const String & name, const NamesAndTypesLis
 	throw Exception("No column " + name + " in table", ErrorCodes::NO_SUCH_COLUMN_IN_TABLE);
 }

+/// одинаковыми считаются имена, вида "name.*"
+static bool namesWithDotEqual(const String & name_with_dot, const DB::NameAndTypePair & name_type)
+{
+	return (name_with_dot == name_type.first.substr(0, name_with_dot.length()));
+}
+
 void MergeTreeData::alter(const ASTAlterQuery::Parameters & params)
 {
 	{
@ -398,7 +414,15 @@ void MergeTreeData::alter(const ASTAlterQuery::Parameters & params)
 	if (params.type == ASTAlterQuery::DROP)
 	{
 		String column_name = dynamic_cast<const ASTIdentifier &>(*params.column).name;
-		removeColumnFiles(column_name);
+
+		/// Если нет колонок вида nested_name.*, то удалим столбцы размера массивов
+		bool remove_array_size_files = false;
+		size_t dot_pos = column_name.find('.');
+		if (dot_pos != std::string::npos)
+		{
+			remove_array_size_files = (columns->end() == std::find_if(columns->begin(), columns->end(), boost::bind(namesWithDotEqual, column_name.substr(0, dot_pos), _1)));
+		}
+		removeColumnFiles(column_name, remove_array_size_files);

 		context.getUncompressedCache()->reset();
 		context.getMarkCache()->reset();
--- a/dbms/tests/queries/0_stateless/00030_1_alter_table.reference
+++ b/dbms/tests/queries/0_stateless/00030_1_alter_table.reference
@ -0,0 +1,10 @@
+CounterID	UInt32
+StartDate	Date
+UserID	UInt32
+VisitID	UInt32
+NestedColumn.A	Array(UInt8)
+NestedColumn.S	Array(String)
+ToDrop	UInt32
+Added0	UInt32
+Added1	UInt32
+Added2	UInt32
--- a/dbms/tests/queries/0_stateless/00030_1_alter_table.sql
+++ b/dbms/tests/queries/0_stateless/00030_1_alter_table.sql
@ -0,0 +1,11 @@
+DROP TABLE IF EXISTS alter_test;
+
+CREATE TABLE alter_test (CounterID UInt32, StartDate Date, UserID UInt32, VisitID UInt32, NestedColumn Nested(A UInt8, S String), ToDrop UInt32) ENGINE = MergeTree(StartDate, intHash32(UserID), (CounterID, StartDate, intHash32(UserID), VisitID), 8192);
+
+INSERT INTO alter_test VALUES (1, '2014-01-01', 2, 3, [1,2,3], ['a','b','c'], 4);
+
+ALTER TABLE alter_test ADD COLUMN Added0 UInt32;
+ALTER TABLE alter_test ADD COLUMN Added2 UInt32;
+ALTER TABLE alter_test ADD COLUMN Added1 UInt32 AFTER Added0;
+
+DESC TABLE alter_test;
--- a/dbms/tests/queries/0_stateless/00030_2_alter_table.reference
+++ b/dbms/tests/queries/0_stateless/00030_2_alter_table.reference
@ -0,0 +1,7 @@
+CounterID	UInt32
+StartDate	Date
+UserID	UInt32
+VisitID	UInt32
+Added0	String
+Added1	UInt32
+Added2	UInt32
--- a/dbms/tests/queries/0_stateless/00030_2_alter_table.sql
+++ b/dbms/tests/queries/0_stateless/00030_2_alter_table.sql
@ -0,0 +1,8 @@
+ALTER TABLE alter_test DROP COLUMN ToDrop;
+
+ALTER TABLE alter_test MODIFY COLUMN Added0 String;
+
+ALTER TABLE alter_test DROP COLUMN `NestedColumn.A`;
+ALTER TABLE alter_test DROP COLUMN `NestedColumn.S`;
+
+DESC TABLE alter_test;
--- a/dbms/tests/queries/0_stateless/00030_3_alter_table.reference
+++ b/dbms/tests/queries/0_stateless/00030_3_alter_table.reference
@ -0,0 +1 @@
+1	2014-01-01	2	3		0	0
--- a/dbms/tests/queries/0_stateless/00030_3_alter_table.sql
+++ b/dbms/tests/queries/0_stateless/00030_3_alter_table.sql
@ -0,0 +1,3 @@
+SELECT * FROM alter_test;
+
+DROP TABLE alter_test;
--- a/dbms/tests/queries/0_stateless/00031_parser_number.reference
+++ b/dbms/tests/queries/0_stateless/00031_parser_number.reference
@ -0,0 +1,3 @@
+0	1	-1	128	-127	-128	255	-128	255	-127	65535	4294967295	12300	4656	-0	-0	0	18446744073709551615	2.09883e+19	-1.84467e+19	-9223372036854775807	-8.98847e+307	-5.56268e-309	inf	-inf	nan	-nan	1e-302	UInt8	UInt8	Int8	UInt8	Int8	Int8	UInt8	Int8	UInt8	Int8	UInt16	UInt32	Float64	Float64	Float64	Float64	UInt8	UInt64	Float64	Float64	Int64	Float64	Float64	Float64	Float64	Float64	Float32	Float64
+1e+308
+-1e-307
--- a/dbms/tests/queries/0_stateless/00031_parser_number.sql
+++ b/dbms/tests/queries/0_stateless/00031_parser_number.sql
@ -0,0 +1,3 @@
+SELECT 0 AS x1, 1 AS x2, -1 AS x3, 128 AS x4, -127 AS x5, -128 AS x6, 0xFF AS x7, -0x80 AS x8, 0377 AS x9, -0177 AS x10, 0xFFFF AS x11, 0xFFFFFFFF AS x12, 123e2 AS x13, 0x123p4 AS x14, -0. AS x15, -.0 AS x16, 0. AS x17, 0xFFFFFFFFFFFFFFFF AS x18, 0x123456789ABCDEF01 AS x19, -0xFFFFFFFFFFFFFFFF AS x20, -0x7FFFFFFFFFFFFFFF AS x21, -0x1P1023 AS x22, -0x1p-1024 AS x23, inf AS x24, -INF AS x25, nan AS x26, 0 / 0 AS x27, 0.01e-300 AS x28, toTypeName(x1), toTypeName(x2), toTypeName(x3), toTypeName(x4), toTypeName(x5), toTypeName(x6), toTypeName(x7), toTypeName(x8), toTypeName(x9), toTypeName(x10), toTypeName(x11), toTypeName(x12), toTypeName(x13), toTypeName(x14), toTypeName(x15), toTypeName(x16), toTypeName(x17), toTypeName(x18), toTypeName(x19), toTypeName(x20), toTypeName(x21), toTypeName(x22), toTypeName(x23), toTypeName(x24), toTypeName(x25), toTypeName(x26), toTypeName(x27), toTypeName(x28);
+SELECT 0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000;
+SELECT -0.0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001;